loc_az_hci/scripts/deploy/deploy-observability.sh

#!/bin/bash
source ~/.bashrc
# Deploy Observability Stack (Prometheus + Grafana) on VM 103 using guest-agent IP discovery

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"

# Load environment variables
if [ -f "$PROJECT_ROOT/.env" ]; then
    set -a
    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
    set +a
fi

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

VM_USER="${VM_USER:-ubuntu}"
SSH_KEY="${SSH_KEY:-$HOME/.ssh/id_ed25519_proxmox}"
VMID=103
VM_NAME="observability"

# Import helper library
if [ -f "$PROJECT_ROOT/scripts/lib/proxmox_vm_helpers.sh" ]; then
    source "$PROJECT_ROOT/scripts/lib/proxmox_vm_helpers.sh"
else
    log_error "Helper library not found"
    exit 1
fi

main() {
    log_info "Deploying Observability Stack on VM $VMID ($VM_NAME)"
    echo ""

    # Get IP using guest agent
    local ip
    ip="$(get_vm_ip_or_warn "$VMID" "$VM_NAME" || true)"
    
    if [[ -z "$ip" ]]; then
        log_error "Cannot get IP for VM $VMID. Ensure SSH is working and QEMU Guest Agent is installed."
        exit 1
    fi

    log_info "Using IP: $ip"
    echo ""

    # Check if Docker is installed
    log_info "Checking Docker installation..."
    if ! ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "${VM_USER}@${ip}" "command -v docker" &>/dev/null; then
        log_warn "Docker not found. Installing Docker..."
        ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "${VM_USER}@${ip}" <<'EOF'
set -e
sudo apt-get update -qq
sudo apt-get install -y docker.io docker-compose
sudo usermod -aG docker $USER
EOF
        log_info "Docker installed. You may need to log out and back in for group changes."
    else
        log_info "Docker is installed"
    fi

    # Create observability directory structure
    log_info "Setting up observability directory..."
    ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "${VM_USER}@${ip}" <<'EOF'
set -e
mkdir -p ~/observability/prometheus
cd ~/observability
EOF

    # Create Prometheus config
    log_info "Creating Prometheus configuration..."
    ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "${VM_USER}@${ip}" "cat > ~/observability/prometheus/prometheus.yml" <<'EOF'
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']
EOF

    # Create docker-compose file
    log_info "Creating docker-compose.yml..."
    ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "${VM_USER}@${ip}" "cat > ~/observability/docker-compose.yml" <<'EOF'
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
    networks:
      - observability

  grafana:
    image: grafana/grafana:latest
    container_name: grafana
    restart: unless-stopped
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin
      - GF_USERS_ALLOW_SIGN_UP=false
      - GF_SERVER_ROOT_URL=http://localhost:3000
    volumes:
      - grafana-data:/var/lib/grafana
    networks:
      - observability
    depends_on:
      - prometheus

volumes:
  prometheus-data:
    driver: local
  grafana-data:
    driver: local

networks:
  observability:
    driver: bridge
EOF

    # Deploy
    log_info "Deploying Observability Stack with Docker Compose..."
    ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "${VM_USER}@${ip}" <<'EOF'
set -e
cd ~/observability
sudo docker-compose up -d
EOF

    # Wait for services to be ready
    log_info "Waiting for services to start..."
    sleep 15

    # Verify
    log_info "Verifying services..."
    local prometheus_ok=false
    local grafana_ok=false

    for i in {1..12}; do
        if curl -s "http://${ip}:9090/-/healthy" &>/dev/null; then
            prometheus_ok=true
        fi
        if curl -s "http://${ip}:3000/api/health" &>/dev/null; then
            grafana_ok=true
        fi
        if [ "$prometheus_ok" = true ] && [ "$grafana_ok" = true ]; then
            break
        fi
        sleep 5
        echo -n "."
    done
    echo ""

    if [ "$prometheus_ok" = true ] && [ "$grafana_ok" = true ]; then
        log_info "✓ Observability Stack is running!"
        echo ""
        log_info "Access services:"
        log_info "  Prometheus: http://${ip}:9090"
        log_info "  Grafana:    http://${ip}:3000 (admin/admin)"
    else
        log_warn "Some services may not be fully ready. Check logs with:"
        log_info "  ssh ${VM_USER}@${ip} 'cd ~/observability && sudo docker-compose logs'"
    fi
}

main "$@"