Sankofa/scripts/verify-guest-agent.sh

#!/bin/bash
# verify-guest-agent.sh
# Verify guest agent status on all VMs via Proxmox API

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"

# Load environment
if [ -f "${PROJECT_ROOT}/.env" ]; then
    set -a
    source "${PROJECT_ROOT}/.env"
    set +a
fi

# Try API tokens first, fall back to password
PROXMOX_1_TOKEN="${PROXMOX_TOKEN_ML110_01:-}"
PROXMOX_2_TOKEN="${PROXMOX_TOKEN_R630_01:-}"
PROXMOX_PASS="${PROXMOX_ROOT_PASS:-L@kers2010}"
PROXMOX_1_URL="https://192.168.11.10:8006"
PROXMOX_2_URL="https://192.168.11.11:8006"

# Colors
GREEN='\033[0;32m'
BLUE='\033[0;34m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
NC='\033[0m'

log() {
    echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $*"
}

log_success() {
    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] ✅${NC} $*"
}

log_error() {
    echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ❌${NC} $*"
}

log_warning() {
    echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] ⚠️${NC} $*"
}

# Get auth - try API token first, fall back to password
get_auth() {
    local api_url=$1
    local api_token=$2
    local response
    
    # Try API token authentication first
    if [ -n "${api_token}" ]; then
        response=$(curl -k -s -X GET \
            -H "Authorization: PVEAuthCookie=${api_token}" \
            "${api_url}/api2/json/version" 2>/dev/null)
        
        if echo "${response}" | grep -q "data\|version"; then
            echo "${api_token}|TOKEN"
            return 0
        fi
    fi
    
    # Fall back to password authentication
    response=$(curl -k -s -X POST \
        -d "username=root@pam&password=${PROXMOX_PASS}" \
        "${api_url}/api2/json/access/ticket" 2>/dev/null)
    
    if echo "${response}" | grep -q "authentication failure"; then
        echo ""
        return 1
    fi
    
    local ticket csrf
    if command -v jq &> /dev/null; then
        ticket=$(echo "${response}" | jq -r '.data.ticket // empty' 2>/dev/null)
        csrf=$(echo "${response}" | jq -r '.data.CSRFPreventionToken // empty' 2>/dev/null)
    else
        ticket=$(echo "${response}" | grep -o '"ticket":"[^"]*' | head -1 | cut -d'"' -f4)
        csrf=$(echo "${response}" | grep -o '"CSRFPreventionToken":"[^"]*' | head -1 | cut -d'"' -f4)
    fi
    
    if [ -z "${ticket}" ] || [ -z "${csrf}" ]; then
        echo ""
        return 1
    fi
    
    echo "${ticket}|${csrf}"
}

# List all nodes in the cluster
list_nodes() {
    local api_url=$1
    local auth_token=$2
    local auth_type=$3
    
    local response
    
    if [ "${auth_type}" = "TOKEN" ]; then
        response=$(curl -k -s -X GET \
            -H "Authorization: PVEAuthCookie=${auth_token}" \
            "${api_url}/api2/json/nodes" 2>/dev/null)
    else
        local ticket csrf
        IFS='|' read -r ticket csrf <<< "${auth_token}"
        response=$(curl -k -s -X GET \
            -H "CSRFPreventionToken: ${csrf}" \
            -b "PVEAuthCookie=${ticket}" \
            "${api_url}/api2/json/nodes" 2>/dev/null)
    fi
    
    if command -v jq &> /dev/null; then
        echo "${response}" | jq -r '.data[]?.node // empty' 2>/dev/null | grep -v '^$' | sort
    else
        echo "${response}" | grep -o '"node":"[^"]*' | cut -d'"' -f4 | sort | uniq
    fi
}

# List all VMs on a node
list_vms() {
    local api_url=$1
    local node=$2
    local auth_token=$3
    local auth_type=$4
    
    local response
    
    if [ "${auth_type}" = "TOKEN" ]; then
        response=$(curl -k -s -X GET \
            -H "Authorization: PVEAuthCookie=${auth_token}" \
            "${api_url}/api2/json/nodes/${node}/qemu" 2>/dev/null)
    else
        local ticket csrf
        IFS='|' read -r ticket csrf <<< "${auth_token}"
        response=$(curl -k -s -X GET \
            -H "CSRFPreventionToken: ${csrf}" \
            -b "PVEAuthCookie=${ticket}" \
            "${api_url}/api2/json/nodes/${node}/qemu" 2>/dev/null)
    fi
    
    if command -v jq &> /dev/null; then
        echo "${response}" | jq -r '.data[]?.vmid // empty' 2>/dev/null | grep -v '^$' | sort -n
    else
        echo "${response}" | grep -o '"vmid":[0-9]*' | grep -o '[0-9]*' | sort -n | uniq
    fi
}

# Get VM name
get_vm_name() {
    local api_url=$1
    local node=$2
    local vmid=$3
    local auth_token=$4
    local auth_type=$5
    
    local response
    
    if [ "${auth_type}" = "TOKEN" ]; then
        response=$(curl -k -s -X GET \
            -H "Authorization: PVEAuthCookie=${auth_token}" \
            "${api_url}/api2/json/nodes/${node}/qemu/${vmid}/config" 2>/dev/null)
    else
        local ticket csrf
        IFS='|' read -r ticket csrf <<< "${auth_token}"
        response=$(curl -k -s -X GET \
            -H "CSRFPreventionToken: ${csrf}" \
            -b "PVEAuthCookie=${ticket}" \
            "${api_url}/api2/json/nodes/${node}/qemu/${vmid}/config" 2>/dev/null)
    fi
    
    if command -v jq &> /dev/null; then
        echo "${response}" | jq -r '.data.name // "unnamed"' 2>/dev/null
    else
        echo "${response}" | grep -o '"name":"[^"]*' | head -1 | cut -d'"' -f4 || echo "unnamed"
    fi
}

# Check guest agent status
check_guest_agent() {
    local api_url=$1
    local node=$2
    local vmid=$3
    local auth_token=$4
    local auth_type=$5
    
    local response
    
    if [ "${auth_type}" = "TOKEN" ]; then
        response=$(curl -k -s -X GET \
            -H "Authorization: PVEAuthCookie=${auth_token}" \
            "${api_url}/api2/json/nodes/${node}/qemu/${vmid}/config" 2>/dev/null)
    else
        local ticket csrf
        IFS='|' read -r ticket csrf <<< "${auth_token}"
        response=$(curl -k -s -X GET \
            -H "CSRFPreventionToken: ${csrf}" \
            -b "PVEAuthCookie=${ticket}" \
            "${api_url}/api2/json/nodes/${node}/qemu/${vmid}/config" 2>/dev/null)
    fi
    
    # Check if agent is enabled
    if echo "${response}" | grep -q '"agent"[[:space:]]*:[[:space:]]*"1"'; then
        return 0  # Enabled
    fi
    return 1  # Not enabled
}

verify_node() {
    local api_url=$1
    local node=$2
    local auth_token=$3
    local auth_type=$4
    
    log "Verifying node: ${node}"
    
    local vmids
    vmids=$(list_vms "${api_url}" "${node}" "${auth_token}" "${auth_type}")
    
    if [ -z "${vmids}" ]; then
        log_warning "  No VMs found on ${node}"
        return 0
    fi
    
    local enabled_count=0
    local disabled_count=0
    
    printf "  %-8s %-30s %-10s\n" "VMID" "Name" "Status"
    printf "  %-8s %-30s %-10s\n" "----" "----" "------"
    
    while IFS= read -r vmid; do
        [ -z "${vmid}" ] && continue
        
        local vm_name
        vm_name=$(get_vm_name "${api_url}" "${node}" "${vmid}" "${auth_token}" "${auth_type}")
        
        if check_guest_agent "${api_url}" "${node}" "${vmid}" "${auth_token}" "${auth_type}"; then
            printf "  %-8s %-30s ${GREEN}%-10s${NC}\n" "${vmid}" "${vm_name}" "ENABLED"
            enabled_count=$((enabled_count + 1))
        else
            printf "  %-8s %-30s ${RED}%-10s${NC}\n" "${vmid}" "${vm_name}" "DISABLED"
            disabled_count=$((disabled_count + 1))
        fi
    done <<< "${vmids}"
    
    log ""
    log "  Summary: ${enabled_count} enabled, ${disabled_count} disabled"
    log ""
}

verify_site() {
    local api_url=$1
    local site_name=$2
    local auth_token=$3
    local auth_type=$4
    
    log "=========================================="
    log "Site: ${site_name}"
    log "=========================================="
    
    local nodes
    nodes=$(list_nodes "${api_url}" "${auth_token}" "${auth_type}")
    
    if [ -z "${nodes}" ]; then
        log_error "Failed to discover nodes on ${site_name}"
        return 1
    fi
    
    log_success "Discovered nodes: $(echo "${nodes}" | tr '\n' ' ')"
    log ""
    
    while IFS= read -r node; do
        [ -z "${node}" ] && continue
        verify_node "${api_url}" "${node}" "${auth_token}" "${auth_type}"
    done <<< "${nodes}"
}

main() {
    log "=========================================="
    log "Verify Guest Agent Status on All VMs"
    log "=========================================="
    log ""
    
    # Site 1
    local auth1
    auth1=$(get_auth "${PROXMOX_1_URL}" "${PROXMOX_1_TOKEN}")
    
    if [ -z "${auth1}" ]; then
        log_error "Failed to authenticate to Site 1"
    else
        IFS='|' read -r auth_token1 auth_type1 <<< "${auth1}"
        log_success "Authenticated to Site 1"
        log ""
        verify_site "${PROXMOX_1_URL}" "Site 1" "${auth_token1}" "${auth_type1}"
    fi
    
    # Site 2
    local auth2
    auth2=$(get_auth "${PROXMOX_2_URL}" "${PROXMOX_2_TOKEN}")
    
    if [ -z "${auth2}" ]; then
        log_error "Failed to authenticate to Site 2"
    else
        IFS='|' read -r auth_token2 auth_type2 <<< "${auth2}"
        log_success "Authenticated to Site 2"
        log ""
        verify_site "${PROXMOX_2_URL}" "Site 2" "${auth_token2}" "${auth_type2}"
    fi
    
    log "=========================================="
    log_success "Verification complete!"
    log "=========================================="
}

main "$@"