loc_az_hci/scripts/deploy/complete-all-with-workarounds.sh

#!/bin/bash
source ~/.bashrc
# Complete All Steps with Workarounds
# Attempts all possible steps, documents what requires manual intervention

set -e

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"

# Load environment variables
if [ -f "$PROJECT_ROOT/.env" ]; then
    set -a
    source <(grep -v '^#' "$PROJECT_ROOT/.env" | grep -v '^$' | sed 's/#.*$//' | grep '=')
    set +a
fi

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() {
    echo -e "${GREEN}[INFO]${NC} $1"
}

log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

log_step() {
    echo -e "\n${BLUE}=== $1 ===${NC}"
}

PVE_USERNAME="${PVE_USERNAME:-root@pam}"
PVE_PASSWORD="${PVE_ROOT_PASS:-}"
PROXMOX_URL="${PROXMOX_ML110_URL:-https://192.168.1.206:8006}"
PROXMOX_NODE="${PROXMOX_NODE:-pve}"

get_api_token() {
    local response=$(curl -s -k --connect-timeout 10 --max-time 15 \
        -d "username=$PVE_USERNAME&password=$PVE_PASSWORD" \
        "$PROXMOX_URL/api2/json/access/ticket" 2>&1)
    
    if echo "$response" | grep -q '"data"'; then
        local ticket=$(echo "$response" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4)
        local csrf_token=$(echo "$response" | grep -o '"CSRFPreventionToken":"[^"]*' | cut -d'"' -f4)
        echo "$ticket|$csrf_token"
    else
        echo ""
    fi
}

# Step 1: Check and attach ISO to template
setup_template_iso() {
    log_step "Step 1: Setting Up Template with ISO"
    
    local tokens=$(get_api_token)
    local ticket=$(echo "$tokens" | cut -d'|' -f1)
    local csrf_token=$(echo "$tokens" | cut -d'|' -f2)
    
    # Check for Ubuntu ISO
    local isos=$(curl -s -k -H "Cookie: PVEAuthCookie=$ticket" \
        -H "CSRFPreventionToken: $csrf_token" \
        "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/storage/local/content" | \
        python3 -c "import sys, json; r=json.load(sys.stdin); isos=[i.get('volid', '') for i in r.get('data', []) if i.get('content')=='iso' and 'ubuntu' in i.get('volid', '').lower()]; print('\n'.join(isos[:1]))" 2>/dev/null)
    
    if [ -n "$isos" ]; then
        local iso_file=$(echo "$isos" | head -1)
        log_info "Found Ubuntu ISO: $iso_file"
        log_info "Attaching to template 9000..."
        
        # Attach ISO and set boot order
        local result=$(curl -s -k -X PUT -H "Cookie: PVEAuthCookie=$ticket" \
            -H "CSRFPreventionToken: $csrf_token" \
            -d "ide2=$iso_file,media=cdrom" \
            -d "boot=order=ide2;scsi0" \
            "$PROXMOX_URL/api2/json/nodes/$PROXMOX_NODE/qemu/9000/config" 2>&1)
        
        if echo "$result" | grep -q '"data"'; then
            log_info "✓ ISO attached successfully"
            log_info "Template 9000 is ready for OS installation"
            log_warn "Next: Start VM 9000 and install Ubuntu via console"
            return 0
        else
            log_warn "Could not attach ISO via API: $result"
            log_info "Manual step: Attach ISO via Proxmox Web UI"
            return 1
        fi
    else
        log_warn "No Ubuntu ISO found in storage"
        log_info "Need to upload Ubuntu 24.04 ISO first"
        log_info "See: scripts/troubleshooting/upload-ubuntu-iso.sh"
        return 1
    fi
}

# Step 2: Attempt infrastructure setup
attempt_infrastructure() {
    log_step "Step 2: Infrastructure Setup"
    
    local tokens=$(get_api_token)
    local ticket=$(echo "$tokens" | cut -d'|' -f1)
    local csrf_token=$(echo "$tokens" | cut -d'|' -f2)
    
    # Check cluster status
    local cluster_status=$(curl -s -k -H "Cookie: PVEAuthCookie=$ticket" \
        -H "CSRFPreventionToken: $csrf_token" \
        "$PROXMOX_URL/api2/json/cluster/status" 2>&1)
    
    if echo "$cluster_status" | grep -q '"data"'; then
        local node_count=$(echo "$cluster_status" | python3 -c "import sys, json; print(len(json.load(sys.stdin).get('data', [])))" 2>/dev/null)
        if [ "$node_count" -gt 1 ]; then
            log_info "✓ Cluster configured with $node_count nodes"
        else
            log_warn "Cluster exists but only has 1 node"
            log_info "Need to join R630 to cluster (requires SSH)"
        fi
    else
        log_warn "No cluster configured"
        log_info "Cluster setup requires SSH access"
    fi
    
    # Check storage
    local storage_status=$(curl -s -k -H "Cookie: PVEAuthCookie=$ticket" \
        -H "CSRFPreventionToken: $csrf_token" \
        "$PROXMOX_URL/api2/json/storage" 2>&1)
    
    local nfs_count=$(echo "$storage_status" | python3 -c "import sys, json; r=json.load(sys.stdin); nfs=[s for s in r.get('data', []) if s.get('type')=='nfs']; print(len(nfs))" 2>/dev/null)
    
    if [ "$nfs_count" -gt 0 ]; then
        log_info "✓ NFS storage configured"
    else
        log_warn "No NFS storage configured"
        log_info "NFS setup requires SSH access or NFS server available"
    fi
}

# Step 3: Monitor and retry VM connectivity
monitor_vms() {
    log_step "Step 3: Monitoring VM Status"
    
    local vms=(
        "100 192.168.1.60 cloudflare-tunnel"
        "101 192.168.1.188 k3s-master"
        "102 192.168.1.121 git-server"
        "103 192.168.1.82 observability"
    )
    
    log_info "Checking VM connectivity (will retry multiple times)..."
    
    for attempt in {1..3}; do
        log_info "Attempt $attempt/3:"
        local any_reachable=false
        
        for vm_spec in "${vms[@]}"; do
            read -r vmid ip name <<< "$vm_spec"
            if ping -c 1 -W 2 "$ip" &>/dev/null; then
                log_info "✓ $name ($ip) is reachable!"
                any_reachable=true
            fi
        done
        
        if [ "$any_reachable" = true ]; then
            log_info "Some VMs are now reachable!"
            break
        fi
        
        if [ $attempt -lt 3 ]; then
            log_warn "VMs not reachable yet, waiting 30 seconds..."
            sleep 30
        fi
    done
}

main() {
    log_info "Completing All Steps with Workarounds"
    echo ""
    
    # Setup template ISO
    setup_template_iso
    
    # Infrastructure
    attempt_infrastructure
    
    # Monitor VMs
    monitor_vms
    
    log_step "Summary"
    log_info "All automated steps attempted"
    log_warn "Template OS installation requires manual step via Web UI"
    log_info "See TROUBLESHOOTING_AND_FIXES.md for template fix instructions"
}

main "$@"