Sankofa/scripts/cleanup-orphaned-vms.sh

#!/bin/bash

# Script to clean up orphaned VMs created during the failed VM creation loop
# VMs that were created: 234, 235, 100, 101, 102

set -e

PROXMOX_ENDPOINT="${PROXMOX_ENDPOINT:-https://192.168.11.10:8006}"
PROXMOX_NODE="${PROXMOX_NODE:-ml110-01}"
PROXMOX_USER="${PROXMOX_USER:-}"
PROXMOX_PASS="${PROXMOX_PASS:-}"

# Orphaned VM IDs from the logs
ORPHANED_VMS=(234 235 100 101 102)

if [ -z "$PROXMOX_USER" ] || [ -z "$PROXMOX_PASS" ]; then
    echo "Error: PROXMOX_USER and PROXMOX_PASS must be set"
    echo "Usage: PROXMOX_USER=user PROXMOX_PASS=pass ./cleanup-orphaned-vms.sh"
    exit 1
fi

echo "Connecting to Proxmox at $PROXMOX_ENDPOINT..."
echo "Node: $PROXMOX_NODE"
echo ""

# Get authentication ticket
TICKET=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.ticket // empty')

if [ -z "$TICKET" ]; then
    echo "Error: Failed to authenticate with Proxmox"
    exit 1
fi

CSRF_TOKEN=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.CSRFPreventionToken // empty')

echo "Authentication successful"
echo ""

# List all VMs on the node first
echo "Listing VMs on node $PROXMOX_NODE..."
VMS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
    "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu" | \
    jq -r '.data[] | "\(.vmid) \(.name) \(.status)"')

echo "Current VMs:"
echo "$VMS"
echo ""

# Delete orphaned VMs
for VMID in "${ORPHANED_VMS[@]}"; do
    echo "Checking VM $VMID..."
    
    # Check if VM exists
    VM_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
        jq -r '.data // empty')
    
    if [ -n "$VM_EXISTS" ] && [ "$VM_EXISTS" != "null" ]; then
        echo "  Found VM $VMID, checking status..."
        
        # Get VM status
        STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
            "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" | \
            jq -r '.data.status // "unknown"')
        
        echo "  Status: $STATUS"
        
        # Stop VM if running
        if [ "$STATUS" = "running" ]; then
            echo "  Stopping VM $VMID..."
            curl -s -k -b "PVEAuthCookie=${TICKET}" \
                -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
                -X POST \
                "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/stop" > /dev/null
            
            # Wait for VM to stop (up to 30 seconds)
            echo "  Waiting for VM to stop..."
            for i in {1..30}; do
                sleep 1
                CURRENT_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                    "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" | \
                    jq -r '.data.status // "unknown"')
                if [ "$CURRENT_STATUS" = "stopped" ]; then
                    echo "  VM stopped"
                    break
                fi
            done
        fi
        
        # Unlock VM if locked (common issue with failed VM creation)
        echo "  Unlocking VM $VMID..."
        UNLOCK_RESULT=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
            -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
            -X POST \
            "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/unlock" 2>/dev/null)
        
        if echo "$UNLOCK_RESULT" | jq -e '.data // empty' > /dev/null 2>&1; then
            echo "  VM unlocked"
            sleep 1
        fi
        
        # Delete VM with purge option to force cleanup
        echo "  Deleting VM $VMID (with purge)..."
        DELETE_RESULT=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
            -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
            -X DELETE \
            "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}?purge=1")
        
        # Check if we got a task ID (UPID)
        TASK_UPID=$(echo "$DELETE_RESULT" | jq -r '.data // empty' 2>/dev/null)
        
        if [ -n "$TASK_UPID" ] && [ "$TASK_UPID" != "null" ]; then
            echo "  Delete task started: $TASK_UPID"
            echo "  Waiting for deletion to complete..."
            
            # Wait for task to complete (up to 60 seconds)
            for i in {1..60}; do
                sleep 1
                TASK_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                    "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" | \
                    jq -r '.data.status // "unknown"')
                
                if [ "$TASK_STATUS" = "stopped" ]; then
                    EXIT_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" | \
                        jq -r '.data.exitstatus // "unknown"')
                    
                    if [ "$EXIT_STATUS" = "OK" ] || [ "$EXIT_STATUS" = "0" ]; then
                        echo "  ✅ Successfully deleted VM $VMID"
                    else
                        echo "  ⚠️  Delete task completed with status: $EXIT_STATUS"
                    fi
                    break
                fi
            done
            
            # Verify VM is actually gone
            sleep 2
            VM_STILL_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
                jq -r '.data // empty')
            
            if [ -z "$VM_STILL_EXISTS" ] || [ "$VM_STILL_EXISTS" = "null" ]; then
                echo "  ✅ Verified: VM $VMID is deleted"
            else
                echo "  ⚠️  Warning: VM $VMID may still exist"
            fi
        else
            echo "  ⚠️  Warning: May have failed to delete VM $VMID"
            echo "     Response: $DELETE_RESULT"
        fi
    else
        echo "  VM $VMID not found (may have been already deleted)"
    fi
    echo ""
done

echo "Cleanup complete!"