Sankofa/scripts/force-remove-all-remaining.sh

#!/bin/bash

# Aggressive force removal of all remaining VMs
set -e

PROXMOX_ENDPOINT="${PROXMOX_ENDPOINT:-https://192.168.11.10:8006}"
PROXMOX_NODE="${PROXMOX_NODE:-ml110-01}"
PROXMOX_USER="${PROXMOX_USER:-}"
PROXMOX_PASS="${PROXMOX_PASS:-}"

if [ -z "$PROXMOX_USER" ] || [ -z "$PROXMOX_PASS" ]; then
    echo "Error: PROXMOX_USER and PROXMOX_PASS must be set"
    exit 1
fi

echo "FORCE REMOVING ALL REMAINING VMs"
echo "================================"
echo ""

# Get authentication
TICKET=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.ticket // empty')

if [ -z "$TICKET" ]; then
    echo "Error: Failed to authenticate"
    exit 1
fi

CSRF_TOKEN=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.CSRFPreventionToken // empty')

# Get list of all VMs
echo "Fetching list of all VMs..."
ALL_VMS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
    "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu" | \
    jq -r '.data[] | .vmid' | sort -n)

if [ -z "$ALL_VMS" ]; then
    echo "No VMs found. All clean!"
    exit 0
fi

VM_COUNT=$(echo "$ALL_VMS" | wc -l)
echo "Found $VM_COUNT VM(s) to remove"
echo ""

SUCCESS_COUNT=0
FAILED_COUNT=0

for VMID in $ALL_VMS; do
    echo "=== Processing VM $VMID ==="
    
    # Aggressive unlock - multiple attempts with longer delays
    echo "  Unlocking VM (multiple attempts)..."
    for i in 1 2 3 4 5 6 7 8 9 10; do
        curl -s -k -b "PVEAuthCookie=${TICKET}" \
            -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
            -X POST \
            "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/unlock" > /dev/null 2>&1
        sleep 1
    done
    
    # Ensure stopped
    echo "  Ensuring VM is stopped..."
    curl -s -k -b "PVEAuthCookie=${TICKET}" \
        -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
        -X POST \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/stop" > /dev/null 2>&1
    sleep 3
    
    # Try delete with all possible parameters
    echo "  Attempting force delete..."
    DELETE_RESULT=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
        -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
        -X DELETE \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}?purge=1&skiplock=1" 2>&1)
    
    TASK_UPID=$(echo "$DELETE_RESULT" | jq -r '.data // empty' 2>/dev/null)
    
    if [ -n "$TASK_UPID" ] && [ "$TASK_UPID" != "null" ]; then
        echo "  Delete task started: $TASK_UPID"
        echo "  Waiting for completion (up to 60 seconds)..."
        
        DELETED=false
        for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60; do
            sleep 1
            TASK_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" 2>/dev/null | \
                jq -r '.data.status // "unknown"')
            
            if [ "$TASK_STATUS" = "stopped" ]; then
                EXIT_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                    "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" 2>/dev/null | \
                    jq -r '.data.exitstatus // "unknown"')
                
                ERROR_MSG=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                    "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" 2>/dev/null | \
                    jq -r '.data.exitstatus // empty')
                
                if [ "$EXIT_STATUS" = "OK" ] || [ "$EXIT_STATUS" = "0" ]; then
                    sleep 3
                    VM_STILL_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
                        jq -r '.data // empty')
                    
                    if [ -z "$VM_STILL_EXISTS" ] || [ "$VM_STILL_EXISTS" = "null" ]; then
                        echo "  ✅ VM $VMID deleted successfully"
                        SUCCESS_COUNT=$((SUCCESS_COUNT + 1))
                        DELETED=true
                    else
                        echo "  ⚠️  Task completed but VM still exists"
                    fi
                else
                    echo "  ⚠️  Task failed with status: $EXIT_STATUS"
                    if echo "$EXIT_STATUS" | grep -q "lock"; then
                        echo "  ❌ Lock file issue - requires manual cleanup"
                    fi
                fi
                break
            fi
        done
        
        if [ "$DELETED" = "false" ]; then
            FAILED_COUNT=$((FAILED_COUNT + 1))
        fi
    else
        echo "  ❌ Failed to start delete task"
        echo "  Response: $DELETE_RESULT"
        FAILED_COUNT=$((FAILED_COUNT + 1))
    fi
    echo ""
done

echo "========================================="
echo "Summary:"
echo "  Successfully deleted: $SUCCESS_COUNT"
echo "  Failed: $FAILED_COUNT"
echo "  Total processed: $VM_COUNT"
echo ""

if [ $FAILED_COUNT -gt 0 ]; then
    echo "⚠️  Some VMs failed to delete due to lock file issues."
    echo "Manual cleanup required via SSH:"
    echo "  ssh root@192.168.11.10"
    echo ""
    echo "For each failed VM, run:"
    echo "  rm -f /var/lock/qemu-server/lock-<VMID>.conf"
    echo "  qm destroy <VMID> --purge"
fi