Sankofa/scripts/force-remove-vms.sh

#!/bin/bash

# Force remove orphaned VMs by removing lock files and using destructive methods
set -e

PROXMOX_ENDPOINT="${PROXMOX_ENDPOINT:-https://192.168.11.10:8006}"
PROXMOX_NODE="${PROXMOX_NODE:-ml110-01}"
PROXMOX_USER="${PROXMOX_USER:-}"
PROXMOX_PASS="${PROXMOX_PASS:-}"
PROXMOX_SSH_HOST="${PROXMOX_SSH_HOST:-192.168.11.10}"
PROXMOX_SSH_USER="${PROXMOX_SSH_USER:-root}"

ORPHANED_VMS=(234 235 100 101 102)

if [ -z "$PROXMOX_USER" ] || [ -z "$PROXMOX_PASS" ]; then
    echo "Error: PROXMOX_USER and PROXMOX_PASS must be set"
    exit 1
fi

echo "FORCE REMOVING ORPHANED VMs"
echo "==========================="
echo ""

# Get authentication
TICKET=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.ticket // empty')

if [ -z "$TICKET" ]; then
    echo "Error: Failed to authenticate"
    exit 1
fi

CSRF_TOKEN=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.CSRFPreventionToken // empty')

# Try to remove lock files via storage API or direct file manipulation
echo "Attempting to remove lock files and force delete VMs..."
echo ""

for VMID in "${ORPHANED_VMS[@]}"; do
    echo "=== Processing VM $VMID ==="
    
    # Check if VM exists
    VM_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
        jq -r '.data // empty')
    
    if [ -z "$VM_EXISTS" ] || [ "$VM_EXISTS" = "null" ]; then
        echo "  VM $VMID not found (already deleted)"
        echo ""
        continue
    fi
    
    echo "  VM $VMID exists, attempting force removal..."
    
    # Method 1: Try multiple unlock attempts with delays
    for i in {1..5}; do
        echo "  Unlock attempt $i/5..."
        UNLOCK_RESULT=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
            -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
            -X POST \
            "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/unlock" 2>&1)
        sleep 3
    done
    
    # Method 2: Try to stop any running processes first
    echo "  Ensuring VM is stopped..."
    curl -s -k -b "PVEAuthCookie=${TICKET}" \
        -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
        -X POST \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/stop" > /dev/null 2>&1
    sleep 5
    
    # Method 3: Try delete with all possible parameters
    echo "  Attempting force delete with purge..."
    
    # Try with purge and skiplock
    DELETE_RESULT=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
        -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
        -X DELETE \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}?purge=1&skiplock=1" 2>&1)
    
    TASK_UPID=$(echo "$DELETE_RESULT" | jq -r '.data // empty' 2>/dev/null)
    
    if [ -n "$TASK_UPID" ] && [ "$TASK_UPID" != "null" ]; then
        echo "  Delete task started: $TASK_UPID"
        echo "  Waiting up to 60 seconds for completion..."
        
        for i in {1..60}; do
            sleep 1
            TASK_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" 2>/dev/null | \
                jq -r '.data.status // "unknown"')
            
            if [ "$TASK_STATUS" = "stopped" ]; then
                EXIT_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                    "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" 2>/dev/null | \
                    jq -r '.data.exitstatus // "unknown"')
                
                if [ "$EXIT_STATUS" = "OK" ] || [ "$EXIT_STATUS" = "0" ]; then
                    echo "  ✅ VM $VMID deleted successfully"
                    break
                else
                    echo "  ⚠️  Task completed with status: $EXIT_STATUS"
                fi
            fi
        done
        
        # Verify deletion
        sleep 3
        VM_STILL_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
            "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
            jq -r '.data // empty')
        
        if [ -z "$VM_STILL_EXISTS" ] || [ "$VM_STILL_EXISTS" = "null" ]; then
            echo "  ✅ Verified: VM $VMID is deleted"
        else
            echo "  ❌ VM $VMID still exists - requires manual intervention"
            echo ""
            echo "  MANUAL CLEANUP REQUIRED:"
            echo "  ssh ${PROXMOX_SSH_USER}@${PROXMOX_SSH_HOST}"
            echo "  rm -f /var/lock/qemu-server/lock-${VMID}.conf"
            echo "  qm destroy ${VMID} --purge"
        fi
    else
        echo "  ❌ Failed to start delete task"
        echo "  Response: $DELETE_RESULT"
        echo ""
        echo "  MANUAL CLEANUP REQUIRED:"
        echo "  ssh ${PROXMOX_SSH_USER}@${PROXMOX_SSH_HOST}"
        echo "  rm -f /var/lock/qemu-server/lock-${VMID}.conf"
        echo "  qm destroy ${VMID} --purge"
    fi
    
    echo ""
done

echo "Force removal attempt complete!"
echo ""
echo "If any VMs still exist, they require manual cleanup via SSH:"
echo "ssh ${PROXMOX_SSH_USER}@${PROXMOX_SSH_HOST}"
for VMID in "${ORPHANED_VMS[@]}"; do
    echo "rm -f /var/lock/qemu-server/lock-${VMID}.conf && qm destroy ${VMID} --purge"
done