Sankofa/scripts/force-remove-vms-batch.sh

#!/bin/bash

# Force remove VMs in batches
set -e

PROXMOX_ENDPOINT="${PROXMOX_ENDPOINT:-https://192.168.11.10:8006}"
PROXMOX_NODE="${PROXMOX_NODE:-ml110-01}"
PROXMOX_USER="${PROXMOX_USER:-}"
PROXMOX_PASS="${PROXMOX_PASS:-}"
PROXMOX_SSH_HOST="${PROXMOX_SSH_HOST:-192.168.11.10}"
PROXMOX_SSH_USER="${PROXMOX_SSH_USER:-root}"

# Generate VM IDs: 103-145 and 211-233
ORPHANED_VMS=()
for i in {103..145}; do
    ORPHANED_VMS+=($i)
done
for i in {211..233}; do
    ORPHANED_VMS+=($i)
done

if [ -z "$PROXMOX_USER" ] || [ -z "$PROXMOX_PASS" ]; then
    echo "Error: PROXMOX_USER and PROXMOX_PASS must be set"
    exit 1
fi

echo "FORCE REMOVING VMs: 103-145 and 211-233"
echo "========================================="
echo "Total VMs to process: ${#ORPHANED_VMS[@]}"
echo ""

# Get authentication
TICKET=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.ticket // empty')

if [ -z "$TICKET" ]; then
    echo "Error: Failed to authenticate"
    exit 1
fi

CSRF_TOKEN=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.CSRFPreventionToken // empty')

SUCCESS_COUNT=0
FAILED_COUNT=0
NOT_FOUND_COUNT=0

echo "Starting batch deletion..."
echo ""

for VMID in "${ORPHANED_VMS[@]}"; do
    echo -n "VM $VMID: "
    
    # Check if VM exists
    VM_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
        jq -r '.data // empty')
    
    if [ -z "$VM_EXISTS" ] || [ "$VM_EXISTS" = "null" ]; then
        echo "not found (already deleted)"
        ((NOT_FOUND_COUNT++))
        continue
    fi
    
    # Multiple unlock attempts
    for i in {1..3}; do
        curl -s -k -b "PVEAuthCookie=${TICKET}" \
            -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
            -X POST \
            "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/unlock" > /dev/null 2>&1
        sleep 1
    done
    
    # Ensure stopped
    curl -s -k -b "PVEAuthCookie=${TICKET}" \
        -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
        -X POST \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/stop" > /dev/null 2>&1
    sleep 2
    
    # Force delete with purge
    DELETE_RESULT=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
        -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
        -X DELETE \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}?purge=1&skiplock=1" 2>&1)
    
    TASK_UPID=$(echo "$DELETE_RESULT" | jq -r '.data // empty' 2>/dev/null)
    
    if [ -n "$TASK_UPID" ] && [ "$TASK_UPID" != "null" ]; then
        # Wait for task completion (shorter timeout for batch processing)
        for i in {1..30}; do
            sleep 1
            TASK_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" 2>/dev/null | \
                jq -r '.data.status // "unknown"')
            
            if [ "$TASK_STATUS" = "stopped" ]; then
                EXIT_STATUS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                    "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/tasks/${TASK_UPID}/status" 2>/dev/null | \
                    jq -r '.data.exitstatus // "unknown"')
                
                if [ "$EXIT_STATUS" = "OK" ] || [ "$EXIT_STATUS" = "0" ]; then
                    # Verify deletion (wait a bit longer for cleanup)
                    sleep 2
                    VM_STILL_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
                        jq -r '.data // empty')
                    
                    if [ -z "$VM_STILL_EXISTS" ] || [ "$VM_STILL_EXISTS" = "null" ]; then
                        echo "✅ deleted"
                        ((SUCCESS_COUNT++))
                    else
                        echo "⚠️  task completed but VM still exists (may need retry)"
                        ((FAILED_COUNT++))
                    fi
                else
                    echo "⚠️  task failed (status: $EXIT_STATUS)"
                    ((FAILED_COUNT++))
                fi
                break
            fi
        done
    else
        echo "❌ failed to start delete task"
        ((FAILED_COUNT++))
    fi
done

echo ""
echo "========================================="
echo "Batch deletion complete!"
echo "  Successfully deleted: $SUCCESS_COUNT"
echo "  Failed: $FAILED_COUNT"
echo "  Not found (already deleted): $NOT_FOUND_COUNT"
echo "  Total processed: ${#ORPHANED_VMS[@]}"
echo ""

if [ $FAILED_COUNT -gt 0 ]; then
    echo "Some VMs failed to delete. Manual cleanup may be required:"
    echo "ssh ${PROXMOX_SSH_USER}@${PROXMOX_SSH_HOST}"
    echo "For each failed VM:"
    echo "  rm -f /var/lock/qemu-server/lock-<VMID>.conf"
    echo "  qm destroy <VMID> --purge"
fi