Sankofa/scripts/force-cleanup-vms.sh

#!/bin/bash

# Force cleanup script for orphaned VMs with lock files
# This script attempts to remove lock files and force delete VMs

set -e

PROXMOX_ENDPOINT="${PROXMOX_ENDPOINT:-https://192.168.11.10:8006}"
PROXMOX_NODE="${PROXMOX_NODE:-ml110-01}"
PROXMOX_USER="${PROXMOX_USER:-}"
PROXMOX_PASS="${PROXMOX_PASS:-}"
PROXMOX_SSH_HOST="${PROXMOX_SSH_HOST:-192.168.11.10}"
PROXMOX_SSH_USER="${PROXMOX_SSH_USER:-root}"

ORPHANED_VMS=(234 235 100 101 102)

if [ -z "$PROXMOX_USER" ] || [ -z "$PROXMOX_PASS" ]; then
    echo "Error: PROXMOX_USER and PROXMOX_PASS must be set"
    exit 1
fi

echo "Force cleanup of orphaned VMs with lock files"
echo "=============================================="
echo ""

# Get authentication ticket
TICKET=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.ticket // empty')

if [ -z "$TICKET" ]; then
    echo "Error: Failed to authenticate with Proxmox"
    exit 1
fi

CSRF_TOKEN=$(curl -s -k -d "username=${PROXMOX_USER}&password=${PROXMOX_PASS}" \
    "${PROXMOX_ENDPOINT}/api2/json/access/ticket" | \
    jq -r '.data.CSRFPreventionToken // empty')

echo "Attempting to remove lock files and delete VMs..."
echo ""

for VMID in "${ORPHANED_VMS[@]}"; do
    echo "Processing VM $VMID..."
    
    # Check if VM exists
    VM_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
        "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
        jq -r '.data // empty')
    
    if [ -n "$VM_EXISTS" ] && [ "$VM_EXISTS" != "null" ]; then
        echo "  VM $VMID exists"
        
        # Try multiple unlock attempts
        for i in {1..3}; do
            echo "  Unlock attempt $i..."
            curl -s -k -b "PVEAuthCookie=${TICKET}" \
                -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
                -X POST \
                "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/unlock" > /dev/null
            sleep 2
        done
        
        # Try to remove lock file via API (if supported) or provide manual instructions
        echo "  Attempting to delete VM $VMID..."
        
        # Try delete with different parameters
        DELETE_RESULT=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
            -H "CSRFPreventionToken: ${CSRF_TOKEN}" \
            -X DELETE \
            "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}?purge=1&skiplock=1" 2>&1)
        
        TASK_UPID=$(echo "$DELETE_RESULT" | jq -r '.data // empty' 2>/dev/null)
        
        if [ -n "$TASK_UPID" ] && [ "$TASK_UPID" != "null" ]; then
            echo "  Delete task started: $TASK_UPID"
            sleep 5
            
            # Check if VM still exists
            VM_STILL_EXISTS=$(curl -s -k -b "PVEAuthCookie=${TICKET}" \
                "${PROXMOX_ENDPOINT}/api2/json/nodes/${PROXMOX_NODE}/qemu/${VMID}/status/current" 2>/dev/null | \
                jq -r '.data // empty')
            
            if [ -z "$VM_STILL_EXISTS" ] || [ "$VM_STILL_EXISTS" = "null" ]; then
                echo "  ✅ VM $VMID deleted successfully"
            else
                echo "  ⚠️  VM $VMID still exists - lock file may need manual removal"
                echo "     Manual cleanup required:"
                echo "     ssh ${PROXMOX_SSH_USER}@${PROXMOX_SSH_HOST}"
                echo "     rm -f /var/lock/qemu-server/lock-${VMID}.conf"
                echo "     qm destroy ${VMID} --purge"
            fi
        else
            echo "  ⚠️  Failed to start delete task"
            echo "     Manual cleanup required:"
            echo "     ssh ${PROXMOX_SSH_USER}@${PROXMOX_SSH_HOST}"
            echo "     rm -f /var/lock/qemu-server/lock-${VMID}.conf"
            echo "     qm destroy ${VMID} --purge"
        fi
    else
        echo "  VM $VMID not found (already deleted)"
    fi
    echo ""
done

echo "Cleanup attempt complete!"
echo ""
echo "If VMs still exist, manual cleanup is required:"
echo "1. SSH into the Proxmox node:"
echo "   ssh ${PROXMOX_SSH_USER}@${PROXMOX_SSH_HOST}"
echo ""
echo "2. For each VM, run:"
for VMID in "${ORPHANED_VMS[@]}"; do
    echo "   rm -f /var/lock/qemu-server/lock-${VMID}.conf"
    echo "   qm destroy ${VMID} --purge"
done