proxmox/scripts/migrate-containers-to-pve2-execute.sh

#!/usr/bin/env bash
# Execute container migrations to pve2 (non-interactive version)
# Migrates priority containers from ml110 to pve2

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"
PROXMOX_PASS="${PROXMOX_PASS:-L@kers2010}"
SOURCE_NODE="ml110"
TARGET_NODE="pve2"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_header() { echo -e "${CYAN}[$1]${NC} $2"; }

# SSH helper
ssh_proxmox() {
    sshpass -p "$PROXMOX_PASS" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@"$PROXMOX_HOST" "$@"
}

# Check if container exists and get its status
check_container() {
    local vmid=$1
    local node=$2
    ssh_proxmox "pvesh get /nodes/$node/lxc/$vmid/status/current --output-format json" 2>&1 | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('status', 'unknown'))" 2>/dev/null || echo "not_found"
}

# Migrate a single container
migrate_container() {
    local vmid=$1
    local name=$2
    
    log_info "Migrating container $vmid ($name) from $SOURCE_NODE to $TARGET_NODE"
    
    # Check source container status
    source_status=$(check_container "$vmid" "$SOURCE_NODE")
    if [[ "$source_status" == "not_found" ]]; then
        log_error "  Container $vmid not found on $SOURCE_NODE"
        return 1
    fi
    
    log_info "  Current status on $SOURCE_NODE: $source_status"
    
    # Perform migration
    log_info "  Starting migration (this may take several minutes)..."
    if ssh_proxmox "pct migrate $vmid $TARGET_NODE --restart" 2>&1; then
        log_success "  Migration command completed"
        
        # Wait and verify
        log_info "  Waiting for migration to complete and verifying..."
        sleep 10
        
        # Check multiple times as migration might take time
        for i in {1..6}; do
            sleep 5
            target_status=$(check_container "$vmid" "$TARGET_NODE")
            if [[ "$target_status" != "not_found" ]]; then
                log_success "  Container $vmid is now on $TARGET_NODE (status: $target_status)"
                return 0
            fi
            log_info "  Still migrating... (attempt $i/6)"
        done
        
        log_warn "  Migration may have succeeded but container not yet visible on target"
        log_info "  Please verify manually: ssh root@$PROXMOX_HOST 'pvesh get /nodes/$TARGET_NODE/lxc'"
        return 0
    else
        log_error "  Migration failed for container $vmid"
        return 1
    fi
}

# Main execution
main() {
    echo "========================================="
    log_header "MIGRATION" "LXC Containers to pve2 - EXECUTING"
    echo "========================================="
    echo ""
    
    # Verify cluster status
    log_info "Verifying cluster status..."
    cluster_status=$(ssh_proxmox "pvecm status" 2>&1 | grep -c "Quorate:.*Yes" || echo "0")
    if [[ "$cluster_status" == "0" ]]; then
        log_error "Cluster is not quorate. Cannot proceed with migration."
        exit 1
    fi
    log_success "Cluster is quorate"
    echo ""
    
    # Verify target node is online
    log_info "Verifying target node ($TARGET_NODE) is online..."
    target_online=$(ssh_proxmox "pvesh get /nodes --output-format json" 2>&1 | python3 -c "import sys, json; nodes=json.load(sys.stdin); target=[n for n in nodes if n['node']=='$TARGET_NODE' and n['status']=='online']; print('online' if target else 'offline')" 2>/dev/null || echo "unknown")
    if [[ "$target_online" != "online" ]]; then
        log_error "Target node $TARGET_NODE is not online (status: $target_online)"
        exit 1
    fi
    log_success "Target node is online"
    echo ""
    
    # Priority containers to migrate (high resource usage)
    PRIORITY_CONTAINERS=(
        "1000:besu-validator-1"
        "1001:besu-validator-2"
        "1002:besu-validator-3"
        "1003:besu-validator-4"
        "1004:besu-validator-5"
        "2500:besu-rpc-1"
        "2501:besu-rpc-2"
        "2502:besu-rpc-3"
        "5000:blockscout-1"
    )
    
    log_info "Will migrate the following containers to $TARGET_NODE:"
    for container in "${PRIORITY_CONTAINERS[@]}"; do
        vmid="${container%%:*}"
        name="${container#*:}"
        echo "  - $vmid: $name"
    done
    echo ""
    
    log_warn "Starting migrations. Each migration may take 2-5 minutes."
    echo ""
    
    failed=0
    success=0
    total=${#PRIORITY_CONTAINERS[@]}
    
    for container in "${PRIORITY_CONTAINERS[@]}"; do
        vmid="${container%%:*}"
        name="${container#*:}"
        
        if migrate_container "$vmid" "$name"; then
            success=$((success + 1))
        else
            failed=$((failed + 1))
        fi
        
        # Small delay between migrations
        if [[ $success -lt $total ]]; then
            log_info "Waiting 10 seconds before next migration..."
            sleep 10
        fi
        echo ""
    done
    
    echo ""
    log_info "========================================="
    log_info "Migration Summary"
    log_info "========================================="
    log_success "Successful: $success/$total"
    if [[ $failed -gt 0 ]]; then
        log_warn "Failed: $failed/$total"
    fi
    echo ""
    
    log_info "Verify containers on target node:"
    echo "  ssh root@$PROXMOX_HOST 'pvesh get /nodes/$TARGET_NODE/lxc'"
    echo ""
    
    log_info "Check container distribution:"
    echo "  ssh root@$PROXMOX_HOST 'pvesh get /nodes/ml110/lxc | wc -l'  # ml110 containers"
    echo "  ssh root@$PROXMOX_HOST 'pvesh get /nodes/pve2/lxc | wc -l'   # pve2 containers"
    echo ""
}

main "$@"