proxmox/scripts/migrate-containers-to-pve2.sh

#!/usr/bin/env bash
# Migrate LXC containers from ml110 to r630-02
# Provides safe migration with verification

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"
# Password defaults: ml110 uses L@kers2010, r630-01/r630-02 use password
# Migration runs from ml110, so use ml110 password
PROXMOX_PASS="${PROXMOX_PASS:-L@kers2010}"
SOURCE_NODE="ml110"
TARGET_NODE="r630-02"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_header() { echo -e "${CYAN}[$1]${NC} $2"; }

# SSH helper (connects to ml110 for cluster management)
# Note: pct migrate command runs on ml110 and handles node-to-node migration
ssh_proxmox() {
    sshpass -p "$PROXMOX_PASS" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@"$PROXMOX_HOST" "$@"
}

# Check if container exists and get its status
check_container() {
    local vmid=$1
    local node=$2
    ssh_proxmox "pvesh get /nodes/$node/lxc/$vmid/status/current --output-format json" 2>&1 | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('status', 'unknown'))" 2>/dev/null || echo "not_found"
}

# Migrate a single container
migrate_container() {
    local vmid=$1
    local name=$2
    local dry_run=${3:-false}
    
    log_info "Migrating container $vmid ($name) from $SOURCE_NODE to $TARGET_NODE"
    
    if [[ "$dry_run" == "true" ]]; then
        log_warn "  [DRY RUN] Would migrate container $vmid"
        return 0
    fi
    
    # Check source container status
    source_status=$(check_container "$vmid" "$SOURCE_NODE")
    if [[ "$source_status" == "not_found" ]]; then
        log_error "  Container $vmid not found on $SOURCE_NODE"
        return 1
    fi
    
    log_info "  Current status on $SOURCE_NODE: $source_status"
    
    # Perform migration
    log_info "  Starting migration..."
    if ssh_proxmox "pct migrate $vmid $TARGET_NODE --restart" 2>&1; then
        log_success "  Migration command completed"
        
        # Wait a bit and verify
        sleep 5
        target_status=$(check_container "$vmid" "$TARGET_NODE")
        if [[ "$target_status" != "not_found" ]]; then
            log_success "  Container $vmid is now on $TARGET_NODE (status: $target_status)"
            return 0
        else
            log_warn "  Migration may have succeeded but container not yet visible on target"
            return 0
        fi
    else
        log_error "  Migration failed for container $vmid"
        return 1
    fi
}

# Main execution
main() {
    echo "========================================="
    log_header "MIGRATION" "LXC Containers to r630-02"
    echo "========================================="
    echo ""
    
    # Check if dry run
    DRY_RUN=false
    if [[ "${1:-}" == "--dry-run" ]] || [[ "${1:-}" == "-n" ]]; then
        DRY_RUN=true
        log_warn "DRY RUN MODE - No containers will be migrated"
        echo ""
    fi
    
    # Verify cluster status
    log_info "Verifying cluster status..."
    cluster_status=$(ssh_proxmox "pvecm status" 2>&1 | grep -c "Quorate:.*Yes" || echo "0")
    if [[ "$cluster_status" == "0" ]]; then
        log_error "Cluster is not quorate. Cannot proceed with migration."
        exit 1
    fi
    log_success "Cluster is quorate"
    echo ""
    
    # Verify target node is online
    log_info "Verifying target node ($TARGET_NODE) is online..."
    target_online=$(ssh_proxmox "pvesh get /nodes --output-format json" 2>&1 | python3 -c "import sys, json; nodes=json.load(sys.stdin); target=[n for n in nodes if n['node']=='$TARGET_NODE' and n['status']=='online']; print('online' if target else 'offline')" 2>/dev/null || echo "unknown")
    if [[ "$target_online" != "online" ]]; then
        log_error "Target node $TARGET_NODE is not online (status: $target_online)"
        exit 1
    fi
    log_success "Target node is online"
    echo ""
    
    # Get list of containers to migrate
    log_info "Getting list of containers on $SOURCE_NODE..."
    
    # Priority containers to migrate (high resource usage)
    PRIORITY_CONTAINERS=(
        "1000:besu-validator-1"
        "1001:besu-validator-2"
        "1002:besu-validator-3"
        "1003:besu-validator-4"
        "1004:besu-validator-5"
        "2500:besu-rpc-1"
        "2501:besu-rpc-2"
        "2502:besu-rpc-3"
        "5000:blockscout-1"
    )
    
    # Secondary containers
    SECONDARY_CONTAINERS=(
        "1500:besu-sentry-1"
        "1501:besu-sentry-2"
        "1502:besu-sentry-3"
        "1503:besu-sentry-4"
        "3500:oracle-publisher-1"
        "3501:ccip-monitor-1"
        "6200:firefly-1"
    )
    
    # Ask which containers to migrate
    echo ""
    log_info "Priority containers (high resource usage):"
    for container in "${PRIORITY_CONTAINERS[@]}"; do
        vmid="${container%%:*}"
        name="${container#*:}"
        echo "  - $vmid: $name"
    done
    echo ""
    
    log_info "Secondary containers (medium resource usage):"
    for container in "${SECONDARY_CONTAINERS[@]}"; do
        vmid="${container%%:*}"
        name="${container#*:}"
        echo "  - $vmid: $name"
    done
    echo ""
    
    if [[ "$DRY_RUN" == "true" ]]; then
        log_warn "DRY RUN: Would migrate priority containers"
        for container in "${PRIORITY_CONTAINERS[@]}"; do
            vmid="${container%%:*}"
            name="${container#*:}"
            migrate_container "$vmid" "$name" "true"
        done
    else
        read -p "Migrate priority containers to $TARGET_NODE? (y/N): " -n 1 -r
        echo
        if [[ $REPLY =~ ^[Yy]$ ]]; then
            failed=0
            for container in "${PRIORITY_CONTAINERS[@]}"; do
                vmid="${container%%:*}"
                name="${container#*:}"
                if ! migrate_container "$vmid" "$name" "false"; then
                    failed=$((failed + 1))
                fi
                echo ""
            done
            
            if [[ $failed -eq 0 ]]; then
                log_success "All priority containers migrated successfully!"
            else
                log_warn "$failed container(s) failed to migrate"
            fi
        else
            log_info "Migration cancelled by user"
        fi
    fi
    
    echo ""
    log_info "Migration complete!"
    echo ""
    log_info "Verify containers on target node:"
    echo "  ssh root@$PROXMOX_HOST 'pvesh get /nodes/$TARGET_NODE/lxc'"
    echo ""
}

main "$@"