proxmox/scripts/execute-all-immediate-actions.sh

#!/usr/bin/env bash
# Execute All Immediate Actions - Final Execution
# 1. Address thin2 capacity (migrate containers using it)
# 2. Execute CPU-intensive workload migrations

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
REPORT_DIR="${PROJECT_ROOT}/reports/status"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
EXECUTION_LOG="${REPORT_DIR}/execution_${TIMESTAMP}.log"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
MAGENTA='\033[0;35m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1" | tee -a "$EXECUTION_LOG"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1" | tee -a "$EXECUTION_LOG"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1" | tee -a "$EXECUTION_LOG"; }
log_error() { echo -e "${RED}[✗]${NC} $1" | tee -a "$EXECUTION_LOG"; }
log_header() { echo -e "${CYAN}=== $1 ===${NC}" | tee -a "$EXECUTION_LOG"; }
log_section() { echo -e "\n${MAGENTA}>>> $1 <<<${NC}\n" | tee -a "$EXECUTION_LOG"; }

mkdir -p "$REPORT_DIR"

declare -A NODES
NODES[ml110]="${PROXMOX_HOST_ML110:-192.168.11.10}:L@kers2010"
NODES[r630-01]="${PROXMOX_HOST_R630_01:-192.168.11.11}:password"
NODES[r630-02]="${PROXMOX_HOST_R630_02:-192.168.11.12}:password"

ssh_node() {
    local hostname="$1"
    shift
    local ip="${NODES[$hostname]%%:*}"
    local password="${NODES[$hostname]#*:}"
    
    if command -v sshpass >/dev/null 2>&1; then
        sshpass -p "$password" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@"$ip" "$@"
    else
        ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@"$ip" "$@"
    fi
}

check_node() {
    local hostname="$1"
    local ip="${NODES[$hostname]%%:*}"
    ping -c 1 -W 2 "$ip" >/dev/null 2>&1
}

# Action 1: Migrate containers from thin2 to other storage
migrate_thin2_containers() {
    log_section "Migrating Containers from thin2 to Free Storage"
    
    local hostname="r630-02"
    if ! check_node "$hostname"; then
        log_error "$hostname is not reachable"
        return 1
    fi
    
    log_info "Checking containers using thin2..."
    
    # Get containers using thin2
    local thin2_containers=$(ssh_node "$hostname" bash <<'ENDSSH'
        for vmid in $(pct list 2>/dev/null | tail -n +2 | awk '{print $1}'); do
            rootfs=$(pct config $vmid 2>/dev/null | grep "^rootfs:" | grep "thin2" || true)
            if [ -n "$rootfs" ]; then
                name=$(pct config $vmid 2>/dev/null | grep "^hostname:" | cut -d: -f2 | xargs || echo "CT-$vmid")
                status=$(pct status $vmid 2>/dev/null | awk '{print $2}')
                echo "$vmid|$name|$status"
            fi
        done
ENDSSH
    )
    
    if [ -z "$thin2_containers" ]; then
        log_info "No containers found using thin2"
        return 0
    fi
    
    log_info "Found containers using thin2:"
    echo "$thin2_containers" | while IFS='|' read -r vmid name status; do
        log_info "  CT $vmid ($name) - Status: $status"
        
        # Check available storage
        local available_storage=$(ssh_node "$hostname" "pvesm status 2>/dev/null | grep -E 'thin[3-6]|thin1-r630-02' | grep active | awk '{print \$1}' | head -1" || echo "")
        
        if [ -n "$available_storage" ]; then
            log_info "  Migrating CT $vmid to $available_storage..."
            
            # Migrate the container
            local migrate_result=$(ssh_node "$hostname" "pct migrate $vmid $hostname --storage $available_storage 2>&1" || echo "Migration failed")
            
            if echo "$migrate_result" | grep -q "error\|failed"; then
                log_error "  Failed to migrate CT $vmid: $migrate_result"
            else
                log_success "  Successfully migrated CT $vmid to $available_storage"
            fi
        else
            log_warn "  No available storage found for migration of CT $vmid"
        fi
    done
    
    log_success "thin2 container migration complete."
}

# Action 2: Execute CPU-intensive workload migrations
execute_cpu_migrations() {
    log_section "Executing CPU-Intensive Workload Migrations"
    
    local source_host="ml110"
    
    if ! check_node "$source_host"; then
        log_error "$source_host is not reachable"
        return 1
    fi
    
    # Migrate to r630-01
    log_info "Migrating workloads to r630-01..."
    local target_host="r630-01"
    
    if check_node "$target_host"; then
        for vmid in 1000 1001 1002 1500 1501 1502 2101; do
            log_info "Migrating CT $vmid to $target_host..."
            
            local migrate_result=$(ssh_node "$source_host" "pct migrate $vmid $target_host --restart 2>&1" || echo "Migration failed")
            
            if echo "$migrate_result" | grep -q "error\|failed\|Error"; then
                log_error "Failed to migrate CT $vmid: $migrate_result"
            else
                log_success "Successfully migrated CT $vmid to $target_host"
                # Wait a bit between migrations
                sleep 5
            fi
        done
    else
        log_error "$target_host is not reachable"
    fi
    
    echo "" | tee -a "$EXECUTION_LOG"
    
    # Migrate to r630-02
    log_info "Migrating workloads to r630-02..."
    target_host="r630-02"
    
    if check_node "$target_host"; then
        for vmid in 1003 1004 1503 1504 2201 2303 2401; do
            log_info "Migrating CT $vmid to $target_host..."
            
            local migrate_result=$(ssh_node "$source_host" "pct migrate $vmid $target_host --restart 2>&1" || echo "Migration failed")
            
            if echo "$migrate_result" | grep -q "error\|failed\|Error"; then
                log_error "Failed to migrate CT $vmid: $migrate_result"
            else
                log_success "Successfully migrated CT $vmid to $target_host"
                # Wait a bit between migrations
                sleep 5
            fi
        done
    else
        log_error "$target_host is not reachable"
    fi
    
    log_success "CPU-intensive workload migrations complete."
}

# Action 3: Verify migrations and check system status
verify_migrations() {
    log_section "Verifying Migrations and System Status"
    
    log_info "Checking container distribution after migrations..."
    
    for hostname in "ml110" "r630-01" "r630-02"; do
        if check_node "$hostname"; then
            log_info "Containers on $hostname:"
            
            local container_count=$(ssh_node "$hostname" "pct list 2>/dev/null | tail -n +2 | wc -l" || echo "0")
            local running_count=$(ssh_node "$hostname" "pct list 2>/dev/null | grep running | wc -l" || echo "0")
            
            log_info "  Total: $container_count containers, $running_count running"
            
            # Get CPU usage
            local cpu_usage=$(ssh_node "$hostname" "top -bn1 | grep 'Cpu(s)' | awk '{print \$2}' | sed 's/%us,//' || echo 'N/A'")
            log_info "  CPU Usage: $cpu_usage%"
        fi
    done
    
    log_success "Migration verification complete."
}

# Main execution
main() {
    log_header "Executing All Immediate Actions"
    echo "Log file: $EXECUTION_LOG" | tee -a "$EXECUTION_LOG"
    echo "Timestamp: $(date)" | tee -a "$EXECUTION_LOG"
    echo "" | tee -a "$EXECUTION_LOG"
    
    log_warn "This will perform actual migrations. Proceeding..."
    echo "" | tee -a "$EXECUTION_LOG"
    
    migrate_thin2_containers
    execute_cpu_migrations
    verify_migrations
    
    log_header "All Immediate Actions Execution Complete"
    log_info "Full log saved to: $EXECUTION_LOG"
    log_success "All actions have been executed!"
}

main "$@"