proxmox/scripts/complete-all-tasks-parallel-comprehensive.sh

#!/usr/bin/env bash
# Complete All Incomplete Tasks - Comprehensive Parallel Execution
# Executes all pending tasks across all 33 containers in parallel

set -uo pipefail

NODE_IP="${PROXMOX_HOST_R630_01}"
MAX_PARALLEL=15  # Increased parallel operations
LOG_DIR="/tmp/parallel-tasks-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$LOG_DIR"

# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_phase() { echo -e "${BLUE}[PHASE]${NC} $1"; }

# Task tracking
declare -A TASK_PIDS=()
TOTAL_TASKS=0
COMPLETED_TASKS=0
FAILED_TASKS=0

# Function to run task in background
run_task() {
    local task_id="$1"
    local task_name="$2"
    local task_command="$3"
    
    (
        eval "$task_command" > "$LOG_DIR/${task_id}.log" 2>&1
        local exit_code=$?
        echo "$exit_code" > "$LOG_DIR/${task_id}.exit"
        exit $exit_code
    ) &
    
    TASK_PIDS[$task_id]=$!
    ((TOTAL_TASKS++))
    log_info "Started: $task_name"
}

# Wait for available slot
wait_for_slot() {
    while [ ${#TASK_PIDS[@]} -ge $MAX_PARALLEL ]; do
        for task_id in "${!TASK_PIDS[@]}"; do
            if ! kill -0 ${TASK_PIDS[$task_id]} 2>/dev/null; then
                wait ${TASK_PIDS[$task_id]}
                exit_code=$(cat "$LOG_DIR/${task_id}.exit" 2>/dev/null || echo "1")
                if [ "$exit_code" = "0" ]; then
                    ((COMPLETED_TASKS++))
                else
                    ((FAILED_TASKS++))
                fi
                unset TASK_PIDS[$task_id]
            fi
        done
        sleep 0.3
    done
}

# Wait for all tasks
wait_all() {
    for task_id in "${!TASK_PIDS[@]}"; do
        wait ${TASK_PIDS[$task_id]} 2>/dev/null || true
        exit_code=$(cat "$LOG_DIR/${task_id}.exit" 2>/dev/null || echo "1")
        if [ "$exit_code" = "0" ]; then
            ((COMPLETED_TASKS++))
        else
            ((FAILED_TASKS++))
        fi
    done
    declare -A TASK_PIDS=()
}

echo "═══════════════════════════════════════════════════════════"
echo "Complete All Tasks - Comprehensive Parallel Execution"
echo "═══════════════════════════════════════════════════════════"
echo "Node: $NODE_IP"
echo "Log Dir: $LOG_DIR"
echo "Max Parallel: $MAX_PARALLEL"
echo ""

# ============================================================================
# PHASE 1: Install Database Services (PostgreSQL)
# ============================================================================

log_phase "PHASE 1: Installing PostgreSQL (4 containers)"

install_postgresql() {
    local vmid="$1"
    local hostname="$2"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq postgresql-15 postgresql-contrib-15 >/dev/null 2>&1 || exit 1
            
            # Configure PostgreSQL
            sed -i \"s/#listen_addresses = .*/listen_addresses = '\''*'\''/\" /etc/postgresql/15/main/postgresql.conf 2>/dev/null || true
            echo \"host all all 0.0.0.0/0 md5\" >> /etc/postgresql/15/main/pg_hba.conf 2>/dev/null || true
            
            systemctl enable postgresql@15-main >/dev/null 2>&1
            systemctl start postgresql@15-main >/dev/null 2>&1
            sleep 3
            systemctl is-active postgresql@15-main >/dev/null && echo \"PostgreSQL installed\" || exit 1
        '
    "
}

for vmid in 10000 10001 10100 10101; do
    wait_for_slot
    hostname=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct config $vmid 2>/dev/null | grep '^hostname:' | sed 's/^hostname: //'" || echo "unknown")
    run_task "pg-$vmid" "PostgreSQL CT $vmid" "install_postgresql $vmid '$hostname'"
done

wait_all
log_info "Phase 1 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 2: Install Redis
# ============================================================================

log_phase "PHASE 2: Installing Redis (2 containers)"

install_redis() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq redis-server >/dev/null 2>&1 || exit 1
            
            sed -i \"s/^bind .*/bind 0.0.0.0/\" /etc/redis/redis.conf 2>/dev/null || true
            systemctl enable redis-server >/dev/null 2>&1
            systemctl restart redis-server >/dev/null 2>&1
            sleep 2
            systemctl is-active redis-server >/dev/null && echo \"Redis installed\" || exit 1
        '
    "
}

for vmid in 10020 10120; do
    wait_for_slot
    run_task "redis-$vmid" "Redis CT $vmid" "install_redis $vmid"
done

wait_all
log_info "Phase 2 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 3: Install Node.js Runtime
# ============================================================================

log_phase "PHASE 3: Installing Node.js (14 containers)"

install_nodejs() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq curl ca-certificates gnupg >/dev/null 2>&1 || exit 1
            
            curl -fsSL https://deb.nodesource.com/setup_18.x | bash - >/dev/null 2>&1 || exit 1
            apt-get install -y -qq nodejs >/dev/null 2>&1 || exit 1
            npm install -g pm2 >/dev/null 2>&1 || exit 1
            
            node --version >/dev/null && npm --version >/dev/null && echo \"Node.js installed\" || exit 1
        '
    "
}

for vmid in 10030 10040 10050 10060 10070 10080 10090 10091 10092 10130 10150 10151; do
    wait_for_slot
    run_task "nodejs-$vmid" "Node.js CT $vmid" "install_nodejs $vmid"
done

wait_all
log_info "Phase 3 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 4: Configure PostgreSQL Databases
# ============================================================================

log_phase "PHASE 4: Configuring PostgreSQL Databases"

configure_order_db() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            sudo -u postgres psql << \"EOF\" >/dev/null 2>&1
                CREATE DATABASE order_db;
                CREATE USER order_user WITH PASSWORD '\''order_password'\'';
                GRANT ALL PRIVILEGES ON DATABASE order_db TO order_user;
                ALTER DATABASE order_db OWNER TO order_user;
EOF
            echo \"Order DB configured\"
        '
    "
}

configure_dbis_db() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            sudo -u postgres psql << \"EOF\" >/dev/null 2>&1
                CREATE DATABASE dbis_core;
                CREATE USER dbis WITH PASSWORD '\''8cba649443f97436db43b34ab2c0e75b5cf15611bef9c099cee6fb22cc3d7771'\'';
                GRANT ALL PRIVILEGES ON DATABASE dbis_core TO dbis;
                ALTER DATABASE dbis_core OWNER TO dbis;
EOF
            echo \"DBIS DB configured\"
        '
    "
}

for vmid in 10000 10001; do
    wait_for_slot
    run_task "pg-config-order-$vmid" "Configure Order DB $vmid" "configure_order_db $vmid"
done

for vmid in 10100 10101; do
    wait_for_slot
    run_task "pg-config-dbis-$vmid" "Configure DBIS DB $vmid" "configure_dbis_db $vmid"
done

wait_all
log_info "Phase 4 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 5: Update Application Configurations
# ============================================================================

log_phase "PHASE 5: Updating Application Configurations (IP addresses)"

update_container_configs() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            # Update .env files
            find /opt /home /root -name \".env\" -type f 2>/dev/null | while read f; do
                [ -r \"\$f\" ] && sed -i \"s|10.200.0.10|${ORDER_POSTGRES_PRIMARY:-${ORDER_POSTGRES_PRIMARY:-192.168.11.44}}|g; s|10.200.0.11|${ORDER_POSTGRES_REPLICA:-${ORDER_POSTGRES_REPLICA:-192.168.11.45}}|g; s|10.200.0.20|${ORDER_REDIS_IP:-192.168.11.38}|g; s|10.200.0.30|${IP_SERVICE_40:-${IP_SERVICE_40:-${IP_SERVICE_40:-192.168.11.40}}}|g; s|10.200.0.40|${IP_SERVICE_41:-${IP_SERVICE_41:-${IP_SERVICE_41:-192.168.11.41}}}|g; s|10.200.0.50|${IP_SERVICE_49:-${IP_SERVICE_49:-${IP_SERVICE_49:-192.168.11.49}}}|g; s|10.200.0.60|${IP_SERVICE_42:-${IP_SERVICE_42:-${IP_SERVICE_42:-192.168.11.42}}}|g; s|10.200.0.70|${IP_SERVICE_50:-${IP_SERVICE_50:-${IP_SERVICE_50:-${IP_SERVICE_50:-${IP_SERVICE_50:-${IP_SERVICE_50:-192.168.11.50}}}}}}|g; s|10.200.0.80|${IP_SERVICE_43:-${IP_SERVICE_43:-${IP_SERVICE_43:-192.168.11.43}}}|g; s|10.200.0.90|${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-192.168.11.36}}}}}}|g; s|10.200.0.91|${IP_SERVICE_35:-${IP_SERVICE_35:-${IP_SERVICE_35:-${IP_SERVICE_35:-${IP_SERVICE_35:-${IP_SERVICE_35:-192.168.11.35}}}}}}|g; s|10.200.0.92|${IP_MIM_WEB:-192.168.11.37}|g; s|10.200.0.200|${ORDER_REDIS_REPLICA:-${ORDER_REDIS_REPLICA:-${ORDER_REDIS_REPLICA:-192.168.11.46}}}|g; s|10.200.0.201|${IP_SERVICE_47:-${IP_SERVICE_47:-${IP_SERVICE_47:-192.168.11.47}}}|g; s|10.200.0.202|${IP_ORDER_OPENSEARCH:-${IP_ORDER_OPENSEARCH:-${IP_ORDER_OPENSEARCH:-192.168.11.48}}}|g; s|10.200.0.210|${IP_ORDER_HAPROXY:-${IP_ORDER_HAPROXY:-192.168.11.39}}|g; s|10.200.0.230|${IP_SERVICE_51:-${IP_SERVICE_51:-${IP_SERVICE_51:-${IP_SERVICE_51:-${IP_SERVICE_51:-${IP_SERVICE_51:-192.168.11.51}}}}}}|g\" \"\$f\" 2>/dev/null || true
            done
            echo \"Configs updated for CT $vmid\"
        '
    "
}

# Update configs for all Order service containers
for vmid in 10000 10001 10020 10030 10040 10050 10060 10070 10080 10090 10091 10092 10200 10201 10202 10210 10230; do
    wait_for_slot
    run_task "update-config-$vmid" "Update configs CT $vmid" "update_container_configs $vmid"
done

wait_all
log_info "Phase 5 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 6: Install Monitoring Services
# ============================================================================

log_phase "PHASE 6: Installing Monitoring Services"

install_prometheus() {
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec 10200 -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq prometheus >/dev/null 2>&1 || exit 1
            systemctl enable prometheus >/dev/null 2>&1
            systemctl start prometheus >/dev/null 2>&1
            sleep 2
            systemctl is-active prometheus >/dev/null && echo \"Prometheus installed\" || exit 1
        '
    "
}

install_grafana() {
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec 10201 -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq apt-transport-https software-properties-common wget >/dev/null 2>&1 || exit 1
            wget -q -O - https://packages.grafana.com/gpg.key | apt-key add - >/dev/null 2>&1
            echo \"deb https://packages.grafana.com/oss/deb stable main\" > /etc/apt/sources.list.d/grafana.list
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq grafana >/dev/null 2>&1 || exit 1
            systemctl enable grafana-server >/dev/null 2>&1
            systemctl start grafana-server >/dev/null 2>&1
            sleep 2
            systemctl is-active grafana-server >/dev/null && echo \"Grafana installed\" || exit 1
        '
    "
}

wait_for_slot
run_task "prometheus-10200" "Prometheus" "install_prometheus"

wait_for_slot
run_task "grafana-10201" "Grafana" "install_grafana"

wait_all
log_info "Phase 6 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 7: Install Infrastructure Services
# ============================================================================

log_phase "PHASE 7: Installing Infrastructure Services"

install_haproxy() {
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec 10210 -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq haproxy >/dev/null 2>&1 || exit 1
            systemctl enable haproxy >/dev/null 2>&1
            systemctl start haproxy >/dev/null 2>&1
            sleep 2
            systemctl is-active haproxy >/dev/null && echo \"HAProxy installed\" || exit 1
        '
    "
}

install_vault() {
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec 10230 -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq curl unzip >/dev/null 2>&1 || exit 1
            VAULT_VERSION=\"1.15.0\"
            wget -q https://releases.hashicorp.com/vault/\${VAULT_VERSION}/vault_\${VAULT_VERSION}_linux_amd64.zip -O /tmp/vault.zip
            unzip -q /tmp/vault.zip -d /usr/local/bin/
            chmod +x /usr/local/bin/vault
            rm /tmp/vault.zip
            echo \"Vault installed\"
        '
    "
}

wait_for_slot
run_task "haproxy-10210" "HAProxy" "install_haproxy"

wait_for_slot
run_task "vault-10230" "Vault" "install_vault"

wait_all
log_info "Phase 7 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 8: Verify Services
# ============================================================================

log_phase "PHASE 8: Verifying Installed Services"

verify_service() {
    local vmid="$1"
    local service_name="$2"
    
    ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- systemctl is-active $service_name >/dev/null 2>&1 && echo \"$service_name active\" || echo \"$service_name inactive\"
    "
}

log_info "Verifying PostgreSQL services..."
for vmid in 10000 10001 10100 10101; do
    wait_for_slot
    run_task "verify-pg-$vmid" "Verify PostgreSQL $vmid" "verify_service $vmid 'postgresql@15-main'"
done

log_info "Verifying Redis services..."
for vmid in 10020 10120; do
    wait_for_slot
    run_task "verify-redis-$vmid" "Verify Redis $vmid" "verify_service $vmid 'redis-server'"
done

wait_for_slot
run_task "verify-prometheus" "Verify Prometheus" "verify_service 10200 'prometheus'"

wait_for_slot
run_task "verify-grafana" "Verify Grafana" "verify_service 10201 'grafana-server'"

wait_for_slot
run_task "verify-haproxy" "Verify HAProxy" "verify_service 10210 'haproxy'"

wait_all

# ============================================================================
# Final Summary
# ============================================================================

echo ""
log_phase "═══════════════════════════════════════════════════════════"
log_phase "PARALLEL EXECUTION COMPLETE"
log_phase "═══════════════════════════════════════════════════════════"
log_info "Total Tasks: $TOTAL_TASKS"
log_info "Completed: $COMPLETED_TASKS"
log_info "Failed: $FAILED_TASKS"
if [ $TOTAL_TASKS -gt 0 ]; then
    success_rate=$(( COMPLETED_TASKS * 100 / TOTAL_TASKS ))
    log_info "Success Rate: ${success_rate}%"
fi
log_info ""
log_info "Logs: $LOG_DIR"
log_phase "═══════════════════════════════════════════════════════════"

if [ $FAILED_TASKS -eq 0 ]; then
    log_info "✅ ALL TASKS COMPLETED SUCCESSFULLY"
    exit 0
else
    log_error "⚠️  $FAILED_TASKS TASK(S) FAILED - Review logs in $LOG_DIR"
    exit 1
fi