#!/bin/bash
# Complete All Incomplete Tasks - Comprehensive Parallel Execution
# Executes all pending tasks across all 33 containers in parallel

set -uo pipefail

NODE_IP="192.168.11.11"
MAX_PARALLEL=15  # Increased parallel operations
LOG_DIR="/tmp/parallel-tasks-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$LOG_DIR"

# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_phase() { echo -e "${BLUE}[PHASE]${NC} $1"; }

# Task tracking
declare -A TASK_PIDS=()
TOTAL_TASKS=0
COMPLETED_TASKS=0
FAILED_TASKS=0

# Function to run task in background
run_task() {
    local task_id="$1"
    local task_name="$2"
    local task_command="$3"
    
    (
        eval "$task_command" > "$LOG_DIR/${task_id}.log" 2>&1
        local exit_code=$?
        echo "$exit_code" > "$LOG_DIR/${task_id}.exit"
        exit $exit_code
    ) &
    
    TASK_PIDS[$task_id]=$!
    ((TOTAL_TASKS++))
    log_info "Started: $task_name"
}

# Wait for available slot
wait_for_slot() {
    while [ ${#TASK_PIDS[@]} -ge $MAX_PARALLEL ]; do
        for task_id in "${!TASK_PIDS[@]}"; do
            if ! kill -0 ${TASK_PIDS[$task_id]} 2>/dev/null; then
                wait ${TASK_PIDS[$task_id]}
                exit_code=$(cat "$LOG_DIR/${task_id}.exit" 2>/dev/null || echo "1")
                if [ "$exit_code" = "0" ]; then
                    ((COMPLETED_TASKS++))
                else
                    ((FAILED_TASKS++))
                fi
                unset TASK_PIDS[$task_id]
            fi
        done
        sleep 0.3
    done
}

# Wait for all tasks
wait_all() {
    for task_id in "${!TASK_PIDS[@]}"; do
        wait ${TASK_PIDS[$task_id]} 2>/dev/null || true
        exit_code=$(cat "$LOG_DIR/${task_id}.exit" 2>/dev/null || echo "1")
        if [ "$exit_code" = "0" ]; then
            ((COMPLETED_TASKS++))
        else
            ((FAILED_TASKS++))
        fi
    done
    declare -A TASK_PIDS=()
}

echo "═══════════════════════════════════════════════════════════"
echo "Complete All Tasks - Comprehensive Parallel Execution"
echo "═══════════════════════════════════════════════════════════"
echo "Node: $NODE_IP"
echo "Log Dir: $LOG_DIR"
echo "Max Parallel: $MAX_PARALLEL"
echo ""

# ============================================================================
# PHASE 1: Install Database Services (PostgreSQL)
# ============================================================================

log_phase "PHASE 1: Installing PostgreSQL (4 containers)"

install_postgresql() {
    local vmid="$1"
    local hostname="$2"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq postgresql-15 postgresql-contrib-15 >/dev/null 2>&1 || exit 1
            
            # Configure PostgreSQL
            sed -i \"s/#listen_addresses = .*/listen_addresses = '\''*'\''/\" /etc/postgresql/15/main/postgresql.conf 2>/dev/null || true
            echo \"host all all 0.0.0.0/0 md5\" >> /etc/postgresql/15/main/pg_hba.conf 2>/dev/null || true
            
            systemctl enable postgresql@15-main >/dev/null 2>&1
            systemctl start postgresql@15-main >/dev/null 2>&1
            sleep 3
            systemctl is-active postgresql@15-main >/dev/null && echo \"PostgreSQL installed\" || exit 1
        '
    "
}

for vmid in 10000 10001 10100 10101; do
    wait_for_slot
    hostname=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct config $vmid 2>/dev/null | grep '^hostname:' | sed 's/^hostname: //'" || echo "unknown")
    run_task "pg-$vmid" "PostgreSQL CT $vmid" "install_postgresql $vmid '$hostname'"
done

wait_all
log_info "Phase 1 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 2: Install Redis
# ============================================================================

log_phase "PHASE 2: Installing Redis (2 containers)"

install_redis() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq redis-server >/dev/null 2>&1 || exit 1
            
            sed -i \"s/^bind .*/bind 0.0.0.0/\" /etc/redis/redis.conf 2>/dev/null || true
            systemctl enable redis-server >/dev/null 2>&1
            systemctl restart redis-server >/dev/null 2>&1
            sleep 2
            systemctl is-active redis-server >/dev/null && echo \"Redis installed\" || exit 1
        '
    "
}

for vmid in 10020 10120; do
    wait_for_slot
    run_task "redis-$vmid" "Redis CT $vmid" "install_redis $vmid"
done

wait_all
log_info "Phase 2 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 3: Install Node.js Runtime
# ============================================================================

log_phase "PHASE 3: Installing Node.js (14 containers)"

install_nodejs() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq curl ca-certificates gnupg >/dev/null 2>&1 || exit 1
            
            curl -fsSL https://deb.nodesource.com/setup_18.x | bash - >/dev/null 2>&1 || exit 1
            apt-get install -y -qq nodejs >/dev/null 2>&1 || exit 1
            npm install -g pm2 >/dev/null 2>&1 || exit 1
            
            node --version >/dev/null && npm --version >/dev/null && echo \"Node.js installed\" || exit 1
        '
    "
}

for vmid in 10030 10040 10050 10060 10070 10080 10090 10091 10092 10130 10150 10151; do
    wait_for_slot
    run_task "nodejs-$vmid" "Node.js CT $vmid" "install_nodejs $vmid"
done

wait_all
log_info "Phase 3 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 4: Configure PostgreSQL Databases
# ============================================================================

log_phase "PHASE 4: Configuring PostgreSQL Databases"

configure_order_db() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            sudo -u postgres psql << \"EOF\" >/dev/null 2>&1
                CREATE DATABASE order_db;
                CREATE USER order_user WITH PASSWORD '\''order_password'\'';
                GRANT ALL PRIVILEGES ON DATABASE order_db TO order_user;
                ALTER DATABASE order_db OWNER TO order_user;
EOF
            echo \"Order DB configured\"
        '
    "
}

configure_dbis_db() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            sudo -u postgres psql << \"EOF\" >/dev/null 2>&1
                CREATE DATABASE dbis_core;
                CREATE USER dbis WITH PASSWORD '\''8cba649443f97436db43b34ab2c0e75b5cf15611bef9c099cee6fb22cc3d7771'\'';
                GRANT ALL PRIVILEGES ON DATABASE dbis_core TO dbis;
                ALTER DATABASE dbis_core OWNER TO dbis;
EOF
            echo \"DBIS DB configured\"
        '
    "
}

for vmid in 10000 10001; do
    wait_for_slot
    run_task "pg-config-order-$vmid" "Configure Order DB $vmid" "configure_order_db $vmid"
done

for vmid in 10100 10101; do
    wait_for_slot
    run_task "pg-config-dbis-$vmid" "Configure DBIS DB $vmid" "configure_dbis_db $vmid"
done

wait_all
log_info "Phase 4 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 5: Update Application Configurations
# ============================================================================

log_phase "PHASE 5: Updating Application Configurations (IP addresses)"

update_container_configs() {
    local vmid="$1"
    
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- bash -c '
            # Update .env files
            find /opt /home /root -name \".env\" -type f 2>/dev/null | while read f; do
                [ -r \"\$f\" ] && sed -i \"s|10.200.0.10|192.168.11.44|g; s|10.200.0.11|192.168.11.45|g; s|10.200.0.20|192.168.11.38|g; s|10.200.0.30|192.168.11.40|g; s|10.200.0.40|192.168.11.41|g; s|10.200.0.50|192.168.11.49|g; s|10.200.0.60|192.168.11.42|g; s|10.200.0.70|192.168.11.50|g; s|10.200.0.80|192.168.11.43|g; s|10.200.0.90|192.168.11.36|g; s|10.200.0.91|192.168.11.35|g; s|10.200.0.92|192.168.11.37|g; s|10.200.0.200|192.168.11.46|g; s|10.200.0.201|192.168.11.47|g; s|10.200.0.202|192.168.11.48|g; s|10.200.0.210|192.168.11.39|g; s|10.200.0.230|192.168.11.51|g\" \"\$f\" 2>/dev/null || true
            done
            echo \"Configs updated for CT $vmid\"
        '
    "
}

# Update configs for all Order service containers
for vmid in 10000 10001 10020 10030 10040 10050 10060 10070 10080 10090 10091 10092 10200 10201 10202 10210 10230; do
    wait_for_slot
    run_task "update-config-$vmid" "Update configs CT $vmid" "update_container_configs $vmid"
done

wait_all
log_info "Phase 5 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 6: Install Monitoring Services
# ============================================================================

log_phase "PHASE 6: Installing Monitoring Services"

install_prometheus() {
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec 10200 -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq prometheus >/dev/null 2>&1 || exit 1
            systemctl enable prometheus >/dev/null 2>&1
            systemctl start prometheus >/dev/null 2>&1
            sleep 2
            systemctl is-active prometheus >/dev/null && echo \"Prometheus installed\" || exit 1
        '
    "
}

install_grafana() {
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec 10201 -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq apt-transport-https software-properties-common wget >/dev/null 2>&1 || exit 1
            wget -q -O - https://packages.grafana.com/gpg.key | apt-key add - >/dev/null 2>&1
            echo \"deb https://packages.grafana.com/oss/deb stable main\" > /etc/apt/sources.list.d/grafana.list
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq grafana >/dev/null 2>&1 || exit 1
            systemctl enable grafana-server >/dev/null 2>&1
            systemctl start grafana-server >/dev/null 2>&1
            sleep 2
            systemctl is-active grafana-server >/dev/null && echo \"Grafana installed\" || exit 1
        '
    "
}

wait_for_slot
run_task "prometheus-10200" "Prometheus" "install_prometheus"

wait_for_slot
run_task "grafana-10201" "Grafana" "install_grafana"

wait_all
log_info "Phase 6 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 7: Install Infrastructure Services
# ============================================================================

log_phase "PHASE 7: Installing Infrastructure Services"

install_haproxy() {
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec 10210 -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq haproxy >/dev/null 2>&1 || exit 1
            systemctl enable haproxy >/dev/null 2>&1
            systemctl start haproxy >/dev/null 2>&1
            sleep 2
            systemctl is-active haproxy >/dev/null && echo \"HAProxy installed\" || exit 1
        '
    "
}

install_vault() {
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec 10230 -- bash -c '
            export DEBIAN_FRONTEND=noninteractive
            apt-get update -qq >/dev/null 2>&1
            apt-get install -y -qq curl unzip >/dev/null 2>&1 || exit 1
            VAULT_VERSION=\"1.15.0\"
            wget -q https://releases.hashicorp.com/vault/\${VAULT_VERSION}/vault_\${VAULT_VERSION}_linux_amd64.zip -O /tmp/vault.zip
            unzip -q /tmp/vault.zip -d /usr/local/bin/
            chmod +x /usr/local/bin/vault
            rm /tmp/vault.zip
            echo \"Vault installed\"
        '
    "
}

wait_for_slot
run_task "haproxy-10210" "HAProxy" "install_haproxy"

wait_for_slot
run_task "vault-10230" "Vault" "install_vault"

wait_all
log_info "Phase 7 complete: $COMPLETED_TASKS completed, $FAILED_TASKS failed"

# ============================================================================
# PHASE 8: Verify Services
# ============================================================================

log_phase "PHASE 8: Verifying Installed Services"

verify_service() {
    local vmid="$1"
    local service_name="$2"
    
    ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} "
        pct exec $vmid -- systemctl is-active $service_name >/dev/null 2>&1 && echo \"$service_name active\" || echo \"$service_name inactive\"
    "
}

log_info "Verifying PostgreSQL services..."
for vmid in 10000 10001 10100 10101; do
    wait_for_slot
    run_task "verify-pg-$vmid" "Verify PostgreSQL $vmid" "verify_service $vmid 'postgresql@15-main'"
done

log_info "Verifying Redis services..."
for vmid in 10020 10120; do
    wait_for_slot
    run_task "verify-redis-$vmid" "Verify Redis $vmid" "verify_service $vmid 'redis-server'"
done

wait_for_slot
run_task "verify-prometheus" "Verify Prometheus" "verify_service 10200 'prometheus'"

wait_for_slot
run_task "verify-grafana" "Verify Grafana" "verify_service 10201 'grafana-server'"

wait_for_slot
run_task "verify-haproxy" "Verify HAProxy" "verify_service 10210 'haproxy'"

wait_all

# ============================================================================
# Final Summary
# ============================================================================

echo ""
log_phase "═══════════════════════════════════════════════════════════"
log_phase "PARALLEL EXECUTION COMPLETE"
log_phase "═══════════════════════════════════════════════════════════"
log_info "Total Tasks: $TOTAL_TASKS"
log_info "Completed: $COMPLETED_TASKS"
log_info "Failed: $FAILED_TASKS"
if [ $TOTAL_TASKS -gt 0 ]; then
    success_rate=$(( COMPLETED_TASKS * 100 / TOTAL_TASKS ))
    log_info "Success Rate: ${success_rate}%"
fi
log_info ""
log_info "Logs: $LOG_DIR"
log_phase "═══════════════════════════════════════════════════════════"

if [ $FAILED_TASKS -eq 0 ]; then
    log_info "✅ ALL TASKS COMPLETED SUCCESSFULLY"
    exit 0
else
    log_error "⚠️  $FAILED_TASKS TASK(S) FAILED - Review logs in $LOG_DIR"
    exit 1
fi