proxmox/scripts/flush-validator-mempools.sh

#!/usr/bin/env bash
# Flush mempools on all validator nodes by restarting Besu services
# Usage: ./flush-validator-mempools.sh

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_detail() { echo -e "${CYAN}[DETAIL]${NC} $1"; }

# Validator nodes (VMID -> IP)
declare -A VALIDATORS=(
    [1000]="${IP_VALIDATOR_0:-${IP_VALIDATOR_0:-${IP_VALIDATOR_0:-${IP_VALIDATOR_0:-${IP_VALIDATOR_0:-${IP_VALIDATOR_0:-192.168.11.100}}}}}}"
    [1001]="${IP_VALIDATOR_1:-${IP_VALIDATOR_1:-${IP_VALIDATOR_1:-${IP_VALIDATOR_1:-${IP_VALIDATOR_1:-${IP_VALIDATOR_1:-192.168.11.101}}}}}}"
    [1002]="${IP_VALIDATOR_2:-${IP_VALIDATOR_2:-${IP_VALIDATOR_2:-${IP_VALIDATOR_2:-${IP_VALIDATOR_2:-${IP_VALIDATOR_2:-192.168.11.102}}}}}}"
    [1003]="${IP_VALIDATOR_3:-${IP_VALIDATOR_3:-${IP_VALIDATOR_3:-${IP_VALIDATOR_3:-${IP_VALIDATOR_3:-${IP_VALIDATOR_3:-192.168.11.103}}}}}}"
    [1004]="${IP_VALIDATOR_4:-${IP_VALIDATOR_4:-${IP_VALIDATOR_4:-${IP_VALIDATOR_4:-${IP_VALIDATOR_4:-${IP_VALIDATOR_4:-192.168.11.104}}}}}}"
)

echo "========================================="
echo "Flush Validator Mempools"
echo "========================================="
echo ""

# Function to check if container is running
check_container() {
    local vmid=$1
    if pct status "$vmid" 2>/dev/null | grep -q "running"; then
        return 0
    else
        return 1
    fi
}

# Function to check if Besu service exists
check_besu_service() {
    local vmid=$1
    if pct exec "$vmid" -- systemctl list-units --type=service 2>/dev/null | grep -q "besu-validator"; then
        return 0
    else
        return 1
    fi
}

# Function to restart Besu service
restart_besu() {
    local vmid=$1
    local ip=$2
    
    log_info "VMID $vmid ($ip): Restarting Besu validator service..."
    
    if pct exec "$vmid" -- systemctl restart besu-validator.service 2>/dev/null; then
        log_success "✓ Service restart command sent"
        sleep 3
        
        # Check if service is active
        if pct exec "$vmid" -- systemctl is-active --quiet besu-validator.service 2>/dev/null; then
            log_success "✓ Service is active"
            return 0
        else
            log_warn "⚠ Service may still be starting..."
            return 1
        fi
    else
        log_error "✗ Failed to restart service"
        return 1
    fi
}

# Function to check Besu process
check_besu_process() {
    local vmid=$1
    if pct exec "$vmid" -- pgrep -f "besu.*validator" >/dev/null 2>&1; then
        return 0
    else
        return 1
    fi
}

# Main execution
SUCCESS_COUNT=0
FAILED_COUNT=0
SKIPPED_COUNT=0

for vmid in "${!VALIDATORS[@]}"; do
    ip="${VALIDATORS[$vmid]}"
    
    echo ""
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "Validator $vmid ($ip)"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    
    # Check if container is running
    if ! check_container "$vmid"; then
        log_warn "⚠ Container not running - skipping"
        ((SKIPPED_COUNT++))
        continue
    fi
    
    log_success "✓ Container is running"
    
    # Check if Besu service exists
    if ! check_besu_service "$vmid"; then
        log_warn "⚠ Besu validator service not found - checking process..."
        if check_besu_process "$vmid"; then
            log_info "Besu process is running (may be running outside systemd)"
            log_warn "⚠ Cannot restart via systemd - manual intervention may be needed"
            ((SKIPPED_COUNT++))
        else
            log_warn "⚠ No Besu process found"
            ((SKIPPED_COUNT++))
        fi
        continue
    fi
    
    # Check current service status
    log_detail "Checking current service status..."
    SERVICE_STATUS=$(pct exec "$vmid" -- systemctl is-active besu-validator.service 2>/dev/null || echo "unknown")
    log_detail "Service status: $SERVICE_STATUS"
    
    # Restart service (this flushes mempool)
    if restart_besu "$vmid" "$ip"; then
        log_success "✓ Mempool flushed (service restarted)"
        ((SUCCESS_COUNT++))
    else
        log_error "✗ Failed to flush mempool"
        ((FAILED_COUNT++))
    fi
    
    # Wait a moment before next node
    sleep 2
done

# Summary
echo ""
echo "========================================="
echo "Summary"
echo "========================================="
echo ""
log_success "✓ Successfully flushed: $SUCCESS_COUNT"
if [ "$FAILED_COUNT" -gt 0 ]; then
    log_error "✗ Failed: $FAILED_COUNT"
fi
if [ "$SKIPPED_COUNT" -gt 0 ]; then
    log_warn "⚠ Skipped: $SKIPPED_COUNT"
fi
echo ""

if [ "$SUCCESS_COUNT" -gt 0 ]; then
    log_info "Waiting 10 seconds for services to stabilize..."
    sleep 10
    
    log_info "Verifying services are running..."
    for vmid in "${!VALIDATORS[@]}"; do
        ip="${VALIDATORS[$vmid]}"
        if check_container "$vmid" && check_besu_process "$vmid"; then
            log_success "✓ VMID $vmid: Besu is running"
        else
            log_warn "⚠ VMID $vmid: Besu may not be running"
        fi
    done
fi

echo ""
log_success "========================================="
log_success "Mempool Flush Complete!"
log_success "========================================="
log_info ""
log_info "Next steps:"
log_info "  1. Wait for all validators to sync"
log_info "  2. Run: ./scripts/configure-ethereum-mainnet-final.sh"
log_info ""