proxmox/scripts/clear-besu-transaction-pools-complete.sh

#!/usr/bin/env bash
# Clear Transaction Pools - Complete Method
# Stops all Besu nodes, clears transaction pools, and restarts in correct order
# For Hyperledger Besu permissioned blockchain

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }
log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; }

# Proxmox hosts
PROXMOX_HOSTS=("ml110" "r630-01")

# RPC nodes (VMID:host)
RPC_NODES=(
    "2101:ml110"  # Core RPC
)

# Validators (VMID:host)
VALIDATORS=(
    "1000:r630-01"
    "1001:r630-01"
    "1002:r630-01"
    "1003:ml110"
    "1004:ml110"
)

log_section "Clear Transaction Pools - Complete Method"

# Step 1: Stop all nodes simultaneously
log_section "Step 1: Stopping All Besu Nodes"

log_info "Stopping RPC nodes..."
for node in "${RPC_NODES[@]}"; do
    IFS=':' read -r vmid host <<< "$node"
    log_info "Stopping RPC node $vmid on $host..."
    ssh -o ConnectTimeout=5 "$host" "pct stop $vmid" 2>/dev/null || log_warn "Could not stop $vmid on $host"
done

log_info "Stopping validators..."
for validator in "${VALIDATORS[@]}"; do
    IFS=':' read -r vmid host <<< "$validator"
    log_info "Stopping validator $vmid on $host..."
    ssh -o ConnectTimeout=5 "$host" "pct stop $vmid" 2>/dev/null || log_warn "Could not stop $vmid on $host"
done

log_info "Waiting 5 seconds for services to fully stop..."
sleep 5

# Step 2: Clear transaction pool databases
log_section "Step 2: Clearing Transaction Pool Databases"

clear_node_pools() {
    local vmid=$1
    local host=$2
    local node_type=$3
    
    log_info "Clearing transaction pools for $node_type $vmid on $host..."
    
    ssh "$host" "pct exec $vmid -- bash -c '
        # Find Besu data directory
        DATA_DIR=\"/data/besu\"
        if [ ! -d \"\$DATA_DIR\" ]; then
            DATA_DIR=\"/var/lib/besu\"
        fi
        
        if [ ! -d \"\$DATA_DIR\" ]; then
            echo \"⚠️  Besu data directory not found\"
            exit 0
        fi
        
        # Clear transaction pool directories
        find \"\$DATA_DIR\" -type d -name \"*pool*\" -exec rm -rf {} + 2>/dev/null || true
        find \"\$DATA_DIR\" -type d -name \"*transaction*\" -exec rm -rf {} + 2>/dev/null || true
        find \"\$DATA_DIR\" -type d -name \"*mempool*\" -exec rm -rf {} + 2>/dev/null || true
        
        # Clear transaction pool files
        find \"\$DATA_DIR\" -type f -name \"*transaction*\" -delete 2>/dev/null || true
        find \"\$DATA_DIR\" -type f -name \"*pool*\" -delete 2>/dev/null || true
        find \"\$DATA_DIR\" -type f -name \"*.ldb\" -delete 2>/dev/null || true
        
        # Clear caches
        if [ -d \"\$DATA_DIR/caches\" ]; then
            rm -rf \"\$DATA_DIR/caches/*\" 2>/dev/null || true
        fi
        
        echo \"✅ Transaction pools cleared for $node_type $vmid\"
    '" 2>&1 || log_warn "Could not clear pools for $vmid on $host"
}

log_info "Clearing RPC node pools..."
for node in "${RPC_NODES[@]}"; do
    IFS=':' read -r vmid host <<< "$node"
    clear_node_pools "$vmid" "$host" "RPC"
done

log_info "Clearing validator pools..."
for validator in "${VALIDATORS[@]}"; do
    IFS=':' read -r vmid host <<< "$validator"
    clear_node_pools "$vmid" "$host" "Validator"
done

# Step 3: Start validators first
log_section "Step 3: Starting Validators"

log_info "Starting validators (must start before RPC nodes)..."
for validator in "${VALIDATORS[@]}"; do
    IFS=':' read -r vmid host <<< "$validator"
    log_info "Starting validator $vmid on $host..."
    ssh -o ConnectTimeout=5 "$host" "pct start $vmid" 2>/dev/null || log_warn "Could not start $vmid on $host"
    sleep 2
done

log_info "Waiting 10 seconds for validators to initialize..."
sleep 10

# Step 4: Start RPC nodes
log_section "Step 4: Starting RPC Nodes"

log_info "Starting RPC nodes..."
for node in "${RPC_NODES[@]}"; do
    IFS=':' read -r vmid host <<< "$node"
    log_info "Starting RPC node $vmid on $host..."
    ssh -o ConnectTimeout=5 "$host" "pct start $vmid" 2>/dev/null || log_warn "Could not start $vmid on $host"
    sleep 2
done

log_info "Waiting 20 seconds for network stabilization..."
sleep 20

# Step 5: Verify
log_section "Step 5: Verification"

log_info "Verifying RPC nodes are online..."
RPC_URL="http://${RPC_CORE_1}:8545"
for i in {1..10}; do
    if timeout 5 cast chain-id --rpc-url "$RPC_URL" 2>/dev/null | grep -q "138"; then
        log_success "RPC node is online"
        break
    fi
    if [ $i -eq 10 ]; then
        log_error "RPC node not responding after 10 attempts"
    else
        log_info "Waiting for RPC node... (attempt $i/10)"
        sleep 3
    fi
done

log_info "Checking block production..."
BLOCK1=$(cast block-number --rpc-url "$RPC_URL" 2>/dev/null || echo "0")
sleep 5
BLOCK2=$(cast block-number --rpc-url "$RPC_URL" 2>/dev/null || echo "0")
if [ "$BLOCK2" -gt "$BLOCK1" ]; then
    log_success "Block production active (blocks advancing)"
else
    log_warn "Block production may be stalled"
fi

log_section "Transaction Pool Clear Complete"

log_success "✅ All nodes restarted"
log_success "✅ Transaction pools cleared"
log_info "⏳ Network stabilizing - wait 30 seconds before deploying"