proxmox/scripts/clear-all-transaction-pools.sh

#!/usr/bin/env bash
# Clear transaction pools on all Besu nodes (RPC and Validators)
# This script clears transaction pool databases to remove stuck transactions

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


PROXMOX_USER="${PROXMOX_USER:-root}"
PROXMOX_ML110="${PROXMOX_ML110:-192.168.11.10}"
PROXMOX_R630="${PROXMOX_R630:-192.168.11.11}"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }
log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; }

echo "=== Clear Transaction Pools on All Nodes ==="
echo ""

# Function to clear transaction pool for a node
clear_node_pool() {
    local VMID=$1
    local HOST=$2
    local NODE_TYPE=$3
    local SSH_TARGET="${PROXMOX_USER}@${HOST}"
    
    log_info "Clearing transaction pool for $NODE_TYPE (VMID $VMID on $HOST)..."
    
    # Stop the service
    log_info "  Stopping service..."
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \
        "pct exec $VMID -- systemctl stop besu-validator 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc.service 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc 2>/dev/null || true" 2>&1 | grep -v "Configuration file" || true
    
    sleep 2
    
    # Find and clear transaction pool database
    log_info "  Clearing transaction pool database..."
    CLEAR_RESULT=$(ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \
        "pct exec $VMID -- bash -c '
        DATA_DIRS=\"/data/besu /var/lib/besu\"
        for DATA_DIR in \$DATA_DIRS; do
            if [ -d \"\$DATA_DIR\" ]; then
                # Find transaction pool database files
                find \"\$DATA_DIR\" -type d -name \"*pool*\" -exec rm -rf {} \; 2>/dev/null || true
                find \"\$DATA_DIR\" -type f -name \"*transaction*\" -delete 2>/dev/null || true
                find \"\$DATA_DIR\" -type f -name \"*txpool*\" -delete 2>/dev/null || true
                echo \"Cleared: \$DATA_DIR\"
            fi
        done
        '" 2>&1 | grep -v "Configuration file" || echo "Cleared")
    
    if [ -n "$CLEAR_RESULT" ]; then
        log_success "  Transaction pool cleared"
    else
        log_warn "  Could not clear transaction pool (may not exist)"
    fi
    
    # Restart the service
    log_info "  Restarting service..."
    ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \
        "pct exec $VMID -- systemctl start besu-validator 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc.service 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc 2>/dev/null || true" 2>&1 | grep -v "Configuration file" || true
    
    sleep 3
    
    # Verify service is running
    STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" \
        "pct exec $VMID -- systemctl is-active besu-validator 2>/dev/null || pct exec $VMID -- systemctl is-active besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl is-active besu-rpc.service 2>/dev/null || pct exec $VMID -- systemctl is-active besu-rpc 2>/dev/null || echo 'unknown'" 2>&1 | grep -v "Configuration file" || echo "unknown")
    
    if [ "$STATUS" = "active" ]; then
        log_success "  Service restarted and active"
    else
        log_warn "  Service status: $STATUS"
    fi
    
    echo ""
}

# Clear validators
log_section "Clearing Validator Transaction Pools"

VALIDATORS=(
    "1000:$PROXMOX_R630:Validator"
    "1001:$PROXMOX_R630:Validator"
    "1002:$PROXMOX_R630:Validator"
    "1003:$PROXMOX_ML110:Validator"
    "1004:$PROXMOX_ML110:Validator"
)

for validator in "${VALIDATORS[@]}"; do
    IFS=':' read -r VMID HOST TYPE <<< "$validator"
    clear_node_pool "$VMID" "$HOST" "$TYPE"
done

# Clear RPC Core (2101)
log_section "Clearing RPC Transaction Pool (2101)"

if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${PROXMOX_ML110}" \
    "pct list | grep -q '2101'" 2>/dev/null; then
    clear_node_pool 2101 "$PROXMOX_ML110" "RPC"
elif ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${PROXMOX_R630}" \
    "pct list | grep -q '2101'" 2>/dev/null; then
    clear_node_pool 2101 "$PROXMOX_R630" "RPC"
else
    log_warn "RPC node (2101) not found on either host"
fi

# Clear RPC Public (2201) — often used when Core is down; ensures deploy txs not stuck
log_section "Clearing RPC Public (2201)"
R630_02="${PROXMOX_R630_02:-${PROXMOX_HOST_R630_02:-192.168.11.12}}"
if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${R630_02}" \
    "pct list | grep -q '2201'" 2>/dev/null; then
    clear_node_pool 2201 "$R630_02" "RPC Public"
else
    log_warn "RPC Public (2201) not found on ${R630_02}"
fi

log_section "Transaction Pool Clear Complete"

echo "Next steps:"
echo "  1. Wait 30-60 seconds for nodes to fully restart"
echo "  2. Check pending transactions: bash scripts/check-pending-transactions.sh"
echo "  3. Monitor health: bash scripts/monitoring/monitor-blockchain-health.sh"