#!/usr/bin/env bash # Clear transaction pools on all Besu nodes (RPC and Validators) # This script clears transaction pool databases to remove stuck transactions set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true PROXMOX_USER="${PROXMOX_USER:-root}" PROXMOX_ML110="${PROXMOX_ML110:-192.168.11.10}" PROXMOX_R630="${PROXMOX_R630:-192.168.11.11}" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } log_error() { echo -e "${RED}[✗]${NC} $1"; } log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; } echo "=== Clear Transaction Pools on All Nodes ===" echo "" # Function to clear transaction pool for a node clear_node_pool() { local VMID=$1 local HOST=$2 local NODE_TYPE=$3 local SSH_TARGET="${PROXMOX_USER}@${HOST}" log_info "Clearing transaction pool for $NODE_TYPE (VMID $VMID on $HOST)..." # Stop the service log_info " Stopping service..." ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \ "pct exec $VMID -- systemctl stop besu-validator 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc.service 2>/dev/null || pct exec $VMID -- systemctl stop besu-rpc 2>/dev/null || true" 2>&1 | grep -v "Configuration file" || true sleep 2 # Find and clear transaction pool database log_info " Clearing transaction pool database..." CLEAR_RESULT=$(ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \ "pct exec $VMID -- bash -c ' DATA_DIRS=\"/data/besu /var/lib/besu\" for DATA_DIR in \$DATA_DIRS; do if [ -d \"\$DATA_DIR\" ]; then # Find transaction pool database files find \"\$DATA_DIR\" -type d -name \"*pool*\" -exec rm -rf {} \; 2>/dev/null || true find \"\$DATA_DIR\" -type f -name \"*transaction*\" -delete 2>/dev/null || true find \"\$DATA_DIR\" -type f -name \"*txpool*\" -delete 2>/dev/null || true echo \"Cleared: \$DATA_DIR\" fi done '" 2>&1 | grep -v "Configuration file" || echo "Cleared") if [ -n "$CLEAR_RESULT" ]; then log_success " Transaction pool cleared" else log_warn " Could not clear transaction pool (may not exist)" fi # Restart the service log_info " Restarting service..." ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$SSH_TARGET" \ "pct exec $VMID -- systemctl start besu-validator 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc.service 2>/dev/null || pct exec $VMID -- systemctl start besu-rpc 2>/dev/null || true" 2>&1 | grep -v "Configuration file" || true sleep 3 # Verify service is running STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" \ "pct exec $VMID -- systemctl is-active besu-validator 2>/dev/null || pct exec $VMID -- systemctl is-active besu-rpc-core 2>/dev/null || pct exec $VMID -- systemctl is-active besu-rpc.service 2>/dev/null || pct exec $VMID -- systemctl is-active besu-rpc 2>/dev/null || echo 'unknown'" 2>&1 | grep -v "Configuration file" || echo "unknown") if [ "$STATUS" = "active" ]; then log_success " Service restarted and active" else log_warn " Service status: $STATUS" fi echo "" } # Clear validators log_section "Clearing Validator Transaction Pools" VALIDATORS=( "1000:$PROXMOX_R630:Validator" "1001:$PROXMOX_R630:Validator" "1002:$PROXMOX_R630:Validator" "1003:$PROXMOX_ML110:Validator" "1004:$PROXMOX_ML110:Validator" ) for validator in "${VALIDATORS[@]}"; do IFS=':' read -r VMID HOST TYPE <<< "$validator" clear_node_pool "$VMID" "$HOST" "$TYPE" done # Clear RPC Core (2101) log_section "Clearing RPC Transaction Pool (2101)" if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${PROXMOX_ML110}" \ "pct list | grep -q '2101'" 2>/dev/null; then clear_node_pool 2101 "$PROXMOX_ML110" "RPC" elif ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${PROXMOX_R630}" \ "pct list | grep -q '2101'" 2>/dev/null; then clear_node_pool 2101 "$PROXMOX_R630" "RPC" else log_warn "RPC node (2101) not found on either host" fi # Clear RPC Public (2201) — often used when Core is down; ensures deploy txs not stuck log_section "Clearing RPC Public (2201)" R630_02="${PROXMOX_R630_02:-${PROXMOX_HOST_R630_02:-192.168.11.12}}" if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${R630_02}" \ "pct list | grep -q '2201'" 2>/dev/null; then clear_node_pool 2201 "$R630_02" "RPC Public" else log_warn "RPC Public (2201) not found on ${R630_02}" fi log_section "Transaction Pool Clear Complete" echo "Next steps:" echo " 1. Wait 30-60 seconds for nodes to fully restart" echo " 2. Check pending transactions: bash scripts/check-pending-transactions.sh" echo " 3. Monitor health: bash scripts/monitoring/monitor-blockchain-health.sh"