#!/usr/bin/env bash # Flush all stuck transactions from all Besu nodes # This script tries multiple methods to clear transaction pools # Usage: ./flush-all-stuck-transactions.sh set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } log_detail() { echo -e "${CYAN}[DETAIL]${NC} $1"; } # Configuration RPC_NODES=(2500 2501 2502) VALIDATORS=(1000 1001 1002 1003 1004) SENTRIES=(1500 1501 1502 1503) RPC_URL="http://${RPC_ALLTRA_1:-${RPC_ALLTRA_1:-192.168.11.250}}:8545" # Use Core RPC (VMID 2500) with TXPOOL enabled # Check if pct is available (for Proxmox host operations) if command -v pct &>/dev/null; then ON_PROXMOX=true else ON_PROXMOX=false fi echo "=========================================" echo "Flush All Stuck Transactions" echo "=========================================" echo "" # Function to check if RPC API method is available check_rpc_method() { local method=$1 local rpc_url=${2:-$RPC_URL} RPC_MODULES=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"rpc_modules","params":[],"id":1}' \ "$rpc_url" 2>/dev/null || echo "") if echo "$RPC_MODULES" | jq -r ".result | keys[]" 2>/dev/null | grep -qi "^${method%_*}"; then return 0 else return 1 fi } # Function to get transaction pool content get_txpool_content() { local rpc_url=${1:-$RPC_URL} # Try txpool_besuTransactions (Besu-specific) TXPOOL=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"txpool_besuTransactions","params":[],"id":1}' \ "$rpc_url" 2>/dev/null || echo "") if echo "$TXPOOL" | jq -e '.result' >/dev/null 2>&1; then echo "$TXPOOL" return 0 fi # Try txpool_content (standard) TXPOOL=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"txpool_content","params":[],"id":1}' \ "$rpc_url" 2>/dev/null || echo "") if echo "$TXPOOL" | jq -e '.result' >/dev/null 2>&1; then echo "$TXPOOL" return 0 fi return 1 } # Method 1: Try to clear using TXPOOL API method_1_txpool_clear() { log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Method 1: Clear via TXPOOL API" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Check if TXPOOL is enabled if ! check_rpc_method "txpool" "$RPC_URL"; then log_warn "⚠ TXPOOL API not enabled on RPC node" log_info "Skipping TXPOOL API method" return 1 fi log_info "TXPOOL API is enabled, attempting to clear..." # Try txpool_besuClear (Besu-specific) log_detail "Trying txpool_besuClear..." CLEAR_RESULT=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"txpool_besuClear","params":[],"id":1}' \ "$RPC_URL" 2>/dev/null || echo "") if echo "$CLEAR_RESULT" | jq -e '.result == true' >/dev/null 2>&1; then log_success "✓ Transaction pool cleared using txpool_besuClear" return 0 elif echo "$CLEAR_RESULT" | jq -e '.error' >/dev/null 2>&1; then ERROR_MSG=$(echo "$CLEAR_RESULT" | jq -r '.error.message // "Unknown error"' 2>/dev/null || echo "Unknown") log_detail "txpool_besuClear failed: $ERROR_MSG" fi # Try txpool_clear (alternative) log_detail "Trying txpool_clear..." CLEAR_RESULT=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"txpool_clear","params":[],"id":1}' \ "$RPC_URL" 2>/dev/null || echo "") if echo "$CLEAR_RESULT" | jq -e '.result == true' >/dev/null 2>&1; then log_success "✓ Transaction pool cleared using txpool_clear" return 0 elif echo "$CLEAR_RESULT" | jq -e '.error' >/dev/null 2>&1; then ERROR_MSG=$(echo "$CLEAR_RESULT" | jq -r '.error.message // "Unknown error"' 2>/dev/null || echo "Unknown") log_detail "txpool_clear failed: $ERROR_MSG" fi log_warn "⚠ TXPOOL clear methods not available or failed" return 1 } # Method 2: Restart all Besu services (clears in-memory pools) method_2_restart_services() { log_info "" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Method 2: Restart All Besu Services" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" if [ "$ON_PROXMOX" != true ]; then log_warn "⚠ Not on Proxmox host - cannot restart services" return 1 fi RESTART_SUCCESS=0 RESTART_TOTAL=0 # Restart validators log_info "Restarting validators..." for vmid in "${VALIDATORS[@]}"; do if pct status "$vmid" 2>/dev/null | grep -q "running"; then ((RESTART_TOTAL++)) if pct exec "$vmid" -- systemctl restart besu-validator.service 2>/dev/null; then log_success "✓ VMID $vmid (validator) restarted" ((RESTART_SUCCESS++)) else log_error "✗ VMID $vmid (validator) restart failed" fi fi done # Restart sentries log_info "Restarting sentries..." for vmid in "${SENTRIES[@]}"; do if pct status "$vmid" 2>/dev/null | grep -q "running"; then ((RESTART_TOTAL++)) if pct exec "$vmid" -- systemctl restart besu-sentry.service 2>/dev/null; then log_success "✓ VMID $vmid (sentry) restarted" ((RESTART_SUCCESS++)) else log_error "✗ VMID $vmid (sentry) restart failed" fi fi done # Restart RPC nodes log_info "Restarting RPC nodes..." for vmid in "${RPC_NODES[@]}"; do if pct status "$vmid" 2>/dev/null | grep -q "running"; then ((RESTART_TOTAL++)) if pct exec "$vmid" -- systemctl restart besu-rpc.service 2>/dev/null; then log_success "✓ VMID $vmid (RPC) restarted" ((RESTART_SUCCESS++)) else log_error "✗ VMID $vmid (RPC) restart failed" fi fi done log_info "Services restarted: $RESTART_SUCCESS/$RESTART_TOTAL" if [ $RESTART_SUCCESS -gt 0 ]; then log_info "Waiting 15 seconds for services to stabilize..." sleep 15 return 0 else return 1 fi } # Method 3: Clear transaction pool database files method_3_clear_database() { log_info "" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Method 3: Clear Transaction Pool Databases" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" if [ "$ON_PROXMOX" != true ]; then log_warn "⚠ Not on Proxmox host - cannot clear database files" return 1 fi log_warn "⚠ WARNING: This will stop Besu nodes and clear transaction pool databases" log_warn "⚠ All pending transactions will be lost" echo "" read -p "Continue with database clear? (yes/no): " CONFIRM if [ "$CONFIRM" != "yes" ]; then log_info "Skipping database clear" return 1 fi CLEAR_SUCCESS=0 CLEAR_TOTAL=0 # Stop all nodes first log_info "Stopping all Besu nodes..." for vmid in "${VALIDATORS[@]}" "${SENTRIES[@]}" "${RPC_NODES[@]}"; do if pct status "$vmid" 2>/dev/null | grep -q "running"; then if pct exec "$vmid" -- systemctl stop besu-validator.service 2>/dev/null || \ pct exec "$vmid" -- systemctl stop besu-sentry.service 2>/dev/null || \ pct exec "$vmid" -- systemctl stop besu-rpc.service 2>/dev/null; then log_detail "Stopped VMID $vmid" fi fi done sleep 5 # Clear transaction pool databases log_info "Clearing transaction pool databases..." for vmid in "${VALIDATORS[@]}" "${SENTRIES[@]}" "${RPC_NODES[@]}"; do if pct status "$vmid" 2>/dev/null | grep -q "running"; then ((CLEAR_TOTAL++)) log_detail "Clearing VMID $vmid..." # Clear caches pct exec "$vmid" -- rm -rf /data/besu/caches/* 2>/dev/null || true # Try to find and clear transaction pool database pct exec "$vmid" -- find /data/besu -type d -name "*pool*" -exec rm -rf {} \; 2>/dev/null || true pct exec "$vmid" -- find /data/besu -type f -name "*pool*" -delete 2>/dev/null || true pct exec "$vmid" -- find /data/besu -type f -name "*transaction*" -delete 2>/dev/null || true log_success "✓ VMID $vmid cleared" ((CLEAR_SUCCESS++)) fi done # Start all nodes log_info "Starting all Besu nodes..." for vmid in "${VALIDATORS[@]}"; do if pct status "$vmid" 2>/dev/null | grep -q "running"; then pct exec "$vmid" -- systemctl start besu-validator.service 2>/dev/null || true fi done for vmid in "${SENTRIES[@]}"; do if pct status "$vmid" 2>/dev/null | grep -q "running"; then pct exec "$vmid" -- systemctl start besu-sentry.service 2>/dev/null || true fi done for vmid in "${RPC_NODES[@]}"; do if pct status "$vmid" 2>/dev/null | grep -q "running"; then pct exec "$vmid" -- systemctl start besu-rpc.service 2>/dev/null || true fi done log_info "Waiting 20 seconds for services to start..." sleep 20 log_info "Database cleared: $CLEAR_SUCCESS/$CLEAR_TOTAL nodes" return 0 } # Verify transaction pool is cleared verify_cleared() { log_info "" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Verification: Check Transaction Pool" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" sleep 3 # Wait for RPC to be ready if TXPOOL_CONTENT=$(get_txpool_content "$RPC_URL" 2>/dev/null); then # Try to count transactions TX_COUNT=$(echo "$TXPOOL_CONTENT" | jq -r '[.result | if type == "array" then .[] else . end] | length' 2>/dev/null || echo "0") if [ "$TX_COUNT" = "0" ] || [ "$TX_COUNT" = "null" ]; then log_success "✓ Transaction pool appears to be empty" return 0 else log_warn "⚠ Transaction pool still contains $TX_COUNT transaction(s)" log_detail "Transactions found in pool (may be new transactions)" return 1 fi else log_warn "⚠ Could not verify transaction pool status" log_info "Pool may be cleared or TXPOOL API not available" return 0 # Assume success if we can't verify fi } # Main execution main() { SUCCESS=false # Try Method 1: TXPOOL API if method_1_txpool_clear; then if verify_cleared; then SUCCESS=true fi fi # If Method 1 failed or verification failed, try Method 2 if [ "$SUCCESS" != true ] && [ "$ON_PROXMOX" = true ]; then if method_2_restart_services; then if verify_cleared; then SUCCESS=true fi fi fi # If still not successful and user wants, try Method 3 if [ "$SUCCESS" != true ] && [ "$ON_PROXMOX" = true ]; then log_info "" log_warn "⚠ Previous methods did not fully clear transactions" log_info "You can try Method 3 (database clear) which requires stopping nodes" log_info "This is more invasive but may be necessary for persistent transactions" echo "" read -p "Try Method 3 (database clear)? (yes/no): " TRY_METHOD3 if [ "$TRY_METHOD3" = "yes" ]; then if method_3_clear_database; then if verify_cleared; then SUCCESS=true fi fi fi fi # Final summary echo "" echo "=========================================" echo "Summary" echo "=========================================" echo "" if [ "$SUCCESS" = true ]; then log_success "✓ Transaction pools have been flushed!" else log_warn "⚠ Transaction pools may still contain transactions" log_info "" log_info "Additional options:" log_info " 1. Wait for transaction retention period to expire" log_info " 2. Use a different account for new transactions" log_info " 3. Manually remove specific transactions if you have their hashes" fi echo "" } # Run main function main