cb47cce074
- Organized 252 files across project - Root directory: 187 → 2 files (98.9% reduction) - Moved configuration guides to docs/04-configuration/ - Moved troubleshooting guides to docs/09-troubleshooting/ - Moved quick start guides to docs/01-getting-started/ - Moved reports to reports/ directory - Archived temporary files - Generated comprehensive reports and documentation - Created maintenance scripts and guides All files organized according to established standards.
372 lines
13 KiB
Bash
Executable File
372 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Flush all stuck transactions from all Besu nodes
|
|
# This script tries multiple methods to clear transaction pools
|
|
# Usage: ./flush-all-stuck-transactions.sh
|
|
|
|
set -euo pipefail
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
|
|
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
|
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
|
log_detail() { echo -e "${CYAN}[DETAIL]${NC} $1"; }
|
|
|
|
# Configuration
|
|
RPC_NODES=(2500 2501 2502)
|
|
VALIDATORS=(1000 1001 1002 1003 1004)
|
|
SENTRIES=(1500 1501 1502 1503)
|
|
RPC_URL="http://192.168.11.250:8545" # Use Core RPC (VMID 2500) with TXPOOL enabled
|
|
|
|
# Check if pct is available (for Proxmox host operations)
|
|
if command -v pct &>/dev/null; then
|
|
ON_PROXMOX=true
|
|
else
|
|
ON_PROXMOX=false
|
|
fi
|
|
|
|
echo "========================================="
|
|
echo "Flush All Stuck Transactions"
|
|
echo "========================================="
|
|
echo ""
|
|
|
|
# Function to check if RPC API method is available
|
|
check_rpc_method() {
|
|
local method=$1
|
|
local rpc_url=${2:-$RPC_URL}
|
|
|
|
RPC_MODULES=$(curl -s -X POST -H "Content-Type: application/json" \
|
|
--data '{"jsonrpc":"2.0","method":"rpc_modules","params":[],"id":1}' \
|
|
"$rpc_url" 2>/dev/null || echo "")
|
|
|
|
if echo "$RPC_MODULES" | jq -r ".result | keys[]" 2>/dev/null | grep -qi "^${method%_*}"; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to get transaction pool content
|
|
get_txpool_content() {
|
|
local rpc_url=${1:-$RPC_URL}
|
|
|
|
# Try txpool_besuTransactions (Besu-specific)
|
|
TXPOOL=$(curl -s -X POST -H "Content-Type: application/json" \
|
|
--data '{"jsonrpc":"2.0","method":"txpool_besuTransactions","params":[],"id":1}' \
|
|
"$rpc_url" 2>/dev/null || echo "")
|
|
|
|
if echo "$TXPOOL" | jq -e '.result' >/dev/null 2>&1; then
|
|
echo "$TXPOOL"
|
|
return 0
|
|
fi
|
|
|
|
# Try txpool_content (standard)
|
|
TXPOOL=$(curl -s -X POST -H "Content-Type: application/json" \
|
|
--data '{"jsonrpc":"2.0","method":"txpool_content","params":[],"id":1}' \
|
|
"$rpc_url" 2>/dev/null || echo "")
|
|
|
|
if echo "$TXPOOL" | jq -e '.result' >/dev/null 2>&1; then
|
|
echo "$TXPOOL"
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
# Method 1: Try to clear using TXPOOL API
|
|
method_1_txpool_clear() {
|
|
log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
log_info "Method 1: Clear via TXPOOL API"
|
|
log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
# Check if TXPOOL is enabled
|
|
if ! check_rpc_method "txpool" "$RPC_URL"; then
|
|
log_warn "⚠ TXPOOL API not enabled on RPC node"
|
|
log_info "Skipping TXPOOL API method"
|
|
return 1
|
|
fi
|
|
|
|
log_info "TXPOOL API is enabled, attempting to clear..."
|
|
|
|
# Try txpool_besuClear (Besu-specific)
|
|
log_detail "Trying txpool_besuClear..."
|
|
CLEAR_RESULT=$(curl -s -X POST -H "Content-Type: application/json" \
|
|
--data '{"jsonrpc":"2.0","method":"txpool_besuClear","params":[],"id":1}' \
|
|
"$RPC_URL" 2>/dev/null || echo "")
|
|
|
|
if echo "$CLEAR_RESULT" | jq -e '.result == true' >/dev/null 2>&1; then
|
|
log_success "✓ Transaction pool cleared using txpool_besuClear"
|
|
return 0
|
|
elif echo "$CLEAR_RESULT" | jq -e '.error' >/dev/null 2>&1; then
|
|
ERROR_MSG=$(echo "$CLEAR_RESULT" | jq -r '.error.message // "Unknown error"' 2>/dev/null || echo "Unknown")
|
|
log_detail "txpool_besuClear failed: $ERROR_MSG"
|
|
fi
|
|
|
|
# Try txpool_clear (alternative)
|
|
log_detail "Trying txpool_clear..."
|
|
CLEAR_RESULT=$(curl -s -X POST -H "Content-Type: application/json" \
|
|
--data '{"jsonrpc":"2.0","method":"txpool_clear","params":[],"id":1}' \
|
|
"$RPC_URL" 2>/dev/null || echo "")
|
|
|
|
if echo "$CLEAR_RESULT" | jq -e '.result == true' >/dev/null 2>&1; then
|
|
log_success "✓ Transaction pool cleared using txpool_clear"
|
|
return 0
|
|
elif echo "$CLEAR_RESULT" | jq -e '.error' >/dev/null 2>&1; then
|
|
ERROR_MSG=$(echo "$CLEAR_RESULT" | jq -r '.error.message // "Unknown error"' 2>/dev/null || echo "Unknown")
|
|
log_detail "txpool_clear failed: $ERROR_MSG"
|
|
fi
|
|
|
|
log_warn "⚠ TXPOOL clear methods not available or failed"
|
|
return 1
|
|
}
|
|
|
|
# Method 2: Restart all Besu services (clears in-memory pools)
|
|
method_2_restart_services() {
|
|
log_info ""
|
|
log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
log_info "Method 2: Restart All Besu Services"
|
|
log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
if [ "$ON_PROXMOX" != true ]; then
|
|
log_warn "⚠ Not on Proxmox host - cannot restart services"
|
|
return 1
|
|
fi
|
|
|
|
RESTART_SUCCESS=0
|
|
RESTART_TOTAL=0
|
|
|
|
# Restart validators
|
|
log_info "Restarting validators..."
|
|
for vmid in "${VALIDATORS[@]}"; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q "running"; then
|
|
((RESTART_TOTAL++))
|
|
if pct exec "$vmid" -- systemctl restart besu-validator.service 2>/dev/null; then
|
|
log_success "✓ VMID $vmid (validator) restarted"
|
|
((RESTART_SUCCESS++))
|
|
else
|
|
log_error "✗ VMID $vmid (validator) restart failed"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Restart sentries
|
|
log_info "Restarting sentries..."
|
|
for vmid in "${SENTRIES[@]}"; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q "running"; then
|
|
((RESTART_TOTAL++))
|
|
if pct exec "$vmid" -- systemctl restart besu-sentry.service 2>/dev/null; then
|
|
log_success "✓ VMID $vmid (sentry) restarted"
|
|
((RESTART_SUCCESS++))
|
|
else
|
|
log_error "✗ VMID $vmid (sentry) restart failed"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Restart RPC nodes
|
|
log_info "Restarting RPC nodes..."
|
|
for vmid in "${RPC_NODES[@]}"; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q "running"; then
|
|
((RESTART_TOTAL++))
|
|
if pct exec "$vmid" -- systemctl restart besu-rpc.service 2>/dev/null; then
|
|
log_success "✓ VMID $vmid (RPC) restarted"
|
|
((RESTART_SUCCESS++))
|
|
else
|
|
log_error "✗ VMID $vmid (RPC) restart failed"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
log_info "Services restarted: $RESTART_SUCCESS/$RESTART_TOTAL"
|
|
|
|
if [ $RESTART_SUCCESS -gt 0 ]; then
|
|
log_info "Waiting 15 seconds for services to stabilize..."
|
|
sleep 15
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Method 3: Clear transaction pool database files
|
|
method_3_clear_database() {
|
|
log_info ""
|
|
log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
log_info "Method 3: Clear Transaction Pool Databases"
|
|
log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
if [ "$ON_PROXMOX" != true ]; then
|
|
log_warn "⚠ Not on Proxmox host - cannot clear database files"
|
|
return 1
|
|
fi
|
|
|
|
log_warn "⚠ WARNING: This will stop Besu nodes and clear transaction pool databases"
|
|
log_warn "⚠ All pending transactions will be lost"
|
|
echo ""
|
|
read -p "Continue with database clear? (yes/no): " CONFIRM
|
|
if [ "$CONFIRM" != "yes" ]; then
|
|
log_info "Skipping database clear"
|
|
return 1
|
|
fi
|
|
|
|
CLEAR_SUCCESS=0
|
|
CLEAR_TOTAL=0
|
|
|
|
# Stop all nodes first
|
|
log_info "Stopping all Besu nodes..."
|
|
for vmid in "${VALIDATORS[@]}" "${SENTRIES[@]}" "${RPC_NODES[@]}"; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q "running"; then
|
|
if pct exec "$vmid" -- systemctl stop besu-validator.service 2>/dev/null || \
|
|
pct exec "$vmid" -- systemctl stop besu-sentry.service 2>/dev/null || \
|
|
pct exec "$vmid" -- systemctl stop besu-rpc.service 2>/dev/null; then
|
|
log_detail "Stopped VMID $vmid"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
sleep 5
|
|
|
|
# Clear transaction pool databases
|
|
log_info "Clearing transaction pool databases..."
|
|
for vmid in "${VALIDATORS[@]}" "${SENTRIES[@]}" "${RPC_NODES[@]}"; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q "running"; then
|
|
((CLEAR_TOTAL++))
|
|
log_detail "Clearing VMID $vmid..."
|
|
|
|
# Clear caches
|
|
pct exec "$vmid" -- rm -rf /data/besu/caches/* 2>/dev/null || true
|
|
|
|
# Try to find and clear transaction pool database
|
|
pct exec "$vmid" -- find /data/besu -type d -name "*pool*" -exec rm -rf {} \; 2>/dev/null || true
|
|
pct exec "$vmid" -- find /data/besu -type f -name "*pool*" -delete 2>/dev/null || true
|
|
pct exec "$vmid" -- find /data/besu -type f -name "*transaction*" -delete 2>/dev/null || true
|
|
|
|
log_success "✓ VMID $vmid cleared"
|
|
((CLEAR_SUCCESS++))
|
|
fi
|
|
done
|
|
|
|
# Start all nodes
|
|
log_info "Starting all Besu nodes..."
|
|
for vmid in "${VALIDATORS[@]}"; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q "running"; then
|
|
pct exec "$vmid" -- systemctl start besu-validator.service 2>/dev/null || true
|
|
fi
|
|
done
|
|
|
|
for vmid in "${SENTRIES[@]}"; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q "running"; then
|
|
pct exec "$vmid" -- systemctl start besu-sentry.service 2>/dev/null || true
|
|
fi
|
|
done
|
|
|
|
for vmid in "${RPC_NODES[@]}"; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q "running"; then
|
|
pct exec "$vmid" -- systemctl start besu-rpc.service 2>/dev/null || true
|
|
fi
|
|
done
|
|
|
|
log_info "Waiting 20 seconds for services to start..."
|
|
sleep 20
|
|
|
|
log_info "Database cleared: $CLEAR_SUCCESS/$CLEAR_TOTAL nodes"
|
|
return 0
|
|
}
|
|
|
|
# Verify transaction pool is cleared
|
|
verify_cleared() {
|
|
log_info ""
|
|
log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
log_info "Verification: Check Transaction Pool"
|
|
log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
|
|
sleep 3 # Wait for RPC to be ready
|
|
|
|
if TXPOOL_CONTENT=$(get_txpool_content "$RPC_URL" 2>/dev/null); then
|
|
# Try to count transactions
|
|
TX_COUNT=$(echo "$TXPOOL_CONTENT" | jq -r '[.result | if type == "array" then .[] else . end] | length' 2>/dev/null || echo "0")
|
|
|
|
if [ "$TX_COUNT" = "0" ] || [ "$TX_COUNT" = "null" ]; then
|
|
log_success "✓ Transaction pool appears to be empty"
|
|
return 0
|
|
else
|
|
log_warn "⚠ Transaction pool still contains $TX_COUNT transaction(s)"
|
|
log_detail "Transactions found in pool (may be new transactions)"
|
|
return 1
|
|
fi
|
|
else
|
|
log_warn "⚠ Could not verify transaction pool status"
|
|
log_info "Pool may be cleared or TXPOOL API not available"
|
|
return 0 # Assume success if we can't verify
|
|
fi
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
SUCCESS=false
|
|
|
|
# Try Method 1: TXPOOL API
|
|
if method_1_txpool_clear; then
|
|
if verify_cleared; then
|
|
SUCCESS=true
|
|
fi
|
|
fi
|
|
|
|
# If Method 1 failed or verification failed, try Method 2
|
|
if [ "$SUCCESS" != true ] && [ "$ON_PROXMOX" = true ]; then
|
|
if method_2_restart_services; then
|
|
if verify_cleared; then
|
|
SUCCESS=true
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# If still not successful and user wants, try Method 3
|
|
if [ "$SUCCESS" != true ] && [ "$ON_PROXMOX" = true ]; then
|
|
log_info ""
|
|
log_warn "⚠ Previous methods did not fully clear transactions"
|
|
log_info "You can try Method 3 (database clear) which requires stopping nodes"
|
|
log_info "This is more invasive but may be necessary for persistent transactions"
|
|
echo ""
|
|
read -p "Try Method 3 (database clear)? (yes/no): " TRY_METHOD3
|
|
if [ "$TRY_METHOD3" = "yes" ]; then
|
|
if method_3_clear_database; then
|
|
if verify_cleared; then
|
|
SUCCESS=true
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Final summary
|
|
echo ""
|
|
echo "========================================="
|
|
echo "Summary"
|
|
echo "========================================="
|
|
echo ""
|
|
|
|
if [ "$SUCCESS" = true ]; then
|
|
log_success "✓ Transaction pools have been flushed!"
|
|
else
|
|
log_warn "⚠ Transaction pools may still contain transactions"
|
|
log_info ""
|
|
log_info "Additional options:"
|
|
log_info " 1. Wait for transaction retention period to expire"
|
|
log_info " 2. Use a different account for new transactions"
|
|
log_info " 3. Manually remove specific transactions if you have their hashes"
|
|
fi
|
|
|
|
echo ""
|
|
}
|
|
|
|
# Run main function
|
|
main
|
|
|