proxmox/scripts/flush-all-stuck-transactions.sh.bak

#!/usr/bin/env bash
# Flush all stuck transactions from all Besu nodes
# This script tries multiple methods to clear transaction pools
# Usage: ./flush-all-stuck-transactions.sh

set -euo pipefail

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_detail() { echo -e "${CYAN}[DETAIL]${NC} $1"; }

# Configuration
RPC_NODES=(2500 2501 2502)
VALIDATORS=(1000 1001 1002 1003 1004)
SENTRIES=(1500 1501 1502 1503)
RPC_URL="http://192.168.11.250:8545"  # Use Core RPC (VMID 2500) with TXPOOL enabled

# Check if pct is available (for Proxmox host operations)
if command -v pct &>/dev/null; then
    ON_PROXMOX=true
else
    ON_PROXMOX=false
fi

echo "========================================="
echo "Flush All Stuck Transactions"
echo "========================================="
echo ""

# Function to check if RPC API method is available
check_rpc_method() {
    local method=$1
    local rpc_url=${2:-$RPC_URL}
    
    RPC_MODULES=$(curl -s -X POST -H "Content-Type: application/json" \
        --data '{"jsonrpc":"2.0","method":"rpc_modules","params":[],"id":1}' \
        "$rpc_url" 2>/dev/null || echo "")
    
    if echo "$RPC_MODULES" | jq -r ".result | keys[]" 2>/dev/null | grep -qi "^${method%_*}"; then
        return 0
    else
        return 1
    fi
}

# Function to get transaction pool content
get_txpool_content() {
    local rpc_url=${1:-$RPC_URL}
    
    # Try txpool_besuTransactions (Besu-specific)
    TXPOOL=$(curl -s -X POST -H "Content-Type: application/json" \
        --data '{"jsonrpc":"2.0","method":"txpool_besuTransactions","params":[],"id":1}' \
        "$rpc_url" 2>/dev/null || echo "")
    
    if echo "$TXPOOL" | jq -e '.result' >/dev/null 2>&1; then
        echo "$TXPOOL"
        return 0
    fi
    
    # Try txpool_content (standard)
    TXPOOL=$(curl -s -X POST -H "Content-Type: application/json" \
        --data '{"jsonrpc":"2.0","method":"txpool_content","params":[],"id":1}' \
        "$rpc_url" 2>/dev/null || echo "")
    
    if echo "$TXPOOL" | jq -e '.result' >/dev/null 2>&1; then
        echo "$TXPOOL"
        return 0
    fi
    
    return 1
}

# Method 1: Try to clear using TXPOOL API
method_1_txpool_clear() {
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "Method 1: Clear via TXPOOL API"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    
    # Check if TXPOOL is enabled
    if ! check_rpc_method "txpool" "$RPC_URL"; then
        log_warn "⚠ TXPOOL API not enabled on RPC node"
        log_info "Skipping TXPOOL API method"
        return 1
    fi
    
    log_info "TXPOOL API is enabled, attempting to clear..."
    
    # Try txpool_besuClear (Besu-specific)
    log_detail "Trying txpool_besuClear..."
    CLEAR_RESULT=$(curl -s -X POST -H "Content-Type: application/json" \
        --data '{"jsonrpc":"2.0","method":"txpool_besuClear","params":[],"id":1}' \
        "$RPC_URL" 2>/dev/null || echo "")
    
    if echo "$CLEAR_RESULT" | jq -e '.result == true' >/dev/null 2>&1; then
        log_success "✓ Transaction pool cleared using txpool_besuClear"
        return 0
    elif echo "$CLEAR_RESULT" | jq -e '.error' >/dev/null 2>&1; then
        ERROR_MSG=$(echo "$CLEAR_RESULT" | jq -r '.error.message // "Unknown error"' 2>/dev/null || echo "Unknown")
        log_detail "txpool_besuClear failed: $ERROR_MSG"
    fi
    
    # Try txpool_clear (alternative)
    log_detail "Trying txpool_clear..."
    CLEAR_RESULT=$(curl -s -X POST -H "Content-Type: application/json" \
        --data '{"jsonrpc":"2.0","method":"txpool_clear","params":[],"id":1}' \
        "$RPC_URL" 2>/dev/null || echo "")
    
    if echo "$CLEAR_RESULT" | jq -e '.result == true' >/dev/null 2>&1; then
        log_success "✓ Transaction pool cleared using txpool_clear"
        return 0
    elif echo "$CLEAR_RESULT" | jq -e '.error' >/dev/null 2>&1; then
        ERROR_MSG=$(echo "$CLEAR_RESULT" | jq -r '.error.message // "Unknown error"' 2>/dev/null || echo "Unknown")
        log_detail "txpool_clear failed: $ERROR_MSG"
    fi
    
    log_warn "⚠ TXPOOL clear methods not available or failed"
    return 1
}

# Method 2: Restart all Besu services (clears in-memory pools)
method_2_restart_services() {
    log_info ""
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "Method 2: Restart All Besu Services"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    
    if [ "$ON_PROXMOX" != true ]; then
        log_warn "⚠ Not on Proxmox host - cannot restart services"
        return 1
    fi
    
    RESTART_SUCCESS=0
    RESTART_TOTAL=0
    
    # Restart validators
    log_info "Restarting validators..."
    for vmid in "${VALIDATORS[@]}"; do
        if pct status "$vmid" 2>/dev/null | grep -q "running"; then
            ((RESTART_TOTAL++))
            if pct exec "$vmid" -- systemctl restart besu-validator.service 2>/dev/null; then
                log_success "✓ VMID $vmid (validator) restarted"
                ((RESTART_SUCCESS++))
            else
                log_error "✗ VMID $vmid (validator) restart failed"
            fi
        fi
    done
    
    # Restart sentries
    log_info "Restarting sentries..."
    for vmid in "${SENTRIES[@]}"; do
        if pct status "$vmid" 2>/dev/null | grep -q "running"; then
            ((RESTART_TOTAL++))
            if pct exec "$vmid" -- systemctl restart besu-sentry.service 2>/dev/null; then
                log_success "✓ VMID $vmid (sentry) restarted"
                ((RESTART_SUCCESS++))
            else
                log_error "✗ VMID $vmid (sentry) restart failed"
            fi
        fi
    done
    
    # Restart RPC nodes
    log_info "Restarting RPC nodes..."
    for vmid in "${RPC_NODES[@]}"; do
        if pct status "$vmid" 2>/dev/null | grep -q "running"; then
            ((RESTART_TOTAL++))
            if pct exec "$vmid" -- systemctl restart besu-rpc.service 2>/dev/null; then
                log_success "✓ VMID $vmid (RPC) restarted"
                ((RESTART_SUCCESS++))
            else
                log_error "✗ VMID $vmid (RPC) restart failed"
            fi
        fi
    done
    
    log_info "Services restarted: $RESTART_SUCCESS/$RESTART_TOTAL"
    
    if [ $RESTART_SUCCESS -gt 0 ]; then
        log_info "Waiting 15 seconds for services to stabilize..."
        sleep 15
        return 0
    else
        return 1
    fi
}

# Method 3: Clear transaction pool database files
method_3_clear_database() {
    log_info ""
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "Method 3: Clear Transaction Pool Databases"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    
    if [ "$ON_PROXMOX" != true ]; then
        log_warn "⚠ Not on Proxmox host - cannot clear database files"
        return 1
    fi
    
    log_warn "⚠ WARNING: This will stop Besu nodes and clear transaction pool databases"
    log_warn "⚠ All pending transactions will be lost"
    echo ""
    read -p "Continue with database clear? (yes/no): " CONFIRM
    if [ "$CONFIRM" != "yes" ]; then
        log_info "Skipping database clear"
        return 1
    fi
    
    CLEAR_SUCCESS=0
    CLEAR_TOTAL=0
    
    # Stop all nodes first
    log_info "Stopping all Besu nodes..."
    for vmid in "${VALIDATORS[@]}" "${SENTRIES[@]}" "${RPC_NODES[@]}"; do
        if pct status "$vmid" 2>/dev/null | grep -q "running"; then
            if pct exec "$vmid" -- systemctl stop besu-validator.service 2>/dev/null || \
               pct exec "$vmid" -- systemctl stop besu-sentry.service 2>/dev/null || \
               pct exec "$vmid" -- systemctl stop besu-rpc.service 2>/dev/null; then
                log_detail "Stopped VMID $vmid"
            fi
        fi
    done
    
    sleep 5
    
    # Clear transaction pool databases
    log_info "Clearing transaction pool databases..."
    for vmid in "${VALIDATORS[@]}" "${SENTRIES[@]}" "${RPC_NODES[@]}"; do
        if pct status "$vmid" 2>/dev/null | grep -q "running"; then
            ((CLEAR_TOTAL++))
            log_detail "Clearing VMID $vmid..."
            
            # Clear caches
            pct exec "$vmid" -- rm -rf /data/besu/caches/* 2>/dev/null || true
            
            # Try to find and clear transaction pool database
            pct exec "$vmid" -- find /data/besu -type d -name "*pool*" -exec rm -rf {} \; 2>/dev/null || true
            pct exec "$vmid" -- find /data/besu -type f -name "*pool*" -delete 2>/dev/null || true
            pct exec "$vmid" -- find /data/besu -type f -name "*transaction*" -delete 2>/dev/null || true
            
            log_success "✓ VMID $vmid cleared"
            ((CLEAR_SUCCESS++))
        fi
    done
    
    # Start all nodes
    log_info "Starting all Besu nodes..."
    for vmid in "${VALIDATORS[@]}"; do
        if pct status "$vmid" 2>/dev/null | grep -q "running"; then
            pct exec "$vmid" -- systemctl start besu-validator.service 2>/dev/null || true
        fi
    done
    
    for vmid in "${SENTRIES[@]}"; do
        if pct status "$vmid" 2>/dev/null | grep -q "running"; then
            pct exec "$vmid" -- systemctl start besu-sentry.service 2>/dev/null || true
        fi
    done
    
    for vmid in "${RPC_NODES[@]}"; do
        if pct status "$vmid" 2>/dev/null | grep -q "running"; then
            pct exec "$vmid" -- systemctl start besu-rpc.service 2>/dev/null || true
        fi
    done
    
    log_info "Waiting 20 seconds for services to start..."
    sleep 20
    
    log_info "Database cleared: $CLEAR_SUCCESS/$CLEAR_TOTAL nodes"
    return 0
}

# Verify transaction pool is cleared
verify_cleared() {
    log_info ""
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "Verification: Check Transaction Pool"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    
    sleep 3  # Wait for RPC to be ready
    
    if TXPOOL_CONTENT=$(get_txpool_content "$RPC_URL" 2>/dev/null); then
        # Try to count transactions
        TX_COUNT=$(echo "$TXPOOL_CONTENT" | jq -r '[.result | if type == "array" then .[] else . end] | length' 2>/dev/null || echo "0")
        
        if [ "$TX_COUNT" = "0" ] || [ "$TX_COUNT" = "null" ]; then
            log_success "✓ Transaction pool appears to be empty"
            return 0
        else
            log_warn "⚠ Transaction pool still contains $TX_COUNT transaction(s)"
            log_detail "Transactions found in pool (may be new transactions)"
            return 1
        fi
    else
        log_warn "⚠ Could not verify transaction pool status"
        log_info "Pool may be cleared or TXPOOL API not available"
        return 0  # Assume success if we can't verify
    fi
}

# Main execution
main() {
    SUCCESS=false
    
    # Try Method 1: TXPOOL API
    if method_1_txpool_clear; then
        if verify_cleared; then
            SUCCESS=true
        fi
    fi
    
    # If Method 1 failed or verification failed, try Method 2
    if [ "$SUCCESS" != true ] && [ "$ON_PROXMOX" = true ]; then
        if method_2_restart_services; then
            if verify_cleared; then
                SUCCESS=true
            fi
        fi
    fi
    
    # If still not successful and user wants, try Method 3
    if [ "$SUCCESS" != true ] && [ "$ON_PROXMOX" = true ]; then
        log_info ""
        log_warn "⚠ Previous methods did not fully clear transactions"
        log_info "You can try Method 3 (database clear) which requires stopping nodes"
        log_info "This is more invasive but may be necessary for persistent transactions"
        echo ""
        read -p "Try Method 3 (database clear)? (yes/no): " TRY_METHOD3
        if [ "$TRY_METHOD3" = "yes" ]; then
            if method_3_clear_database; then
                if verify_cleared; then
                    SUCCESS=true
                fi
            fi
        fi
    fi
    
    # Final summary
    echo ""
    echo "========================================="
    echo "Summary"
    echo "========================================="
    echo ""
    
    if [ "$SUCCESS" = true ]; then
        log_success "✓ Transaction pools have been flushed!"
    else
        log_warn "⚠ Transaction pools may still contain transactions"
        log_info ""
        log_info "Additional options:"
        log_info "  1. Wait for transaction retention period to expire"
        log_info "  2. Use a different account for new transactions"
        log_info "  3. Manually remove specific transactions if you have their hashes"
    fi
    
    echo ""
}

# Run main function
main