proxmox/scripts/clear-transaction-pool-all-nodes.sh

#!/usr/bin/env bash
# Clear Transaction Pool Database on All Besu Nodes
# Stops services, clears transaction pool databases, restarts

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }
log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; }

# RPC Nodes
RPC_NODES=(
    "2101:besu-rpc.service"
    "2201:besu-rpc.service"
)

# Validators
VALIDATORS=(
    "1000:besu-validator.service"
    "1001:besu-validator.service"
    "1002:besu-validator.service"
    "1003:besu-validator.service"
    "1004:besu-validator.service"
)

log_section "Clear Transaction Pool Database on All Nodes"
log_warn "⚠️  WARNING: This will stop Besu nodes and clear transaction pools"
log_warn "⚠️  All pending transactions will be lost"
log_info "Proxmox Host: $PROXMOX_HOST"
echo ""

# Function to clear transaction pool for a node
clear_node_pool() {
    local vmid=$1
    local service=$2
    local name=$3
    
    log_info "Processing $name (VMID $vmid)..."
    
    # Check if VM is running
    VM_STATUS=$(ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" || echo "")
    if ! echo "$VM_STATUS" | grep -q "running"; then
        log_warn "$name: Container not running, skipping"
        return 1
    fi
    
    # Stop service
    log_info "$name: Stopping $service..."
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl stop $service 2>&1"; then
        log_success "$name: Service stopped"
        sleep 2
    else
        log_warn "$name: Failed to stop service (may not be running)"
    fi
    
    # Clear transaction pool database files
    log_info "$name: Clearing transaction pool database..."
    CLEARED=0
    
    # Clear transaction pool directories
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type d -name '*pool*' -exec rm -rf {} \; 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    # Clear transaction pool files
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type f -name '*transaction*' -delete 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type f -name '*pool*' -delete 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    # Clear caches
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- rm -rf /data/besu/caches/* 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    if [ $CLEARED -gt 0 ]; then
        log_success "$name: Transaction pool database cleared"
    else
        log_warn "$name: No transaction pool files found (may already be clear)"
    fi
    
    # Start service
    log_info "$name: Starting $service..."
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl start $service 2>&1"; then
        log_success "$name: Service started"
        sleep 2
    else
        log_error "$name: Failed to start service"
        return 1
    fi
    
    return 0
}

# Clear RPC nodes
log_section "Clearing RPC Nodes"

RPC_SUCCESS=0
RPC_TOTAL=0

for node in "${RPC_NODES[@]}"; do
    IFS=':' read -r vmid service <<< "$node"
    ((RPC_TOTAL++))
    
    if clear_node_pool "$vmid" "$service" "RPC-$vmid"; then
        ((RPC_SUCCESS++))
    fi
    
    echo ""
done

log_info "RPC nodes cleared: $RPC_SUCCESS/$RPC_TOTAL"

# Clear validators
log_section "Clearing Validators"

VALIDATOR_SUCCESS=0
VALIDATOR_TOTAL=0

for validator in "${VALIDATORS[@]}"; do
    IFS=':' read -r vmid service <<< "$validator"
    ((VALIDATOR_TOTAL++))
    
    if clear_node_pool "$vmid" "$service" "Validator-$vmid"; then
        ((VALIDATOR_SUCCESS++))
    fi
    
    echo ""
done

log_info "Validators cleared: $VALIDATOR_SUCCESS/$VALIDATOR_TOTAL"

# Wait for services to stabilize
log_section "Waiting for Services to Stabilize"
log_info "Waiting 15 seconds for all services to start..."
sleep 15

# Verify RPC nodes are back online
log_section "Verifying RPC Nodes"

RPC_IPS=(
    "2101:${RPC_CORE_1}"
    "2201:${RPC_PUBLIC_1}"
)

RPC_ONLINE=0
for rpc in "${RPC_IPS[@]}"; do
    IFS=':' read -r vmid ip <<< "$rpc"
    log_info "Checking RPC-$vmid ($ip)..."
    
    if cast block-number --rpc-url "http://${ip}:8545" >/dev/null 2>&1; then
        log_success "RPC-$vmid: Online"
        ((RPC_ONLINE++))
    else
        log_warn "RPC-$vmid: Not responding (may need more time)"
    fi
done

# Final summary
log_section "Summary"

TOTAL_SUCCESS=$((RPC_SUCCESS + VALIDATOR_SUCCESS))
TOTAL_NODES=$((RPC_TOTAL + VALIDATOR_TOTAL))

echo "Transaction Pool Clear Results:"
echo "  • RPC nodes: $RPC_SUCCESS/$RPC_TOTAL cleared"
echo "  • Validators: $VALIDATOR_SUCCESS/$VALIDATOR_TOTAL cleared"
echo "  • Total: $TOTAL_SUCCESS/$TOTAL_NODES cleared"
echo ""
echo "RPC Status:"
echo "  • Online: $RPC_ONLINE/${#RPC_IPS[@]} RPC nodes"

if [ $TOTAL_SUCCESS -eq $TOTAL_NODES ] && [ $RPC_ONLINE -eq ${#RPC_IPS[@]} ]; then
    log_success "\n✅ All nodes cleared and back online!"
    log_info "Next: Verify pending transactions are cleared"
else
    log_warn "\n⚠️  Some nodes may need attention"
    log_info "Check individual node status if needed"
fi

log_success "\nTransaction pool database clear complete!"