proxmox/scripts/fix-all-validators-and-txpool.sh

#!/usr/bin/env bash
# Fix all validators: remove legacy tx-pool options, set layered pool with aggressive eviction,
# restart besu-validator. Run from project root (sources config/ip-addresses.conf).
#
# Eviction: Besu layered pool has no "drop after N blocks". We use tx-pool-min-score=0
# so transactions that are penalized (not included) get evicted once score drops to 0 or below,
# reducing stuck transactions. See docs/06-besu/TXPOOL_EVICTION_PREVENT_STUCK.md.
#
# Usage: bash scripts/fix-all-validators-and-txpool.sh [--dry-run]

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_ROOT"
[ -f config/ip-addresses.conf ] && source config/ip-addresses.conf 2>/dev/null || true

PROXMOX_USER="${PROXMOX_USER:-root}"
R630_01="${PROXMOX_HOST_R630_01:-${PROXMOX_R630_01:-192.168.11.11}}"
ML110="${PROXMOX_HOST_ML110:-${PROXMOX_ML110:-192.168.11.10}}"

# VMID -> Proxmox host
# 1000,1001,1002 on r630-01; 1003,1004 on ml110
VALIDATORS=(
  "1000:$R630_01"
  "1001:$R630_01"
  "1002:$R630_01"
  "1003:$ML110"
  "1004:$ML110"
)

CONFIG_PATH="/etc/besu/config-validator.toml"
LAYERED_BLOCK="# Layered Transaction Pool (Besu 23.10+); evict penalized txs (min-score=0)
tx-pool-max-future-by-sender=200
tx-pool-layer-max-capacity=12500000
tx-pool-max-prioritized=2000
tx-pool-price-bump=10
tx-pool-min-score=0
"

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }

DRY_RUN=false
[[ "${1:-}" = "--dry-run" ]] && DRY_RUN=true

echo ""
echo "=== Fix All Validators + Tx-Pool (Evict Stuck) ==="
echo ""

fix_one() {
  local vmid="$1"
  local host="$2"
  local ssh_target="${PROXMOX_USER}@${host}"

  log_info "Validator $vmid on $host"

  # Check container running
  local status
  status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct status $vmid 2>/dev/null" | awk '{print $2}' || echo "unknown")
  if [[ "$status" != "running" ]]; then
    log_warn "  VMID $vmid not running (status: $status) — skip"
    return 0
  fi

  if "$DRY_RUN"; then
    log_info "  [dry-run] Would remove legacy tx-pool lines and add layered + tx-pool-min-score=0, then restart besu-validator"
    return 0
  fi

  # Remove legacy options (cause crash with layered pool); add layered + min-score
  ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct exec $vmid -- bash -c '
    set -e
    CFG=/etc/besu/config-validator.toml
    [ -f /config/config-validator.toml ] && CFG=/config/config-validator.toml
    if [ ! -f \"\$CFG\" ]; then echo \"Config not found: \$CFG\"; exit 1; fi
    sed -i \"/^tx-pool-max-size=/d\" \"\$CFG\" 2>/dev/null || true
    sed -i \"/^tx-pool-limit-by-account-percentage=/d\" \"\$CFG\" 2>/dev/null || true
    sed -i \"/^tx-pool-retention-hours=/d\" \"\$CFG\" 2>/dev/null || true
    if ! grep -q \"tx-pool-max-future-by-sender\" \"\$CFG\"; then
      echo \"\" >> \"\$CFG\"
      echo \"# Layered Transaction Pool (Besu 23.10+)\" >> \"\$CFG\"
      echo \"tx-pool-max-future-by-sender=200\" >> \"\$CFG\"
      echo \"tx-pool-layer-max-capacity=12500000\" >> \"\$CFG\"
      echo \"tx-pool-max-prioritized=2000\" >> \"\$CFG\"
      echo \"tx-pool-price-bump=10\" >> \"\$CFG\"
    fi
    # tx-pool-min-score=0 not added: unsupported in some Besu builds (causes Unknown option and crash loop)
    sed -i \"/^tx-pool-min-score=/d\" \"\$CFG\" 2>/dev/null || true
  '" 2>/dev/null || { log_error "  SSH/config update failed"; return 1; }

  log_success "  Config updated"

  # Restart
  if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct exec $vmid -- systemctl restart besu-validator" 2>/dev/null; then
    log_error "  Restart failed"
    return 1
  fi
  log_success "  besu-validator restarted"
  return 0
}

FAILED=0
for entry in "${VALIDATORS[@]}"; do
  IFS=: read -r vmid host <<< "$entry"
  if ! fix_one "$vmid" "$host"; then
    ((FAILED++)) || true
  fi
  echo ""
done

echo "=== Summary ==="
if [[ $FAILED -eq 0 ]]; then
  log_success "All validators updated and restarted (or skipped if not running)."
  log_info "Block production should resume once quorum is met. Check: bash scripts/monitoring/monitor-blockchain-health.sh"
else
  log_error "$FAILED validator(s) failed. Check SSH and container status."
  exit 1
fi