proxmox/scripts/analyze-all-rpc-peers.sh

#!/usr/bin/env bash
set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


# Comprehensive RPC node peer count analysis
# Usage: ./analyze-all-rpc-peers.sh [proxmox-host]

set -e

PROXMOX_HOST="${1:-pve2}"

echo "=========================================="
echo "RPC Node Peer Count Analysis"
echo "=========================================="
echo ""

# Function to execute command on Proxmox host
exec_proxmox() {
    if command -v pct &>/dev/null; then
        eval "$@"
    else
        ssh root@$PROXMOX_HOST "$@"
    fi
}

declare -A NODES=(
    ["2101"]="${RPC_CORE_1}"
    ["2201"]="${RPC_PUBLIC_1}"
    ["2303"]="${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-192.168.11.233}}}}}}}"
    ["2304"]="${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-192.168.11.234}}}}}}}"
    ["2305"]="${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-192.168.11.235}}}}}}}"
    ["2306"]="${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-192.168.11.236}}}}}}}"
    ["2307"]="${IP_RPC_237:-${IP_RPC_237:-${IP_RPC_237:-192.168.11.237}}}"
    ["2308"]="${IP_RPC_238:-${IP_RPC_238:-${IP_RPC_238:-192.168.11.238}}}"
    ["2400"]="${RPC_THIRDWEB_PRIMARY}"
    ["2401"]="${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-192.168.11.241}}}}}}}"
    ["2402"]="${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-192.168.11.242}}}}}}}"
    ["2403"]="${RPC_THIRDWEB_3:-${RPC_THIRDWEB_3:-${RPC_THIRDWEB_3:-192.168.11.243}}}"
)

declare -A PEER_COUNTS
declare -A BLOCK_NUMBERS
declare -A STATUS

echo "Gathering data from all RPC nodes..."
echo ""

for VMID in "${!NODES[@]}"; do
    IP="${NODES[$VMID]}"
    
    # Get block number
    BLOCK_RESPONSE=$(exec_proxmox "curl -s -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}' http://$IP:8545 2>/dev/null" || echo "")
    if [ -n "$BLOCK_RESPONSE" ]; then
        BLOCK_HEX=$(echo "$BLOCK_RESPONSE" | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
        if [ -n "$BLOCK_HEX" ]; then
            BLOCK_NUM=$(printf "%d" $BLOCK_HEX 2>/dev/null || echo "0")
            BLOCK_NUMBERS[$VMID]=$BLOCK_NUM
        else
            BLOCK_NUMBERS[$VMID]="N/A"
        fi
    else
        BLOCK_NUMBERS[$VMID]="N/A"
        STATUS[$VMID]="❌ Not responding"
        continue
    fi
    
    # Get peer count
    PEER_RESPONSE=$(exec_proxmox "curl -s -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"net_peerCount\",\"params\":[],\"id\":1}' http://$IP:8545 2>/dev/null" || echo "")
    if [ -n "$PEER_RESPONSE" ]; then
        PEER_HEX=$(echo "$PEER_RESPONSE" | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
        if [ -n "$PEER_HEX" ]; then
            PEER_NUM=$(printf "%d" $PEER_HEX 2>/dev/null || echo "0")
            PEER_COUNTS[$VMID]=$PEER_NUM
        else
            PEER_COUNTS[$VMID]="0"
        fi
    else
        PEER_COUNTS[$VMID]="N/A"
    fi
    
    # Determine status
    if [ "${PEER_COUNTS[$VMID]}" = "N/A" ] || [ "${BLOCK_NUMBERS[$VMID]}" = "N/A" ]; then
        STATUS[$VMID]="❌ Not responding"
    elif [ "${PEER_COUNTS[$VMID]}" -ge 7 ]; then
        STATUS[$VMID]="✅ Excellent"
    elif [ "${PEER_COUNTS[$VMID]}" -ge 5 ]; then
        STATUS[$VMID]="✅ Good"
    elif [ "${PEER_COUNTS[$VMID]}" -ge 3 ]; then
        STATUS[$VMID]="⚠️  Acceptable"
    elif [ "${PEER_COUNTS[$VMID]}" -ge 1 ]; then
        STATUS[$VMID]="⚠️  Warning"
    else
        STATUS[$VMID]="❌ Critical"
    fi
done

# Display results
echo "=========================================="
echo "Peer Count Analysis Results"
echo "=========================================="
printf "%-6s | %-15s | %-12s | %-6s | %-15s\n" "VMID" "IP Address" "Block Height" "Peers" "Status"
echo "----------------------------------------------------------------------------"

for VMID in "${!NODES[@]}"; do
    IP="${NODES[$VMID]}"
    BLOCK="${BLOCK_NUMBERS[$VMID]}"
    PEERS="${PEER_COUNTS[$VMID]}"
    STAT="${STATUS[$VMID]}"
    
    printf "%-6s | %-15s | %-12s | %-6s | %-15s\n" "$VMID" "$IP" "$BLOCK" "$PEERS" "$STAT"
done

echo ""
echo "=========================================="
echo "Summary"
echo "=========================================="
echo ""

# Group by peer count
EXCELLENT=0
GOOD=0
ACCEPTABLE=0
WARNING=0
CRITICAL=0
NON_RESPONDING=0

for VMID in "${!STATUS[@]}"; do
    case "${STATUS[$VMID]}" in
        *Excellent*) ((EXCELLENT++)) ;;
        *Good*) ((GOOD++)) ;;
        *Acceptable*) ((ACCEPTABLE++)) ;;
        *Warning*) ((WARNING++)) ;;
        *Critical*) ((CRITICAL++)) ;;
        *Not responding*) ((NON_RESPONDING++)) ;;
    esac
done

echo "✅ Excellent (7+ peers): $EXCELLENT nodes"
echo "✅ Good (5-6 peers): $GOOD nodes"
echo "⚠️  Acceptable (3-4 peers): $ACCEPTABLE nodes"
echo "⚠️  Warning (1-2 peers): $WARNING nodes"
echo "❌ Critical (0 peers): $CRITICAL nodes"
echo "❌ Not responding: $NON_RESPONDING nodes"
echo ""

# Expected peer count
echo "=========================================="
echo "Expected Peer Count"
echo "=========================================="
echo ""
echo "Network Size: ~19-20 active nodes"
echo ""
echo "Recommended Peer Counts:"
echo "  - Minimum healthy: 2-3 peers"
echo "  - Recommended: 5-7 peers ✅"
echo "  - Maximum: 20-25 peers (max-peers setting)"
echo ""

# Analysis
echo "=========================================="
echo "Analysis"
echo "=========================================="
echo ""

# Check for block height mismatches
MAIN_BLOCK=""
MAIN_COUNT=0

for VMID in "${!BLOCK_NUMBERS[@]}"; do
    BLOCK="${BLOCK_NUMBERS[$VMID]}"
    if [ "$BLOCK" != "N/A" ] && [[ "$BLOCK" =~ ^[0-9]+$ ]]; then
        # Find most common block height
        COUNT=0
        for VMID2 in "${!BLOCK_NUMBERS[@]}"; do
            if [ "${BLOCK_NUMBERS[$VMID2]}" = "$BLOCK" ]; then
                ((COUNT++))
            fi
        done
        if [ $COUNT -gt $MAIN_COUNT ]; then
            MAIN_COUNT=$COUNT
            MAIN_BLOCK=$BLOCK
        fi
    fi
done

echo "Main network block height: $MAIN_BLOCK (${MAIN_COUNT} nodes)"
echo ""

# Check for nodes ahead or behind
AHEAD=()
BEHIND=()

for VMID in "${!BLOCK_NUMBERS[@]}"; do
    BLOCK="${BLOCK_NUMBERS[$VMID]}"
    if [ "$BLOCK" != "N/A" ] && [[ "$BLOCK" =~ ^[0-9]+$ ]] && [ "$MAIN_BLOCK" != "" ]; then
        DIFF=$((BLOCK - MAIN_BLOCK))
        if [ $DIFF -gt 1000 ]; then
            AHEAD+=("$VMID (block $BLOCK, +$DIFF)")
        elif [ $DIFF -lt -1000 ]; then
            BEHIND+=("$VMID (block $BLOCK, $DIFF)")
        fi
    fi
done

if [ ${#AHEAD[@]} -gt 0 ]; then
    echo "⚠️  Nodes ahead of main network:"
    for node in "${AHEAD[@]}"; do
        echo "   - VMID $node"
    done
    echo ""
fi

if [ ${#BEHIND[@]} -gt 0 ]; then
    echo "⏳ Nodes behind main network (syncing):"
    for node in "${BEHIND[@]}"; do
        echo "   - VMID $node"
    done
    echo ""
fi

# Recommendations
echo "=========================================="
echo "Recommendations"
echo "=========================================="
echo ""

# Nodes with 2 peers (ThirdWeb)
LOW_PEER_NODES=()
for VMID in "${!PEER_COUNTS[@]}"; do
    PEERS="${PEER_COUNTS[$VMID]}"
    if [ "$PEERS" != "N/A" ] && [ "$PEERS" -lt 3 ] && [ "$PEERS" -gt 0 ]; then
        LOW_PEER_NODES+=("$VMID")
    fi
done

if [ ${#LOW_PEER_NODES[@]} -gt 0 ]; then
    echo "⚠️  Nodes with low peer count (${#LOW_PEER_NODES[@]} nodes):"
    for vmid in "${LOW_PEER_NODES[@]}"; do
        echo "   - VMID $vmid: ${PEER_COUNTS[$vmid]} peers (should have 5-7)"
    done
    echo ""
    echo "Actions needed:"
    echo "  1. Verify static-nodes.json contains all 15 nodes"
    echo "  2. Check discovery-enabled=true"
    echo "  3. Verify permissions-nodes.toml is correct"
    echo "  4. Restart Besu services"
    echo ""
    echo "Run: ./scripts/fix-thirdweb-peer-connectivity.sh"
    echo ""
fi

# Nodes with 0 peers
ZERO_PEER_NODES=()
for VMID in "${!PEER_COUNTS[@]}"; do
    PEERS="${PEER_COUNTS[$VMID]}"
    if [ "$PEERS" = "0" ]; then
        ZERO_PEER_NODES+=("$VMID")
    fi
done

if [ ${#ZERO_PEER_NODES[@]} -gt 0 ]; then
    echo "⏳ Nodes with 0 peers (${#ZERO_PEER_NODES[@]} nodes):"
    for vmid in "${ZERO_PEER_NODES[@]}"; do
        BLOCK="${BLOCK_NUMBERS[$vmid]}"
        echo "   - VMID $vmid: Block $BLOCK"
        if [ "$BLOCK" != "N/A" ] && [ "$BLOCK" -lt 1000000 ]; then
            echo "     Status: Syncing (expected during initial sync)"
        else
            echo "     Status: May be isolated - investigate"
        fi
    done
    echo ""
fi

echo "=========================================="
echo "Complete"
echo "=========================================="