#!/usr/bin/env bash # Comprehensive RPC Node Testing Script # Tests all RPC nodes for connectivity, block sync, and peer connections set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true PROXMOX_HOST="${1:-192.168.11.10}" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } log_error() { echo -e "${RED}[✗]${NC} $1"; } # RPC Node mappings declare -A RPC_NODES=( ["2101"]="${RPC_CORE_1:-${IP_SERVICE_21:-${IP_SERVICE_21:-${IP_SERVICE_21:-192.168.11.21}}}1}:besu-rpc-core-1" ["2201"]="${RPC_PUBLIC_1:-192.168.11.221}:besu-rpc-public-1" ["2301"]="${RPC_PRIVATE_1:-192.168.11.232}:besu-rpc-private-1" ["2303"]="${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-192.168.11.233}}}}:besu-rpc-ali-0x8a" ["2304"]="${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-192.168.11.234}}}}:besu-rpc-ali-0x1" ["2305"]="${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-192.168.11.235}}}}:besu-rpc-luis-0x8a" ["2306"]="${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-192.168.11.236}}}}:besu-rpc-luis-0x1" ["2307"]="192.168.11.237:besu-rpc-putu-0x8a" ["2308"]="192.168.11.238:besu-rpc-putu-0x1" ["2401"]="${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-192.168.11.241}}}}:besu-rpc-thirdweb-0x8a-1" ["2402"]="${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-192.168.11.242}}}}:besu-rpc-thirdweb-0x8a-2" ["2403"]="192.168.11.243:besu-rpc-thirdweb-0x8a-3" ) # Test RPC call test_rpc() { local ip=$1 local method=$2 local params="${3:-[]}" timeout 5 curl -s -X POST -H "Content-Type: application/json" \ --data "{\"jsonrpc\":\"2.0\",\"method\":\"$method\",\"params\":$params,\"id\":1}" \ http://$ip:8545 2>/dev/null || echo "" } # Get block number get_block_number() { local ip=$1 local result=$(test_rpc "$ip" "eth_blockNumber" "[]") if [ -n "$result" ]; then echo "$result" | grep -o '"result":"[^"]*"' | cut -d'"' -f4 else echo "" fi } # Get peer count get_peer_count() { local ip=$1 local result=$(test_rpc "$ip" "net_peerCount" "[]") if [ -n "$result" ]; then local hex=$(echo "$result" | grep -o '"result":"[^"]*"' | cut -d'"' -f4) if [ -n "$hex" ]; then printf "%d" $hex 2>/dev/null || echo "0" else echo "0" fi else echo "0" fi } # Get sync status get_sync_status() { local ip=$1 local result=$(test_rpc "$ip" "eth_syncing" "[]") if [ -n "$result" ]; then local syncing=$(echo "$result" | grep -o '"result":[^,}]*' | cut -d':' -f2) if [ "$syncing" = "false" ]; then echo "synced" else echo "syncing" fi else echo "unknown" fi } # Get chain ID get_chain_id() { local ip=$1 local result=$(test_rpc "$ip" "eth_chainId" "[]") if [ -n "$result" ]; then echo "$result" | grep -o '"result":"[^"]*"' | cut -d'"' -f4 else echo "" fi } echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "🔍 Comprehensive RPC Node Testing" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" # Test 1: Container Status log_info "Test 1: Container Status" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" running_count=0 stopped_count=0 for vmid in "${!RPC_NODES[@]}"; do status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" \ "pct status $vmid 2>/dev/null | awk '{print \$2}'" || echo "unknown") if [ "$status" = "running" ]; then log_success "VMID $vmid: Running" running_count=$((running_count + 1)) else log_warn "VMID $vmid: $status" stopped_count=$((stopped_count + 1)) fi done echo "" echo "Summary: $running_count running, $stopped_count stopped/unknown" echo "" # Test 2: RPC Connectivity log_info "Test 2: RPC Connectivity (eth_blockNumber)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" responsive_count=0 unresponsive_count=0 declare -A block_numbers for vmid in "${!RPC_NODES[@]}"; do IFS=':' read -r ip name <<< "${RPC_NODES[$vmid]}" block_hex=$(get_block_number "$ip") if [ -n "$block_hex" ] && [ "$block_hex" != "" ]; then block_dec=$(printf "%d" $block_hex 2>/dev/null || echo "0") block_numbers[$vmid]=$block_dec log_success "VMID $vmid ($ip): Block $block_dec ($block_hex)" responsive_count=$((responsive_count + 1)) else log_error "VMID $vmid ($ip): Not responding" block_numbers[$vmid]="N/A" unresponsive_count=$((unresponsive_count + 1)) fi done echo "" echo "Summary: $responsive_count responsive, $unresponsive_count unresponsive" echo "" # Test 3: Block Synchronization log_info "Test 3: Block Synchronization Check" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Find highest and lowest block numbers max_block=0 min_block=999999999 valid_blocks=() for vmid in "${!block_numbers[@]}"; do block=${block_numbers[$vmid]} if [ "$block" != "N/A" ] && [ -n "$block" ]; then valid_blocks+=($block) if [ $block -gt $max_block ]; then max_block=$block fi if [ $block -lt $min_block ]; then min_block=$block fi fi done if [ ${#valid_blocks[@]} -gt 0 ]; then block_diff=$((max_block - min_block)) echo "Highest block: $max_block" echo "Lowest block: $min_block" echo "Block difference: $block_diff" if [ $block_diff -le 2 ]; then log_success "All nodes are well synchronized (difference ≤ 2 blocks)" elif [ $block_diff -le 10 ]; then log_warn "Nodes are slightly out of sync (difference: $block_diff blocks)" else log_error "Nodes are significantly out of sync (difference: $block_diff blocks)" fi # Show block numbers per node echo "" echo "Block numbers by node:" for vmid in $(printf '%s\n' "${!block_numbers[@]}" | sort -n); do block=${block_numbers[$vmid]} IFS=':' read -r ip name <<< "${RPC_NODES[$vmid]}" if [ "$block" != "N/A" ]; then diff=$((max_block - block)) if [ $diff -eq 0 ]; then echo " VMID $vmid ($ip): $block (current)" else echo " VMID $vmid ($ip): $block (-$diff)" fi fi done else log_error "No valid block numbers found" fi echo "" # Test 4: Peer Connections log_info "Test 4: Peer Connection Status" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" total_peers=0 nodes_with_peers=0 for vmid in "${!RPC_NODES[@]}"; do IFS=':' read -r ip name <<< "${RPC_NODES[$vmid]}" peer_count=$(get_peer_count "$ip") total_peers=$((total_peers + peer_count)) if [ "$peer_count" -gt 0 ]; then nodes_with_peers=$((nodes_with_peers + 1)) log_success "VMID $vmid ($ip): $peer_count peers" else log_warn "VMID $vmid ($ip): 0 peers" fi done echo "" echo "Summary: $nodes_with_peers nodes with peers, total $total_peers peer connections" echo "" # Test 5: Sync Status log_info "Test 5: Sync Status (eth_syncing)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" synced_count=0 syncing_count=0 for vmid in "${!RPC_NODES[@]}"; do IFS=':' read -r ip name <<< "${RPC_NODES[$vmid]}" sync_status=$(get_sync_status "$ip") if [ "$sync_status" = "synced" ]; then log_success "VMID $vmid ($ip): Synced" synced_count=$((synced_count + 1)) elif [ "$sync_status" = "syncing" ]; then log_warn "VMID $vmid ($ip): Syncing" syncing_count=$((syncing_count + 1)) else log_error "VMID $vmid ($ip): Status unknown" fi done echo "" echo "Summary: $synced_count synced, $syncing_count syncing" echo "" # Test 6: Chain ID Verification log_info "Test 6: Chain ID Verification" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" declare -A chain_ids for vmid in "${!RPC_NODES[@]}"; do IFS=':' read -r ip name <<< "${RPC_NODES[$vmid]}" chain_id=$(get_chain_id "$ip") if [ -n "$chain_id" ]; then chain_ids[$vmid]=$chain_id log_success "VMID $vmid ($ip): Chain ID $chain_id" else log_error "VMID $vmid ($ip): Could not get Chain ID" fi done # Check if all chain IDs match unique_chain_ids=$(printf '%s\n' "${chain_ids[@]}" | sort -u | wc -l) if [ $unique_chain_ids -eq 1 ] && [ ${#chain_ids[@]} -gt 0 ]; then log_success "All nodes on the same chain" else log_warn "Chain IDs may differ (found $unique_chain_ids unique chain IDs)" fi echo "" # Test 7: Additional RPC Methods log_info "Test 7: Additional RPC Method Tests" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Test a few nodes with additional methods test_nodes=("2101" "2201" "2303") for vmid in "${test_nodes[@]}"; do IFS=':' read -r ip name <<< "${RPC_NODES[$vmid]}" if [ -n "$(get_block_number "$ip")" ]; then echo "VMID $vmid ($ip):" # Test eth_getBalance (use zero address) balance_result=$(test_rpc "$ip" "eth_getBalance" '["0x0000000000000000000000000000000000000000","latest"]') if echo "$balance_result" | grep -q '"result"'; then echo " ✓ eth_getBalance: Working" else echo " ✗ eth_getBalance: Failed" fi # Test net_version net_version=$(test_rpc "$ip" "net_version" "[]") if echo "$net_version" | grep -q '"result"'; then version=$(echo "$net_version" | grep -o '"result":"[^"]*"' | cut -d'"' -f4) echo " ✓ net_version: $version" else echo " ✗ net_version: Failed" fi fi done echo "" # Final Summary echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Final Summary" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Containers Running: $running_count/12" echo "RPC Responsive: $responsive_count/12" echo "Nodes Synced: $synced_count/12" echo "Nodes with Peers: $nodes_with_peers/12" echo "" if [ $responsive_count -eq 12 ] && [ $block_diff -le 2 ] && [ $synced_count -eq 12 ]; then log_success "✅ All tests passed! All nodes are running and synchronized." exit 0 elif [ $responsive_count -ge 9 ] && [ $block_diff -le 10 ]; then log_warn "⚠ Most nodes are working, but some issues detected." exit 1 else log_error "❌ Significant issues detected. Review the output above." exit 2 fi