#!/usr/bin/env bash # Comprehensive Blockchain Health Monitoring Script # Monitors block production, transaction inclusion, and node health set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true RPC_CORE_1="${RPC_CORE_1:-192.168.11.211}" RPC_URL="${RPC_URL:-http://${RPC_CORE_1}:8545}" DEPLOYER="${DEPLOYER:-0x4A666F96fC8764181194447A7dFdb7d471b301C8}" PROXMOX_USER="${PROXMOX_USER:-root}" PROXMOX_ML110="${PROXMOX_ML110:-${PROXMOX_HOST_ML110:-192.168.11.10}}" PROXMOX_R630="${PROXMOX_R630:-${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}}" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } log_error() { echo -e "${RED}[✗]${NC} $1"; } log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; } echo "=== Blockchain Health Monitor ===" echo "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')" echo "" # Check RPC connectivity log_section "RPC Node Status" if timeout 5 cast chain-id --rpc-url "$RPC_URL" >/dev/null 2>&1; then CHAIN_ID=$(cast chain-id --rpc-url "$RPC_URL" 2>/dev/null) BLOCK_NUM=$(cast block-number --rpc-url "$RPC_URL" 2>/dev/null) BLOCK_DEC=$(cast --to-dec "$BLOCK_NUM" 2>/dev/null || echo "0") log_success "RPC accessible" echo " Chain ID: $CHAIN_ID" echo " Latest block: $BLOCK_DEC ($BLOCK_NUM)" else log_error "RPC not accessible" exit 1 fi # Check block production log_section "Block Production" BLOCK1=$(cast block-number --rpc-url "$RPC_URL" 2>/dev/null) sleep 5 BLOCK2=$(cast block-number --rpc-url "$RPC_URL" 2>/dev/null) BLOCK1_DEC=$(cast --to-dec "$BLOCK1" 2>/dev/null || echo "0") BLOCK2_DEC=$(cast --to-dec "$BLOCK2" 2>/dev/null || echo "0") BLOCK_DIFF=$((BLOCK2_DEC - BLOCK1_DEC)) if [ "$BLOCK_DIFF" -gt 0 ]; then log_success "Blocks being produced ($BLOCK_DIFF blocks in 5s)" else log_error "Block production stalled (no new blocks in 5s)" # If validators are all active, they may still be syncing (QBFT does not produce until sync completes) SYNC_HINT=$(ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${PROXMOX_R630}" \ "pct exec 1000 -- journalctl -u besu-validator --no-pager -n 30 2>/dev/null" 2>/dev/null | grep -c "Full sync\|initial sync in progress\|QBFT mining coordinator not starting" || true) if [ "${SYNC_HINT:-0}" -gt 0 ]; then echo " → Validators may be syncing; block production will resume when sync completes (see docs/06-besu/CRITICAL_ISSUE_BLOCK_PRODUCTION_STOPPED.md)." fi fi # Check transaction inclusion log_section "Transaction Inclusion" TX_COUNT_TOTAL=0 EMPTY_BLOCKS=0 for i in 0 1 2 3 4 5; do BLOCK_NUM=$((BLOCK2_DEC - i)) BLOCK_HEX=$(printf '0x%x' $BLOCK_NUM) TX_COUNT=$(cast rpc eth_getBlockTransactionCountByNumber "$BLOCK_HEX" --rpc-url "$RPC_URL" 2>/dev/null | tr -d '"') TX_COUNT_DEC=$(cast --to-dec "$TX_COUNT" 2>/dev/null || echo "0") TX_COUNT_TOTAL=$((TX_COUNT_TOTAL + TX_COUNT_DEC)) if [ "$TX_COUNT_DEC" -eq 0 ]; then EMPTY_BLOCKS=$((EMPTY_BLOCKS + 1)) fi done if [ "$TX_COUNT_TOTAL" -gt 0 ]; then log_success "Transactions being included ($TX_COUNT_TOTAL txs in last 6 blocks)" else log_warn "No transactions in last 6 blocks ($EMPTY_BLOCKS empty blocks)" fi # Check pending transactions log_section "Pending Transactions" LATEST_HEX=$(cast rpc eth_getTransactionCount "$DEPLOYER" latest --rpc-url "$RPC_URL" 2>/dev/null | tr -d '"') PENDING_HEX=$(cast rpc eth_getTransactionCount "$DEPLOYER" pending --rpc-url "$RPC_URL" 2>/dev/null | tr -d '"') LATEST_DEC=$(cast --to-dec "$LATEST_HEX" 2>/dev/null || echo "0") PENDING_DEC=$(cast --to-dec "$PENDING_HEX" 2>/dev/null || echo "0") PENDING_COUNT=$((PENDING_DEC - LATEST_DEC)) if [ "$PENDING_COUNT" -eq 0 ]; then log_success "No pending transactions" else log_warn "$PENDING_COUNT pending transactions (nonces $((LATEST_DEC + 1))-$PENDING_DEC)" fi # Check validator status log_section "Validator Status" VALIDATORS=( "1000:$PROXMOX_R630" "1001:$PROXMOX_R630" "1002:$PROXMOX_R630" "1003:$PROXMOX_ML110" "1004:$PROXMOX_ML110" ) ACTIVE_COUNT=0 for validator in "${VALIDATORS[@]}"; do IFS=':' read -r VMID HOST <<< "$validator" SSH_TARGET="${PROXMOX_USER}@${HOST}" STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$SSH_TARGET" \ "pct exec $VMID -- systemctl is-active besu-validator" 2>/dev/null || echo "unknown") if [ "$STATUS" = "active" ]; then ACTIVE_COUNT=$((ACTIVE_COUNT + 1)) echo " Validator $VMID: $STATUS" else log_warn "Validator $VMID: $STATUS" fi done if [ "$ACTIVE_COUNT" -eq 5 ]; then log_success "All 5 validators active" else log_error "Only $ACTIVE_COUNT/5 validators active" fi # Check peer connections log_section "Peer Connections" PEER_COUNT=$(cast rpc admin_peers --rpc-url "$RPC_URL" 2>/dev/null | jq '. | length' 2>/dev/null || echo "N/A") if [ "$PEER_COUNT" != "N/A" ] && [ "$PEER_COUNT" -ge 5 ]; then log_success "RPC has $PEER_COUNT peer connections" else log_warn "RPC has $PEER_COUNT peer connections (expected >= 5)" fi # Summary log_section "Health Summary" ISSUES=0 if [ "$BLOCK_DIFF" -eq 0 ]; then log_error "❌ Block production stalled" ISSUES=$((ISSUES + 1)) # Hint when validators are syncing (all active but no new blocks yet) if [ "$ACTIVE_COUNT" -eq 5 ]; then echo " (If validators recently restarted: they are likely in full sync; blocks will resume when sync completes.)" fi else log_success "✓ Block production active" fi if [ "$TX_COUNT_TOTAL" -eq 0 ] && [ "$PENDING_COUNT" -gt 0 ]; then log_error "❌ Transactions not being included" ISSUES=$((ISSUES + 1)) elif [ "$TX_COUNT_TOTAL" -gt 0 ]; then log_success "✓ Transactions being included" fi if [ "$ACTIVE_COUNT" -lt 5 ]; then log_error "❌ Not all validators active" ISSUES=$((ISSUES + 1)) else log_success "✓ All validators active" fi if [ "$PENDING_COUNT" -gt 10 ]; then log_warn "⚠ High number of pending transactions ($PENDING_COUNT)" ISSUES=$((ISSUES + 1)) fi echo "" if [ "$ISSUES" -eq 0 ]; then log_success "Overall Status: HEALTHY" exit 0 else log_error "Overall Status: $ISSUES issue(s) detected" exit 1 fi