284 lines
8.9 KiB
Bash
Executable File
284 lines
8.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Validate Validator Set Script
|
|
# Validates that all validators are properly configured and can participate in consensus
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
source "$PROJECT_ROOT/lib/common.sh"
|
|
|
|
# Load configuration
|
|
load_config
|
|
load_config "$PROJECT_ROOT/config/network.conf" || true
|
|
|
|
# VMID ranges
|
|
VALIDATORS_START="${VALIDATOR_START:-1000}"
|
|
VALIDATORS_COUNT="${VALIDATOR_COUNT:-${VALIDATORS_COUNT:-5}}"
|
|
VALIDATORS_END=$((VALIDATORS_START + VALIDATORS_COUNT - 1))
|
|
|
|
VALIDATORS=()
|
|
for ((vmid=VALIDATORS_START; vmid<=VALIDATORS_END; vmid++)); do
|
|
VALIDATORS+=($vmid)
|
|
done
|
|
|
|
log_info "========================================="
|
|
log_info "Validator Set Validation"
|
|
log_info "========================================="
|
|
log_info ""
|
|
log_info "Validating ${#VALIDATORS[@]} validators (${VALIDATORS_START}-${VALIDATORS_END})"
|
|
log_info ""
|
|
|
|
VALIDATION_ERRORS=0
|
|
VALIDATION_WARNINGS=0
|
|
|
|
# Function to check if container is running
|
|
check_container_running() {
|
|
local vmid=$1
|
|
if pct status "$vmid" 2>/dev/null | grep -q running; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to check if service is running
|
|
check_service_running() {
|
|
local vmid=$1
|
|
if pct exec "$vmid" -- systemctl is-active --quiet besu-validator 2>/dev/null; then
|
|
return 0
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to check validator keys exist
|
|
check_validator_keys() {
|
|
local vmid=$1
|
|
local keys_dir="/keys/validators"
|
|
|
|
if pct exec "$vmid" -- test -d "$keys_dir" 2>/dev/null; then
|
|
local key_count
|
|
key_count=$(pct exec "$vmid" -- find "$keys_dir" -name "*.priv" -o -name "key.pem" 2>/dev/null | wc -l || echo "0")
|
|
if [[ $key_count -gt 0 ]]; then
|
|
return 0
|
|
fi
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Function to check validator address exists
|
|
check_validator_address() {
|
|
local vmid=$1
|
|
local address_file="/keys/validators/validator-*/address.txt"
|
|
|
|
if pct exec "$vmid" -- sh -c "test -f $address_file" 2>/dev/null; then
|
|
local address
|
|
address=$(pct exec "$vmid" -- sh -c "cat $address_file 2>/dev/null | head -1" 2>/dev/null | tr -d '\n\r ' || echo "")
|
|
if [[ -n "$address" ]] && [[ ${#address} -eq 42 ]] && [[ "$address" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
|
|
echo "$address"
|
|
return 0
|
|
fi
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Function to check Besu process is using validator keys
|
|
check_besu_using_keys() {
|
|
local vmid=$1
|
|
|
|
# Check if Besu process is running and has key directory mounted
|
|
if pct exec "$vmid" -- pgrep -f "besu.*validator" >/dev/null 2>&1; then
|
|
# Check if Besu config references validator keys
|
|
if pct exec "$vmid" -- grep -q "validator-keys" /etc/besu/config-validator.toml 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Function to check consensus participation (via block production)
|
|
check_consensus_participation() {
|
|
local vmid=$1
|
|
|
|
# Try to get block number from RPC (validators typically don't have RPC enabled)
|
|
# Instead, check logs for consensus activity
|
|
if pct exec "$vmid" -- journalctl -u besu-validator --no-pager -n 50 2>/dev/null | grep -qiE "(consensus|qbft|block.*produced|proposing)" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
|
|
# Alternative: Check if process is running and healthy
|
|
if pct exec "$vmid" -- pgrep -f "besu.*validator" >/dev/null 2>&1; then
|
|
# If process is running, assume it's participating (may not have produced blocks yet)
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
# Function to check validator can connect to peers
|
|
check_peer_connectivity() {
|
|
local vmid=$1
|
|
|
|
# Validators typically don't have RPC, so check if process is running
|
|
# Peer connectivity is validated during network bootstrap
|
|
if pct exec "$vmid" -- pgrep -f "besu.*validator" >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
# Main validation loop
|
|
log_info "=== Validation Phase 1: Container & Service Status ==="
|
|
for vmid in "${VALIDATORS[@]}"; do
|
|
log_info "Validating validator $vmid..."
|
|
|
|
# Check container running
|
|
if check_container_running "$vmid"; then
|
|
log_success " ✓ Container $vmid is running"
|
|
else
|
|
log_error " ✗ Container $vmid is not running"
|
|
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
|
|
continue
|
|
fi
|
|
|
|
# Check service running
|
|
if check_service_running "$vmid"; then
|
|
log_success " ✓ Besu validator service is running"
|
|
else
|
|
log_error " ✗ Besu validator service is not running"
|
|
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
|
|
fi
|
|
done
|
|
|
|
log_info ""
|
|
log_info "=== Validation Phase 2: Validator Keys ==="
|
|
for vmid in "${VALIDATORS[@]}"; do
|
|
if ! check_container_running "$vmid"; then
|
|
continue
|
|
fi
|
|
|
|
log_info "Checking validator keys for $vmid..."
|
|
|
|
# Check validator keys exist
|
|
if check_validator_keys "$vmid"; then
|
|
log_success " ✓ Validator keys directory exists and contains keys"
|
|
else
|
|
log_error " ✗ Validator keys not found in /keys/validators/"
|
|
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
|
|
fi
|
|
|
|
# Check validator address
|
|
address=$(check_validator_address "$vmid")
|
|
if [[ -n "$address" ]]; then
|
|
log_success " ✓ Validator address found: $address"
|
|
else
|
|
log_warn " ⚠ Validator address file not found or invalid"
|
|
VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1))
|
|
fi
|
|
done
|
|
|
|
log_info ""
|
|
log_info "=== Validation Phase 3: Besu Configuration ==="
|
|
for vmid in "${VALIDATORS[@]}"; do
|
|
if ! check_container_running "$vmid"; then
|
|
continue
|
|
fi
|
|
|
|
log_info "Checking Besu configuration for $vmid..."
|
|
|
|
# Check config file exists
|
|
if pct exec "$vmid" -- test -f /etc/besu/config-validator.toml 2>/dev/null; then
|
|
log_success " ✓ Config file exists"
|
|
else
|
|
log_error " ✗ Config file missing: /etc/besu/config-validator.toml"
|
|
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
|
|
continue
|
|
fi
|
|
|
|
# Check genesis file exists
|
|
if pct exec "$vmid" -- test -f /etc/besu/genesis.json 2>/dev/null; then
|
|
log_success " ✓ Genesis file exists"
|
|
else
|
|
log_error " ✗ Genesis file missing: /etc/besu/genesis.json"
|
|
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
|
|
fi
|
|
|
|
# Check static-nodes.json exists
|
|
if pct exec "$vmid" -- test -f /etc/besu/static-nodes.json 2>/dev/null; then
|
|
log_success " ✓ static-nodes.json exists"
|
|
else
|
|
log_warn " ⚠ static-nodes.json missing (may not be critical)"
|
|
VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1))
|
|
fi
|
|
|
|
# Check permissions-nodes.toml exists
|
|
if pct exec "$vmid" -- test -f /etc/besu/permissions-nodes.toml 2>/dev/null; then
|
|
log_success " ✓ permissions-nodes.toml exists"
|
|
else
|
|
log_warn " ⚠ permissions-nodes.toml missing (may not be critical)"
|
|
VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1))
|
|
fi
|
|
|
|
# Check Besu is using validator keys
|
|
if check_besu_using_keys "$vmid"; then
|
|
log_success " ✓ Besu configured with validator keys"
|
|
else
|
|
log_warn " ⚠ Cannot verify Besu is using validator keys"
|
|
VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1))
|
|
fi
|
|
done
|
|
|
|
log_info ""
|
|
log_info "=== Validation Phase 4: Consensus Participation ==="
|
|
for vmid in "${VALIDATORS[@]}"; do
|
|
if ! check_container_running "$vmid"; then
|
|
continue
|
|
fi
|
|
|
|
log_info "Checking consensus participation for $vmid..."
|
|
|
|
# Check consensus participation
|
|
if check_consensus_participation "$vmid"; then
|
|
log_success " ✓ Validator appears to be participating in consensus"
|
|
else
|
|
log_warn " ⚠ Cannot verify consensus participation (may still be starting)"
|
|
VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1))
|
|
fi
|
|
|
|
# Check peer connectivity
|
|
if check_peer_connectivity "$vmid"; then
|
|
log_success " ✓ Validator process is running (peer connectivity checked during bootstrap)"
|
|
else
|
|
log_warn " ⚠ Validator process not running or not responding"
|
|
VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1))
|
|
fi
|
|
done
|
|
|
|
# Summary
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_info "Validation Summary"
|
|
log_info "========================================="
|
|
log_info "Validators checked: ${#VALIDATORS[@]}"
|
|
log_info "Errors: $VALIDATION_ERRORS"
|
|
log_info "Warnings: $VALIDATION_WARNINGS"
|
|
log_info ""
|
|
|
|
if [[ $VALIDATION_ERRORS -eq 0 ]]; then
|
|
if [[ $VALIDATION_WARNINGS -eq 0 ]]; then
|
|
log_success "✓ All validators validated successfully!"
|
|
exit 0
|
|
else
|
|
log_warn "⚠ Validation complete with $VALIDATION_WARNINGS warning(s)"
|
|
log_info "Warnings are non-critical but should be reviewed"
|
|
exit 0
|
|
fi
|
|
else
|
|
log_error "✗ Validation failed with $VALIDATION_ERRORS error(s)"
|
|
log_info "Please fix the errors before proceeding"
|
|
exit 1
|
|
fi
|
|
|