proxmox/scripts/archive/consolidated/verify/verify-post-deployment.sh

#!/usr/bin/env bash
# Post-Deployment Verification Script
# Verifies all deployed Besu nodes after configuration deployment

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Configuration
PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"
SSH_KEY="${SSH_KEY:-~/.ssh/id_ed25519_proxmox}"

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Deployed nodes
VALIDATORS=(1000 1001 1002 1003 1004)
SENTRIES=(1500 1501 1502 1503)
RPC_NODES=(2101)

CHECKS_PASSED=0
CHECKS_FAILED=0
CHECKS_WARN=0

echo -e "${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║  POST-DEPLOYMENT VERIFICATION                                ║${NC}"
echo -e "${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}"
echo ""

# Check SSH access
log_info "Checking SSH access to Proxmox host ($PROXMOX_HOST)..."
if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "echo 'SSH OK'" &>/dev/null 2>&1; then
    log_success "SSH access confirmed"
    SSH_AVAILABLE=1
else
    log_error "Cannot access Proxmox host via SSH"
    log_error "Run this script from a machine with SSH access to Proxmox host"
    SSH_AVAILABLE=0
    exit 1
fi

echo ""

# Function to check service status
check_service_status() {
    local vmid=$1
    local service=$2
    local node_type=$3
    
    local status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $vmid -- systemctl is-active $service 2>&1" || echo "unknown")
    
    if [ "$status" = "active" ]; then
        log_success "VMID $vmid ($node_type): Service is active"
        ((CHECKS_PASSED++))
        return 0
    else
        log_warn "VMID $vmid ($node_type): Service status: $status"
        ((CHECKS_WARN++))
        return 1
    fi
}

# Function to check for errors in logs
check_service_errors() {
    local vmid=$1
    local service=$2
    local node_type=$3
    
    local errors=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $vmid -- journalctl -u $service --since '10 minutes ago' --no-pager 2>&1 | grep -iE 'error|exception|failed|unknown option' | head -5" || echo "")
    
    if [ -z "$errors" ]; then
        log_success "VMID $vmid ($node_type): No errors in recent logs"
        ((CHECKS_PASSED++))
        return 0
    else
        log_warn "VMID $vmid ($node_type): Found errors in logs:"
        echo "$errors" | while IFS= read -r line; do
            echo "    $line"
        done
        ((CHECKS_WARN++))
        return 1
    fi
}

# Function to verify logging level
verify_logging_level() {
    local vmid=$1
    local config_file=$2
    local expected_level=$3
    local node_type=$4
    
    local actual_level=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $vmid -- grep -E '^[[:space:]]*logging=' $config_file 2>/dev/null | grep -oE '\"[A-Z]+\"' | tr -d '\"'" || echo "")
    
    if [ "$actual_level" = "$expected_level" ]; then
        log_success "VMID $vmid ($node_type): Logging level is $expected_level (correct)"
        ((CHECKS_PASSED++))
        return 0
    else
        log_warn "VMID $vmid ($node_type): Logging level is '$actual_level' (expected: $expected_level)"
        ((CHECKS_WARN++))
        return 1
    fi
}

# Phase 1: Service Status Verification
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BLUE}Phase 1: Service Status Verification${NC}"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo ""

log_info "Waiting 30 seconds for services to stabilize..."
sleep 30

for vmid in "${VALIDATORS[@]}"; do
    check_service_status "$vmid" "besu-validator.service" "validator"
done

for vmid in "${SENTRIES[@]}"; do
    check_service_status "$vmid" "besu-sentry.service" "sentry"
done

for vmid in "${RPC_NODES[@]}"; do
    check_service_status "$vmid" "besu-rpc.service" "RPC"
done

echo ""

# Phase 2: Configuration Error Checking
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BLUE}Phase 2: Configuration Error Checking${NC}"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo ""

for vmid in "${VALIDATORS[@]}"; do
    check_service_errors "$vmid" "besu-validator.service" "validator"
done

for vmid in "${SENTRIES[@]}"; do
    check_service_errors "$vmid" "besu-sentry.service" "sentry"
done

for vmid in "${RPC_NODES[@]}"; do
    check_service_errors "$vmid" "besu-rpc.service" "RPC"
done

echo ""

# Phase 3: Logging Level Verification
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BLUE}Phase 3: Logging Level Verification${NC}"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo ""

for vmid in "${VALIDATORS[@]}"; do
    verify_logging_level "$vmid" "/etc/besu/config-validator.toml" "WARN" "validator"
done

for vmid in "${SENTRIES[@]}"; do
    verify_logging_level "$vmid" "/etc/besu/config-sentry.toml" "INFO" "sentry"
done

for vmid in "${RPC_NODES[@]}"; do
    verify_logging_level "$vmid" "/etc/besu/config-rpc-core.toml" "WARN" "RPC"
done

echo ""

# Summary
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BLUE}Verification Summary${NC}"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo ""

TOTAL_CHECKS=$((CHECKS_PASSED + CHECKS_FAILED + CHECKS_WARN))

echo "Checks passed:  $CHECKS_PASSED"
echo "Checks failed:  $CHECKS_FAILED"
echo "Warnings:       $CHECKS_WARN"
echo "Total checks:   $TOTAL_CHECKS"
echo ""

if [ "$CHECKS_FAILED" -eq 0 ]; then
    if [ "$CHECKS_WARN" -eq 0 ]; then
        log_success "✅ All verification checks passed!"
        echo ""
        echo "Deployment Status: SUCCESS"
        echo "All services are running correctly with new configurations."
        exit 0
    else
        log_warn "⚠️  Verification completed with warnings"
        echo ""
        echo "Deployment Status: SUCCESS WITH WARNINGS"
        echo "Review warnings above. Services are running but may need attention."
        exit 0
    fi
else
    log_error "❌ Verification found critical issues"
    echo ""
    echo "Deployment Status: NEEDS ATTENTION"
    echo "Review failed checks above and take corrective action."
    exit 1
fi