Files
proxmox/scripts/validate-deployment-ml110.sh

556 lines
20 KiB
Bash
Raw Normal View History

#!/bin/bash
# Comprehensive Deployment Validation Script for ml110-01
# Validates all prerequisites, configurations, and deployment readiness
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$SCRIPT_DIR"
DEPLOY_DIR="$PROJECT_ROOT/smom-dbis-138-proxmox"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'
# Status counters
PASSED=0
FAILED=0
WARNINGS=0
# Functions
pass() {
echo -e "${GREEN}${NC} $1"
((PASSED++))
}
fail() {
echo -e "${RED}${NC} $1"
((FAILED++))
}
warn() {
echo -e "${YELLOW}⚠️${NC} $1"
((WARNINGS++))
}
info() {
echo -e "${BLUE}${NC} $1"
}
section() {
echo ""
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${CYAN}$1${NC}"
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""
}
# Load environment
ENV_FILE="$HOME/.env"
if [ -f "$ENV_FILE" ]; then
source <(grep -E "^PROXMOX_" "$ENV_FILE" 2>/dev/null | sed 's/^/export /' || true)
fi
TARGET_HOST="${PROXMOX_HOST:-192.168.11.10}"
TARGET_NODE="${TARGET_NODE:-ml110-01}"
echo ""
echo -e "${BLUE}╔════════════════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}${NC} Deployment Validation for ml110-01 (${TARGET_HOST}) ${BLUE}${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════════════════════╝${NC}"
echo ""
# ============================================================================
# SECTION 1: PREREQUISITES
# ============================================================================
section "1. PREREQUISITES CHECK"
# Check if running on Proxmox host or remote
if command -v pct &> /dev/null; then
info "Running on Proxmox host - can execute pct commands directly"
ON_PROXMOX_HOST=true
else
info "Running remotely - will use Proxmox API"
ON_PROXMOX_HOST=false
fi
# Check required tools
REQUIRED_TOOLS=("curl" "jq" "git" "bash")
for tool in "${REQUIRED_TOOLS[@]}"; do
if command -v "$tool" &> /dev/null; then
pass "$tool installed"
else
fail "$tool not found (required)"
fi
done
# Check deployment directory
if [ -d "$DEPLOY_DIR" ]; then
pass "Deployment directory exists: $DEPLOY_DIR"
else
fail "Deployment directory not found: $DEPLOY_DIR"
fi
# ============================================================================
# SECTION 2: PROXMOX CONNECTION
# ============================================================================
section "2. PROXMOX CONNECTION VALIDATION"
# Test API connectivity
info "Testing connection to $TARGET_HOST:8006"
API_RESPONSE=$(curl -k -s -w "\n%{http_code}" -H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/version" 2>&1 || echo "ERROR")
HTTP_CODE=$(echo "$API_RESPONSE" | tail -1)
RESPONSE_BODY=$(echo "$API_RESPONSE" | sed '$d')
if [ "$HTTP_CODE" = "200" ]; then
VERSION=$(echo "$RESPONSE_BODY" | jq -r '.data.version' 2>/dev/null || echo "unknown")
pass "API connection successful (Proxmox version: $VERSION)"
else
fail "API connection failed (HTTP $HTTP_CODE)"
if [ -n "$RESPONSE_BODY" ]; then
info "Response: $(echo "$RESPONSE_BODY" | head -c 200)"
fi
fi
# Get node list
info "Retrieving node list..."
NODES_RESPONSE=$(curl -k -s -H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes" 2>&1)
if echo "$NODES_RESPONSE" | jq -e '.data' &>/dev/null; then
NODE_COUNT=$(echo "$NODES_RESPONSE" | jq '.data | length')
NODE_NAMES=$(echo "$NODES_RESPONSE" | jq -r '.data[].node' | tr '\n' ' ')
pass "Retrieved $NODE_COUNT node(s): $NODE_NAMES"
# Check if target node exists
if echo "$NODES_RESPONSE" | jq -e ".data[] | select(.node==\"$TARGET_NODE\")" &>/dev/null; then
pass "Target node '$TARGET_NODE' found"
else
FIRST_NODE=$(echo "$NODES_RESPONSE" | jq -r '.data[0].node')
warn "Target node '$TARGET_NODE' not found, using first node: $FIRST_NODE"
TARGET_NODE="$FIRST_NODE"
fi
else
fail "Failed to retrieve nodes"
fi
# Get node status
info "Checking node status for $TARGET_NODE..."
NODE_STATUS=$(curl -k -s -H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/status" 2>&1)
if echo "$NODE_STATUS" | jq -e '.data.status' &>/dev/null; then
STATUS=$(echo "$NODE_STATUS" | jq -r '.data.status')
if [ "$STATUS" = "online" ]; then
pass "Node $TARGET_NODE is online"
else
fail "Node $TARGET_NODE status: $STATUS"
fi
else
warn "Could not retrieve node status"
fi
# ============================================================================
# SECTION 3: STORAGE VALIDATION
# ============================================================================
section "3. STORAGE VALIDATION"
# Get storage list
info "Checking storage availability..."
STORAGE_RESPONSE=$(curl -k -s -H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/storage" 2>&1)
if echo "$STORAGE_RESPONSE" | jq -e '.data' &>/dev/null; then
STORAGE_COUNT=$(echo "$STORAGE_RESPONSE" | jq '.data | length')
pass "Found $STORAGE_COUNT storage pool(s)"
# Check for common storage types
STORAGE_TYPES=$(echo "$STORAGE_RESPONSE" | jq -r '.data[].type' | sort -u | tr '\n' ' ')
info "Storage types available: $STORAGE_TYPES"
# Check for container storage (rootdir)
CONTAINER_STORAGE=$(echo "$STORAGE_RESPONSE" | jq -r '.data[] | select(.content | contains("rootdir")) | .storage' | head -1)
if [ -n "$CONTAINER_STORAGE" ]; then
pass "Container storage found: $CONTAINER_STORAGE"
# Check storage capacity
STORAGE_INFO=$(echo "$STORAGE_RESPONSE" | jq ".[] | select(.storage==\"$CONTAINER_STORAGE\")")
if echo "$STORAGE_INFO" | jq -e '.content' &>/dev/null; then
TOTAL=$(echo "$STORAGE_INFO" | jq -r '.total // 0' | numfmt --to=iec-i --suffix=B 2>/dev/null || echo "unknown")
FREE=$(echo "$STORAGE_INFO" | jq -r '.avail // 0' | numfmt --to=iec-i --suffix=B 2>/dev/null || echo "unknown")
USED=$(echo "$STORAGE_INFO" | jq -r '.used // 0' | numfmt --to=iec-i --suffix=B 2>/dev/null || echo "unknown")
info "Storage $CONTAINER_STORAGE: Total=$TOTAL, Free=$FREE, Used=$USED"
fi
else
fail "No container storage (rootdir) found"
fi
# Check for template storage (vztmpl)
TEMPLATE_STORAGE=$(echo "$STORAGE_RESPONSE" | jq -r '.data[] | select(.content | contains("vztmpl")) | .storage' | head -1)
if [ -n "$TEMPLATE_STORAGE" ]; then
pass "Template storage found: $TEMPLATE_STORAGE"
else
warn "No template storage (vztmpl) found"
fi
else
fail "Failed to retrieve storage information"
fi
# ============================================================================
# SECTION 4: TEMPLATE VALIDATION
# ============================================================================
section "4. LXC TEMPLATE VALIDATION"
if [ -n "$TEMPLATE_STORAGE" ]; then
info "Checking templates on storage: $TEMPLATE_STORAGE"
TEMPLATES_RESPONSE=$(curl -k -s -H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/storage/${TEMPLATE_STORAGE}/content?content=vztmpl" 2>&1)
if echo "$TEMPLATES_RESPONSE" | jq -e '.data' &>/dev/null; then
TEMPLATE_COUNT=$(echo "$TEMPLATES_RESPONSE" | jq '.data | length')
pass "Found $TEMPLATE_COUNT template(s)"
# Check for Debian 12 template
DEBIAN_TEMPLATE=$(echo "$TEMPLATES_RESPONSE" | jq -r '.data[] | select(.volid | contains("debian-12")) | .volid' | head -1)
if [ -n "$DEBIAN_TEMPLATE" ]; then
pass "Debian 12 template found: $DEBIAN_TEMPLATE"
else
fail "Debian 12 template not found (required for deployment)"
info "Available templates:"
echo "$TEMPLATES_RESPONSE" | jq -r '.data[].volid' | head -10 | sed 's/^/ - /'
fi
else
warn "Could not retrieve template list"
fi
else
warn "Cannot check templates - no template storage found"
fi
# ============================================================================
# SECTION 5: CONFIGURATION FILES
# ============================================================================
section "5. CONFIGURATION FILES VALIDATION"
# Check if config files exist
CONFIG_DIR="$DEPLOY_DIR/config"
if [ -d "$CONFIG_DIR" ]; then
pass "Config directory exists: $CONFIG_DIR"
# Check proxmox.conf
if [ -f "$CONFIG_DIR/proxmox.conf" ]; then
pass "proxmox.conf exists"
source "$CONFIG_DIR/proxmox.conf" 2>/dev/null || true
# Validate config values
if [ -n "${PROXMOX_HOST:-}" ]; then
if [ "$PROXMOX_HOST" = "$TARGET_HOST" ] || [ "$PROXMOX_HOST" = "proxmox.example.com" ]; then
if [ "$PROXMOX_HOST" = "$TARGET_HOST" ]; then
pass "PROXMOX_HOST configured: $PROXMOX_HOST"
else
warn "PROXMOX_HOST still set to example: $PROXMOX_HOST"
fi
else
warn "PROXMOX_HOST mismatch: configured=$PROXMOX_HOST, expected=$TARGET_HOST"
fi
else
fail "PROXMOX_HOST not set in proxmox.conf"
fi
if [ -n "${PROXMOX_NODE:-}" ]; then
pass "PROXMOX_NODE configured: $PROXMOX_NODE"
else
warn "PROXMOX_NODE not set (will use default)"
fi
if [ -n "${PROXMOX_STORAGE:-}" ]; then
pass "PROXMOX_STORAGE configured: $PROXMOX_STORAGE"
else
warn "PROXMOX_STORAGE not set (will use default)"
fi
if [ -n "${CONTAINER_OS_TEMPLATE:-}" ]; then
pass "CONTAINER_OS_TEMPLATE configured: $CONTAINER_OS_TEMPLATE"
else
warn "CONTAINER_OS_TEMPLATE not set (will use default)"
fi
else
if [ -f "$CONFIG_DIR/proxmox.conf.example" ]; then
fail "proxmox.conf not found (example exists - needs to be copied and configured)"
else
fail "proxmox.conf not found"
fi
fi
# Check network.conf
if [ -f "$CONFIG_DIR/network.conf" ]; then
pass "network.conf exists"
source "$CONFIG_DIR/network.conf" 2>/dev/null || true
if [ -n "${SUBNET_BASE:-}" ]; then
pass "SUBNET_BASE configured: $SUBNET_BASE"
else
warn "SUBNET_BASE not set"
fi
if [ -n "${GATEWAY:-}" ]; then
pass "GATEWAY configured: $GATEWAY"
else
warn "GATEWAY not set"
fi
else
if [ -f "$CONFIG_DIR/network.conf.example" ]; then
warn "network.conf not found (example exists)"
else
warn "network.conf not found"
fi
fi
else
fail "Config directory not found: $CONFIG_DIR"
fi
# ============================================================================
# SECTION 6: DEPLOYMENT SCRIPTS
# ============================================================================
section "6. DEPLOYMENT SCRIPTS VALIDATION"
SCRIPTS_DIR="$DEPLOY_DIR/scripts/deployment"
if [ -d "$SCRIPTS_DIR" ]; then
pass "Deployment scripts directory exists"
REQUIRED_SCRIPTS=(
"deploy-all.sh"
"deploy-besu-nodes.sh"
"deploy-services.sh"
"deploy-hyperledger-services.sh"
"deploy-monitoring.sh"
"deploy-explorer.sh"
)
for script in "${REQUIRED_SCRIPTS[@]}"; do
SCRIPT_PATH="$SCRIPTS_DIR/$script"
if [ -f "$SCRIPT_PATH" ]; then
if [ -x "$SCRIPT_PATH" ]; then
pass "$script exists and is executable"
else
warn "$script exists but is not executable"
fi
# Check for syntax errors
if bash -n "$SCRIPT_PATH" 2>&1 | grep -q "error"; then
fail "$script has syntax errors"
fi
else
warn "$script not found"
fi
done
else
fail "Deployment scripts directory not found: $SCRIPTS_DIR"
fi
# Check common library
LIB_DIR="$DEPLOY_DIR/lib"
if [ -f "$LIB_DIR/common.sh" ]; then
pass "common.sh library exists"
else
fail "common.sh library not found"
fi
if [ -f "$LIB_DIR/proxmox-api.sh" ]; then
pass "proxmox-api.sh library exists"
else
warn "proxmox-api.sh library not found"
fi
# ============================================================================
# SECTION 7: RESOURCE REQUIREMENTS
# ============================================================================
section "7. RESOURCE REQUIREMENTS VALIDATION"
# Calculate total resource requirements
TOTAL_MEMORY=0
TOTAL_CORES=0
TOTAL_DISK=0
# Load resource config if available
if [ -f "$CONFIG_DIR/proxmox.conf" ]; then
source "$CONFIG_DIR/proxmox.conf" 2>/dev/null || true
# Calculate based on default counts
VALIDATOR_COUNT="${VALIDATORS_COUNT:-4}"
SENTRY_COUNT="${SENTRIES_COUNT:-3}"
RPC_COUNT="${RPC_COUNT:-3}"
SERVICE_COUNT="${SERVICES_COUNT:-4}"
TOTAL_MEMORY=$((VALIDATOR_MEMORY * VALIDATOR_COUNT + SENTRY_MEMORY * SENTRY_COUNT + RPC_MEMORY * RPC_COUNT + SERVICE_MEMORY * SERVICE_COUNT + MONITORING_MEMORY))
TOTAL_CORES=$((VALIDATOR_CORES * VALIDATOR_COUNT + SENTRY_CORES * SENTRY_COUNT + RPC_CORES * RPC_COUNT + SERVICE_CORES * SERVICE_COUNT + MONITORING_CORES))
TOTAL_DISK=$((VALIDATOR_DISK * VALIDATOR_COUNT + SENTRY_DISK * SENTRY_COUNT + RPC_DISK * RPC_COUNT + SERVICE_DISK * SERVICE_COUNT + MONITORING_DISK))
info "Estimated resource requirements:"
info " Memory: $((TOTAL_MEMORY / 1024))GB"
info " CPU Cores: $TOTAL_CORES"
info " Disk: ${TOTAL_DISK}GB"
# Get node resources
if [ -n "$NODE_STATUS" ]; then
NODE_MEMORY=$(echo "$NODE_STATUS" | jq -r '.data.memory.total // 0')
NODE_FREE_MEMORY=$(echo "$NODE_STATUS" | jq -r '.data.memory.free // 0')
NODE_CPU_COUNT=$(echo "$NODE_STATUS" | jq -r '.data.cpuinfo.cpus // 0')
if [ "$NODE_MEMORY" != "0" ] && [ -n "$NODE_MEMORY" ]; then
MEMORY_GB=$((NODE_MEMORY / 1024 / 1024 / 1024))
FREE_MEMORY_GB=$((NODE_FREE_MEMORY / 1024 / 1024 / 1024))
info "Node resources:"
info " Total Memory: ${MEMORY_GB}GB (Free: ${FREE_MEMORY_GB}GB)"
info " CPU Cores: $NODE_CPU_COUNT"
if [ $FREE_MEMORY_GB -lt $((TOTAL_MEMORY / 1024)) ]; then
warn "Insufficient free memory: need $((TOTAL_MEMORY / 1024))GB, have ${FREE_MEMORY_GB}GB"
else
pass "Sufficient memory available"
fi
if [ $NODE_CPU_COUNT -lt $TOTAL_CORES ]; then
warn "Limited CPU cores: need $TOTAL_CORES, have $NODE_CPU_COUNT (containers can share CPU)"
else
pass "Sufficient CPU cores available"
fi
fi
fi
fi
# ============================================================================
# SECTION 8: NETWORK CONFIGURATION
# ============================================================================
section "8. NETWORK CONFIGURATION VALIDATION"
# Get network interfaces
info "Checking network configuration..."
NETWORK_RESPONSE=$(curl -k -s -H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/network" 2>&1)
if echo "$NETWORK_RESPONSE" | jq -e '.data' &>/dev/null; then
BRIDGES=$(echo "$NETWORK_RESPONSE" | jq -r '.data[] | select(.type=="bridge") | .iface' | tr '\n' ' ')
if [ -n "$BRIDGES" ]; then
pass "Network bridges found: $BRIDGES"
# Check for vmbr0
if echo "$BRIDGES" | grep -q "vmbr0"; then
pass "vmbr0 bridge exists (required)"
else
warn "vmbr0 bridge not found (may cause network issues)"
fi
else
warn "No network bridges found"
fi
else
warn "Could not retrieve network configuration"
fi
# ============================================================================
# SECTION 9: EXISTING CONTAINERS
# ============================================================================
section "9. EXISTING CONTAINERS CHECK"
info "Checking for existing containers..."
CONTAINERS_RESPONSE=$(curl -k -s -H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/lxc" 2>&1)
if echo "$CONTAINERS_RESPONSE" | jq -e '.data' &>/dev/null; then
EXISTING_COUNT=$(echo "$CONTAINERS_RESPONSE" | jq '.data | length')
if [ "$EXISTING_COUNT" -gt 0 ]; then
info "Found $EXISTING_COUNT existing container(s)"
# Check for VMID conflicts
EXPECTED_VMIDS=(106 107 108 109 110 111 112 115 116 117 120 121 122 130 140 150 151 152 153)
CONFLICTS=()
for vmid in "${EXPECTED_VMIDS[@]}"; do
if echo "$CONTAINERS_RESPONSE" | jq -e ".data[] | select(.vmid==$vmid)" &>/dev/null; then
CONFLICTS+=("$vmid")
fi
done
if [ ${#CONFLICTS[@]} -gt 0 ]; then
warn "VMID conflicts detected: ${CONFLICTS[*]} (containers already exist)"
else
pass "No VMID conflicts detected"
fi
else
pass "No existing containers (clean slate)"
fi
else
warn "Could not retrieve container list"
fi
# ============================================================================
# SECTION 10: INSTALLATION SCRIPTS
# ============================================================================
section "10. INSTALLATION SCRIPTS VALIDATION"
INSTALL_DIR="$DEPLOY_DIR/install"
if [ -d "$INSTALL_DIR" ]; then
pass "Install scripts directory exists"
REQUIRED_INSTALL_SCRIPTS=(
"besu-validator-install.sh"
"besu-sentry-install.sh"
"besu-rpc-install.sh"
"oracle-publisher-install.sh"
"ccip-monitor-install.sh"
"keeper-install.sh"
)
for script in "${REQUIRED_INSTALL_SCRIPTS[@]}"; do
if [ -f "$INSTALL_DIR/$script" ]; then
pass "$script exists"
else
warn "$script not found"
fi
done
else
fail "Install scripts directory not found: $INSTALL_DIR"
fi
# ============================================================================
# SUMMARY
# ============================================================================
section "VALIDATION SUMMARY"
echo -e "${CYAN}Results:${NC}"
echo -e " ${GREEN}Passed:${NC} $PASSED"
echo -e " ${RED}Failed:${NC} $FAILED"
echo -e " ${YELLOW}Warnings:${NC} $WARNINGS"
echo ""
if [ $FAILED -eq 0 ]; then
echo -e "${GREEN}✅ Deployment validation PASSED${NC}"
echo ""
echo "Next steps:"
echo " 1. Review configuration files in $DEPLOY_DIR/config/"
echo " 2. Run deployment: cd $DEPLOY_DIR && ./scripts/deployment/deploy-all.sh"
exit 0
else
echo -e "${RED}❌ Deployment validation FAILED${NC}"
echo ""
echo "Please fix the issues above before proceeding with deployment."
exit 1
fi