Files
proxmox/scripts/validate-ml110-deployment.sh

450 lines
18 KiB
Bash
Executable File
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# Comprehensive Deployment Validation for ml110-01
# Validates all prerequisites, configurations, and deployment readiness
set +e # Don't exit on errors - we want to collect all results
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
DEPLOY_DIR="$PROJECT_ROOT/smom-dbis-138-proxmox"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'
# Status tracking
declare -a PASSED_ITEMS
declare -a FAILED_ITEMS
declare -a WARNING_ITEMS
pass() {
echo -e "${GREEN}${NC} $1"
PASSED_ITEMS+=("$1")
}
fail() {
echo -e "${RED}${NC} $1"
FAILED_ITEMS+=("$1")
}
warn() {
echo -e "${YELLOW}⚠️${NC} $1"
WARNING_ITEMS+=("$1")
}
info() {
echo -e "${BLUE}${NC} $1"
}
section() {
echo ""
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${CYAN}$1${NC}"
echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""
}
# Load environment
ENV_FILE="$HOME/.env"
if [ -f "$ENV_FILE" ]; then
source <(grep -E "^PROXMOX_" "$ENV_FILE" 2>/dev/null | sed 's/^/export /' || true)
fi
TARGET_HOST="${PROXMOX_HOST:-192.168.11.10}"
TARGET_NODE="${PROXMOX_NODE:-ml110-01}"
echo ""
echo -e "${BLUE}╔════════════════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}${NC} Deployment Validation for ml110-01 (${TARGET_HOST}) ${BLUE}${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════════════════════╝${NC}"
echo ""
# ============================================================================
# SECTION 1: PREREQUISITES
# ============================================================================
section "1. PREREQUISITES CHECK"
# Check required tools
REQUIRED_TOOLS=("curl" "jq" "git" "bash")
for tool in "${REQUIRED_TOOLS[@]}"; do
if command -v "$tool" &> /dev/null; then
VERSION=$($tool --version 2>&1 | head -1 | cut -d' ' -f1-3)
pass "$tool installed ($VERSION)"
else
fail "$tool not found (required)"
fi
done
# Check deployment directory
if [ -d "$DEPLOY_DIR" ]; then
pass "Deployment directory exists: $DEPLOY_DIR"
else
fail "Deployment directory not found: $DEPLOY_DIR"
exit 1
fi
# ============================================================================
# SECTION 2: PROXMOX CONNECTION
# ============================================================================
section "2. PROXMOX CONNECTION VALIDATION"
if [ -z "${PROXMOX_USER:-}" ] || [ -z "${PROXMOX_TOKEN_NAME:-}" ] || [ -z "${PROXMOX_TOKEN_VALUE:-}" ]; then
fail "Proxmox credentials not configured in .env"
info "Required: PROXMOX_USER, PROXMOX_TOKEN_NAME, PROXMOX_TOKEN_VALUE"
else
pass "Proxmox credentials found in .env"
fi
# Test API connectivity
info "Testing connection to $TARGET_HOST:8006"
API_RESPONSE=$(curl -k -s -w "\n%{http_code}" -m 10 \
-H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/version" 2>&1)
HTTP_CODE=$(echo "$API_RESPONSE" | tail -1)
RESPONSE_BODY=$(echo "$API_RESPONSE" | sed '$d')
if [ "$HTTP_CODE" = "200" ]; then
VERSION=$(echo "$RESPONSE_BODY" | python3 -c "import sys, json; print(json.load(sys.stdin)['data']['version'])" 2>/dev/null || echo "unknown")
pass "API connection successful (Proxmox version: $VERSION)"
else
fail "API connection failed (HTTP $HTTP_CODE)"
if [ -n "$RESPONSE_BODY" ]; then
info "Response: $(echo "$RESPONSE_BODY" | head -c 200)"
fi
fi
# Get node list
info "Retrieving node list..."
NODES_RESPONSE=$(curl -k -s -m 10 \
-H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes" 2>&1)
if echo "$NODES_RESPONSE" | python3 -c "import sys, json; json.load(sys.stdin)['data']" 2>/dev/null; then
NODE_COUNT=$(echo "$NODES_RESPONSE" | python3 -c "import sys, json; print(len(json.load(sys.stdin)['data']))" 2>/dev/null)
NODE_NAMES=$(echo "$NODES_RESPONSE" | python3 -c "import sys, json; print(' '.join([n['node'] for n in json.load(sys.stdin)['data']]))" 2>/dev/null)
pass "Retrieved $NODE_COUNT node(s): $NODE_NAMES"
# Check if target node exists
if echo "$NODES_RESPONSE" | python3 -c "import sys, json; data=json.load(sys.stdin)['data']; exit(0 if any(n['node']=='${TARGET_NODE}' for n in data) else 1)" 2>/dev/null; then
pass "Target node '$TARGET_NODE' found"
else
FIRST_NODE=$(echo "$NODES_RESPONSE" | python3 -c "import sys, json; print(json.load(sys.stdin)['data'][0]['node'])" 2>/dev/null)
warn "Target node '$TARGET_NODE' not found, will use: $FIRST_NODE"
TARGET_NODE="$FIRST_NODE"
fi
else
fail "Failed to retrieve nodes"
fi
# ============================================================================
# SECTION 3: STORAGE VALIDATION
# ============================================================================
section "3. STORAGE VALIDATION"
STORAGE_RESPONSE=$(curl -k -s -m 10 \
-H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/storage" 2>&1)
if echo "$STORAGE_RESPONSE" | python3 -c "import sys, json; json.load(sys.stdin)['data']" 2>/dev/null; then
STORAGE_COUNT=$(echo "$STORAGE_RESPONSE" | python3 -c "import sys, json; print(len(json.load(sys.stdin)['data']))" 2>/dev/null)
pass "Found $STORAGE_COUNT storage pool(s)"
# Check for container storage
CONTAINER_STORAGE=$(echo "$STORAGE_RESPONSE" | python3 -c "import sys, json; data=json.load(sys.stdin)['data']; result=[s['storage'] for s in data if 'rootdir' in s.get('content', '').split(',')]; print(result[0] if result else '')" 2>/dev/null)
if [ -n "$CONTAINER_STORAGE" ]; then
pass "Container storage found: $CONTAINER_STORAGE"
else
fail "No container storage (rootdir) found"
fi
# Check for template storage
TEMPLATE_STORAGE=$(echo "$STORAGE_RESPONSE" | python3 -c "import sys, json; data=json.load(sys.stdin)['data']; result=[s['storage'] for s in data if 'vztmpl' in s.get('content', '').split(',')]; print(result[0] if result else '')" 2>/dev/null)
if [ -n "$TEMPLATE_STORAGE" ]; then
pass "Template storage found: $TEMPLATE_STORAGE"
else
warn "No template storage (vztmpl) found"
fi
else
fail "Failed to retrieve storage information"
fi
# ============================================================================
# SECTION 4: TEMPLATE VALIDATION
# ============================================================================
section "4. LXC TEMPLATE VALIDATION"
if [ -n "$TEMPLATE_STORAGE" ]; then
info "Checking templates on storage: $TEMPLATE_STORAGE"
TEMPLATES_RESPONSE=$(curl -k -s -m 10 \
-H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/storage/${TEMPLATE_STORAGE}/content?content=vztmpl" 2>&1)
if echo "$TEMPLATES_RESPONSE" | python3 -c "import sys, json; json.load(sys.stdin)['data']" 2>/dev/null; then
TEMPLATE_COUNT=$(echo "$TEMPLATES_RESPONSE" | python3 -c "import sys, json; print(len(json.load(sys.stdin)['data']))" 2>/dev/null)
pass "Found $TEMPLATE_COUNT template(s)"
# Check for Debian 12 template
DEBIAN_TEMPLATE=$(echo "$TEMPLATES_RESPONSE" | python3 -c "import sys, json; data=json.load(sys.stdin)['data']; result=[t['volid'] for t in data if 'debian-12' in t.get('volid', '')]; print(result[0] if result else '')" 2>/dev/null)
if [ -n "$DEBIAN_TEMPLATE" ]; then
pass "Debian 12 template found: $DEBIAN_TEMPLATE"
else
fail "Debian 12 template not found (required)"
info "Available templates:"
echo "$TEMPLATES_RESPONSE" | python3 -c "import sys, json; [print(' -', t['volid']) for t in json.load(sys.stdin)['data'][:10]]" 2>/dev/null
fi
else
warn "Could not retrieve template list"
fi
else
warn "Cannot check templates - no template storage configured"
fi
# ============================================================================
# SECTION 5: CONFIGURATION FILES
# ============================================================================
section "5. CONFIGURATION FILES VALIDATION"
CONFIG_DIR="$DEPLOY_DIR/config"
if [ -d "$CONFIG_DIR" ]; then
pass "Config directory exists: $CONFIG_DIR"
# Check proxmox.conf
if [ -f "$CONFIG_DIR/proxmox.conf" ]; then
pass "proxmox.conf exists"
source "$CONFIG_DIR/proxmox.conf" 2>/dev/null || true
if [ -n "${PROXMOX_HOST:-}" ]; then
if [ "$PROXMOX_HOST" = "$TARGET_HOST" ]; then
pass "PROXMOX_HOST configured correctly: $PROXMOX_HOST"
elif [ "$PROXMOX_HOST" = "proxmox.example.com" ]; then
warn "PROXMOX_HOST still set to example value"
else
warn "PROXMOX_HOST mismatch: $PROXMOX_HOST (expected: $TARGET_HOST)"
fi
else
fail "PROXMOX_HOST not set in proxmox.conf"
fi
[ -n "${PROXMOX_NODE:-}" ] && pass "PROXMOX_NODE configured: $PROXMOX_NODE" || warn "PROXMOX_NODE not set"
[ -n "${PROXMOX_STORAGE:-}" ] && pass "PROXMOX_STORAGE configured: $PROXMOX_STORAGE" || warn "PROXMOX_STORAGE not set"
[ -n "${CONTAINER_OS_TEMPLATE:-}" ] && pass "CONTAINER_OS_TEMPLATE configured: $CONTAINER_OS_TEMPLATE" || warn "CONTAINER_OS_TEMPLATE not set"
else
if [ -f "$CONFIG_DIR/proxmox.conf.example" ]; then
fail "proxmox.conf not found - copy from proxmox.conf.example and configure"
else
fail "proxmox.conf not found"
fi
fi
# Check network.conf
if [ -f "$CONFIG_DIR/network.conf" ]; then
pass "network.conf exists"
source "$CONFIG_DIR/network.conf" 2>/dev/null || true
[ -n "${SUBNET_BASE:-}" ] && pass "SUBNET_BASE configured: $SUBNET_BASE" || warn "SUBNET_BASE not set"
[ -n "${GATEWAY:-}" ] && pass "GATEWAY configured: $GATEWAY" || warn "GATEWAY not set"
else
if [ -f "$CONFIG_DIR/network.conf.example" ]; then
warn "network.conf not found (example exists)"
else
warn "network.conf not found"
fi
fi
else
fail "Config directory not found: $CONFIG_DIR"
fi
# ============================================================================
# SECTION 6: DEPLOYMENT SCRIPTS
# ============================================================================
section "6. DEPLOYMENT SCRIPTS VALIDATION"
SCRIPTS_DIR="$DEPLOY_DIR/scripts/deployment"
if [ -d "$SCRIPTS_DIR" ]; then
pass "Deployment scripts directory exists"
REQUIRED_SCRIPTS=(
"deploy-all.sh"
"deploy-besu-nodes.sh"
"deploy-services.sh"
"deploy-hyperledger-services.sh"
"deploy-monitoring.sh"
"deploy-explorer.sh"
)
for script in "${REQUIRED_SCRIPTS[@]}"; do
SCRIPT_PATH="$SCRIPTS_DIR/$script"
if [ -f "$SCRIPT_PATH" ]; then
if [ -x "$SCRIPT_PATH" ]; then
pass "$script exists and is executable"
# Check syntax
if bash -n "$SCRIPT_PATH" 2>&1 | grep -q "error"; then
fail "$script has syntax errors"
fi
else
warn "$script exists but is not executable"
fi
else
warn "$script not found"
fi
done
else
fail "Deployment scripts directory not found: $SCRIPTS_DIR"
fi
# Check libraries
LIB_DIR="$DEPLOY_DIR/lib"
[ -f "$LIB_DIR/common.sh" ] && pass "common.sh library exists" || fail "common.sh library not found"
[ -f "$LIB_DIR/proxmox-api.sh" ] && pass "proxmox-api.sh library exists" || warn "proxmox-api.sh library not found"
# ============================================================================
# SECTION 7: INSTALLATION SCRIPTS
# ============================================================================
section "7. INSTALLATION SCRIPTS VALIDATION"
INSTALL_DIR="$DEPLOY_DIR/install"
if [ -d "$INSTALL_DIR" ]; then
pass "Install scripts directory exists"
REQUIRED_INSTALL_SCRIPTS=(
"besu-validator-install.sh"
"besu-sentry-install.sh"
"besu-rpc-install.sh"
"oracle-publisher-install.sh"
"ccip-monitor-install.sh"
"keeper-install.sh"
"monitoring-stack-install.sh"
"blockscout-install.sh"
)
for script in "${REQUIRED_INSTALL_SCRIPTS[@]}"; do
[ -f "$INSTALL_DIR/$script" ] && pass "$script exists" || warn "$script not found"
done
else
fail "Install scripts directory not found: $INSTALL_DIR"
fi
# ============================================================================
# SECTION 8: RESOURCE REQUIREMENTS
# ============================================================================
section "8. RESOURCE REQUIREMENTS VALIDATION"
# Load config if available
if [ -f "$CONFIG_DIR/proxmox.conf" ]; then
source "$CONFIG_DIR/proxmox.conf" 2>/dev/null || true
# Estimate resources
VALIDATOR_COUNT="${VALIDATORS_COUNT:-4}"
SENTRY_COUNT="${SENTRIES_COUNT:-3}"
RPC_COUNT="${RPC_COUNT:-3}"
ESTIMATED_MEMORY=$(( (VALIDATOR_MEMORY * VALIDATOR_COUNT) + (SENTRY_MEMORY * SENTRY_COUNT) + (RPC_MEMORY * RPC_COUNT) + MONITORING_MEMORY ))
ESTIMATED_DISK=$(( (VALIDATOR_DISK * VALIDATOR_COUNT) + (SENTRY_DISK * SENTRY_COUNT) + (RPC_DISK * RPC_COUNT) + MONITORING_DISK ))
info "Estimated requirements:"
info " Memory: ~$((ESTIMATED_MEMORY / 1024))GB"
info " Disk: ~${ESTIMATED_DISK}GB"
# Get node resources
NODE_STATUS=$(curl -k -s -m 10 \
-H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/status" 2>&1)
if echo "$NODE_STATUS" | python3 -c "import sys, json; json.load(sys.stdin)['data']" 2>/dev/null; then
NODE_MEMORY=$(echo "$NODE_STATUS" | python3 -c "import sys, json; print(json.load(sys.stdin)['data']['memory']['total'])" 2>/dev/null || echo "0")
NODE_FREE_MEMORY=$(echo "$NODE_STATUS" | python3 -c "import sys, json; print(json.load(sys.stdin)['data']['memory']['free'])" 2>/dev/null || echo "0")
if [ "$NODE_MEMORY" != "0" ] && [ -n "$NODE_MEMORY" ]; then
MEMORY_GB=$((NODE_MEMORY / 1024 / 1024 / 1024))
FREE_MEMORY_GB=$((NODE_FREE_MEMORY / 1024 / 1024 / 1024))
info "Node resources: ${MEMORY_GB}GB total, ${FREE_MEMORY_GB}GB free"
if [ $FREE_MEMORY_GB -ge $((ESTIMATED_MEMORY / 1024)) ]; then
pass "Sufficient memory available"
else
warn "Limited free memory: need ~$((ESTIMATED_MEMORY / 1024))GB, have ${FREE_MEMORY_GB}GB"
fi
fi
fi
fi
# ============================================================================
# SECTION 9: EXISTING CONTAINERS
# ============================================================================
section "9. EXISTING CONTAINERS CHECK"
CONTAINERS_RESPONSE=$(curl -k -s -m 10 \
-H "Authorization: PVEAPIToken=${PROXMOX_USER}!${PROXMOX_TOKEN_NAME}=${PROXMOX_TOKEN_VALUE}" \
"https://${TARGET_HOST}:8006/api2/json/nodes/${TARGET_NODE}/lxc" 2>&1)
if echo "$CONTAINERS_RESPONSE" | python3 -c "import sys, json; json.load(sys.stdin)['data']" 2>/dev/null; then
EXISTING_COUNT=$(echo "$CONTAINERS_RESPONSE" | python3 -c "import sys, json; print(len(json.load(sys.stdin)['data']))" 2>/dev/null)
if [ "$EXISTING_COUNT" -gt 0 ]; then
info "Found $EXISTING_COUNT existing container(s)"
# Check for VMID conflicts
EXPECTED_VMIDS=(106 107 108 109 110 111 112 115 116 117 120 121 122 130 140 150 151 152 153)
CONFLICTS=$(echo "$CONTAINERS_RESPONSE" | python3 -c "
import sys, json
data = json.load(sys.stdin)['data']
existing = [c['vmid'] for c in data]
expected = [106, 107, 108, 109, 110, 111, 112, 115, 116, 117, 120, 121, 122, 130, 140, 150, 151, 152, 153]
conflicts = [v for v in expected if v in existing]
print(' '.join(map(str, conflicts)))" 2>/dev/null)
if [ -n "$CONFLICTS" ]; then
warn "VMID conflicts: $CONFLICTS (containers already exist)"
else
pass "No VMID conflicts detected"
fi
else
pass "No existing containers (clean slate)"
fi
else
warn "Could not retrieve container list"
fi
# ============================================================================
# SUMMARY
# ============================================================================
section "VALIDATION SUMMARY"
TOTAL_PASSED=${#PASSED_ITEMS[@]}
TOTAL_FAILED=${#FAILED_ITEMS[@]}
TOTAL_WARNINGS=${#WARNING_ITEMS[@]}
echo -e "${CYAN}Results:${NC}"
echo -e " ${GREEN}Passed:${NC} $TOTAL_PASSED"
echo -e " ${RED}Failed:${NC} $TOTAL_FAILED"
echo -e " ${YELLOW}Warnings:${NC} $TOTAL_WARNINGS"
echo ""
if [ $TOTAL_FAILED -eq 0 ]; then
echo -e "${GREEN}✅ Deployment validation PASSED${NC}"
echo ""
echo "Next steps:"
echo " 1. Review and update configuration files in $CONFIG_DIR/"
echo " 2. Run deployment: cd $DEPLOY_DIR && ./scripts/deployment/deploy-all.sh"
exit 0
else
echo -e "${RED}❌ Deployment validation FAILED${NC}"
echo ""
echo "Critical issues found. Please fix the failures above before deployment."
exit 1
fi