proxmox/scripts/verify-r630-02-services.sh

#!/usr/bin/env bash
# Verify all services on r630-02 are running and accessible
# Usage: ./scripts/verify-r630-02-services.sh

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# Configuration
NODE_IP="192.168.11.12"
NODE_NAME="r630-02"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_section() { echo -e "\n${CYAN}=== $1 ===${NC}\n"; }

echo ""
log_info "═══════════════════════════════════════════════════════════"
log_info "  VERIFYING ALL SERVICES ON $NODE_NAME"
log_info "═══════════════════════════════════════════════════════════"
echo ""

# Get all containers
log_section "Container Status"
CONTAINERS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
    "pct list 2>/dev/null | tail -n +2" || echo "")

if [[ -z "$CONTAINERS" ]]; then
    log_error "No containers found"
    exit 1
fi

printf "%-8s | %-12s | %-30s | %-15s | %s\n" "VMID" "Status" "Hostname" "IP Address" "Connectivity"
echo "---------|--------------|------------------------------|----------------|-------------"

declare -a SERVICE_INFO=()

while IFS= read -r line; do
    if [[ -z "$line" ]]; then
        continue
    fi
    
    vmid=$(echo "$line" | awk '{print $1}')
    status=$(echo "$line" | awk '{print $2}')
    
    # Get container details
    hostname=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct config $vmid 2>/dev/null | grep -oP 'hostname=\\K[^,]+' | head -1" 2>/dev/null || echo "unknown")
    
    ip=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct config $vmid 2>/dev/null | grep -oP 'ip=\\K[^,]+' | head -1" 2>/dev/null || echo "")
    
    if [[ "$ip" == "dhcp" ]] || [[ -z "$ip" ]]; then
        ip_display="DHCP"
        ip_base=""
    else
        ip_base="${ip%/*}"
        ip_display="$ip_base"
    fi
    
    # Test connectivity
    connectivity="N/A"
    if [[ -n "$ip_base" ]]; then
        if ping -c 1 -W 2 "$ip_base" &>/dev/null; then
            connectivity="${GREEN}✓ Reachable${NC}"
        else
            connectivity="${RED}✗ Unreachable${NC}"
        fi
    else
        connectivity="${YELLOW}-${NC}"
    fi
    
    # Format status
    if [[ "$status" == "running" ]]; then
        status_display="${GREEN}running${NC}"
    else
        status_display="${RED}$status${NC}"
    fi
    
    printf "%-8s | %b | %-30s | %-15s | %b\n" "$vmid" "$status_display" "$hostname" "$ip_display" "$connectivity"
    
    if [[ "$status" == "running" ]] && [[ -n "$ip_base" ]]; then
        SERVICE_INFO+=("$vmid:$hostname:$ip_base")
    fi
    
done <<< "$CONTAINERS"

log_section "Service Port Verification"

# Check common service ports
declare -A SERVICE_PORTS=(
    ["nginxproxymanager"]="8443"
    ["monitoring"]="3000"
    ["blockscout"]="4000"
    ["gitea"]="3000"
    ["omada"]="8043"
    ["cloudflared"]="N/A"
    ["mail-gateway"]="25,587,993"
    ["datacenter-manager"]="8006"
    ["firefly"]="5000"
    ["mim-api"]="8080"
)

for service_info in "${SERVICE_INFO[@]}"; do
    IFS=':' read -r vmid hostname ip <<< "$service_info"
    
    log_info "Checking VMID $vmid ($hostname) at $ip..."
    
    # Try to get listening ports
    ports=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct exec $vmid -- netstat -tlnp 2>/dev/null | grep LISTEN | awk '{print \$4}' | cut -d':' -f2 | sort -u" 2>/dev/null || echo "")
    
    if [[ -n "$ports" ]]; then
        log_success "  Listening ports: $(echo $ports | tr '\n' ' ')"
    else
        log_warn "  Could not determine listening ports"
    fi
    
    # Test HTTP/HTTPS if applicable
    for port in 80 443 3000 4000 5000 8080 8443; do
        if echo "$ports" | grep -q "^$port$"; then
            protocol="http"
            if [[ $port -eq 443 ]] || [[ $port -eq 8443 ]]; then
                protocol="https"
            fi
            
            response=$(curl -k -s -o /dev/null -w "%{http_code}" --connect-timeout 3 "${protocol}://${ip}:${port}" 2>/dev/null || echo "000")
            if [[ "$response" =~ ^[23] ]]; then
                log_success "  ✓ $protocol://$ip:$port - HTTP $response"
            elif [[ "$response" != "000" ]]; then
                log_warn "  ⚠ $protocol://$ip:$port - HTTP $response"
            fi
        fi
    done
    
    echo ""
done

log_section "Service Health Checks"

# Check container health
for service_info in "${SERVICE_INFO[@]}"; do
    IFS=':' read -r vmid hostname ip <<< "$service_info"
    
    log_info "Health check for VMID $vmid ($hostname)..."
    
    # Check if container is responsive
    if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct exec $vmid -- echo 'OK' 2>/dev/null" &>/dev/null; then
        log_success "  ✓ Container is responsive"
    else
        log_error "  ✗ Container is not responsive"
    fi
    
    # Check systemd services (if applicable)
    services=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct exec $vmid -- systemctl list-units --type=service --state=running 2>/dev/null | grep -E '^[^ ]+\.service' | wc -l" 2>/dev/null || echo "0")
    
    if [[ "$services" -gt 0 ]]; then
        log_info "  Running services: $services"
    fi
    
    echo ""
done

log_section "Summary"

RUNNING_COUNT=$(echo "$CONTAINERS" | grep -c running || echo 0)
TOTAL_COUNT=$(echo "$CONTAINERS" | wc -l)

log_info "Total containers: $TOTAL_COUNT"
log_info "Running: $RUNNING_COUNT"
log_info "Services verified: ${#SERVICE_INFO[@]}"

if [[ $RUNNING_COUNT -eq $TOTAL_COUNT ]]; then
    log_success "✅ All containers are running"
else
    log_warn "⚠️  Some containers are not running"
fi

echo ""
log_success "═══════════════════════════════════════════════════════════"
log_success "  VERIFICATION COMPLETE"
log_success "═══════════════════════════════════════════════════════════"
echo ""