proxmox/scripts/verify/generate-source-of-truth.sh

#!/usr/bin/env bash
# Generate source-of-truth JSON from verification outputs
# Combines all verification results into canonical data model

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
EVIDENCE_DIR="$PROJECT_ROOT/docs/04-configuration/verification-evidence"
OUTPUT_FILE="$PROJECT_ROOT/docs/04-configuration/INGRESS_SOURCE_OF_TRUTH.json"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }

cd "$PROJECT_ROOT"

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "🔍 Generate Source-of-Truth JSON"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Find latest verification outputs
LATEST_DNS_DIR=$(ls -td "$EVIDENCE_DIR"/dns-verification-* 2>/dev/null | head -1 || echo "")
LATEST_UDM_DIR=$(ls -td "$EVIDENCE_DIR"/udm-pro-verification-* 2>/dev/null | head -1 || echo "")
LATEST_NPM_DIR=$(ls -td "$EVIDENCE_DIR"/npmplus-verification-* 2>/dev/null | head -1 || echo "")
LATEST_VM_DIR=$(ls -td "$EVIDENCE_DIR"/backend-vms-verification-* 2>/dev/null | head -1 || echo "")
LATEST_E2E_DIR=$(ls -td "$EVIDENCE_DIR"/e2e-verification-* 2>/dev/null | head -1 || echo "")

# Validate that source files exist
log_info "Validating source files..."
MISSING_FILES=()

if [ -z "$LATEST_DNS_DIR" ] || [ ! -f "$LATEST_DNS_DIR/all_dns_records.json" ]; then
    log_warn "DNS verification results not found. Run: bash scripts/verify/export-cloudflare-dns-records.sh"
    MISSING_FILES+=("DNS verification")
fi

if [ -z "$LATEST_NPM_DIR" ] || [ ! -f "$LATEST_NPM_DIR/proxy_hosts.json" ]; then
    log_warn "NPMplus verification results not found. Run: bash scripts/verify/export-npmplus-config.sh"
    MISSING_FILES+=("NPMplus verification")
fi

if [ -z "$LATEST_VM_DIR" ] || [ ! -f "$LATEST_VM_DIR/all_vms_verification.json" ]; then
    log_warn "Backend VM verification results not found. Run: bash scripts/verify/verify-backend-vms.sh"
    MISSING_FILES+=("Backend VM verification")
fi

if [ ${#MISSING_FILES[@]} -gt 0 ]; then
    log_warn "Some verification results are missing. Source of truth will be incomplete."
    log_info "Missing: ${MISSING_FILES[*]}"
    if [ "${CONTINUE_PARTIAL:-0}" = "1" ] || [ "${CONTINUE_PARTIAL}" = "true" ]; then
        log_info "Continuing (CONTINUE_PARTIAL=1)"
    else
        log_info "You can still generate a partial source of truth, or run full verification first."
        echo ""
        read -p "Continue with partial source of truth? (y/N): " -n 1 -r
        echo
        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
            log_info "Exiting. Run full verification first."
            exit 0
        fi
    fi
fi

# Allow partial generation if at least DNS or NPM data exists
if [ -z "$LATEST_DNS_DIR" ] && [ -z "$LATEST_NPM_DIR" ]; then
    log_error "No verification outputs found. Run verification scripts first."
    log_info "Required: DNS verification OR NPMplus verification"
    exit 1
fi

log_info "Using verification outputs:"
[ -n "$LATEST_DNS_DIR" ] && log_info "  DNS: $(basename "$LATEST_DNS_DIR")"
[ -n "$LATEST_UDM_DIR" ] && log_info "  UDM Pro: $(basename "$LATEST_UDM_DIR")"
[ -n "$LATEST_NPM_DIR" ] && log_info "  NPMplus: $(basename "$LATEST_NPM_DIR")"
[ -n "$LATEST_VM_DIR" ] && log_info "  Backend VMs: $(basename "$LATEST_VM_DIR")"
[ -n "$LATEST_E2E_DIR" ] && log_info "  E2E: $(basename "$LATEST_E2E_DIR")"
echo ""

# Validate and load DNS records
log_info "Loading DNS records..."
DNS_RECORDS="[]"
if [ -f "$LATEST_DNS_DIR/all_dns_records.json" ]; then
    if jq empty "$LATEST_DNS_DIR/all_dns_records.json" 2>/dev/null; then
        DNS_RECORDS=$(cat "$LATEST_DNS_DIR/all_dns_records.json" 2>/dev/null || echo "[]")
    else
        log_error "Invalid JSON in DNS records file"
        DNS_RECORDS="[]"
    fi
fi

# Validate and load NPMplus config
log_info "Loading NPMplus configuration..."
PROXY_HOSTS="[]"
CERTIFICATES="[]"
if [ -f "$LATEST_NPM_DIR/proxy_hosts.json" ]; then
    if jq empty "$LATEST_NPM_DIR/proxy_hosts.json" 2>/dev/null; then
        PROXY_HOSTS=$(cat "$LATEST_NPM_DIR/proxy_hosts.json" 2>/dev/null || echo "[]")
    else
        log_error "Invalid JSON in proxy hosts file"
        PROXY_HOSTS="[]"
    fi
fi
if [ -f "$LATEST_NPM_DIR/certificates.json" ]; then
    if jq empty "$LATEST_NPM_DIR/certificates.json" 2>/dev/null; then
        CERTIFICATES=$(cat "$LATEST_NPM_DIR/certificates.json" 2>/dev/null || echo "[]")
    else
        log_error "Invalid JSON in certificates file"
        CERTIFICATES="[]"
    fi
fi

# Validate and load backend VMs
log_info "Loading backend VMs..."
BACKEND_VMS="[]"
if [ -f "$LATEST_VM_DIR/all_vms_verification.json" ]; then
    if jq empty "$LATEST_VM_DIR/all_vms_verification.json" 2>/dev/null; then
        BACKEND_VMS=$(cat "$LATEST_VM_DIR/all_vms_verification.json" 2>/dev/null || echo "[]")
    else
        log_error "Invalid JSON in backend VMs file"
        BACKEND_VMS="[]"
    fi
fi

# Validate and load UDM Pro config
log_info "Loading UDM Pro configuration..."
UDM_CONFIG="{}"
if [ -f "$LATEST_UDM_DIR/verification_results.json" ]; then
    if jq empty "$LATEST_UDM_DIR/verification_results.json" 2>/dev/null; then
        UDM_CONFIG=$(cat "$LATEST_UDM_DIR/verification_results.json" 2>/dev/null || echo "{}")
    else
        log_error "Invalid JSON in UDM Pro config file"
        UDM_CONFIG="{}"
    fi
fi

# Build source-of-truth JSON
log_info "Generating source-of-truth JSON..."

# Transform DNS records
dns_records_array=$(echo "$DNS_RECORDS" | jq -c '.[] | {
    zone: (.zone // ""),
    hostname: .name,
    record_type: .type,
    record_value: .content,
    proxied: (.proxied // false),
    ttl: (.ttl // 1),
    status: "verified",
    verified_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ")),
    notes: ""
}' 2>/dev/null || echo "[]")

# Transform proxy hosts
proxy_hosts_array=$(echo "$PROXY_HOSTS" | jq -c '.[] | {
    id: .id,
    domain_names: (.domain_names // []),
    forward_scheme: (.forward_scheme // "http"),
    forward_host: (.forward_host // ""),
    forward_port: (.forward_port // 80),
    ssl_certificate_id: (.certificate_id // null),
    force_ssl: (.ssl_forced // false),
    allow_websocket_upgrade: (.allow_websocket_upgrade // false),
    access_list_id: (.access_list_id // null),
    advanced_config: (.advanced_config // ""),
    status: "verified",
    verified_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ"))
}' 2>/dev/null || echo "[]")

# Transform certificates
certificates_array=$(echo "$CERTIFICATES" | jq -c '.[] | {
    id: .id,
    provider_name: (.provider // "letsencrypt"),
    nice_name: (.nice_name // ""),
    domain_names: (.domain_names // []),
    expires_at: (if .expires then (.expires | strftime("%Y-%m-%dT%H:%M:%SZ")) else "" end),
    enabled: (.enabled // true),
    auto_renewal: (.auto_renewal // true),
    certificate_files: {
        fullchain: "/data/tls/certbot/live/npm-\(.id)/fullchain.pem",
        privkey: "/data/tls/certbot/live/npm-\(.id)/privkey.pem"
    },
    status: "verified",
    verified_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ"))
}' 2>/dev/null || echo "[]")

# Transform backend VMs (keep as-is but ensure status)
backend_vms_array=$(echo "$BACKEND_VMS" | jq -c '.[] | . + {
    status: (if .status then .status else "verified" end),
    verified_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ"))
}' 2>/dev/null || echo "[]")

# Extract UDM Pro info
udm_wan_ip=$(echo "$UDM_CONFIG" | jq -r '.expected_configuration.public_ip // "76.53.10.36"' 2>/dev/null || echo "76.53.10.36")
udm_port_forwarding=$(echo "$UDM_CONFIG" | jq -c '.expected_configuration.port_forwarding_rules // []' 2>/dev/null || echo "[]")

# Build complete JSON structure
SOURCE_OF_TRUTH=$(jq -n \
    --argjson dns_records "$(echo "$dns_records_array" | jq -s '.')" \
    --argjson proxy_hosts "$(echo "$proxy_hosts_array" | jq -s '.')" \
    --argjson certificates "$(echo "$certificates_array" | jq -s '.')" \
    --argjson backend_vms "$(echo "$backend_vms_array" | jq -s '.')" \
    --argjson port_forwarding "$udm_port_forwarding" \
    --arg wan_ip "$udm_wan_ip" \
    '{
        metadata: {
            version: "1.0.0",
            last_verified: (now | strftime("%Y-%m-%dT%H:%M:%SZ")),
            verifier: (env.USER // "unknown"),
            baseline_docs: [
                "docs/04-configuration/DNS_NPMPLUS_VM_COMPREHENSIVE_ARCHITECTURE.md",
                "docs/04-configuration/DNS_NPMPLUS_VM_STREAMLINED_TABLE.md"
            ]
        },
        dns_records: $dns_records,
        edge_routing: {
            wan_ip: $wan_ip,
            port_forwarding_rules: $port_forwarding
        },
        npmplus: {
            container: {
                vmid: 10233,
                host: "r630-01",
                host_ip: "${PROXMOX_HOST_R630_01:-192.168.11.11}",
                internal_ips: {
                    eth0: "${IP_NPMPLUS_ETH0:-${IP_NPMPLUS_ETH0:-192.168.11.166}}",
                    eth1: "${IP_NPMPLUS:-${IP_NPMPLUS:-192.168.11.167}}"
                },
                management_ui: "https://${IP_NPMPLUS_ETH0:-${IP_NPMPLUS_ETH0:-192.168.11.166}}:81",
                status: "running"
            },
            proxy_hosts: $proxy_hosts,
            certificates: $certificates
        },
        backend_vms: $backend_vms,
        issues: [
            {
                severity: "critical",
                component: "backend",
                domain: "sankofa.nexus",
                description: "Sankofa services not deployed, routing to Blockscout",
                status: "known",
                action_required: "Deploy Sankofa services and update NPMplus routing"
            }
        ]
    }' 2>/dev/null || echo "{}")

# Write to file
# Validate final JSON before writing
if echo "$SOURCE_OF_TRUTH" | jq empty 2>/dev/null; then
    echo "$SOURCE_OF_TRUTH" | jq '.' > "$OUTPUT_FILE"
    log_success "Source of truth JSON validated and written"
else
    log_error "Generated JSON is invalid - not writing file"
    exit 1
fi

log_success "Source-of-truth JSON generated: $OUTPUT_FILE"

# Show summary
DNS_COUNT=$(echo "$SOURCE_OF_TRUTH" | jq '.dns_records | length' 2>/dev/null || echo "0")
PROXY_COUNT=$(echo "$SOURCE_OF_TRUTH" | jq '.npmplus.proxy_hosts | length' 2>/dev/null || echo "0")
CERT_COUNT=$(echo "$SOURCE_OF_TRUTH" | jq '.npmplus.certificates | length' 2>/dev/null || echo "0")
VM_COUNT=$(echo "$SOURCE_OF_TRUTH" | jq '.backend_vms | length' 2>/dev/null || echo "0")

log_info ""
log_info "Summary:"
log_info "  DNS Records: $DNS_COUNT"
log_info "  Proxy Hosts: $PROXY_COUNT"
log_info "  Certificates: $CERT_COUNT"
log_info "  Backend VMs: $VM_COUNT"