proxmox/scripts/deployment/bootstrap-sankofa-it-read-api-lan.sh

#!/usr/bin/env bash
# One-shot LAN bootstrap: IT read API on PVE, repo .env, portal CT 7801, optional weekly timer.
#
# - Refreshes inventory JSON locally, rsyncs minimal tree to /opt/proxmox on the seed host
# - Generates IT_READ_API_KEY if missing in repo .env; sets IT_READ_API_URL for portal
# - Installs /etc/sankofa-it-read-api.env + systemd unit (bind 0.0.0.0:8787)
# - Runs sankofa-portal-merge-it-read-api-env-from-repo.sh
# - Optional: weekly systemd timer for export-live-inventory-and-drift.sh on PVE
#
# Usage:
#   ./scripts/deployment/bootstrap-sankofa-it-read-api-lan.sh [--dry-run] [--no-timer] [--no-portal-merge]
#
# Env: PROXMOX_HOST (default 192.168.11.11), IT_READ_API_PORT (8787), IT_BOOTSTRAP_REMOTE_ROOT (/opt/proxmox)
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
# shellcheck source=/dev/null
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
# shellcheck source=/dev/null
source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true

PROXMOX_HOST="${PROXMOX_HOST:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
REMOTE_ROOT="${IT_BOOTSTRAP_REMOTE_ROOT:-/opt/proxmox}"
PORT="${IT_READ_API_PORT:-8787}"
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=20 -o StrictHostKeyChecking=accept-new)
RSYNC_SSH="ssh -o BatchMode=yes -o ConnectTimeout=20 -o StrictHostKeyChecking=accept-new"

DRY=false
NO_TIMER=false
NO_PORTAL=false
for a in "$@"; do
  case "$a" in
    --dry-run) DRY=true ;;
    --no-timer) NO_TIMER=true ;;
    --no-portal-merge) NO_PORTAL=true ;;
  esac
done

PUBLIC_URL="http://${PROXMOX_HOST}:${PORT}"
CORS_ORIGIN="${IT_READ_API_CORS_ORIGINS:-https://portal.sankofa.nexus}"

log() { echo "[bootstrap-it-read-api] $*"; }

upsert_env_file() {
  local f="$1"
  shift
  python3 - "$f" "$@" <<'PY'
import os, re, sys
path = sys.argv[1]
pairs = list(zip(sys.argv[2::2], sys.argv[3::2]))

def upsert(text: str, k: str, v: str) -> str:
    line = f"{k}={v}"
    if re.search(rf"^{re.escape(k)}=", text, flags=re.M):
        return re.sub(rf"^{re.escape(k)}=.*$", line, text, flags=re.M, count=1)
    if text and not text.endswith("\n"):
        text += "\n"
    return text + line + "\n"

text = open(path).read() if os.path.isfile(path) else ""
for k, v in pairs:
    text = upsert(text, k, v)
open(path, "w").write(text)
PY
}

if $DRY; then
  log "dry-run: would export inventory, rsync → root@${PROXMOX_HOST}:${REMOTE_ROOT}, systemd, portal merge"
  exit 0
fi

log "refresh inventory + drift (local → reports/status)"
set +e
bash "${PROJECT_ROOT}/scripts/it-ops/export-live-inventory-and-drift.sh"
EX_INV=$?
set -e
if [[ "$EX_INV" -eq 2 ]]; then
  log "warning: export exited 2 (duplicate guest IPs on cluster); JSON still written — continuing"
elif [[ "$EX_INV" -ne 0 ]]; then
  exit "$EX_INV"
fi

API_KEY="${IT_READ_API_KEY:-}"
if [[ -z "$API_KEY" ]]; then
  API_KEY="$(openssl rand -hex 32)"
  log "generated IT_READ_API_KEY (openssl rand -hex 32)"
fi

ENV_LOCAL="${PROJECT_ROOT}/.env"
touch "$ENV_LOCAL"
upsert_env_file "$ENV_LOCAL" "IT_READ_API_URL" "$PUBLIC_URL" "IT_READ_API_KEY" "$API_KEY"
chmod 600 "$ENV_LOCAL" 2>/dev/null || true
log "upserted IT_READ_API_URL and IT_READ_API_KEY in repo .env (gitignored)"

log "rsync minimal repo tree → root@${PROXMOX_HOST}:${REMOTE_ROOT}"
ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" \
  "mkdir -p '${REMOTE_ROOT}/config' '${REMOTE_ROOT}/scripts/it-ops' '${REMOTE_ROOT}/services/sankofa-it-read-api' '${REMOTE_ROOT}/docs/04-configuration' '${REMOTE_ROOT}/reports/status'"
cd "$PROJECT_ROOT"
rsync -az --delete -e "$RSYNC_SSH" ./config/ip-addresses.conf "root@${PROXMOX_HOST}:${REMOTE_ROOT}/config/"
rsync -az --delete -e "$RSYNC_SSH" ./scripts/it-ops/ "root@${PROXMOX_HOST}:${REMOTE_ROOT}/scripts/it-ops/"
rsync -az --delete -e "$RSYNC_SSH" ./services/sankofa-it-read-api/server.py "root@${PROXMOX_HOST}:${REMOTE_ROOT}/services/sankofa-it-read-api/"
rsync -az --delete -e "$RSYNC_SSH" ./docs/04-configuration/ALL_VMIDS_ENDPOINTS.md "root@${PROXMOX_HOST}:${REMOTE_ROOT}/docs/04-configuration/"
rsync -az -e "$RSYNC_SSH" ./reports/status/live_inventory.json ./reports/status/drift.json "root@${PROXMOX_HOST}:${REMOTE_ROOT}/reports/status/"

ENV_REMOTE="/etc/sankofa-it-read-api.env"

# shellcheck disable=SC2087
ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" bash -s <<REMOTE
set -euo pipefail
umask 077
cat > ${ENV_REMOTE} <<ENVEOF
IT_READ_API_HOST=0.0.0.0
IT_READ_API_PORT=${PORT}
IT_READ_API_KEY=${API_KEY}
IT_READ_API_CORS_ORIGINS=${CORS_ORIGIN}
ENVEOF
chmod 600 ${ENV_REMOTE}
cat > /etc/systemd/system/sankofa-it-read-api.service <<SVCEOF
[Unit]
Description=Sankofa IT read API (live inventory JSON)
After=network.target

[Service]
Type=simple
User=root
WorkingDirectory=${REMOTE_ROOT}
EnvironmentFile=-${ENV_REMOTE}
ExecStart=/usr/bin/python3 ${REMOTE_ROOT}/services/sankofa-it-read-api/server.py
Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target
SVCEOF
systemctl daemon-reload
systemctl enable --now sankofa-it-read-api
systemctl is-active sankofa-it-read-api
REMOTE

log "verify read API (localhost on PVE — WAN/WSL may be firewalled)"
if ! ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" "curl -sS -f -m 5 http://127.0.0.1:${PORT}/health" | head -c 200; then
  echo "health check failed on PVE" >&2
  exit 1
fi
echo ""
if ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" "pct status 7801 &>/dev/null"; then
  log "verify from portal CT 7801 → ${PUBLIC_URL}/health"
  ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" \
    "pct exec 7801 -- sh -c 'curl -sS -f -m 8 http://${PROXMOX_HOST}:${PORT}/health || true'" | head -c 200 || true
  echo ""
fi

if ! $NO_TIMER; then
  log "install weekly inventory timer on PVE"
  # shellcheck disable=SC2087
  ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" bash -s <<REMOTE
set -euo pipefail
cat > /etc/systemd/system/sankofa-it-inventory-export.service <<EOF
[Unit]
Description=Export Proxmox live inventory and IPAM drift
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
User=root
WorkingDirectory=${REMOTE_ROOT}
ExecStart=/usr/bin/bash ${REMOTE_ROOT}/scripts/it-ops/export-live-inventory-and-drift.sh
EOF
cat > /etc/systemd/system/sankofa-it-inventory-export.timer <<'EOF'
[Unit]
Description=Timer — Proxmox live inventory + drift export (weekly)

[Timer]
OnCalendar=Sun *-*-* 03:30:00
Persistent=true

[Install]
WantedBy=timers.target
EOF
systemctl daemon-reload
systemctl enable --now sankofa-it-inventory-export.timer
systemctl list-timers sankofa-it-inventory-export.timer --no-pager || true
REMOTE
fi

if ! $NO_PORTAL; then
  log "merge IT_READ_API_* into portal CT 7801"
  export IT_READ_API_URL="$PUBLIC_URL"
  export IT_READ_API_KEY="$API_KEY"
  bash "${SCRIPT_DIR}/sankofa-portal-merge-it-read-api-env-from-repo.sh"
fi

log "done. Portal uses ${PUBLIC_URL} (server-side key in CT .env). Optional: NPM + DNS for it-api hostname → same upstream."