feat(it-ops): LAN bootstrap for read API, NPM proxy, Cloudflare DNS

- bootstrap-sankofa-it-read-api-lan.sh: rsync /opt/proxmox, systemd + env file, repo .env keys, portal CT 7801 merge, weekly export timer; tolerate export exit 2 - upsert-it-read-api-proxy-host.sh, add-it-api-sankofa-dns.sh - systemd example uses EnvironmentFile; docs, spec, AGENTS, read API README Made-with: Cursor
2026-04-09 01:50:14 -07:00
parent bd3424d78b
commit a41c3adea0
8 changed files with 388 additions and 14 deletions
--- a/scripts/cloudflare/add-it-api-sankofa-dns.sh
+++ b/scripts/cloudflare/add-it-api-sankofa-dns.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Cloudflare DNS: it-api.sankofa.nexus → PUBLIC_IP (A record, proxied).
+# Pair with scripts/nginx-proxy-manager/upsert-it-read-api-proxy-host.sh (NPM upstream r630-01:8787).
+#
+# Usage: bash scripts/cloudflare/add-it-api-sankofa-dns.sh [--dry-run]
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+cd "$PROJECT_ROOT"
+source config/ip-addresses.conf 2>/dev/null || true
+[ -f .env ] && set +u && source .env 2>/dev/null || true && set -u
+
+ZONE_ID="${CLOUDFLARE_ZONE_ID_SANKOFA_NEXUS:-}"
+PUBLIC_IP="${PUBLIC_IP:-76.53.10.36}"
+NAME="${IT_READ_API_DNS_NAME:-it-api}"
+DRY=false
+[[ "${1:-}" == "--dry-run" ]] && DRY=true
+
+if [ -n "${CLOUDFLARE_API_TOKEN:-}" ]; then
+  AUTH_H=(-H "Authorization: Bearer $CLOUDFLARE_API_TOKEN")
+elif [ -n "${CLOUDFLARE_API_KEY:-}" ] && [ -n "${CLOUDFLARE_EMAIL:-}" ]; then
+  AUTH_H=(-H "X-Auth-Email: $CLOUDFLARE_EMAIL" -H "X-Auth-Key: $CLOUDFLARE_API_KEY")
+else
+  echo "Set CLOUDFLARE_API_TOKEN or (CLOUDFLARE_EMAIL + CLOUDFLARE_API_KEY) in .env" >&2
+  exit 1
+fi
+
+[ -z "$ZONE_ID" ] && { echo "Set CLOUDFLARE_ZONE_ID_SANKOFA_NEXUS in .env" >&2; exit 1; }
+
+FQDN="${NAME}.sankofa.nexus"
+echo "DNS ${FQDN} → ${PUBLIC_IP} (zone sankofa.nexus)"
+
+DATA=$(jq -n --arg name "$NAME" --arg content "$PUBLIC_IP" \
+  '{type:"A",name:$name,content:$content,ttl:1,proxied:true}')
+
+if $DRY; then
+  echo "[dry-run] would POST/PUT Cloudflare record"
+  exit 0
+fi
+
+EXISTING=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${FQDN}" \
+  "${AUTH_H[@]}" -H "Content-Type: application/json")
+RECORD_ID=$(echo "$EXISTING" | jq -r '.result[0].id // empty')
+
+if [ -n "$RECORD_ID" ] && [ "$RECORD_ID" != "null" ]; then
+  UPD=$(curl -s -X PUT "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" \
+    "${AUTH_H[@]}" -H "Content-Type: application/json" -d "$DATA")
+  echo "$UPD" | jq -e '.success == true' >/dev/null 2>&1 && echo "Updated ${FQDN}" || { echo "$UPD" | jq . >&2; exit 1; }
+else
+  CR=$(curl -s -X POST "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" \
+    "${AUTH_H[@]}" -H "Content-Type: application/json" -d "$DATA")
+  echo "$CR" | jq -e '.success == true' >/dev/null 2>&1 && echo "Created ${FQDN}" || { echo "$CR" | jq . >&2; exit 1; }
+fi
--- a/scripts/deployment/bootstrap-sankofa-it-read-api-lan.sh
+++ b/scripts/deployment/bootstrap-sankofa-it-read-api-lan.sh
@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+# One-shot LAN bootstrap: IT read API on PVE, repo .env, portal CT 7801, optional weekly timer.
+#
+# - Refreshes inventory JSON locally, rsyncs minimal tree to /opt/proxmox on the seed host
+# - Generates IT_READ_API_KEY if missing in repo .env; sets IT_READ_API_URL for portal
+# - Installs /etc/sankofa-it-read-api.env + systemd unit (bind 0.0.0.0:8787)
+# - Runs sankofa-portal-merge-it-read-api-env-from-repo.sh
+# - Optional: weekly systemd timer for export-live-inventory-and-drift.sh on PVE
+#
+# Usage:
+#   ./scripts/deployment/bootstrap-sankofa-it-read-api-lan.sh [--dry-run] [--no-timer] [--no-portal-merge]
+#
+# Env: PROXMOX_HOST (default 192.168.11.11), IT_READ_API_PORT (8787), IT_BOOTSTRAP_REMOTE_ROOT (/opt/proxmox)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true
+
+PROXMOX_HOST="${PROXMOX_HOST:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
+REMOTE_ROOT="${IT_BOOTSTRAP_REMOTE_ROOT:-/opt/proxmox}"
+PORT="${IT_READ_API_PORT:-8787}"
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=20 -o StrictHostKeyChecking=accept-new)
+RSYNC_SSH="ssh -o BatchMode=yes -o ConnectTimeout=20 -o StrictHostKeyChecking=accept-new"
+
+DRY=false
+NO_TIMER=false
+NO_PORTAL=false
+for a in "$@"; do
+  case "$a" in
+    --dry-run) DRY=true ;;
+    --no-timer) NO_TIMER=true ;;
+    --no-portal-merge) NO_PORTAL=true ;;
+  esac
+done
+
+PUBLIC_URL="http://${PROXMOX_HOST}:${PORT}"
+CORS_ORIGIN="${IT_READ_API_CORS_ORIGINS:-https://portal.sankofa.nexus}"
+
+log() { echo "[bootstrap-it-read-api] $*"; }
+
+upsert_env_file() {
+  local f="$1"
+  shift
+  python3 - "$f" "$@" <<'PY'
+import os, re, sys
+path = sys.argv[1]
+pairs = list(zip(sys.argv[2::2], sys.argv[3::2]))
+
+def upsert(text: str, k: str, v: str) -> str:
+    line = f"{k}={v}"
+    if re.search(rf"^{re.escape(k)}=", text, flags=re.M):
+        return re.sub(rf"^{re.escape(k)}=.*$", line, text, flags=re.M, count=1)
+    if text and not text.endswith("\n"):
+        text += "\n"
+    return text + line + "\n"
+
+text = open(path).read() if os.path.isfile(path) else ""
+for k, v in pairs:
+    text = upsert(text, k, v)
+open(path, "w").write(text)
+PY
+}
+
+if $DRY; then
+  log "dry-run: would export inventory, rsync → root@${PROXMOX_HOST}:${REMOTE_ROOT}, systemd, portal merge"
+  exit 0
+fi
+
+log "refresh inventory + drift (local → reports/status)"
+set +e
+bash "${PROJECT_ROOT}/scripts/it-ops/export-live-inventory-and-drift.sh"
+EX_INV=$?
+set -e
+if [[ "$EX_INV" -eq 2 ]]; then
+  log "warning: export exited 2 (duplicate guest IPs on cluster); JSON still written — continuing"
+elif [[ "$EX_INV" -ne 0 ]]; then
+  exit "$EX_INV"
+fi
+
+API_KEY="${IT_READ_API_KEY:-}"
+if [[ -z "$API_KEY" ]]; then
+  API_KEY="$(openssl rand -hex 32)"
+  log "generated IT_READ_API_KEY (openssl rand -hex 32)"
+fi
+
+ENV_LOCAL="${PROJECT_ROOT}/.env"
+touch "$ENV_LOCAL"
+upsert_env_file "$ENV_LOCAL" "IT_READ_API_URL" "$PUBLIC_URL" "IT_READ_API_KEY" "$API_KEY"
+chmod 600 "$ENV_LOCAL" 2>/dev/null || true
+log "upserted IT_READ_API_URL and IT_READ_API_KEY in repo .env (gitignored)"
+
+log "rsync minimal repo tree → root@${PROXMOX_HOST}:${REMOTE_ROOT}"
+ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" \
+  "mkdir -p '${REMOTE_ROOT}/config' '${REMOTE_ROOT}/scripts/it-ops' '${REMOTE_ROOT}/services/sankofa-it-read-api' '${REMOTE_ROOT}/docs/04-configuration' '${REMOTE_ROOT}/reports/status'"
+cd "$PROJECT_ROOT"
+rsync -az --delete -e "$RSYNC_SSH" ./config/ip-addresses.conf "root@${PROXMOX_HOST}:${REMOTE_ROOT}/config/"
+rsync -az --delete -e "$RSYNC_SSH" ./scripts/it-ops/ "root@${PROXMOX_HOST}:${REMOTE_ROOT}/scripts/it-ops/"
+rsync -az --delete -e "$RSYNC_SSH" ./services/sankofa-it-read-api/server.py "root@${PROXMOX_HOST}:${REMOTE_ROOT}/services/sankofa-it-read-api/"
+rsync -az --delete -e "$RSYNC_SSH" ./docs/04-configuration/ALL_VMIDS_ENDPOINTS.md "root@${PROXMOX_HOST}:${REMOTE_ROOT}/docs/04-configuration/"
+rsync -az -e "$RSYNC_SSH" ./reports/status/live_inventory.json ./reports/status/drift.json "root@${PROXMOX_HOST}:${REMOTE_ROOT}/reports/status/"
+
+ENV_REMOTE="/etc/sankofa-it-read-api.env"
+
+# shellcheck disable=SC2087
+ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" bash -s <<REMOTE
+set -euo pipefail
+umask 077
+cat > ${ENV_REMOTE} <<ENVEOF
+IT_READ_API_HOST=0.0.0.0
+IT_READ_API_PORT=${PORT}
+IT_READ_API_KEY=${API_KEY}
+IT_READ_API_CORS_ORIGINS=${CORS_ORIGIN}
+ENVEOF
+chmod 600 ${ENV_REMOTE}
+cat > /etc/systemd/system/sankofa-it-read-api.service <<SVCEOF
+[Unit]
+Description=Sankofa IT read API (live inventory JSON)
+After=network.target
+
+[Service]
+Type=simple
+User=root
+WorkingDirectory=${REMOTE_ROOT}
+EnvironmentFile=-${ENV_REMOTE}
+ExecStart=/usr/bin/python3 ${REMOTE_ROOT}/services/sankofa-it-read-api/server.py
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+SVCEOF
+systemctl daemon-reload
+systemctl enable --now sankofa-it-read-api
+systemctl is-active sankofa-it-read-api
+REMOTE
+
+log "verify read API (localhost on PVE — WAN/WSL may be firewalled)"
+if ! ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" "curl -sS -f -m 5 http://127.0.0.1:${PORT}/health" | head -c 200; then
+  echo "health check failed on PVE" >&2
+  exit 1
+fi
+echo ""
+if ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" "pct status 7801 &>/dev/null"; then
+  log "verify from portal CT 7801 → ${PUBLIC_URL}/health"
+  ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" \
+    "pct exec 7801 -- sh -c 'curl -sS -f -m 8 http://${PROXMOX_HOST}:${PORT}/health || true'" | head -c 200 || true
+  echo ""
+fi
+
+if ! $NO_TIMER; then
+  log "install weekly inventory timer on PVE"
+  # shellcheck disable=SC2087
+  ssh "${SSH_OPTS[@]}" "root@${PROXMOX_HOST}" bash -s <<REMOTE
+set -euo pipefail
+cat > /etc/systemd/system/sankofa-it-inventory-export.service <<EOF
+[Unit]
+Description=Export Proxmox live inventory and IPAM drift
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=root
+WorkingDirectory=${REMOTE_ROOT}
+ExecStart=/usr/bin/bash ${REMOTE_ROOT}/scripts/it-ops/export-live-inventory-and-drift.sh
+EOF
+cat > /etc/systemd/system/sankofa-it-inventory-export.timer <<'EOF'
+[Unit]
+Description=Timer — Proxmox live inventory + drift export (weekly)
+
+[Timer]
+OnCalendar=Sun *-*-* 03:30:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+systemctl daemon-reload
+systemctl enable --now sankofa-it-inventory-export.timer
+systemctl list-timers sankofa-it-inventory-export.timer --no-pager || true
+REMOTE
+fi
+
+if ! $NO_PORTAL; then
+  log "merge IT_READ_API_* into portal CT 7801"
+  export IT_READ_API_URL="$PUBLIC_URL"
+  export IT_READ_API_KEY="$API_KEY"
+  bash "${SCRIPT_DIR}/sankofa-portal-merge-it-read-api-env-from-repo.sh"
+fi
+
+log "done. Portal uses ${PUBLIC_URL} (server-side key in CT .env). Optional: NPM + DNS for it-api hostname → same upstream."
--- a/scripts/nginx-proxy-manager/upsert-it-read-api-proxy-host.sh
+++ b/scripts/nginx-proxy-manager/upsert-it-read-api-proxy-host.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+# Upsert NPMplus proxy host for the IT read API (Phase 0).
+# Point DNS (e.g. it-api.sankofa.nexus) at your NPM public IP before using TLS.
+#
+# Env: NPM_URL, NPM_EMAIL, NPM_PASSWORD; optional:
+#   IT_READ_API_PUBLIC_HOST (default it-api.sankofa.nexus)
+#   IT_READ_API_UPSTREAM_IP (default PROXMOX_HOST_R630_01 or 192.168.11.11)
+#   IT_READ_API_UPSTREAM_PORT (default 8787)
+#   NPM_CURL_MAX_TIME (default 300)
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+_orig_npm_url="${NPM_URL:-}"
+_orig_npm_email="${NPM_EMAIL:-}"
+_orig_npm_password="${NPM_PASSWORD:-}"
+if [ -f "$PROJECT_ROOT/.env" ]; then set +u; source "$PROJECT_ROOT/.env"; set -u; fi
+[ -n "$_orig_npm_url" ] && NPM_URL="$_orig_npm_url"
+[ -n "$_orig_npm_email" ] && NPM_EMAIL="$_orig_npm_email"
+[ -n "$_orig_npm_password" ] && NPM_PASSWORD="$_orig_npm_password"
+source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+NPM_URL="${NPM_URL:-https://${IP_NPMPLUS:-192.168.11.167}:81}"
+NPM_EMAIL="${NPM_EMAIL:-}"
+NPM_PASSWORD="${NPM_PASSWORD:-}"
+[ -z "$NPM_PASSWORD" ] && { echo "NPM_PASSWORD required (.env or export)" >&2; exit 1; }
+
+DOMAIN="${IT_READ_API_PUBLIC_HOST:-it-api.sankofa.nexus}"
+UP_IP="${IT_READ_API_UPSTREAM_IP:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
+UP_PORT="${IT_READ_API_UPSTREAM_PORT:-8787}"
+
+NPM_CURL_MAX_TIME="${NPM_CURL_MAX_TIME:-300}"
+curl_npm() { curl -s -k -L --http1.1 --connect-timeout 30 --max-time "$NPM_CURL_MAX_TIME" "$@"; }
+
+try_connect() { curl -s -k -L -o /dev/null --connect-timeout 5 --max-time 20 "$1" 2>/dev/null; }
+if ! try_connect "$NPM_URL/"; then
+  http_url="${NPM_URL/https:/http:}"
+  try_connect "$http_url/" && NPM_URL="$http_url"
+fi
+
+AUTH_JSON=$(jq -n --arg identity "$NPM_EMAIL" --arg secret "$NPM_PASSWORD" '{identity:$identity,secret:$secret}')
+TOKEN=$(curl_npm -X POST "$NPM_URL/api/tokens" -H "Content-Type: application/json" -d "$AUTH_JSON" | jq -r '.token // empty')
+[ -n "$TOKEN" ] && [ "$TOKEN" != "null" ] || { echo "NPM auth failed" >&2; exit 1; }
+
+ADV='add_header Referrer-Policy "strict-origin-when-cross-origin" always;'
+PAYLOAD_ADD=$(jq -n \
+  --arg domain "$DOMAIN" \
+  --arg host "$UP_IP" \
+  --argjson port "$UP_PORT" \
+  --arg adv "$ADV" \
+  '{domain_names:[$domain],forward_scheme:"http",forward_host:$host,forward_port:$port,allow_websocket_upgrade:false,block_exploits:true,certificate_id:null,ssl_forced:false,advanced_config:$adv}')
+
+echo "Trying create (POST) for $DOMAIN -> http://${UP_IP}:${UP_PORT}"
+RESP=$(curl_npm -X POST "$NPM_URL/api/nginx/proxy-hosts" -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD_ADD")
+if echo "$RESP" | jq -e '.id' >/dev/null 2>&1; then
+  echo "OK created id=$(echo "$RESP" | jq -r .id)"
+  exit 0
+fi
+
+ERR_MSG=$(echo "$RESP" | jq -r '.message // .error.message // .error // empty' 2>/dev/null || echo "")
+if ! echo "$ERR_MSG" | grep -qiE 'already|in use|exist|duplicate|unique'; then
+  echo "Create failed (not a duplicate case): $ERR_MSG" >&2
+  echo "$RESP" | jq . 2>/dev/null || echo "$RESP"
+  exit 1
+fi
+
+echo "Host exists; fetching proxy list for PUT ($ERR_MSG)"
+PROXY_JSON=$(curl_npm -X GET "$NPM_URL/api/nginx/proxy-hosts" -H "Authorization: Bearer $TOKEN")
+HOST_ID=$(echo "$PROXY_JSON" | jq -r --arg d "$DOMAIN" '.[] | select(.domain_names | type == "array") | select(any(.domain_names[]; (. | tostring | ascii_downcase) == ($d | ascii_downcase))) | .id' | head -n1)
+
+if [ -z "$HOST_ID" ] || [ "$HOST_ID" = "null" ]; then
+  echo "Could not resolve proxy host id for $DOMAIN." >&2
+  exit 1
+fi
+
+echo "Updating proxy host $DOMAIN (id=$HOST_ID) -> http://${UP_IP}:${UP_PORT}"
+PAYLOAD_PUT=$(jq -n \
+  --arg host "$UP_IP" \
+  --argjson port "$UP_PORT" \
+  --arg adv "$ADV" \
+  '{forward_scheme:"http",forward_host:$host,forward_port:$port,allow_websocket_upgrade:false,block_exploits:true,advanced_config:$adv}')
+RESP=$(curl_npm -X PUT "$NPM_URL/api/nginx/proxy-hosts/$HOST_ID" -H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" -d "$PAYLOAD_PUT")
+echo "$RESP" | jq -e '.id' >/dev/null && echo "OK updated" || { echo "$RESP" | jq . 2>/dev/null || echo "$RESP"; exit 1; }