Align E2E profile workflow across scripts and runbooks

2026-03-06 08:46:55 -08:00
parent e4c9dda0fd
commit d38174dc25
18 changed files with 345 additions and 53 deletions
--- a/scripts/deployment/run-all-next-steps-chain138.sh
+++ b/scripts/deployment/run-all-next-steps-chain138.sh
@@ -1,9 +1,13 @@
 #!/usr/bin/env bash
-# Run all deployment next steps for Chain 138 in order: preflight → mirror+pool (or pool-only) → register c* as GRU → verify.
+# Run all deployment next steps for Chain 138 in order:
+# preflight → (optional mirror+seed pool) → PMM mesh (default) → register c* as GRU → verify.
 #
-# Usage: ./scripts/deployment/run-all-next-steps-chain138.sh [--dry-run] [--skip-mirror] [--skip-register-gru] [--skip-verify]
+# Usage: ./scripts/deployment/run-all-next-steps-chain138.sh [--dry-run] [--skip-mirror] [--skip-mesh] [--legacy-pools-only] [--mesh-only] [--skip-register-gru] [--skip-verify]
 #   --dry-run         Print steps only; do not run deploy/scripts.
-#   --skip-mirror     Do not deploy TransactionMirror (pool-only; requires TRANSACTION_MIRROR_ADDRESS in smom-dbis-138/.env).
+#   --skip-mirror     Do not deploy TransactionMirror + seed pool step.
+#   --skip-mesh       Do not run full PMM mesh creation script.
+#   --legacy-pools-only  Equivalent to --skip-mesh (keeps legacy mirror+seed behavior only).
+#   --mesh-only       Skip mirror+seed step and run mesh creation only.
 #   --skip-register-gru  Skip RegisterGRUCompliantTokens (e.g. if already registered).
 #   --skip-verify     Skip final on-chain verification.
 #
@@ -17,17 +21,22 @@ SMOM="$PROJECT_ROOT/smom-dbis-138"

 DRY_RUN=""
 SKIP_MIRROR=""
+SKIP_MESH=""
+MESH_ONLY=""
 SKIP_REGISTER_GRU=""
 SKIP_VERIFY=""
 for a in "$@"; do
  [[ "$a" == "--dry-run" ]] && DRY_RUN=1
  [[ "$a" == "--skip-mirror" ]] && SKIP_MIRROR=1
+  [[ "$a" == "--skip-mesh" ]] && SKIP_MESH=1
+  [[ "$a" == "--legacy-pools-only" ]] && SKIP_MESH=1
+  [[ "$a" == "--mesh-only" ]] && MESH_ONLY=1 && SKIP_MIRROR=1
  [[ "$a" == "--skip-register-gru" ]] && SKIP_REGISTER_GRU=1
  [[ "$a" == "--skip-verify" ]] && SKIP_VERIFY=1
 done

 echo "=== Chain 138 — run all next steps ==="
-echo "  dry-run: $DRY_RUN  skip-mirror: $SKIP_MIRROR  skip-register-gru: $SKIP_REGISTER_GRU  skip-verify: $SKIP_VERIFY"
+echo "  dry-run: $DRY_RUN  skip-mirror: $SKIP_MIRROR  skip-mesh: $SKIP_MESH  mesh-only: $MESH_ONLY  skip-register-gru: $SKIP_REGISTER_GRU  skip-verify: $SKIP_VERIFY"
 echo ""

 # 1) Preflight
@@ -39,26 +48,37 @@ else
 fi
 echo ""

-# 2) TransactionMirror + PMM pool (or pool-only)
-echo "--- Step 2: TransactionMirror + PMM pool ---"
-if [[ -n "$DRY_RUN" ]]; then
-  if [[ -n "$SKIP_MIRROR" ]]; then
-    echo "[DRY-RUN] $PROJECT_ROOT/scripts/deployment/deploy-transaction-mirror-and-pmm-pool-after-txpool-clear.sh --skip-mirror"
-  else
+# 2) TransactionMirror + seed pool (legacy step; optional)
+if [[ -z "$SKIP_MIRROR" ]]; then
+  echo "--- Step 2: TransactionMirror + seed pool ---"
+  if [[ -n "$DRY_RUN" ]]; then
    echo "[DRY-RUN] $PROJECT_ROOT/scripts/deployment/deploy-transaction-mirror-and-pmm-pool-after-txpool-clear.sh"
-  fi
-else
-  if [[ -n "$SKIP_MIRROR" ]]; then
-    "$PROJECT_ROOT/scripts/deployment/deploy-transaction-mirror-and-pmm-pool-after-txpool-clear.sh" --skip-mirror || { echo "Deploy (pool-only) failed." >&2; exit 1; }
  else
    "$PROJECT_ROOT/scripts/deployment/deploy-transaction-mirror-and-pmm-pool-after-txpool-clear.sh" || { echo "Deploy failed." >&2; exit 1; }
  fi
+  echo ""
+else
+  echo "--- Step 2: TransactionMirror + seed pool (skipped) ---"
+  echo ""
 fi
-echo ""

-# 3) Register c* as GRU (optional)
+# 3) PMM full mesh (default on Chain 138)
+if [[ -z "$SKIP_MESH" ]]; then
+  echo "--- Step 3: PMM full mesh (Chain 138) ---"
+  if [[ -n "$DRY_RUN" ]]; then
+    echo "[DRY-RUN] $PROJECT_ROOT/scripts/create-pmm-full-mesh-chain138.sh"
+  else
+    "$PROJECT_ROOT/scripts/create-pmm-full-mesh-chain138.sh" || { echo "PMM full mesh failed." >&2; exit 1; }
+  fi
+  echo ""
+else
+  echo "--- Step 3: PMM full mesh (skipped; legacy-only mode) ---"
+  echo ""
+fi
+
+# 4) Register c* as GRU (optional)
 if [[ -z "$SKIP_REGISTER_GRU" ]]; then
-  echo "--- Step 3: Register c* as GRU (UniversalAssetRegistry) ---"
+  echo "--- Step 4: Register c* as GRU (UniversalAssetRegistry) ---"
  if [[ -n "$DRY_RUN" ]]; then
    echo "[DRY-RUN] cd $SMOM && forge script script/deploy/RegisterGRUCompliantTokens.s.sol --rpc-url \$RPC_URL_138 --broadcast --private-key \$PRIVATE_KEY --with-gas-price 1000000000"
  else
@@ -78,13 +98,13 @@ if [[ -z "$SKIP_REGISTER_GRU" ]]; then
  fi
  echo ""
 else
-  echo "--- Step 3: Register c* as GRU (skipped) ---"
+  echo "--- Step 4: Register c* as GRU (skipped) ---"
  echo ""
 fi

-# 4) Verify
+# 5) Verify
 if [[ -z "$SKIP_VERIFY" ]]; then
-  echo "--- Step 4: On-chain verification ---"
+  echo "--- Step 5: On-chain verification ---"
  if [[ -n "$DRY_RUN" ]]; then
    echo "[DRY-RUN] $PROJECT_ROOT/scripts/verify/check-contracts-on-chain-138.sh"
  else
@@ -93,7 +113,7 @@ if [[ -z "$SKIP_VERIFY" ]]; then
  fi
  echo ""
 else
-  echo "--- Step 4: Verify (skipped) ---"
+  echo "--- Step 5: Verify (skipped) ---"
  echo ""
 fi

--- a/scripts/deployment/run-sankofa-studio-e2e.sh
+++ b/scripts/deployment/run-sankofa-studio-e2e.sh
@@ -53,7 +53,7 @@ echo "   If using tunnel: add Public Hostname studio.sankofa.nexus → https://1
 echo ""
 echo "4. Verify:"
 echo "   curl -s http://${IP}:8000/health"
-echo "   bash scripts/verify/verify-end-to-end-routing.sh"
+echo "   bash scripts/verify/verify-end-to-end-routing.sh --profile=public"
 echo "   https://studio.sankofa.nexus/studio/"
 echo ""
 echo "Full flow: docs/03-deployment/SANKOFA_STUDIO_E2E_FLOW.md"
--- a/scripts/run-all-next-steps.sh
+++ b/scripts/run-all-next-steps.sh
@@ -80,12 +80,12 @@ echo "" >> "$REPORT_FILE"

 # 4. E2E routing (may have RPC/Blockscout skip when off-LAN)
 log_info "4. End-to-end routing verification"
-if E2E_SUCCESS_IF_ONLY_RPC_BLOCKED=1 bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" >> "$REPORT_FILE" 2>&1; then
+if E2E_SUCCESS_IF_ONLY_RPC_BLOCKED=1 bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=public >> "$REPORT_FILE" 2>&1; then
  log_ok "E2E routing"
-  echo "| E2E routing | OK | \`verify-end-to-end-routing.sh\` (RPC may skip off-LAN) |" >> "$REPORT_FILE"
+  echo "| E2E routing | OK | \`verify-end-to-end-routing.sh --profile=public\` (RPC may skip off-LAN) |" >> "$REPORT_FILE"
 else
  log_skip "E2E routing (check report in verification-evidence/e2e-verification-*)"
-  echo "| E2E routing | WARN/FAIL | \`verify-end-to-end-routing.sh\` — see latest e2e-verification-* |" >> "$REPORT_FILE"
+  echo "| E2E routing | WARN/FAIL | \`verify-end-to-end-routing.sh --profile=public\` — see latest e2e-verification-* |" >> "$REPORT_FILE"
 fi
 echo "" >> "$REPORT_FILE"

--- a/scripts/verify/run-full-connection-and-fastly-tests.sh
+++ b/scripts/verify/run-full-connection-and-fastly-tests.sh
@@ -91,7 +91,7 @@ echo ""
 # 5) End-to-end routing (full domain list: DNS, SSL, HTTPS, RPC where applicable)
 # When only RPC fails (edge blocks POST), treat as success so full run passes
 info "5. End-to-end routing (all domains)"
-if E2E_SUCCESS_IF_ONLY_RPC_BLOCKED=1 bash scripts/verify/verify-end-to-end-routing.sh 2>&1; then
+if E2E_SUCCESS_IF_ONLY_RPC_BLOCKED=1 bash scripts/verify/verify-end-to-end-routing.sh --profile=public 2>&1; then
  ok "E2E routing completed"
 else
  warn "E2E routing had failures (see above)"
--- a/scripts/verify/run-full-verification.sh
+++ b/scripts/verify/run-full-verification.sh
@@ -102,7 +102,7 @@ log_info "Progress: 5/$TOTAL_STEPS steps"
 log_info ""
 log_info "Step 5/$TOTAL_STEPS: End-to-End Routing Verification"
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-if bash "$SCRIPT_DIR/verify-end-to-end-routing.sh"; then
+if bash "$SCRIPT_DIR/verify-end-to-end-routing.sh" --profile=public; then
    log_success "E2E verification complete"
 else
    log_warn "E2E verification completed with warnings"
--- a/scripts/verify/verify-end-to-end-routing.sh
+++ b/scripts/verify/verify-end-to-end-routing.sh
@@ -32,6 +32,12 @@ PUBLIC_IP="${PUBLIC_IP:-76.53.10.36}"
 PUBLIC_IP_FOURTH="${PUBLIC_IP_FOURTH:-76.53.10.40}"
 # Set ACCEPT_ANY_DNS=1 to pass DNS if domain resolves to any IP (e.g. Fastly CNAME or Cloudflare Tunnel)
 ACCEPT_ANY_DNS="${ACCEPT_ANY_DNS:-0}"
+# Use system resolver (e.g. /etc/hosts) instead of dig @8.8.8.8 — set when running from LAN with generate-e2e-hosts.sh entries
+E2E_USE_SYSTEM_RESOLVER="${E2E_USE_SYSTEM_RESOLVER:-0}"
+if [ "$E2E_USE_SYSTEM_RESOLVER" = "1" ]; then
+    ACCEPT_ANY_DNS=1
+    log_info "E2E_USE_SYSTEM_RESOLVER=1: using getent (respects /etc/hosts); ACCEPT_ANY_DNS=1"
+fi
 # When using Option B (RPC via Cloudflare Tunnel), RPC hostnames resolve to Cloudflare IPs; auto-enable if tunnel ID set
 if [ "$ACCEPT_ANY_DNS" = "0" ] && [ -n "${CLOUDFLARE_TUNNEL_ID:-}" ]; then
    ACCEPT_ANY_DNS=1
@@ -46,8 +52,8 @@ if [ "$ACCEPT_ANY_DNS" = "0" ] && [ -f "$PROJECT_ROOT/.env" ]; then
    fi
 fi

-# Expected domains and their types (all Cloudflare/DNS-facing public endpoints)
-declare -A DOMAIN_TYPES=(
+# Expected domains and their types (full combined inventory)
+declare -A DOMAIN_TYPES_ALL=(
    ["explorer.d-bis.org"]="web"
    ["rpc-http-pub.d-bis.org"]="rpc-http"
    ["rpc-ws-pub.d-bis.org"]="rpc-ws"
@@ -94,10 +100,77 @@ declare -A DOMAIN_TYPES=(
    ["dev.d-bis.org"]="web"
    ["codespaces.d-bis.org"]="web"
 )
+# Private/admin profile domains (private RPC + Fireblocks RPC only).
+declare -a PRIVATE_PROFILE_DOMAINS=(
+    "rpc-http-prv.d-bis.org"
+    "rpc-ws-prv.d-bis.org"
+    "rpc-fireblocks.d-bis.org"
+    "ws.rpc-fireblocks.d-bis.org"
+)
+
+PRIVATE_PROFILE_SET=" ${PRIVATE_PROFILE_DOMAINS[*]} "
+PROFILE="${E2E_PROFILE:-public}"
+LIST_ENDPOINTS=0
+
+for arg in "$@"; do
+    case "$arg" in
+        --list-endpoints) LIST_ENDPOINTS=1 ;;
+        --profile=*) PROFILE="${arg#*=}" ;;
+        --profile-public) PROFILE="public" ;;
+        --profile-private) PROFILE="private" ;;
+        --profile-all) PROFILE="all" ;;
+        *)
+            if [[ "$arg" != "--list-endpoints" ]]; then
+                echo "Unknown argument: $arg" >&2
+                echo "Usage: $0 [--list-endpoints] [--profile=public|private|all]" >&2
+                exit 2
+            fi
+            ;;
+    esac
+done
+
+declare -A DOMAIN_TYPES=()
+for domain in "${!DOMAIN_TYPES_ALL[@]}"; do
+    is_private=0
+    [[ "$PRIVATE_PROFILE_SET" == *" $domain "* ]] && is_private=1
+    case "$PROFILE" in
+        public)
+            [[ "$is_private" -eq 0 ]] && DOMAIN_TYPES["$domain"]="${DOMAIN_TYPES_ALL[$domain]}"
+            ;;
+        private)
+            [[ "$is_private" -eq 1 ]] && DOMAIN_TYPES["$domain"]="${DOMAIN_TYPES_ALL[$domain]}"
+            ;;
+        all)
+            DOMAIN_TYPES["$domain"]="${DOMAIN_TYPES_ALL[$domain]}"
+            ;;
+        *)
+            echo "Invalid profile: $PROFILE (expected public|private|all)" >&2
+            exit 2
+            ;;
+    esac
+done
+
 # Domains that are optional (not yet configured); no DNS = skip instead of fail. Space-separated.
-E2E_OPTIONAL_DOMAINS="${E2E_OPTIONAL_DOMAINS:-dapp.d-bis.org}"
-# Domains that are optional when any test fails (off-LAN, 502, unreachable); fail → skip so run passes. Set to empty for strict.
-E2E_OPTIONAL_WHEN_FAIL="${E2E_OPTIONAL_WHEN_FAIL:-dapp.d-bis.org mifos.d-bis.org explorer.d-bis.org dbis-admin.d-bis.org dbis-api.d-bis.org dbis-api-2.d-bis.org secure.d-bis.org sankofa.nexus www.sankofa.nexus phoenix.sankofa.nexus www.phoenix.sankofa.nexus the-order.sankofa.nexus studio.sankofa.nexus mim4u.org www.mim4u.org secure.mim4u.org training.mim4u.org rpc-http-pub.d-bis.org rpc.d-bis.org rpc2.d-bis.org rpc-http-prv.d-bis.org rpc-fireblocks.d-bis.org ws.rpc-fireblocks.d-bis.org rpc.public-0138.defi-oracle.io rpc.defi-oracle.io ws.rpc.d-bis.org rpc-ws-prv.d-bis.org ws.rpc2.d-bis.org}"
+if [[ -z "${E2E_OPTIONAL_DOMAINS:-}" ]]; then
+    if [[ "$PROFILE" == "private" ]]; then
+        E2E_OPTIONAL_DOMAINS=""
+    else
+        E2E_OPTIONAL_DOMAINS="dapp.d-bis.org"
+    fi
+else
+    E2E_OPTIONAL_DOMAINS="${E2E_OPTIONAL_DOMAINS}"
+fi
+
+# Domains that are optional when any test fails (off-LAN, 502, unreachable); fail → skip so run passes.
+if [[ -z "${E2E_OPTIONAL_WHEN_FAIL:-}" ]]; then
+    if [[ "$PROFILE" == "private" ]]; then
+        E2E_OPTIONAL_WHEN_FAIL="rpc-http-prv.d-bis.org rpc-ws-prv.d-bis.org rpc-fireblocks.d-bis.org ws.rpc-fireblocks.d-bis.org"
+    else
+        E2E_OPTIONAL_WHEN_FAIL="dapp.d-bis.org mifos.d-bis.org explorer.d-bis.org dbis-admin.d-bis.org dbis-api.d-bis.org dbis-api-2.d-bis.org secure.d-bis.org sankofa.nexus www.sankofa.nexus phoenix.sankofa.nexus www.phoenix.sankofa.nexus the-order.sankofa.nexus studio.sankofa.nexus mim4u.org www.mim4u.org secure.mim4u.org training.mim4u.org rpc-http-pub.d-bis.org rpc.d-bis.org rpc2.d-bis.org rpc.public-0138.defi-oracle.io rpc.defi-oracle.io ws.rpc.d-bis.org ws.rpc2.d-bis.org"
+    fi
+else
+    E2E_OPTIONAL_WHEN_FAIL="${E2E_OPTIONAL_WHEN_FAIL}"
+fi

 # Per-domain expected DNS IP (optional). Unset = use PUBLIC_IP.
 declare -A EXPECTED_IP=(
@@ -106,11 +179,34 @@ declare -A EXPECTED_IP=(
    ["codespaces.d-bis.org"]="$PUBLIC_IP_FOURTH"
 )

+# --list-endpoints: print selected profile endpoints and exit (no tests)
+if [[ "$LIST_ENDPOINTS" == "1" ]]; then
+    echo ""
+    echo "E2E endpoints (${#DOMAIN_TYPES[@]} total, profile: $PROFILE) — verify-end-to-end-routing.sh"
+    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+    echo ""
+    printf "%-40s %-12s %s\n" "Domain" "Type" "URL"
+    printf "%-40s %-12s %s\n" "------" "----" "---"
+    for domain in $(echo "${!DOMAIN_TYPES[@]}" | tr ' ' '\n' | sort); do
+        dtype="${DOMAIN_TYPES[$domain]:-unknown}"
+        if [[ "$dtype" == "rpc-http" || "$dtype" == "rpc-ws" ]]; then
+            url="https://$domain (RPC)"
+        else
+            url="https://$domain"
+        fi
+        printf "%-40s %-12s %s\n" "$domain" "$dtype" "$url"
+    done
+    echo ""
+    exit 0
+fi
+
 echo ""
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
 echo "🔍 End-to-End Routing Verification"
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
 echo ""
+echo "Profile: $PROFILE"
+echo ""

 E2E_RESULTS=()

@@ -126,7 +222,11 @@ test_domain() {
    
    # Test 1: DNS Resolution
    log_info "Test 1: DNS Resolution"
-    dns_result=$(dig +short "$domain" @8.8.8.8 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -1 || echo "")
+    if [ "${E2E_USE_SYSTEM_RESOLVER:-0}" = "1" ]; then
+        dns_result=$(getent hosts "$domain" 2>/dev/null | awk '{print $1}' | head -1 || echo "")
+    else
+        dns_result=$(dig +short "$domain" @8.8.8.8 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -1 || echo "")
+    fi
    expected_ip="${EXPECTED_IP[$domain]:-$PUBLIC_IP}"
    
    if [ "$dns_result" = "$expected_ip" ]; then
@@ -372,8 +472,21 @@ cat > "$REPORT_FILE" <<EOF

 **Date**: $(date -Iseconds)
 **Public IP**: $PUBLIC_IP
+**Profile**: $PROFILE
 **Verifier**: $(whoami)

+## All endpoints ($TOTAL_TESTS)
+
+| Domain | Type | URL |
+|--------|------|-----|
+EOF
+for domain in $(echo "${!DOMAIN_TYPES[@]}" | tr ' ' '\n' | sort); do
+    dtype="${DOMAIN_TYPES[$domain]:-unknown}"
+    echo "| $domain | $dtype | https://$domain |" >> "$REPORT_FILE"
+done
+
+cat >> "$REPORT_FILE" <<EOF
+
 ## Summary

 - **Total domains tested**: $TOTAL_TESTS
@@ -383,7 +496,25 @@ cat > "$REPORT_FILE" <<EOF
 - **Skipped / optional (not configured or unreachable)**: $SKIPPED_OPTIONAL
 - **Average response time**: ${AVG_RESPONSE_TIME}s

-## Test Results by Domain
+## Results overview
+
+| Domain | Type | DNS | SSL | HTTPS | RPC |
+|--------|------|-----|-----|-------|-----|
+EOF
+
+for result in "${E2E_RESULTS[@]}"; do
+    domain=$(echo "$result" | jq -r '.domain' 2>/dev/null || echo "")
+    domain_type=$(echo "$result" | jq -r '.domain_type' 2>/dev/null || echo "")
+    dns_status=$(echo "$result" | jq -r '.tests.dns.status // "-"' 2>/dev/null || echo "-")
+    ssl_status=$(echo "$result" | jq -r '.tests.ssl.status // "-"' 2>/dev/null || echo "-")
+    https_status=$(echo "$result" | jq -r '.tests.https.status // "-"' 2>/dev/null || echo "-")
+    rpc_status=$(echo "$result" | jq -r '.tests.rpc_http.status // "-"' 2>/dev/null || echo "-")
+    echo "| $domain | $domain_type | $dns_status | $ssl_status | $https_status | $rpc_status |" >> "$REPORT_FILE"
+done
+
+cat >> "$REPORT_FILE" <<EOF
+
+## Test Results by Domain (detail)

 EOF