proxmox/scripts/run-full-operator-completion-from-lan.sh

#!/usr/bin/env bash
# Run the full remaining operator checklist from a LAN-connected host.
# Order:
#   1. Fix token-aggregation DB + explorer /api/v1 proxy
#   2. Wave 0 (NPMplus RPC fix + backup)
#   3. Blockscout verification
#   4. Public/private E2E
#   5. Optional E2E remediation
#   6. Optional config-ready chains + LINK funding
#   7. Optional Chain 138 next steps
#   8. Optional real sendCrossChain
#   9. Optional local security / cron
#  10. Final completion summary
#
# Usage:
#   ./scripts/run-full-operator-completion-from-lan.sh --dry-run
#   ./scripts/run-full-operator-completion-from-lan.sh
#   ./scripts/run-full-operator-completion-from-lan.sh --fix-e2e-if-needed --install-cron
#   ./scripts/run-full-operator-completion-from-lan.sh --include-config-ready-chains --include-chain138-next-steps
#   ./scripts/run-full-operator-completion-from-lan.sh --include-send-cross-chain --send-amount 0.01 [--send-recipient 0x...]
#   ./scripts/run-full-operator-completion-from-lan.sh --force-nginx-reset

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_ROOT"

if [[ -f "$PROJECT_ROOT/config/ip-addresses.conf" ]]; then
  # shellcheck source=config/ip-addresses.conf
  source "$PROJECT_ROOT/config/ip-addresses.conf" 2>/dev/null || true
fi

DRY_RUN=false
SKIP_TOKEN_API_FIX=false
SKIP_WAVE0=false
SKIP_VERIFY=false
SKIP_E2E=false
SKIP_BACKUP=false
FIX_E2E_IF_NEEDED=false
INCLUDE_CONFIG_READY_CHAINS=false
INCLUDE_CHAIN138_NEXT_STEPS=false
INCLUDE_SEND_CROSS_CHAIN=false
INSTALL_CRON=false
APPLY_LOCAL_SECURITY=false
FORCE_NGINX_RESET=false
TOKEN_AGG_PORT_OVERRIDE=""
LINK_AMOUNT=""
SEND_AMOUNT=""
SEND_RECIPIENT=""

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run) DRY_RUN=true ;;
    --skip-token-api-fix) SKIP_TOKEN_API_FIX=true ;;
    --skip-wave0) SKIP_WAVE0=true ;;
    --skip-verify) SKIP_VERIFY=true ;;
    --skip-e2e) SKIP_E2E=true ;;
    --skip-backup) SKIP_BACKUP=true ;;
    --fix-e2e-if-needed) FIX_E2E_IF_NEEDED=true ;;
    --include-config-ready-chains) INCLUDE_CONFIG_READY_CHAINS=true ;;
    --include-chain138-next-steps) INCLUDE_CHAIN138_NEXT_STEPS=true ;;
    --include-send-cross-chain) INCLUDE_SEND_CROSS_CHAIN=true ;;
    --install-cron) INSTALL_CRON=true ;;
    --apply-local-security) APPLY_LOCAL_SECURITY=true ;;
    --force-nginx-reset) FORCE_NGINX_RESET=true ;;
    --token-agg-port)
      shift
      TOKEN_AGG_PORT_OVERRIDE="${1:-}"
      ;;
    --link)
      shift
      LINK_AMOUNT="${1:-}"
      ;;
    --send-amount)
      shift
      SEND_AMOUNT="${1:-}"
      ;;
    --send-recipient)
      shift
      SEND_RECIPIENT="${1:-}"
      ;;
    -h|--help)
      sed -n '1,28p' "$0"
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      exit 1
      ;;
  esac
  shift
done

if [[ "$INCLUDE_SEND_CROSS_CHAIN" == true && -z "$SEND_AMOUNT" ]]; then
  echo "ERROR: --include-send-cross-chain requires --send-amount <eth>" >&2
  exit 1
fi

PROXMOX_HOST="${PROXMOX_HOST_OVERRIDE:-${PROXMOX_HOST_R630_02:-192.168.11.12}}"

log_info() { printf '\033[0;34m[INFO]\033[0m %s\n' "$1"; }
log_ok() { printf '\033[0;32m[OK]\033[0m %s\n' "$1"; }
log_warn() { printf '\033[0;33m[WARN]\033[0m %s\n' "$1"; }
log_err() { printf '\033[0;31m[ERR]\033[0m %s\n' "$1"; }
section() { printf '\n=== %s ===\n' "$1"; }

print_cmd() {
  printf '  '
  printf '%q ' "$@"
  printf '\n'
}

run_cmd() {
  if [[ "$DRY_RUN" == true ]]; then
    printf '[DRY-RUN]\n'
    print_cmd "$@"
    return 0
  fi
  "$@"
}

run_ssh() {
  if [[ "$DRY_RUN" == true ]]; then
    printf '[DRY-RUN]\n'
    print_cmd ssh "$@"
    return 0
  fi
  ssh "$@"
}

run_scp() {
  if [[ "$DRY_RUN" == true ]]; then
    printf '[DRY-RUN]\n'
    print_cmd scp "$@"
    return 0
  fi
  scp "$@"
}

push_script_to_vmid5000() {
  local local_script="$1"
  local remote_tmp="/tmp/$(basename "$local_script")"
  local remote_vm="/root/$(basename "$local_script")"
  run_scp "$local_script" "root@${PROXMOX_HOST}:${remote_tmp}"
  run_ssh "root@${PROXMOX_HOST}" "pct push 5000 ${remote_tmp} ${remote_vm}"
  run_ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- chmod +x ${remote_vm}"
}

detect_token_agg_port() {
  if [[ -n "$TOKEN_AGG_PORT_OVERRIDE" ]]; then
    printf '%s\n' "$TOKEN_AGG_PORT_OVERRIDE"
    return 0
  fi

  local attempt
  for attempt in $(seq 1 12); do
    if ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- bash -lc '
      ss -lntp 2>/dev/null | grep -q \"*:3001\" && { echo 3001; exit 0; }
      ss -lntp 2>/dev/null | grep -q \"*:3000\" && { echo 3000; exit 0; }
      for p in 3001 3000; do
        curl -fsS --max-time 5 http://127.0.0.1:\$p/api/v1/networks >/dev/null 2>&1 && { echo \$p; exit 0; }
      done
      exit 1
    '" 2>/dev/null; then
      return 0
    fi
    sleep 2
  done
  return 1
}

fix_public_report_api() {
  section "Step 1: Token-Aggregation DB + /api/v1 Proxy"

  if [[ "$DRY_RUN" == true ]]; then
    run_cmd bash "$SCRIPT_DIR/apply-token-aggregation-fix.sh" --dry-run
  else
    run_cmd bash "$SCRIPT_DIR/apply-token-aggregation-fix.sh"
  fi

  local token_agg_port="${TOKEN_AGG_PORT_OVERRIDE:-auto}"
  if [[ "$DRY_RUN" != true ]]; then
    token_agg_port="$(detect_token_agg_port)" || {
      log_err "Could not detect token-aggregation port inside VMID 5000. Re-run with --token-agg-port 3001 or 3000."
      exit 1
    }
    log_info "Detected token-aggregation port: ${token_agg_port}"
  else
    log_info "Would auto-detect token-aggregation port inside VMID 5000 (prefers 3001, then 3000)."
  fi

  if [[ "$FORCE_NGINX_RESET" == true ]]; then
    log_warn "Using full nginx reset for VMID 5000."
    push_script_to_vmid5000 "$PROJECT_ROOT/explorer-monorepo/scripts/fix-nginx-conflicts-vmid5000.sh"
    run_ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- bash -lc '/root/fix-nginx-conflicts-vmid5000.sh'"
  else
    push_script_to_vmid5000 "$PROJECT_ROOT/explorer-monorepo/scripts/apply-nginx-token-aggregation-proxy.sh"
    run_ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- bash -lc 'TOKEN_AGG_PORT=${token_agg_port} CONFIG_FILE=/etc/nginx/sites-available/blockscout /root/apply-nginx-token-aggregation-proxy.sh'"
  fi

  if [[ "$DRY_RUN" == true ]]; then
    run_cmd bash "$SCRIPT_DIR/verify/check-public-report-api.sh"
  else
    if ! bash "$SCRIPT_DIR/verify/check-public-report-api.sh"; then
      log_warn "Public report API still failing after HTTPS proxy patch. Applying HTTP /api/v1/ fallback on VMID 5000..."
      push_script_to_vmid5000 "$PROJECT_ROOT/scripts/fix-explorer-http-api-v1-proxy.sh"
      run_ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- bash -lc 'TOKEN_AGG_PORT=${token_agg_port} /root/fix-explorer-http-api-v1-proxy.sh'"
      bash "$SCRIPT_DIR/verify/check-public-report-api.sh"
    fi
  fi
  run_cmd bash "$PROJECT_ROOT/metamask-integration/chain138-snap/scripts/verify-snap-api-and-icons.sh" "https://explorer.d-bis.org"
}

run_wave0() {
  section "Step 2: Wave 0"
  if [[ "$SKIP_BACKUP" == true ]]; then
    run_cmd bash "$SCRIPT_DIR/run-wave0-from-lan.sh" --skip-backup
  else
    run_cmd bash "$SCRIPT_DIR/run-wave0-from-lan.sh"
  fi
}

run_blockscout_verify() {
  section "Step 3: Blockscout Verification"
  if [[ "$DRY_RUN" == true ]]; then
    run_cmd bash -lc "set -a; source '$PROJECT_ROOT/smom-dbis-138/.env' 2>/dev/null || true; set +a; '$SCRIPT_DIR/verify/run-contract-verification-with-proxy.sh'"
  else
    bash -lc "set -a; source '$PROJECT_ROOT/smom-dbis-138/.env' 2>/dev/null || true; set +a; '$SCRIPT_DIR/verify/run-contract-verification-with-proxy.sh'"
  fi
}

run_e2e_with_optional_fix() {
  section "Step 4: Public/Private E2E"

  local public_ok=0
  local private_ok=0

  if [[ "$DRY_RUN" == true ]]; then
    run_cmd bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=public
    run_cmd bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=private
    if [[ "$FIX_E2E_IF_NEEDED" == true ]]; then
      run_cmd bash "$SCRIPT_DIR/maintenance/address-all-remaining-502s.sh" --run-besu-fix --e2e --dry-run
    fi
    return 0
  fi

  if bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=public; then
    public_ok=1
    log_ok "Public E2E passed."
  else
    log_warn "Public E2E failed."
  fi

  if bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=private; then
    private_ok=1
    log_ok "Private E2E passed."
  else
    log_warn "Private E2E failed."
  fi

  if [[ "$public_ok" == 1 && "$private_ok" == 1 ]]; then
    return 0
  fi

  if [[ "$FIX_E2E_IF_NEEDED" != true ]]; then
    log_warn "E2E remediation not requested. Re-run with --fix-e2e-if-needed to attempt backend/NPM/Besu fixes."
    return 0
  fi

  section "Step 4b: E2E Remediation"
  bash "$SCRIPT_DIR/maintenance/address-all-remaining-502s.sh" --run-besu-fix --e2e
  bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=public || log_warn "Public E2E still failing after remediation."
  bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=private || log_warn "Private E2E still failing after remediation."
}

run_config_ready_chains() {
  section "Step 5: Config-Ready Chains + LINK Funding"
  if [[ "$DRY_RUN" == true ]]; then
    run_cmd bash -lc "cd '$PROJECT_ROOT/smom-dbis-138' && DRY_RUN=1 ./scripts/deployment/complete-config-ready-chains.sh"
    if [[ -n "$LINK_AMOUNT" ]]; then
      run_cmd bash -lc "cd '$PROJECT_ROOT/smom-dbis-138' && ./scripts/deployment/fund-ccip-bridges-with-link.sh --dry-run --link '$LINK_AMOUNT'"
    else
      run_cmd bash -lc "cd '$PROJECT_ROOT/smom-dbis-138' && ./scripts/deployment/fund-ccip-bridges-with-link.sh --dry-run"
    fi
    return 0
  fi

  (cd "$PROJECT_ROOT/smom-dbis-138" && ./scripts/deployment/complete-config-ready-chains.sh)
  if [[ -n "$LINK_AMOUNT" ]]; then
    (cd "$PROJECT_ROOT/smom-dbis-138" && ./scripts/deployment/fund-ccip-bridges-with-link.sh --link "$LINK_AMOUNT")
  else
    (cd "$PROJECT_ROOT/smom-dbis-138" && ./scripts/deployment/fund-ccip-bridges-with-link.sh)
  fi
}

run_chain138_next_steps() {
  section "Step 6: Chain 138 Next Steps"
  if [[ "$DRY_RUN" == true ]]; then
    run_cmd bash "$SCRIPT_DIR/deployment/run-all-next-steps-chain138.sh" --dry-run
  else
    run_cmd bash "$SCRIPT_DIR/deployment/run-all-next-steps-chain138.sh"
  fi
}

run_send_cross_chain() {
  section "Step 7: Real sendCrossChain"
  if [[ -n "$SEND_RECIPIENT" ]]; then
    if [[ "$DRY_RUN" == true ]]; then
      run_cmd bash "$SCRIPT_DIR/bridge/run-send-cross-chain.sh" "$SEND_AMOUNT" "$SEND_RECIPIENT" --dry-run
    else
      run_cmd bash "$SCRIPT_DIR/bridge/run-send-cross-chain.sh" "$SEND_AMOUNT" "$SEND_RECIPIENT"
    fi
  else
    if [[ "$DRY_RUN" == true ]]; then
      run_cmd bash "$SCRIPT_DIR/bridge/run-send-cross-chain.sh" "$SEND_AMOUNT" --dry-run
    else
      run_cmd bash "$SCRIPT_DIR/bridge/run-send-cross-chain.sh" "$SEND_AMOUNT"
    fi
  fi
}

run_local_security_and_cron() {
  section "Step 8: Local Security + Cron"

  if [[ "$DRY_RUN" == true ]]; then
    [[ -f "$PROJECT_ROOT/.env" ]] && run_cmd chmod 600 "$PROJECT_ROOT/.env"
    [[ -f "$PROJECT_ROOT/smom-dbis-138/.env" ]] && run_cmd chmod 600 "$PROJECT_ROOT/smom-dbis-138/.env"
  else
    [[ -f "$PROJECT_ROOT/.env" ]] && chmod 600 "$PROJECT_ROOT/.env"
    [[ -f "$PROJECT_ROOT/smom-dbis-138/.env" ]] && chmod 600 "$PROJECT_ROOT/smom-dbis-138/.env"
    log_ok "Local .env permissions tightened where present."
  fi

  if [[ "$APPLY_LOCAL_SECURITY" == true ]]; then
    if [[ "$DRY_RUN" == true ]]; then
      run_cmd bash "$SCRIPT_DIR/security/setup-ssh-key-auth.sh" --apply
      run_cmd bash "$SCRIPT_DIR/security/firewall-proxmox-8006.sh" --apply "192.168.11.0/24"
    else
      run_cmd bash "$SCRIPT_DIR/security/setup-ssh-key-auth.sh" --apply
      run_cmd bash "$SCRIPT_DIR/security/firewall-proxmox-8006.sh" --apply "192.168.11.0/24"
    fi
  else
    run_cmd bash "$SCRIPT_DIR/security/setup-ssh-key-auth.sh"
    run_cmd bash "$SCRIPT_DIR/security/firewall-proxmox-8006.sh" --dry-run "192.168.11.0/24"
  fi

  if [[ "$INSTALL_CRON" == true ]]; then
    run_cmd bash "$SCRIPT_DIR/maintenance/schedule-npmplus-backup-cron.sh" --install
    run_cmd bash "$SCRIPT_DIR/maintenance/schedule-daily-weekly-cron.sh" --install
  else
    run_cmd bash "$SCRIPT_DIR/maintenance/schedule-npmplus-backup-cron.sh" --show
    run_cmd bash "$SCRIPT_DIR/maintenance/schedule-daily-weekly-cron.sh" --show
  fi
}

print_external_remainder() {
  section "Still Manual / External"
  cat <<'EOF'
- LINK support on the mainnet relay: docs/07-ccip/RELAY_BRIDGE_ADD_LINK_SUPPORT_RUNBOOK.md
- WEMIX verification if policy changes: docs/07-ccip/WEMIX_TOKEN_VERIFICATION.md
- Phase 2-4 infra expansion and observability: docs/00-meta/OPERATOR_AND_EXTERNAL_COMPLETION_CHECKLIST.md
- Ledger, Trust Wallet, Consensys, CoinGecko, CMC submissions: docs/00-meta/STILL_NOT_DONE_EXECUTION_CHECKLIST.md
EOF
}

section "Run Full Operator Completion"
printf '  dry-run=%s skip-token-api-fix=%s skip-wave0=%s skip-verify=%s skip-e2e=%s fix-e2e-if-needed=%s include-config-ready-chains=%s include-chain138-next-steps=%s include-send-cross-chain=%s install-cron=%s apply-local-security=%s force-nginx-reset=%s\n' \
  "$DRY_RUN" "$SKIP_TOKEN_API_FIX" "$SKIP_WAVE0" "$SKIP_VERIFY" "$SKIP_E2E" "$FIX_E2E_IF_NEEDED" "$INCLUDE_CONFIG_READY_CHAINS" "$INCLUDE_CHAIN138_NEXT_STEPS" "$INCLUDE_SEND_CROSS_CHAIN" "$INSTALL_CRON" "$APPLY_LOCAL_SECURITY" "$FORCE_NGINX_RESET"
printf '  proxmox-host=%s token-agg-port=%s\n' "$PROXMOX_HOST" "${TOKEN_AGG_PORT_OVERRIDE:-auto}"

if [[ "$SKIP_TOKEN_API_FIX" != true ]]; then
  fix_public_report_api
else
  log_warn "Skipping token-aggregation/API fix."
fi

if [[ "$SKIP_WAVE0" != true ]]; then
  run_wave0
else
  log_warn "Skipping Wave 0."
fi

if [[ "$SKIP_VERIFY" != true ]]; then
  run_blockscout_verify
else
  log_warn "Skipping Blockscout verification."
fi

if [[ "$SKIP_E2E" != true ]]; then
  run_e2e_with_optional_fix
else
  log_warn "Skipping E2E."
fi

if [[ "$INCLUDE_CONFIG_READY_CHAINS" == true ]]; then
  run_config_ready_chains
fi

if [[ "$INCLUDE_CHAIN138_NEXT_STEPS" == true ]]; then
  run_chain138_next_steps
fi

if [[ "$INCLUDE_SEND_CROSS_CHAIN" == true ]]; then
  run_send_cross_chain
fi

run_local_security_and_cron

section "Step 9: Completion Summary"
run_cmd bash "$SCRIPT_DIR/verify/check-completion-status.sh"
print_external_remainder