#!/usr/bin/env bash # Run the full remaining operator checklist from a LAN-connected host. # Order: # 1. Fix token-aggregation DB + explorer /api/v1 proxy # 2. Wave 0 (NPMplus RPC fix + backup) # 3. Blockscout verification # 4. Public/private E2E # 5. Optional E2E remediation # 6. Optional config-ready chains + LINK funding # 7. Optional Chain 138 next steps # 8. Optional real sendCrossChain # 9. Optional local security / cron # 10. Final completion summary # # Usage: # ./scripts/run-full-operator-completion-from-lan.sh --dry-run # ./scripts/run-full-operator-completion-from-lan.sh # ./scripts/run-full-operator-completion-from-lan.sh --fix-e2e-if-needed --install-cron # ./scripts/run-full-operator-completion-from-lan.sh --include-config-ready-chains --include-chain138-next-steps # ./scripts/run-full-operator-completion-from-lan.sh --include-send-cross-chain --send-amount 0.01 [--send-recipient 0x...] # ./scripts/run-full-operator-completion-from-lan.sh --force-nginx-reset set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$PROJECT_ROOT" if [[ -f "$PROJECT_ROOT/config/ip-addresses.conf" ]]; then # shellcheck source=config/ip-addresses.conf source "$PROJECT_ROOT/config/ip-addresses.conf" 2>/dev/null || true fi DRY_RUN=false SKIP_TOKEN_API_FIX=false SKIP_WAVE0=false SKIP_VERIFY=false SKIP_E2E=false SKIP_BACKUP=false FIX_E2E_IF_NEEDED=false INCLUDE_CONFIG_READY_CHAINS=false INCLUDE_CHAIN138_NEXT_STEPS=false INCLUDE_SEND_CROSS_CHAIN=false INSTALL_CRON=false APPLY_LOCAL_SECURITY=false FORCE_NGINX_RESET=false TOKEN_AGG_PORT_OVERRIDE="" LINK_AMOUNT="" SEND_AMOUNT="" SEND_RECIPIENT="" while [[ $# -gt 0 ]]; do case "$1" in --dry-run) DRY_RUN=true ;; --skip-token-api-fix) SKIP_TOKEN_API_FIX=true ;; --skip-wave0) SKIP_WAVE0=true ;; --skip-verify) SKIP_VERIFY=true ;; --skip-e2e) SKIP_E2E=true ;; --skip-backup) SKIP_BACKUP=true ;; --fix-e2e-if-needed) FIX_E2E_IF_NEEDED=true ;; --include-config-ready-chains) INCLUDE_CONFIG_READY_CHAINS=true ;; --include-chain138-next-steps) INCLUDE_CHAIN138_NEXT_STEPS=true ;; --include-send-cross-chain) INCLUDE_SEND_CROSS_CHAIN=true ;; --install-cron) INSTALL_CRON=true ;; --apply-local-security) APPLY_LOCAL_SECURITY=true ;; --force-nginx-reset) FORCE_NGINX_RESET=true ;; --token-agg-port) shift TOKEN_AGG_PORT_OVERRIDE="${1:-}" ;; --link) shift LINK_AMOUNT="${1:-}" ;; --send-amount) shift SEND_AMOUNT="${1:-}" ;; --send-recipient) shift SEND_RECIPIENT="${1:-}" ;; -h|--help) sed -n '1,28p' "$0" exit 0 ;; *) echo "Unknown argument: $1" >&2 exit 1 ;; esac shift done if [[ "$INCLUDE_SEND_CROSS_CHAIN" == true && -z "$SEND_AMOUNT" ]]; then echo "ERROR: --include-send-cross-chain requires --send-amount " >&2 exit 1 fi PROXMOX_HOST="${PROXMOX_HOST_OVERRIDE:-${PROXMOX_HOST_R630_02:-192.168.11.12}}" log_info() { printf '\033[0;34m[INFO]\033[0m %s\n' "$1"; } log_ok() { printf '\033[0;32m[OK]\033[0m %s\n' "$1"; } log_warn() { printf '\033[0;33m[WARN]\033[0m %s\n' "$1"; } log_err() { printf '\033[0;31m[ERR]\033[0m %s\n' "$1"; } section() { printf '\n=== %s ===\n' "$1"; } print_cmd() { printf ' ' printf '%q ' "$@" printf '\n' } run_cmd() { if [[ "$DRY_RUN" == true ]]; then printf '[DRY-RUN]\n' print_cmd "$@" return 0 fi "$@" } run_ssh() { if [[ "$DRY_RUN" == true ]]; then printf '[DRY-RUN]\n' print_cmd ssh "$@" return 0 fi ssh "$@" } run_scp() { if [[ "$DRY_RUN" == true ]]; then printf '[DRY-RUN]\n' print_cmd scp "$@" return 0 fi scp "$@" } push_script_to_vmid5000() { local local_script="$1" local remote_tmp="/tmp/$(basename "$local_script")" local remote_vm="/root/$(basename "$local_script")" run_scp "$local_script" "root@${PROXMOX_HOST}:${remote_tmp}" run_ssh "root@${PROXMOX_HOST}" "pct push 5000 ${remote_tmp} ${remote_vm}" run_ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- chmod +x ${remote_vm}" } detect_token_agg_port() { if [[ -n "$TOKEN_AGG_PORT_OVERRIDE" ]]; then printf '%s\n' "$TOKEN_AGG_PORT_OVERRIDE" return 0 fi local attempt for attempt in $(seq 1 12); do if ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- bash -lc ' ss -lntp 2>/dev/null | grep -q \"*:3001\" && { echo 3001; exit 0; } ss -lntp 2>/dev/null | grep -q \"*:3000\" && { echo 3000; exit 0; } for p in 3001 3000; do curl -fsS --max-time 5 http://127.0.0.1:\$p/api/v1/networks >/dev/null 2>&1 && { echo \$p; exit 0; } done exit 1 '" 2>/dev/null; then return 0 fi sleep 2 done return 1 } fix_public_report_api() { section "Step 1: Token-Aggregation DB + /api/v1 Proxy" if [[ "$DRY_RUN" == true ]]; then run_cmd bash "$SCRIPT_DIR/apply-token-aggregation-fix.sh" --dry-run else run_cmd bash "$SCRIPT_DIR/apply-token-aggregation-fix.sh" fi local token_agg_port="${TOKEN_AGG_PORT_OVERRIDE:-auto}" if [[ "$DRY_RUN" != true ]]; then token_agg_port="$(detect_token_agg_port)" || { log_err "Could not detect token-aggregation port inside VMID 5000. Re-run with --token-agg-port 3001 or 3000." exit 1 } log_info "Detected token-aggregation port: ${token_agg_port}" else log_info "Would auto-detect token-aggregation port inside VMID 5000 (prefers 3001, then 3000)." fi if [[ "$FORCE_NGINX_RESET" == true ]]; then log_warn "Using full nginx reset for VMID 5000." push_script_to_vmid5000 "$PROJECT_ROOT/explorer-monorepo/scripts/fix-nginx-conflicts-vmid5000.sh" run_ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- bash -lc '/root/fix-nginx-conflicts-vmid5000.sh'" else push_script_to_vmid5000 "$PROJECT_ROOT/explorer-monorepo/scripts/apply-nginx-token-aggregation-proxy.sh" run_ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- bash -lc 'TOKEN_AGG_PORT=${token_agg_port} CONFIG_FILE=/etc/nginx/sites-available/blockscout /root/apply-nginx-token-aggregation-proxy.sh'" fi if [[ "$DRY_RUN" == true ]]; then run_cmd bash "$SCRIPT_DIR/verify/check-public-report-api.sh" else if ! bash "$SCRIPT_DIR/verify/check-public-report-api.sh"; then log_warn "Public report API still failing after HTTPS proxy patch. Applying HTTP /api/v1/ fallback on VMID 5000..." push_script_to_vmid5000 "$PROJECT_ROOT/scripts/fix-explorer-http-api-v1-proxy.sh" run_ssh "root@${PROXMOX_HOST}" "pct exec 5000 -- bash -lc 'TOKEN_AGG_PORT=${token_agg_port} /root/fix-explorer-http-api-v1-proxy.sh'" bash "$SCRIPT_DIR/verify/check-public-report-api.sh" fi fi run_cmd bash "$PROJECT_ROOT/metamask-integration/chain138-snap/scripts/verify-snap-api-and-icons.sh" "https://explorer.d-bis.org" } run_wave0() { section "Step 2: Wave 0" if [[ "$SKIP_BACKUP" == true ]]; then run_cmd bash "$SCRIPT_DIR/run-wave0-from-lan.sh" --skip-backup else run_cmd bash "$SCRIPT_DIR/run-wave0-from-lan.sh" fi } run_blockscout_verify() { section "Step 3: Blockscout Verification" if [[ "$DRY_RUN" == true ]]; then run_cmd bash -lc "set -a; source '$PROJECT_ROOT/smom-dbis-138/.env' 2>/dev/null || true; set +a; '$SCRIPT_DIR/verify/run-contract-verification-with-proxy.sh'" else bash -lc "set -a; source '$PROJECT_ROOT/smom-dbis-138/.env' 2>/dev/null || true; set +a; '$SCRIPT_DIR/verify/run-contract-verification-with-proxy.sh'" fi } run_e2e_with_optional_fix() { section "Step 4: Public/Private E2E" local public_ok=0 local private_ok=0 if [[ "$DRY_RUN" == true ]]; then run_cmd bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=public run_cmd bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=private if [[ "$FIX_E2E_IF_NEEDED" == true ]]; then run_cmd bash "$SCRIPT_DIR/maintenance/address-all-remaining-502s.sh" --run-besu-fix --e2e --dry-run fi return 0 fi if bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=public; then public_ok=1 log_ok "Public E2E passed." else log_warn "Public E2E failed." fi if bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=private; then private_ok=1 log_ok "Private E2E passed." else log_warn "Private E2E failed." fi if [[ "$public_ok" == 1 && "$private_ok" == 1 ]]; then return 0 fi if [[ "$FIX_E2E_IF_NEEDED" != true ]]; then log_warn "E2E remediation not requested. Re-run with --fix-e2e-if-needed to attempt backend/NPM/Besu fixes." return 0 fi section "Step 4b: E2E Remediation" bash "$SCRIPT_DIR/maintenance/address-all-remaining-502s.sh" --run-besu-fix --e2e bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=public || log_warn "Public E2E still failing after remediation." bash "$SCRIPT_DIR/verify/verify-end-to-end-routing.sh" --profile=private || log_warn "Private E2E still failing after remediation." } run_config_ready_chains() { section "Step 5: Config-Ready Chains + LINK Funding" if [[ "$DRY_RUN" == true ]]; then run_cmd bash -lc "cd '$PROJECT_ROOT/smom-dbis-138' && DRY_RUN=1 ./scripts/deployment/complete-config-ready-chains.sh" if [[ -n "$LINK_AMOUNT" ]]; then run_cmd bash -lc "cd '$PROJECT_ROOT/smom-dbis-138' && ./scripts/deployment/fund-ccip-bridges-with-link.sh --dry-run --link '$LINK_AMOUNT'" else run_cmd bash -lc "cd '$PROJECT_ROOT/smom-dbis-138' && ./scripts/deployment/fund-ccip-bridges-with-link.sh --dry-run" fi return 0 fi (cd "$PROJECT_ROOT/smom-dbis-138" && ./scripts/deployment/complete-config-ready-chains.sh) if [[ -n "$LINK_AMOUNT" ]]; then (cd "$PROJECT_ROOT/smom-dbis-138" && ./scripts/deployment/fund-ccip-bridges-with-link.sh --link "$LINK_AMOUNT") else (cd "$PROJECT_ROOT/smom-dbis-138" && ./scripts/deployment/fund-ccip-bridges-with-link.sh) fi } run_chain138_next_steps() { section "Step 6: Chain 138 Next Steps" if [[ "$DRY_RUN" == true ]]; then run_cmd bash "$SCRIPT_DIR/deployment/run-all-next-steps-chain138.sh" --dry-run else run_cmd bash "$SCRIPT_DIR/deployment/run-all-next-steps-chain138.sh" fi } run_send_cross_chain() { section "Step 7: Real sendCrossChain" if [[ -n "$SEND_RECIPIENT" ]]; then if [[ "$DRY_RUN" == true ]]; then run_cmd bash "$SCRIPT_DIR/bridge/run-send-cross-chain.sh" "$SEND_AMOUNT" "$SEND_RECIPIENT" --dry-run else run_cmd bash "$SCRIPT_DIR/bridge/run-send-cross-chain.sh" "$SEND_AMOUNT" "$SEND_RECIPIENT" fi else if [[ "$DRY_RUN" == true ]]; then run_cmd bash "$SCRIPT_DIR/bridge/run-send-cross-chain.sh" "$SEND_AMOUNT" --dry-run else run_cmd bash "$SCRIPT_DIR/bridge/run-send-cross-chain.sh" "$SEND_AMOUNT" fi fi } run_local_security_and_cron() { section "Step 8: Local Security + Cron" if [[ "$DRY_RUN" == true ]]; then [[ -f "$PROJECT_ROOT/.env" ]] && run_cmd chmod 600 "$PROJECT_ROOT/.env" [[ -f "$PROJECT_ROOT/smom-dbis-138/.env" ]] && run_cmd chmod 600 "$PROJECT_ROOT/smom-dbis-138/.env" else [[ -f "$PROJECT_ROOT/.env" ]] && chmod 600 "$PROJECT_ROOT/.env" [[ -f "$PROJECT_ROOT/smom-dbis-138/.env" ]] && chmod 600 "$PROJECT_ROOT/smom-dbis-138/.env" log_ok "Local .env permissions tightened where present." fi if [[ "$APPLY_LOCAL_SECURITY" == true ]]; then if [[ "$DRY_RUN" == true ]]; then run_cmd bash "$SCRIPT_DIR/security/setup-ssh-key-auth.sh" --apply run_cmd bash "$SCRIPT_DIR/security/firewall-proxmox-8006.sh" --apply "192.168.11.0/24" else run_cmd bash "$SCRIPT_DIR/security/setup-ssh-key-auth.sh" --apply run_cmd bash "$SCRIPT_DIR/security/firewall-proxmox-8006.sh" --apply "192.168.11.0/24" fi else run_cmd bash "$SCRIPT_DIR/security/setup-ssh-key-auth.sh" run_cmd bash "$SCRIPT_DIR/security/firewall-proxmox-8006.sh" --dry-run "192.168.11.0/24" fi if [[ "$INSTALL_CRON" == true ]]; then run_cmd bash "$SCRIPT_DIR/maintenance/schedule-npmplus-backup-cron.sh" --install run_cmd bash "$SCRIPT_DIR/maintenance/schedule-daily-weekly-cron.sh" --install else run_cmd bash "$SCRIPT_DIR/maintenance/schedule-npmplus-backup-cron.sh" --show run_cmd bash "$SCRIPT_DIR/maintenance/schedule-daily-weekly-cron.sh" --show fi } print_external_remainder() { section "Still Manual / External" cat <<'EOF' - LINK support on the mainnet relay: docs/07-ccip/RELAY_BRIDGE_ADD_LINK_SUPPORT_RUNBOOK.md - WEMIX verification if policy changes: docs/07-ccip/WEMIX_TOKEN_VERIFICATION.md - Phase 2-4 infra expansion and observability: docs/00-meta/OPERATOR_AND_EXTERNAL_COMPLETION_CHECKLIST.md - Ledger, Trust Wallet, Consensys, CoinGecko, CMC submissions: docs/00-meta/STILL_NOT_DONE_EXECUTION_CHECKLIST.md EOF } section "Run Full Operator Completion" printf ' dry-run=%s skip-token-api-fix=%s skip-wave0=%s skip-verify=%s skip-e2e=%s fix-e2e-if-needed=%s include-config-ready-chains=%s include-chain138-next-steps=%s include-send-cross-chain=%s install-cron=%s apply-local-security=%s force-nginx-reset=%s\n' \ "$DRY_RUN" "$SKIP_TOKEN_API_FIX" "$SKIP_WAVE0" "$SKIP_VERIFY" "$SKIP_E2E" "$FIX_E2E_IF_NEEDED" "$INCLUDE_CONFIG_READY_CHAINS" "$INCLUDE_CHAIN138_NEXT_STEPS" "$INCLUDE_SEND_CROSS_CHAIN" "$INSTALL_CRON" "$APPLY_LOCAL_SECURITY" "$FORCE_NGINX_RESET" printf ' proxmox-host=%s token-agg-port=%s\n' "$PROXMOX_HOST" "${TOKEN_AGG_PORT_OVERRIDE:-auto}" if [[ "$SKIP_TOKEN_API_FIX" != true ]]; then fix_public_report_api else log_warn "Skipping token-aggregation/API fix." fi if [[ "$SKIP_WAVE0" != true ]]; then run_wave0 else log_warn "Skipping Wave 0." fi if [[ "$SKIP_VERIFY" != true ]]; then run_blockscout_verify else log_warn "Skipping Blockscout verification." fi if [[ "$SKIP_E2E" != true ]]; then run_e2e_with_optional_fix else log_warn "Skipping E2E." fi if [[ "$INCLUDE_CONFIG_READY_CHAINS" == true ]]; then run_config_ready_chains fi if [[ "$INCLUDE_CHAIN138_NEXT_STEPS" == true ]]; then run_chain138_next_steps fi if [[ "$INCLUDE_SEND_CROSS_CHAIN" == true ]]; then run_send_cross_chain fi run_local_security_and_cron section "Step 9: Completion Summary" run_cmd bash "$SCRIPT_DIR/verify/check-completion-status.sh" print_external_remainder