chore: sync all changes to Gitea

- Config, docs, scripts, and backup manifests - Submodule refs unchanged (m = modified content in submodules) Made-with: Cursor
2026-03-02 11:37:34 -08:00
parent ed85135249
commit b3a8fe4496
883 changed files with 73580 additions and 4796 deletions
--- a/scripts/maintenance/fix-storage-r630-01-and-thin5.sh
+++ b/scripts/maintenance/fix-storage-r630-01-and-thin5.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# Fix high storage: r630-01 data/local-lvm (72%) and r630-02 thin5 (84.6%).
+# - thin5: Only VMID 5000 (Blockscout) uses it; prune logs/Docker inside 5000. Optional: migrate 5000 to thin2/thin6.
+# - r630-01 data: Prune logs/journal in CTs on local-lvm to free space.
+#
+# Usage:
+#   ./scripts/maintenance/fix-storage-r630-01-and-thin5.sh [--dry-run] [--prune-only] [--migrate-5000 TARGET]
+#   --dry-run       Print actions only.
+#   --prune-only    Only prune logs (no migration).
+#   --migrate-5000  Migrate VMID 5000 from thin5 to TARGET (e.g. thin2 or thin6). Does backup/restore.
+# Requires: SSH key-based access to r630-01 and r630-02. Run from project root (LAN).
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+[[ -f "${PROJECT_ROOT}/config/ip-addresses.conf" ]] && source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+R630_01="${PROXMOX_HOST_R630_01:-192.168.11.11}"
+R630_02="${PROXMOX_HOST_R630_02:-192.168.11.12}"
+SSH_OPTS="-o ConnectTimeout=10 -o StrictHostKeyChecking=no"
+
+DRY_RUN=false
+PRUNE_ONLY=true
+MIGRATE_5000_TARGET=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --dry-run)        DRY_RUN=true; shift ;;
+    --prune-only)     PRUNE_ONLY=true; shift ;;
+    --migrate-5000)   MIGRATE_5000_TARGET="${2:-}"; PRUNE_ONLY=false; shift 2 ;;
+    *) shift ;;
+  esac
+done
+
+run_r630_01() { ssh $SSH_OPTS "root@$R630_01" "$@"; }
+run_r630_02() { ssh $SSH_OPTS "root@$R630_02" "$@"; }
+
+log_info() { echo -e "\033[0;34m[INFO]\033[0m $1"; }
+log_ok()   { echo -e "\033[0;32m[✓]\033[0m $1"; }
+log_warn() { echo -e "\033[0;33m[⚠]\033[0m $1"; }
+
+echo ""
+echo "=== Fix storage: r630-01 data (72%) + r630-02 thin5 (84.6%) ==="
+echo "  dry-run=$DRY_RUN  prune-only=$PRUNE_ONLY  migrate-5000=${MIGRATE_5000_TARGET:-no}"
+echo ""
+
+# ----- Phase 1: Prune VMID 5000 (thin5) on r630-02 -----
+echo "-------- Phase 1: Free space in VMID 5000 (thin5 on r630-02) --------"
+if $DRY_RUN; then
+  echo "  Would run: vmid5000-free-disk-and-logs.sh (journal vacuum, Docker prune, logrotate)"
+else
+  if bash "${SCRIPT_DIR}/vmid5000-free-disk-and-logs.sh" 2>&1; then
+    log_ok "VMID 5000 prune done."
+  else
+    log_warn "VMID 5000 prune had warnings or failed (check above)."
+  fi
+fi
+echo ""
+
+# ----- Phase 2: Prune logs in CTs on r630-01 (data) -----
+echo "-------- Phase 2: Prune logs in CTs on r630-01 (local-lvm/data) --------"
+VMIDS_R630_01=$(run_r630_01 "pct list 2>/dev/null | awk 'NR>1 && \$2==\"running\" {print \$1}'" 2>/dev/null || true)
+if [[ -z "$VMIDS_R630_01" ]]; then
+  log_warn "Could not list running CTs on r630-01 (SSH failed?)."
+else
+  for vmid in $VMIDS_R630_01; do
+    if $DRY_RUN; then
+      echo "  Would prune in VMID $vmid: journalctl vacuum-time=3d vacuum-size=200M, logrotate -f"
+      continue
+    fi
+    run_r630_01 "pct exec $vmid -- journalctl --vacuum-time=3d 2>/dev/null; pct exec $vmid -- journalctl --vacuum-size=200M 2>/dev/null" 2>/dev/null || true
+    run_r630_01 "pct exec $vmid -- logrotate -f /etc/logrotate.conf 2>/dev/null" 2>/dev/null || true
+    run_r630_01 "pct exec $vmid -- sh -c ': > /var/log/syslog 2>/dev/null'" 2>/dev/null || true
+    log_ok "Pruned VMID $vmid"
+  done
+fi
+echo ""
+
+# ----- Phase 3: Optional migrate 5000 from thin5 to thin2/thin6 -----
+if [[ -n "$MIGRATE_5000_TARGET" ]]; then
+  echo "-------- Phase 3: Migrate VMID 5000 from thin5 to $MIGRATE_5000_TARGET --------"
+  if $DRY_RUN; then
+    echo "  Would: stop 5000 -> vzdump to local -> destroy -> pct restore 5000 --storage $MIGRATE_5000_TARGET -> start"
+    echo "  Target must be thin2 or thin6 (have free space on r630-02)."
+  else
+    run_r630_02 "pct stop 5000" 2>/dev/null || true
+    sleep 3
+    BACKUP_OUT=$(run_r630_02 "vzdump 5000 --storage local --compress gzip --mode stop --remove 0 2>&1" || true)
+    if echo "$BACKUP_OUT" | grep -qE "error|Error|failed|Failed"; then
+      log_warn "Backup failed: $BACKUP_OUT"
+      run_r630_02 "pct start 5000" 2>/dev/null || true
+    else
+      BACKUP_FILE=$(run_r630_02 "ls -t /var/lib/vz/dump/vzdump-lxc-5000-*.tar.gz 2>/dev/null | head -1" || true)
+      if [[ -z "$BACKUP_FILE" ]]; then
+        log_warn "Backup file not found."
+        run_r630_02 "pct start 5000" 2>/dev/null || true
+      else
+        run_r630_02 "pct destroy 5000 --force 2>/dev/null" || true
+        sleep 2
+        run_r630_02 "pct restore 5000 $BACKUP_FILE --storage $MIGRATE_5000_TARGET 2>&1" || { log_warn "Restore failed"; run_r630_02 "pct start 5000" 2>/dev/null; exit 1; }
+        run_r630_02 "rm -f $BACKUP_FILE"
+        run_r630_02 "pct start 5000"
+        log_ok "Migrated 5000 to $MIGRATE_5000_TARGET and started."
+      fi
+    fi
+  fi
+  echo ""
+fi
+
+echo "=== Re-check storage (run audit after a few minutes for thin pool reclaim) ==="
+if ! $DRY_RUN; then
+  echo "  r630-01: ssh root@$R630_01 'pvesm status | grep -E \"data|local-lvm\"'"
+  echo "  r630-02: ssh root@$R630_02 'pvesm status | grep thin5'"
+  echo "  Or: bash scripts/audit-proxmox-rpc-storage.sh"
+fi
+echo ""
--- a/scripts/maintenance/health-check-rpc-2101.sh
+++ b/scripts/maintenance/health-check-rpc-2101.sh
@@ -77,6 +77,14 @@ else
  ((FAIL++)) || true
 fi

+# 6. Database writable (required for deployments — Besu must write to /data/besu/database)
+if run_ssh "pct exec $VMID -- sh -c 'mkdir -p /data/besu/database && touch /data/besu/database/.write_test && rm -f /data/besu/database/.write_test'" 2>/dev/null; then
+  log_ok "Database path writable (/data/besu/database)"
+else
+  log_warn "Database path not writable (read-only?). Run: ./scripts/maintenance/make-rpc-vmids-writable-via-ssh.sh"
+  ((FAIL++)) || true
+fi
+
 echo ""
 if [[ "${FAIL:-0}" -gt 0 ]]; then
  log_warn "Health check had $FAIL issue(s). Fix: ./scripts/maintenance/fix-core-rpc-2101.sh or see docs/09-troubleshooting/RPC_NODES_BLOCK_PRODUCTION_FIX.md"
--- a/scripts/maintenance/migrate-ct-r630-01-data-to-thin1.sh
+++ b/scripts/maintenance/migrate-ct-r630-01-data-to-thin1.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Migrate one LXC container from r630-01 data pool to thin1 (same host).
+# Use to free space on data (currently 72%). See docs/04-configuration/MIGRATION_PLAN_R630_01_DATA.md
+#
+# Usage: bash scripts/maintenance/migrate-ct-r630-01-data-to-thin1.sh <VMID> [--dry-run]
+# Example: bash scripts/maintenance/migrate-ct-r630-01-data-to-thin1.sh 10232
+# Requires: SSH to r630-01. Run from project root (LAN).
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+[[ -f "${PROJECT_ROOT}/config/ip-addresses.conf" ]] && source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+R630_01="${PROXMOX_HOST_R630_01:-192.168.11.11}"
+TARGET="thin1"
+SSH_OPTS="-o ConnectTimeout=10 -o StrictHostKeyChecking=no"
+
+VMID="${1:-}"
+DRY_RUN=false
+[[ "${2:-}" == "--dry-run" ]] && DRY_RUN=true
+
+if [[ -z "$VMID" ]] || ! [[ "$VMID" =~ ^[0-9]+$ ]]; then
+  echo "Usage: $0 <VMID> [--dry-run]"
+  echo "  Migrates CT VMID from data to thin1 on r630-01."
+  echo "  Suggested VMIDs (small first): 10232 10233 10120 10100 10101 10235 10236 7804 8640 8642"
+  exit 1
+fi
+
+run() { ssh $SSH_OPTS "root@$R630_01" "$@"; }
+
+echo "=== Migrate CT $VMID from data to $TARGET on r630-01 ==="
+echo "  dry-run=$DRY_RUN"
+echo ""
+
+# Check CT exists and is on data
+ROOTFS=$(run "pct config $VMID 2>/dev/null | grep '^rootfs:'" || true)
+if [[ -z "$ROOTFS" ]]; then
+  echo "ERROR: CT $VMID not found on r630-01."
+  exit 1
+fi
+if ! echo "$ROOTFS" | grep -q 'local-lvm\|data'; then
+  echo "WARNING: CT $VMID rootfs does not look like data pool: $ROOTFS"
+  read -p "Continue anyway? [y/N] " -n 1 -r; echo
+  [[ "${REPLY:-}" =~ ^[yY]$ ]] || exit 1
+fi
+
+if $DRY_RUN; then
+  echo "Would: stop $VMID -> vzdump -> destroy -> restore --storage $TARGET -> start"
+  exit 0
+fi
+
+echo "1. Stopping CT $VMID..."
+run "pct stop $VMID" 2>/dev/null || true
+sleep 3
+
+echo "2. Backup (vzdump)..."
+run "vzdump $VMID --storage local --compress gzip --mode stop --remove 0 2>&1" || true
+
+BACKUP_FILE=$(run "ls -t /var/lib/vz/dump/vzdump-lxc-$VMID-*.tar.gz 2>/dev/null | head -1" || true)
+if [[ -z "$BACKUP_FILE" ]]; then
+  echo "ERROR: Backup not found. Start CT back: ssh root@$R630_01 'pct start $VMID'"
+  exit 1
+fi
+echo "   Backup: $BACKUP_FILE"
+
+echo "3. Destroy CT..."
+run "pct destroy $VMID --force 2>/dev/null" || true
+sleep 2
+
+echo "4. Restore to $TARGET..."
+run "pct restore $VMID $BACKUP_FILE --storage $TARGET 2>&1" || {
+  echo "ERROR: Restore failed. Restore backup manually."
+  exit 1
+}
+
+echo "5. Remove backup..."
+run "rm -f $BACKUP_FILE" 2>/dev/null || true
+
+echo "6. Start CT..."
+run "pct start $VMID"
+echo ""
+echo "Done. CT $VMID is now on $TARGET. Verify: ssh root@$R630_01 'pct config $VMID | grep rootfs'"
--- a/scripts/maintenance/migrate-vmid-5000-to-thin2.sh
+++ b/scripts/maintenance/migrate-vmid-5000-to-thin2.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Migrate VMID 5000 (Blockscout) from thin5 to thin2 on r630-02. No prune steps.
+# Usage: bash scripts/maintenance/migrate-vmid-5000-to-thin2.sh
+# Requires: SSH to r630-02. Run from project root (LAN).
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+[[ -f "${PROJECT_ROOT}/config/ip-addresses.conf" ]] && source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
+
+R630_02="${PROXMOX_HOST_R630_02:-192.168.11.12}"
+TARGET="thin2"
+VMID=5000
+SSH_OPTS="-o ConnectTimeout=10 -o StrictHostKeyChecking=no"
+
+run() { ssh $SSH_OPTS "root@$R630_02" "$@"; }
+
+echo "=== Migrate VMID $VMID from thin5 to $TARGET on r630-02 ==="
+echo ""
+
+echo "1. Stopping CT $VMID..."
+run "pct stop $VMID" 2>/dev/null || true
+sleep 5
+
+echo "2. Backup (vzdump)..."
+run "vzdump $VMID --storage local --compress gzip --mode stop --remove 0 2>&1" || true
+
+BACKUP_FILE=$(run "ls -t /var/lib/vz/dump/vzdump-lxc-$VMID-*.tar.gz 2>/dev/null | head -1" || true)
+if [[ -z "$BACKUP_FILE" ]]; then
+  echo "ERROR: Backup file not found. Starting CT back."
+  run "pct start $VMID" 2>/dev/null || true
+  exit 1
+fi
+echo "   Backup: $BACKUP_FILE"
+
+echo "3. Destroy CT..."
+run "pct destroy $VMID --force 2>/dev/null" || true
+sleep 3
+
+echo "4. Restore to $TARGET..."
+run "pct restore $VMID $BACKUP_FILE --storage $TARGET 2>&1" || {
+  echo "ERROR: Restore failed."
+  exit 1
+}
+
+echo "5. Remove backup..."
+run "rm -f $BACKUP_FILE" 2>/dev/null || true
+
+echo "6. Start CT..."
+run "pct start $VMID"
+echo ""
+echo "Done. VMID $VMID is now on $TARGET. Verify: ssh root@$R630_02 'pct config 5000 | grep rootfs'"