d-bis/proxmox
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finalize DBIS infra verification and runtime baselines
All checks were successful
Deploy to Phoenix / deploy (push) Successful in 6s
Details

Browse Source
This commit is contained in:
defiQUG
2026-03-28 19:18:32 -07:00
parent 266a8ae30f
commit 6f53323eae
22 changed files with 1924 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

204
scripts/besu/upgrade-public-rpc-vmid2201.sh Executable file
View File

@@ -0,0 +1,204 @@
#!/usr/bin/env bash
# Upgrade the public Chain 138 RPC node (VMID 2201) to a Besu version that supports
# eth_maxPriorityFeePerGas. Default target is the current fleet baseline.
#
# Usage:
# bash scripts/besu/upgrade-public-rpc-vmid2201.sh
# bash scripts/besu/upgrade-public-rpc-vmid2201.sh --dry-run
# BESU_VERSION=25.12.0 bash scripts/besu/upgrade-public-rpc-vmid2201.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
VMID="${RPC_VMID_2201:-2201}"
RPC_HOST="${RPC_VM_2201_HOST:-root@${PROXMOX_R630_02:-192.168.11.12}}"
[[ "$RPC_HOST" != *"@"* ]] && RPC_HOST="root@$RPC_HOST"
BESU_VERSION="${BESU_VERSION:-25.12.0}"
BESU_TAR="besu-${BESU_VERSION}.tar.gz"
BESU_DIR="/opt/besu-${BESU_VERSION}"
DOWNLOAD_URL="${BESU_DOWNLOAD_URL:-https://github.com/hyperledger/besu/releases/download/${BESU_VERSION}/${BESU_TAR}}"
JAVA21_FALLBACK_URL="${JAVA21_FALLBACK_URL:-https://api.adoptium.net/v3/binary/latest/21/ga/linux/x64/jre/hotspot/normal/eclipse}"
RPC_HTTP_MAX_ACTIVE_CONNECTIONS="${RPC_HTTP_MAX_ACTIVE_CONNECTIONS:-256}"
RPC_WS_MAX_ACTIVE_CONNECTIONS="${RPC_WS_MAX_ACTIVE_CONNECTIONS:-256}"
LOCAL_CACHE="${LOCAL_CACHE:-/tmp}"
DRY_RUN=false
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=true
SSH_OPTS=(-o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
RPC_IP="${RPC_PUBLIC_1:-192.168.11.221}"
run_on_host() {
ssh "${SSH_OPTS[@]}" "$RPC_HOST" "$@"
}
run_in_vmid() {
local cmd="$1"
if command -v pct >/dev/null 2>&1 && pct list 2>/dev/null | grep -q "^${VMID} "; then
pct exec "$VMID" -- bash -lc "$cmd"
else
run_on_host "pct exec ${VMID} -- bash -lc $(printf '%q' "$cmd")"
fi
}
push_to_vmid() {
local src="$1"
local dest="$2"
if command -v pct >/dev/null 2>&1 && pct list 2>/dev/null | grep -q "^${VMID} "; then
pct push "$VMID" "$src" "$dest"
else
local host_tmp="/tmp/$(basename "$src")"
scp "${SSH_OPTS[@]}" "$src" "${RPC_HOST}:${host_tmp}"
run_on_host "pct push ${VMID} $(printf '%q' "$host_tmp") $(printf '%q' "$dest") && rm -f $(printf '%q' "$host_tmp")"
fi
}
rpc_request() {
local method="$1"
local params="${2:-[]}"
curl -sS --max-time 20 -X POST "http://${RPC_IP}:8545" \
-H "Content-Type: application/json" \
-d "{\"jsonrpc\":\"2.0\",\"method\":\"${method}\",\"params\":${params},\"id\":1}"
}
echo "=============================================="
echo "Upgrade public Chain 138 RPC (VMID ${VMID})"
echo "Host: ${RPC_HOST}"
echo "Target Besu version: ${BESU_VERSION}"
echo "=============================================="
if $DRY_RUN; then
echo "[dry-run] No changes will be made."
fi
run_on_host "echo connected >/dev/null"
run_in_vmid "
set -euo pipefail
if [[ ! -e /opt/besu ]]; then
fallback=\$(find /opt -maxdepth 1 -type d -name 'besu-*' | sort -V | tail -1)
if [[ -n \"\${fallback:-}\" ]]; then
ln -sfn \"\$fallback\" /opt/besu
chown -h besu:besu /opt/besu 2>/dev/null || true
fi
fi
"
CURRENT_VERSION="$(run_in_vmid '/opt/besu/bin/besu --version 2>/dev/null || besu --version 2>/dev/null || true' | head -1 || true)"
JAVA_VERSION_RAW="$(run_in_vmid 'java -version 2>&1 | head -1' || true)"
echo "Current version: ${CURRENT_VERSION:-unknown}"
echo "Current Java: ${JAVA_VERSION_RAW:-unknown}"
if $DRY_RUN; then
echo "[dry-run] Would download ${DOWNLOAD_URL}"
echo "[dry-run] Would stage ${BESU_TAR} in VMID ${VMID}, extract to ${BESU_DIR}, switch /opt/besu, restart besu-rpc."
exit 0
fi
mkdir -p "$LOCAL_CACHE"
if [[ ! -f "${LOCAL_CACHE}/${BESU_TAR}" ]]; then
echo "Downloading ${DOWNLOAD_URL} ..."
curl -fsSL -o "${LOCAL_CACHE}/${BESU_TAR}" "${DOWNLOAD_URL}"
fi
echo "Pushing tarball into VMID ${VMID} ..."
push_to_vmid "${LOCAL_CACHE}/${BESU_TAR}" "/tmp/${BESU_TAR}"
echo "Ensuring Java 21 runtime is present ..."
run_in_vmid "
set -euo pipefail
java_major=\$(java -version 2>&1 | sed -n '1s/.*version \"\\([0-9][0-9]*\\).*/\\1/p')
if [[ -z \"\${java_major:-}\" || \"\$java_major\" -lt 21 ]]; then
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y -qq openjdk-21-jre-headless || true
java_major=\$(java -version 2>&1 | sed -n '1s/.*version \"\\([0-9][0-9]*\\).*/\\1/p')
if [[ -z \"\${java_major:-}\" || \"\$java_major\" -lt 21 ]]; then
command -v curl >/dev/null 2>&1 || apt-get install -y -qq curl ca-certificates
tmp_jre=/tmp/java21-jre.tar.gz
curl -fsSL -o \"\$tmp_jre\" '${JAVA21_FALLBACK_URL}'
tar -tzf \"\$tmp_jre\" > /tmp/java21-jre.list
extracted_dir=\$(head -1 /tmp/java21-jre.list | cut -d/ -f1)
rm -f /tmp/java21-jre.list
tar -xzf \"\$tmp_jre\" -C /opt
rm -f \"\$tmp_jre\"
ln -sfn \"/opt/\${extracted_dir}\" /opt/java-21
update-alternatives --install /usr/bin/java java /opt/java-21/bin/java 2100
fi
fi
config_file=\$(systemctl cat besu-rpc.service | sed -n 's/.*--config-file=\\([^ ]*\\).*/\\1/p' | tail -1)
if [[ -n \"\${config_file:-}\" && -f \"\$config_file\" ]]; then
find /etc/besu -maxdepth 1 -type f -name '*.toml' -print0 2>/dev/null | while IFS= read -r -d '' toml; do
sed -i \
-e '/^[[:space:]]*miner-enabled[[:space:]]*=.*/d' \
-e '/^[[:space:]]*privacy-enabled[[:space:]]*=.*/d' \
\"\$toml\"
if grep -q '^rpc-http-enabled=true' \"\$toml\" && ! grep -q '^rpc-http-max-active-connections=' \"\$toml\"; then
tmp=\$(mktemp)
awk '1; /^rpc-http-port=/{print \"rpc-http-max-active-connections=${RPC_HTTP_MAX_ACTIVE_CONNECTIONS}\"}' \"\$toml\" > \"\$tmp\"
cat \"\$tmp\" > \"\$toml\"
rm -f \"\$tmp\"
fi
if grep -q '^rpc-ws-enabled=true' \"\$toml\" && ! grep -q '^rpc-ws-max-active-connections=' \"\$toml\"; then
tmp=\$(mktemp)
awk '1; /^rpc-ws-port=/{print \"rpc-ws-max-active-connections=${RPC_WS_MAX_ACTIVE_CONNECTIONS}\"}' \"\$toml\" > \"\$tmp\"
cat \"\$tmp\" > \"\$toml\"
rm -f \"\$tmp\"
fi
done
if ! grep -q '^data-storage-format=' \"\$config_file\"; then
tmp=\$(mktemp)
awk '1; /^sync-mode=/{print \"data-storage-format=\\\"FOREST\\\"\"}' \"\$config_file\" > \"\$tmp\"
cat \"\$tmp\" > \"\$config_file\"
rm -f \"\$tmp\"
fi
fi
"
echo "Installing Besu ${BESU_VERSION} inside VMID ${VMID} ..."
run_in_vmid "
set -euo pipefail
cd /opt
if [[ -L /opt/besu ]]; then
current_target=\$(readlink -f /opt/besu)
current_version=\$(basename \"\$current_target\")
else
current_version=\$(/opt/besu/bin/besu --version 2>/dev/null | head -1 | sed -E 's#^.*/(v)?([0-9.]+).*\$#besu-\\2#')
[[ -z \"\$current_version\" ]] && current_version=besu-backup-pre-${BESU_VERSION}
mv /opt/besu \"/opt/\${current_version}\"
fi
rm -rf '${BESU_DIR}'
tar -xzf '/tmp/${BESU_TAR}' -C /opt
rm -f '/tmp/${BESU_TAR}'
ln -sfn '${BESU_DIR}' /opt/besu
chown -h besu:besu /opt/besu
chown -R besu:besu '${BESU_DIR}' /opt/besu-* 2>/dev/null || true
"
echo "Restarting besu-rpc.service ..."
run_in_vmid "systemctl restart besu-rpc.service"
for _ in $(seq 1 24); do
ACTIVE_STATE="$(run_in_vmid 'systemctl is-active besu-rpc.service' || true)"
[[ "$ACTIVE_STATE" == "active" ]] && break
sleep 5
done
NEW_VERSION="$(run_in_vmid '/opt/besu/bin/besu --version 2>/dev/null | grep -m1 "besu/" || true' | head -1 || true)"
echo "Service state: ${ACTIVE_STATE:-unknown}"
echo "New version: ${NEW_VERSION:-unknown}"
echo "Verifying live RPC methods ..."
CHAIN_ID="$(rpc_request eth_chainId | jq -r '.result // empty' 2>/dev/null || true)"
PRIORITY_FEE="$(curl -sS --max-time 20 -X POST 'https://rpc-http-pub.d-bis.org' -H 'Content-Type: application/json' -d '{\"jsonrpc\":\"2.0\",\"method\":\"eth_maxPriorityFeePerGas\",\"params\":[],\"id\":1}' | jq -r '.result // empty' 2>/dev/null || true)"
TRACE_OK="$(rpc_request trace_block '[\"0x1\"]' | jq -r 'has(\"result\")' 2>/dev/null || true)"
if [[ "$ACTIVE_STATE" != "active" || -z "$CHAIN_ID" || "$TRACE_OK" != "true" || -z "$PRIORITY_FEE" ]]; then
echo "ERROR: post-upgrade verification failed."
echo " eth_chainId result: ${CHAIN_ID:-missing}"
echo " trace_block result present: ${TRACE_OK:-false}"
echo " eth_maxPriorityFeePerGas result: ${PRIORITY_FEE:-missing}"
exit 1
fi
echo "OK: VMID ${VMID} upgraded successfully and public RPC now exposes eth_maxPriorityFeePerGas."

148
scripts/docs/generate-dbis-node-role-matrix-md.sh Executable file
View File

@@ -0,0 +1,148 @@
#!/usr/bin/env bash
# Regenerate docs/02-architecture/DBIS_NODE_ROLE_MATRIX.md body tables from
# config/proxmox-operational-template.json (run from repo root).
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
JSON="$ROOT/config/proxmox-operational-template.json"
OUT="$ROOT/docs/02-architecture/DBIS_NODE_ROLE_MATRIX.md"
if ! command -v jq &>/dev/null; then
echo "jq required" >&2
exit 1
fi
TMP="$(mktemp)"
trap 'rm -f "$TMP"' EXIT
jq -r '
def vstatus:
if .category == "besu_validator" then "QBFT signer"
elif .category == "besu_sentry" then "Sentry (no signer)"
elif (.category | test("^rpc")) then "RPC only"
else "N/A"
end;
def ntype:
if .category == "besu_validator" then "Besu validator"
elif .category == "besu_sentry" then "Besu sentry"
elif .category == "rpc_core" or .category == "rpc_public" or .category == "rpc_private" or .category == "rpc_named" or .category == "rpc_thirdweb" or .category == "rpc_alltra_hybx" then "Besu RPC (\(.category))"
elif .category == "dlt" and (.hostname | test("fabric")) then "Fabric"
elif .category == "dlt" and (.hostname | test("indy")) then "Indy"
elif .category == "firefly" then "FireFly"
elif .category == "explorer" then "Blockscout"
elif .category == "npmplus" then "NPMplus ingress"
elif .category == "infra" then "Infra LXC"
elif .category == "monitoring" and (.hostname | test("cacti")) then "Cacti"
elif .category == "monitoring" then "Monitoring"
elif .category == "oracle" then "Oracle publisher"
elif .category == "ccip" then "CCIP monitor"
elif .category == "tunnel" then "Cloudflare tunnel"
elif .category == "ml" then "ML node"
elif .category == "vault" then "HashiCorp Vault"
elif .category == "order" then "The Order service"
elif .category == "sankofa_phoenix" then "Sankofa / Phoenix"
elif .category == "mim4u" then "MIM4U"
elif .category == "dbis" then "DBIS stack"
elif .category == "mifos" then "Mifos"
elif .category == "dapp" then "DApp"
elif .category == "dev" then "Dev"
elif .category == "ai_infra" then "AI infra"
elif .category == "defi" then "DeFi"
elif .category == "general" then "General CT"
elif .category == "legacy_proxy" then "Legacy NPM"
else .category
end;
def stier:
if .category == "besu_validator" or .category == "besu_sentry" then "validator-tier"
elif (.category | test("^rpc")) then "DMZ / RPC exposure"
elif .category == "npmplus" or .category == "tunnel" then "edge ingress"
elif .category == "dlt" or .category == "firefly" then "identity / workflow DLT"
elif .category == "vault" or .category == "infra" then "management / secrets"
elif .category == "order" or .category == "sankofa_phoenix" or .category == "dbis" then "application"
else "standard internal"
end;
([.services[] | select(.ipv4 != null) | .ipv4] | group_by(.) | map(select(length > 1) | .[0])) as $dup_ips
| .services[]
| (.ipv4) as $ip
| [(.vmid // "—"), .hostname, ($ip // "—"), (if ($ip != null and ($dup_ips | index($ip))) then "shared / non-concurrent mapping — verify live owner" else "unique in template" end), ntype, "TBD", "TBD", (.preferred_node // "—"), vstatus, stier]
| @tsv
' "$JSON" | sort -t$'\t' -k1,1n > "$TMP"
UPDATED="$(date -u +%Y-%m-%d)"
{
cat <<EOF
# DBIS Node Role Matrix
**Last updated:** ${UPDATED} (UTC) — regenerate machine-derived rows: \`bash scripts/docs/generate-dbis-node-role-matrix-md.sh\`
**Status:** Active — infrastructure constitution for DBIS Chain 138 and colocated workloads.
## Purpose
This matrix assigns **node type**, **preferred host placement**, **validator/signing role** (for Besu), and **security tier** per workload. It implements the entity-placement model in [dbis_chain_138_technical_master_plan.md](../../dbis_chain_138_technical_master_plan.md) (Sections 6–7) in a form operators can maintain.
**Canonical pairs (keep in sync):**
- Human detail and status: [ALL_VMIDS_ENDPOINTS.md](../04-configuration/ALL_VMIDS_ENDPOINTS.md)
- Machine-readable services: [config/proxmox-operational-template.json](../../config/proxmox-operational-template.json)
When you change VMID, IP, hostname, or placement, update **ALL_VMIDS** and **operational-template.json** first, then regenerate the table below with this script (or edit the static sections manually).
## Columns
| Column | Meaning |
|--------|---------|
| **Entity owner** | DBIS Core, Central Bank, IFI, Regional Operator, etc. — use **TBD** until governance assigns. |
| **Region** | Geographic or site label — **TBD** until multi-site is formalized. |
| **IP note** | Flags duplicate IPv4 entries in the planning template. A duplicate means **shared or historical mapping**, not concurrent ownership — verify live owner in ALL_VMIDS or on-cluster. |
| **Preferred host** | Preferred Proxmox node (\`r630-01\`, \`r630-02\`, \`ml110\`, \`any\`). This is a planning target, not an assertion of current placement. |
| **Validator / signing** | For Chain 138 Besu: QBFT signer, sentry (no signer), RPC-only, or N/A. |
| **Security tier** | High-level zone: validator-tier, DMZ/RPC, edge ingress, identity/DLT, application, etc. |
## Proxmox hypervisor nodes
| Hostname | MGMT IP | Cluster | Role (summary) |
|----------|---------|---------|------------------|
EOF
jq -r '.proxmox_nodes[] | "| \(.hostname) | \(.mgmt_ipv4) | \(.cluster_name // "h — verify") | \(.role) |"' "$JSON"
cat <<'MID'
## Workloads (from operational template)
Machine-derived rows below come from `services[]` in `config/proxmox-operational-template.json`. Duplicate IPv4 notes are warnings that the planning template still contains alternative or legacy ownership for the same address; they must not be read as concurrent live allocations.
| VMID | Hostname | IPv4 | IP note | Node type | Entity owner | Region | Preferred host | Validator / signing | Security tier |
|------|----------|------|---------|-----------|--------------|--------|----------------|---------------------|---------------|
MID
while IFS=$'\t' read -r vmid host ip ipnote ntype ent reg hw vst stier; do
echo "| $vmid | $host | $ip | $ipnote | $ntype | $ent | $reg | $hw | $vst | $stier |"
done < "$TMP"
cat <<'FOOT'
## Supplementary rows (not in template JSON)
These appear in [ALL_VMIDS_ENDPOINTS.md](../04-configuration/ALL_VMIDS_ENDPOINTS.md) but are not modeled as `services[]` entries in `proxmox-operational-template.json`. They are **manual supplements**, not generator-backed source of truth.
| VMID | Hostname | IPv4 | IP note | Node type | Entity owner | Region | Preferred host | Validator / signing | Security tier |
|------|----------|------|---------|-----------|--------------|--------|----------------|---------------------|---------------|
| 106 | redis-rpc-translator | 192.168.11.110 | manual supplement | RPC translator (Redis) | TBD | TBD | r630-01 (per ALL_VMIDS) | N/A | DMZ / RPC exposure |
| 107 | web3signer-rpc-translator | 192.168.11.111 | manual supplement | RPC translator (Web3Signer) | TBD | TBD | r630-01 | N/A | DMZ / RPC exposure |
| 108 | vault-rpc-translator | 192.168.11.112 | manual supplement | RPC translator (Vault) | TBD | TBD | r630-01 | N/A | management / secrets |
## Host-level services (no VMID)
| Name | Location | Node type | Notes |
|------|----------|-----------|-------|
| CCIP relay | r630-01 host `/opt/smom-dbis-138/services/relay` | Cross-chain relay | Uses RPC (e.g. VMID 2201); see [NETWORK_CONFIGURATION_MASTER.md](../11-references/NETWORK_CONFIGURATION_MASTER.md), [docs/07-ccip/](../07-ccip/). |
## Related
- [dbis_chain_138_technical_master_plan.md](../../dbis_chain_138_technical_master_plan.md)
- [CHAIN138_CANONICAL_NETWORK_ROLES_VALIDATORS_SENTRY_AND_RPC.md](CHAIN138_CANONICAL_NETWORK_ROLES_VALIDATORS_SENTRY_AND_RPC.md)
- [VMID_ALLOCATION_FINAL.md](VMID_ALLOCATION_FINAL.md)
FOOT
} > "$OUT"
echo "Wrote $OUT"

360
scripts/upgrade-besu-all-nodes.sh
View File

@@ -1,52 +1,31 @@
#!/usr/bin/env bash
# Upgrade all Besu nodes to the latest (or specified) version.
# Requires: SSH to Proxmox host, curl/wget, enough disk in containers.
# Upgrade all running Besu containers to the requested version.
# Installs Java 21 where needed, preserves the previous /opt/besu-* directory for rollback,
# and restarts the detected Besu systemd unit in each container.
#
# Usage:
# ./scripts/upgrade-besu-all-nodes.sh # upgrade to latest (25.12.0)
# ./scripts/upgrade-besu-all-nodes.sh --dry-run # show what would be done
# BESU_VERSION=25.11.0 ./scripts/upgrade-besu-all-nodes.sh
# Optional: pre-download to avoid long run (script uses $LOCAL_CACHE/besu-${BESU_VERSION}.tar.gz):
# curl -sSL -o /tmp/besu-25.12.0.tar.gz https://github.com/hyperledger/besu/releases/download/25.12.0/besu-25.12.0.tar.gz
# bash scripts/upgrade-besu-all-nodes.sh
# bash scripts/upgrade-besu-all-nodes.sh --dry-run
# BESU_VERSION=25.12.0 bash scripts/upgrade-besu-all-nodes.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
if [ -f "$PROJECT_ROOT/config/ip-addresses.conf" ]; then
# shellcheck source=../config/ip-addresses.conf
source "$PROJECT_ROOT/config/ip-addresses.conf"
fi
PROXMOX_HOST="${PROXMOX_HOST:-${PROXMOX_HOST_ML110:-192.168.11.10}}"
# Latest stable as of 2025-12; EIP-7702 requires >= 24.1.0
BESU_VERSION="${BESU_VERSION:-25.12.0}"
BESU_TAR="besu-${BESU_VERSION}.tar.gz"
BESU_DIR="besu-${BESU_VERSION}"
BESU_DIR="/opt/besu-${BESU_VERSION}"
DOWNLOAD_URL="${BESU_DOWNLOAD_URL:-https://github.com/hyperledger/besu/releases/download/${BESU_VERSION}/${BESU_TAR}}"
JAVA21_FALLBACK_URL="${JAVA21_FALLBACK_URL:-https://api.adoptium.net/v3/binary/latest/21/ga/linux/x64/jre/hotspot/normal/eclipse}"
RPC_HTTP_MAX_ACTIVE_CONNECTIONS="${RPC_HTTP_MAX_ACTIVE_CONNECTIONS:-256}"
RPC_WS_MAX_ACTIVE_CONNECTIONS="${RPC_WS_MAX_ACTIVE_CONNECTIONS:-256}"
LOCAL_CACHE="${LOCAL_CACHE:-/tmp}"
DRY_RUN=false
for arg in "$@"; do
[ "$arg" = "--dry-run" ] && DRY_RUN=true
done
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=true
# Same node list and services as restart-all-besu-services.sh
declare -A NODE_SERVICES=(
["1000"]="besu-validator"
["1001"]="besu-validator"
["1002"]="besu-validator"
["1003"]="besu-validator"
["1004"]="besu-validator"
["1500"]="besu-sentry"
["1501"]="besu-sentry"
["1502"]="besu-sentry"
["1503"]="besu-sentry"
["2101"]="besu-rpc"
["2400"]="besu-rpc"
["2401"]="besu-rpc"
["2402"]="besu-rpc"
)
SSH_OPTS=(-o ConnectTimeout=20 -o ServerAliveInterval=15 -o ServerAliveCountMax=3 -o StrictHostKeyChecking=accept-new)
RED='\033[0;31m'
GREEN='\033[0;32m'
@@ -54,120 +33,245 @@ YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_ok() { echo -e "${GREEN}[OK]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_err() { echo -e "${RED}[ERROR]${NC} $1"; }
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_ok() { echo -e "${GREEN}[OK]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_err() { echo -e "${RED}[ERROR]${NC} $1"; }
is_running() {
local vmid=$1
ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new root@"$PROXMOX_HOST" \
"pct status $vmid 2>/dev/null" | grep -q running
declare -A HOST_BY_VMID
for v in 1000 1001 1002 1500 1501 1502 2101; do HOST_BY_VMID[$v]="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"; done
for v in 2201 2303 2401; do HOST_BY_VMID[$v]="${PROXMOX_R630_02:-${PROXMOX_HOST_R630_02:-192.168.11.12}}"; done
for v in 1003 1004 1503 1504 1505 1506 1507 1508 2102 2301 2304 2305 2306 2307 2308 2400 2402 2403; do HOST_BY_VMID[$v]="${PROXMOX_ML110:-${PROXMOX_HOST_ML110:-192.168.11.10}}"; done
BESU_VMIDS=(
1000 1001 1002 1003 1004
1500 1501 1502 1503 1504 1505 1506 1507 1508
2101 2102 2201 2301 2303 2304 2305 2306 2307 2308
2400 2401 2402 2403
)
host_ssh() {
local host="$1"
shift
ssh "${SSH_OPTS[@]}" "root@${host}" "$@"
}
# Ensure tarball exists (download to host or use cache)
ensure_tarball() {
local path="$LOCAL_CACHE/$BESU_TAR"
if [ -f "$path" ]; then
log_ok "Using existing $path"
echo "$path"
return
fi
log_info "Downloading $DOWNLOAD_URL ..."
if $DRY_RUN; then
echo ""
return
fi
(cd "$LOCAL_CACHE" && curl -sSfL -o "$BESU_TAR" "$DOWNLOAD_URL") || {
log_err "Download failed"
return 1
}
log_ok "Downloaded $path"
echo "$path"
local path="${LOCAL_CACHE}/${BESU_TAR}"
mkdir -p "$LOCAL_CACHE"
if [[ -f "$path" ]]; then
log_ok "Using existing $path" >&2
printf '%s\n' "$path"
return 0
fi
if $DRY_RUN; then
printf '%s\n' "$path"
return 0
fi
log_info "Downloading ${DOWNLOAD_URL}" >&2
curl -fsSL -o "$path" "$DOWNLOAD_URL"
log_ok "Downloaded $path" >&2
printf '%s\n' "$path"
}
detect_service() {
local host="$1"
local vmid="$2"
host_ssh "$host" "pct exec ${vmid} -- bash -lc 'systemctl list-units --type=service --no-legend 2>/dev/null | awk \"{print \\\$1}\" | grep -iE \"^besu-(validator|sentry|rpc|rpc-core)\\.service$|^besu\\.service$\" | head -1'" 2>/dev/null || true
}
is_running() {
local host="$1"
local vmid="$2"
host_ssh "$host" "pct status ${vmid} 2>/dev/null | awk '{print \$2}'" 2>/dev/null | grep -q '^running$'
}
prepare_host_tarball() {
local host="$1"
local local_path="$2"
local host_tmp="/tmp/${BESU_TAR}"
if $DRY_RUN; then
log_info " [dry-run] would copy ${BESU_TAR} to ${host}:${host_tmp}"
return 0
fi
scp "${SSH_OPTS[@]}" "$local_path" "root@${host}:${host_tmp}" >/dev/null
}
upgrade_node() {
local vmid=$1
local service="${NODE_SERVICES[$vmid]:-besu-rpc}"
local tarball_path="$2"
local host="$1"
local vmid="$2"
local service="$3"
if ! is_running "$vmid"; then
log_warn "VMID $vmid not running — skip"
return 0
if ! is_running "$host" "$vmid"; then
log_warn "VMID ${vmid} @ ${host}: not running, skipping"
return 0
fi
if [[ -z "$service" ]]; then
log_warn "VMID ${vmid} @ ${host}: no Besu service detected, skipping"
return 0
fi
log_info "VMID ${vmid} @ ${host}: upgrading ${service} to Besu ${BESU_VERSION}"
if $DRY_RUN; then
log_info " [dry-run] would install Java 21, extract ${BESU_TAR}, switch /opt/besu, restart ${service}"
return 0
fi
host_ssh "$host" "pct push ${vmid} /tmp/${BESU_TAR} /tmp/${BESU_TAR}" >/dev/null
host_ssh "$host" "pct exec ${vmid} -- bash -lc '
set -euo pipefail
if [[ ! -e /opt/besu ]]; then
fallback=\$(find /opt -maxdepth 1 -type d -name \"besu-*\" | sort -V | tail -1)
if [[ -n \"\${fallback:-}\" ]]; then
ln -sfn \"\$fallback\" /opt/besu
chown -h besu:besu /opt/besu 2>/dev/null || true
fi
elif [[ ! -L /opt/besu ]]; then
current_semver=\$(/opt/besu/bin/besu --version 2>/dev/null | grep -Eo \"[0-9]+\\.[0-9]+\\.[0-9]+\" | head -1)
current_version=\"besu-\${current_semver:-}\"
[[ -z \"\${current_version:-}\" ]] && current_version=besu-backup-pre-${BESU_VERSION}
if [[ ! -d \"/opt/\${current_version}\" ]]; then
mv /opt/besu \"/opt/\${current_version}\"
else
rm -rf /opt/besu
fi
ln -sfn \"/opt/\${current_version}\" /opt/besu
chown -h besu:besu /opt/besu 2>/dev/null || true
fi
log_info "VMID $vmid: upgrade to Besu $BESU_VERSION ($service) ..."
if $DRY_RUN; then
log_info " [dry-run] would push $BESU_TAR and extract, switch /opt/besu, restart $service"
return 0
java_major=\$(java -version 2>&1 | sed -n \"1s/.*version \\\"\\([0-9][0-9]*\\).*/\\1/p\")
if [[ -z \"\${java_major:-}\" || \"\$java_major\" -lt 21 ]]; then
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq
apt-get install -y -qq openjdk-21-jre-headless || true
java_major=\$(java -version 2>&1 | sed -n \"1s/.*version \\\"\\([0-9][0-9]*\\).*/\\1/p\")
if [[ -z \"\${java_major:-}\" || \"\$java_major\" -lt 21 ]]; then
command -v curl >/dev/null 2>&1 || apt-get install -y -qq curl ca-certificates
tmp_jre=/tmp/java21-jre.tar.gz
curl -fsSL -o \"\$tmp_jre\" '${JAVA21_FALLBACK_URL}'
tar -tzf \"\$tmp_jre\" > /tmp/java21-jre.list
extracted_dir=\$(head -1 /tmp/java21-jre.list | cut -d/ -f1)
rm -f /tmp/java21-jre.list
tar -xzf \"\$tmp_jre\" -C /opt
rm -f \"\$tmp_jre\"
ln -sfn \"/opt/\${extracted_dir}\" /opt/java-21
update-alternatives --install /usr/bin/java java /opt/java-21/bin/java 2100
fi
fi
# Copy tarball into container
if ! ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$PROXMOX_HOST" \
"pct push $vmid $tarball_path /tmp/$BESU_TAR" 2>/dev/null; then
log_err " Failed to push tarball to $vmid"
return 1
config_file=\$(systemctl cat ${service} | sed -n \"s/.*--config-file=\\\\([^ ]*\\\\).*/\\\\1/p\" | tail -1)
if [[ -n \"\${config_file:-}\" && -f \"\$config_file\" ]]; then
find /etc/besu -maxdepth 1 -type f -name \"*.toml\" -print0 2>/dev/null | while IFS= read -r -d \"\" toml; do
sed -i \
-e \"/^[[:space:]]*miner-enabled[[:space:]]*=.*/d\" \
-e \"/^[[:space:]]*privacy-enabled[[:space:]]*=.*/d\" \
\"\$toml\"
if grep -q \"^rpc-http-enabled=true\" \"\$toml\" && ! grep -q \"^rpc-http-max-active-connections=\" \"\$toml\"; then
tmp=\$(mktemp)
awk \"1; /^rpc-http-port=/{print \\\"rpc-http-max-active-connections=${RPC_HTTP_MAX_ACTIVE_CONNECTIONS}\\\"}\" \"\$toml\" > \"\$tmp\"
cat \"\$tmp\" > \"\$toml\"
rm -f \"\$tmp\"
fi
if grep -q \"^rpc-ws-enabled=true\" \"\$toml\" && ! grep -q \"^rpc-ws-max-active-connections=\" \"\$toml\"; then
tmp=\$(mktemp)
awk \"1; /^rpc-ws-port=/{print \\\"rpc-ws-max-active-connections=${RPC_WS_MAX_ACTIVE_CONNECTIONS}\\\"}\" \"\$toml\" > \"\$tmp\"
cat \"\$tmp\" > \"\$toml\"
rm -f \"\$tmp\"
fi
done
if ! grep -q \"^data-storage-format=\" \"\$config_file\"; then
tmp=\$(mktemp)
awk \"1; /^sync-mode=/{print \\\"data-storage-format=\\\\\\\"FOREST\\\\\\\"\\\"}\" \"\$config_file\" > \"\$tmp\"
cat \"\$tmp\" > \"\$config_file\"
rm -f \"\$tmp\"
fi
fi
# Extract, switch symlink, fix ownership, restart (each step via pct exec to avoid quoting issues)
ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=accept-new root@"$PROXMOX_HOST" \
"pct exec $vmid -- bash -c 'cd /opt && tar -xzf /tmp/$BESU_TAR && rm -f /tmp/$BESU_TAR'" || {
log_err " VMID $vmid: extract failed"
return 1
}
ssh -o ConnectTimeout=10 root@"$PROXMOX_HOST" \
"pct exec $vmid -- bash -c 'cd /opt && rm -f besu && ln -sf $BESU_DIR besu && chown -R besu:besu $BESU_DIR besu 2>/dev/null || true'" || true
ssh -o ConnectTimeout=15 root@"$PROXMOX_HOST" \
"pct exec $vmid -- systemctl restart ${service}.service" || {
log_err " VMID $vmid: restart failed"
return 1
}
sleep 3
local active
active=$(ssh -o ConnectTimeout=5 root@"$PROXMOX_HOST" "pct exec $vmid -- systemctl is-active ${service}.service 2>/dev/null" || echo "unknown")
if [ "$active" = "active" ]; then
log_ok " VMID $vmid upgraded and $service active"
return 0
cd /opt
if [[ ! -d ${BESU_DIR} ]]; then
tar -xzf /tmp/${BESU_TAR} -C /opt
fi
log_err " VMID $vmid: service status after restart: $active"
return 1
rm -f /tmp/${BESU_TAR}
ln -sfn ${BESU_DIR} /opt/besu
chown -h besu:besu /opt/besu 2>/dev/null || true
chown -R besu:besu ${BESU_DIR} /opt/besu-* 2>/dev/null || true
systemctl restart ${service}
'" || return 1
local active version
active=""
for _ in $(seq 1 24); do
active="$(host_ssh "$host" "pct exec ${vmid} -- systemctl is-active ${service}" 2>/dev/null || true)"
[[ "$active" == "active" ]] && break
sleep 5
done
version="$(host_ssh "$host" "pct exec ${vmid} -- bash -lc '/opt/besu/bin/besu --version 2>/dev/null | grep -m1 \"besu/\" || true'" 2>/dev/null || true)"
if [[ "$active" == "active" ]]; then
log_ok " VMID ${vmid}: ${service} active (${version:-version unavailable})"
return 0
fi
log_err " VMID ${vmid}: ${service} state=${active:-unknown}"
host_ssh "$host" "pct exec ${vmid} -- journalctl -u ${service} -n 30 --no-pager" 2>/dev/null || true
return 1
}
# --- main ---
log_info "Upgrade Besu on all nodes to $BESU_VERSION (host: $PROXMOX_HOST)"
[ "$DRY_RUN" = true ] && log_warn "DRY RUN — no changes will be made"
echo ""
log_info "Upgrade Besu fleet to ${BESU_VERSION}"
$DRY_RUN && log_warn "DRY RUN: no changes will be made"
echo
# Check SSH
if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new root@"$PROXMOX_HOST" "echo OK" &>/dev/null; then
log_err "Cannot SSH to $PROXMOX_HOST"
exit 1
fi
TARBALL_PATH="$(ensure_tarball)"
tarball_path=""
if ! $DRY_RUN; then
tarball_path=$(ensure_tarball) || exit 1
[ -z "$tarball_path" ] && exit 1
fi
declare -A VMIDS_ON_HOST
for vmid in "${BESU_VMIDS[@]}"; do
host="${HOST_BY_VMID[$vmid]:-}"
[[ -n "$host" ]] || continue
VMIDS_ON_HOST[$host]+=" ${vmid}"
done
PASS=0
FAIL=0
VMIDS_SORTED=$(echo "${!NODE_SERVICES[@]}" | tr ' ' '\n' | sort -n)
for vmid in $VMIDS_SORTED; do
if upgrade_node "$vmid" "$tarball_path"; then
((PASS++)) || true
else
((FAIL++)) || true
SKIP=0
for host in "${!VMIDS_ON_HOST[@]}"; do
log_info "Host ${host}"
if ! host_ssh "$host" "echo OK" >/dev/null 2>&1; then
log_err " Cannot SSH to ${host}"
((FAIL++)) || true
continue
fi
prepare_host_tarball "$host" "$TARBALL_PATH"
for vmid in ${VMIDS_ON_HOST[$host]}; do
service="$(detect_service "$host" "$vmid")"
if ! is_running "$host" "$vmid"; then
log_warn "VMID ${vmid} @ ${host}: not running, skipping"
((SKIP++)) || true
continue
fi
echo ""
if [[ -z "$service" ]]; then
log_warn "VMID ${vmid} @ ${host}: no Besu unit found, skipping"
((SKIP++)) || true
continue
fi
if upgrade_node "$host" "$vmid" "$service"; then
((PASS++)) || true
else
((FAIL++)) || true
fi
echo
done
if ! $DRY_RUN; then
host_ssh "$host" "rm -f /tmp/${BESU_TAR}" >/dev/null 2>&1 || true
fi
done
echo "────────────────────────────────────────────────────────────"
log_info "Upgrade summary: $PASS succeeded, $FAIL failed"
echo "────────────────────────────────────────────────────────────"
echo "------------------------------------------------------------"
log_info "Upgrade summary: passed=${PASS} skipped=${SKIP} failed=${FAIL}"
echo "------------------------------------------------------------"
if [ "$FAIL" -gt 0 ]; then
exit 1
if [[ "$FAIL" -gt 0 ]]; then
exit 1
fi
exit 0

18
scripts/verify/check-chain138-rpc-health.sh
View File

@@ -122,28 +122,12 @@ check_supported_method() {
return 1
}
check_expected_missing_method() {
local method="$1"
local params="${2:-[]}"
local response code message
response="$(rpc_request "$method" "$params" || printf '%s' '{"error":"curl"}')"
code="$(printf '%s' "$response" | jq -r '.error.code // empty' 2>/dev/null || true)"
message="$(printf '%s' "$response" | jq -r '.error.message // empty' 2>/dev/null || true)"
if [[ "$code" == "-32601" || "$message" == "Method not found" ]]; then
printf ' %-32s %s\n' "$method" "EXPECTED_MISSING"
return 0
fi
printf ' %-32s %s\n' "$method" "UNEXPECTED"
((fail++)) || true
return 1
}
check_supported_method "eth_chainId"
check_supported_method "eth_gasPrice"
check_supported_method "eth_maxPriorityFeePerGas"
check_supported_method "eth_feeHistory" "[\"0x1\", \"latest\", []]"
check_supported_method "trace_block" "[\"0x1\"]"
check_supported_method "trace_replayBlockTransactions" "[\"0x1\", [\"trace\"]]"
check_expected_missing_method "eth_maxPriorityFeePerGas"
if [[ "$fail" -eq 0 ]]; then
echo "OK: node health and public RPC capability checks passed"

11
scripts/verify/print-caliper-chain138-stub.sh Executable file
View File

@@ -0,0 +1,11 @@
#!/usr/bin/env bash
# Print Caliper integration hints for Chain 138 (no network I/O).
# See docs/03-deployment/CALIPER_CHAIN138_PERF_HOOK.md
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
echo "Caliper is not bundled in this repo."
echo "Read: $ROOT/docs/03-deployment/CALIPER_CHAIN138_PERF_HOOK.md"
echo ""
echo "Suggested SUT URL for benchmarks (lab): \${RPC_URL_138:-http://192.168.11.211:8545}"
echo "Chain ID: 138 (verify with eth_chainId)."

72
scripts/verify/run-dbis-phase3-e2e-simulation.sh Executable file
View File

@@ -0,0 +1,72 @@
#!/usr/bin/env bash
# DBIS Phase 3 — liveness / availability wrapper: Besu RPC liveness + optional FireFly HTTP + optional full RPC health.
# This does NOT execute Indy issuance, Aries verification, Fabric chaincode, or cross-chain business workflow steps.
#
# Usage: bash scripts/verify/run-dbis-phase3-e2e-simulation.sh
# Env: RPC_URL_138 (default http://192.168.11.211:8545)
# FIREFLY_URL (default http://192.168.11.35:5000)
# RUN_CHAIN138_RPC_HEALTH=1 to run check-chain138-rpc-health.sh (slower)
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
# shellcheck source=/dev/null
source "$PROJECT_ROOT/config/ip-addresses.conf" 2>/dev/null || true
RPC_URL="${RPC_URL_138:-http://${IP_BESU_RPC_CORE_1:-192.168.11.211}:8545}"
FIREFLY_URL="${FIREFLY_URL:-http://192.168.11.35:5000}"
fail=0
echo "=== DBIS Phase 3 liveness wrapper (partial) ==="
echo "RPC: $RPC_URL"
echo ""
if command -v curl &>/dev/null; then
echo "--- Besu eth_chainId / eth_blockNumber ---"
if ! out=$(curl -sS --connect-timeout 5 -X POST -H 'Content-Type: application/json' \
-d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' "$RPC_URL"); then
echo "[FAIL] curl chainId"
fail=1
else
echo "$out"
fi
if ! out=$(curl -sS --connect-timeout 5 -X POST -H 'Content-Type: application/json' \
-d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' "$RPC_URL"); then
echo "[FAIL] curl blockNumber"
fail=1
else
echo "$out"
fi
else
echo "[SKIP] curl not installed"
fail=1
fi
echo ""
echo "--- FireFly HTTP (optional) ---"
if command -v curl &>/dev/null; then
code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 4 "$FIREFLY_URL/api/v1/status" || true)
if [[ "$code" =~ ^(200|401|403)$ ]]; then
echo "[OK] $FIREFLY_URL/api/v1/status HTTP $code"
else
echo "[WARN] $FIREFLY_URL/api/v1/status HTTP ${code:-000} (FireFly may be down or path differs)"
fi
else
echo "[SKIP] curl not installed"
fi
if [[ "${RUN_CHAIN138_RPC_HEALTH:-}" == "1" ]]; then
echo ""
echo "--- check-chain138-rpc-health.sh ---"
bash "$PROJECT_ROOT/scripts/verify/check-chain138-rpc-health.sh" || fail=1
fi
echo ""
echo "--- Manual follow-ups (Section 18) ---"
echo "This script proves only liveness / availability for the automated checks above."
echo "Indy 6400 / Fabric 6000 / CCIP relay on r630-01: see docs/03-deployment/DBIS_PHASE3_E2E_PRODUCTION_SIMULATION_RUNBOOK.md"
echo "Caliper: docs/03-deployment/CALIPER_CHAIN138_PERF_HOOK.md"
echo ""
exit "$fail"

218
scripts/verify/run-phase1-discovery.sh Executable file
View File

@@ -0,0 +1,218 @@
#!/usr/bin/env bash
# Phase 1 — Reality mapping (read-only): compose Proxmox/Besu audits and optional
# Hyperledger CT probes into a timestamped report under reports/phase1-discovery/.
#
# Usage (repo root, LAN + SSH to Proxmox recommended):
# bash scripts/verify/run-phase1-discovery.sh
# HYPERLEDGER_PROBE=1 bash scripts/verify/run-phase1-discovery.sh # SSH pct exec smoke checks on r630-02
#
# Env: PROXMOX_HOSTS, SSH_USER, SSH_OPTS (same as audit-proxmox-operational-template.sh)
# HYPERLEDGER_PROBE=1 to run optional Fabric/Indy/FireFly container checks (requires SSH to r630-02)
set -uo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
# shellcheck source=/dev/null
source "$PROJECT_ROOT/config/ip-addresses.conf" 2>/dev/null || true
REPORT_DIR="${REPORT_DIR:-$PROJECT_ROOT/reports/phase1-discovery}"
STAMP="$(date -u +%Y%m%d_%H%M%S)"
MD="$REPORT_DIR/phase1-discovery-${STAMP}.md"
LOG="$REPORT_DIR/phase1-discovery-${STAMP}.log"
mkdir -p "$REPORT_DIR"
SSH_USER="${SSH_USER:-root}"
SSH_OPTS="${SSH_OPTS:--o BatchMode=yes -o ConnectTimeout=6 -o StrictHostKeyChecking=accept-new}"
R630_02="${PROXMOX_HOST_R630_02:-192.168.11.12}"
append_cmd() {
local title="$1"
local severity="${2:-info}"
shift 2 || true
local rc=0
local tmp
tmp="$(mktemp)"
"$@" >"$tmp" 2>&1
rc=$?
{
echo ""
echo "## $title"
echo ""
echo '```text'
cat "$tmp"
if (( rc != 0 )); then
echo "[exit $rc]"
fi
echo '```'
} | tee -a "$MD" >>"$LOG"
rm -f "$tmp"
if (( rc != 0 )) && [[ "$severity" == "critical" ]]; then
PHASE1_CRITICAL_FAILURES+=("$title (exit $rc)")
fi
}
PHASE1_CRITICAL_FAILURES=()
{
echo "# Phase 1 discovery report"
echo ""
echo "**Generated (UTC):** $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo ""
echo "**Runbook:** [docs/03-deployment/PHASE1_DISCOVERY_RUNBOOK.md](../../docs/03-deployment/PHASE1_DISCOVERY_RUNBOOK.md)"
echo ""
echo "**Doctrine:** [dbis_chain_138_technical_master_plan.md](../../dbis_chain_138_technical_master_plan.md) (Sections 3, 19–20)"
echo ""
echo "## Dependency graph (logical)"
echo ""
echo "Same diagram as the runbook; edges reflect documented traffic flow, not live packet capture."
echo ""
cat <<'MERMAID'
```mermaid
flowchart TB
subgraph edge [EdgeIngress]
CF[Cloudflare_DNS]
NPM[NPMplus_LXC]
end
subgraph besu [Chain138_Besu]
RPCpub[RPC_public_2201]
RPCcore[RPC_core_2101]
Val[Validators_1000_1004]
Sen[Sentries_1500_1508]
end
subgraph observe [Observability]
BS[Blockscout_5000]
end
subgraph relay [CrossChain]
CCIP[CCIP_relay_r63001_host]
end
subgraph dlt [Hyperledger_optional]
FF[FireFly_6200_6201]
Fab[Fabric_6000_plus]
Indy[Indy_6400_plus]
end
CF --> NPM
NPM --> RPCpub
NPM --> RPCcore
NPM --> BS
RPCpub --> Sen
RPCcore --> Sen
Sen --> Val
CCIP --> RPCpub
FF --> Fab
FF --> Indy
```
MERMAID
} >"$MD"
touch "$LOG"
append_cmd "Proxmox template vs live VMID audit" critical bash "$PROJECT_ROOT/scripts/verify/audit-proxmox-operational-template.sh"
PROXMOX_HOSTS="${PROXMOX_HOSTS:-${PROXMOX_HOST_ML110:-192.168.11.10} ${PROXMOX_HOST_R630_01:-192.168.11.11} $R630_02}"
append_cmd "Proxmox cluster status (pvecm) per host" critical bash -c "
fail=0
for h in $PROXMOX_HOSTS; do
echo '=== '"\$h"' ==='
ssh $SSH_OPTS ${SSH_USER}@\"\$h\" 'pvecm status 2>&1' || fail=1
echo ''
done
exit \$fail
"
append_cmd "Proxmox storage (pvesm status) per host" critical bash -c "
fail=0
for h in $PROXMOX_HOSTS; do
echo '=== '"\$h"' ==='
ssh $SSH_OPTS ${SSH_USER}@\"\$h\" 'pvesm status 2>&1 | head -80' || fail=1
echo ''
done
exit \$fail
"
append_cmd "Live pct/qm lists per host" critical bash -c "
fail=0
for h in $PROXMOX_HOSTS; do
echo '=== '"\$h"' ==='
ssh $SSH_OPTS ${SSH_USER}@\"\$h\" 'echo PCT:; pct list 2>&1; echo VM:; qm list 2>&1' || fail=1
echo ''
done
exit \$fail
"
if command -v curl &>/dev/null; then
append_cmd "Chain 138 RPC quick probe (core, LAN)" critical bash -c "
curl -sS --connect-timeout 4 -X POST -H 'Content-Type: application/json' \
--data '{\"jsonrpc\":\"2.0\",\"method\":\"eth_chainId\",\"params\":[],\"id\":1}' \
\"http://${IP_BESU_RPC_CORE_1:-192.168.11.211}:8545\" || echo 'curl failed'
"
fi
append_cmd "Besu RPC health script (may fail off-LAN)" critical bash -c "
bash \"$PROJECT_ROOT/scripts/verify/check-chain138-rpc-health.sh\"
"
append_cmd "Besu enodes / IPs verify (may fail off-LAN)" critical bash -c "
bash \"$PROJECT_ROOT/scripts/verify/verify-besu-enodes-and-ips.sh\"
"
if [[ "${HYPERLEDGER_PROBE:-}" == "1" ]]; then
append_cmd "Hyperledger CT smoke (r630-02; pct exec)" critical bash -c "
ssh $SSH_OPTS ${SSH_USER}@$R630_02 '
for id in 6200 6201 6000 6001 6002 6400 6401 6402; do
echo \"=== VMID \$id status ===\"
pct status \$id 2>&1 || true
if pct status \$id 2>/dev/null | grep -q running; then
pct exec \$id -- bash -lc \"command -v docker >/dev/null && docker ps --format 'table {{.Names}}\t{{.Status}}' 2>/dev/null | head -10 || true; command -v systemctl >/dev/null && systemctl list-units --type=service --state=running --no-pager 2>/dev/null | head -20 || true; ss -ltnp 2>/dev/null | head -20 || true\" 2>&1 || echo \"[exec failed]\"
fi
echo \"\"
done
'
"
else
{
echo ""
echo "## Hyperledger CT smoke (skipped)"
echo ""
echo "Set \`HYPERLEDGER_PROBE=1\` to SSH to r630-02 and run \`pct status/exec\` on 6200, 6201, 6000, 6001, 6002, 6400, 6401, 6402."
echo ""
} >>"$MD"
fi
{
echo ""
echo "## Configuration snapshot pointers (no secrets in repo)"
echo ""
echo "- \`config/proxmox-operational-template.json\`"
echo "- \`config/ip-addresses.conf\`"
echo "- \`docs/04-configuration/ALL_VMIDS_ENDPOINTS.md\`"
echo ""
echo "## Next steps"
echo ""
echo "1. Reconcile **Entity owner** / **Region** in [DBIS_NODE_ROLE_MATRIX.md](../../docs/02-architecture/DBIS_NODE_ROLE_MATRIX.md)."
echo "2. If ML110 row shows Proxmox + workloads, update [PHYSICAL_HARDWARE_INVENTORY.md](../../docs/02-architecture/PHYSICAL_HARDWARE_INVENTORY.md) vs [NETWORK_CONFIGURATION_MASTER.md](../../docs/11-references/NETWORK_CONFIGURATION_MASTER.md)."
echo ""
if ((${#PHASE1_CRITICAL_FAILURES[@]} > 0)); then
echo "## Critical failure summary"
echo ""
for failure in "${PHASE1_CRITICAL_FAILURES[@]}"; do
echo "- $failure"
done
echo ""
echo "This report is complete as evidence capture, but the discovery run is **not** a pass. Re-run from LAN with working SSH/RPC access until the critical failures clear."
else
echo "## Critical failure summary"
echo ""
echo "- none"
echo ""
echo "All critical discovery checks completed successfully for this run."
fi
echo ""
} >>"$MD"
echo "Wrote $MD"
echo "Full log mirror: $LOG"
ls -la "$MD" "$LOG"
if ((${#PHASE1_CRITICAL_FAILURES[@]} > 0)); then
exit 1
fi