proxmox/scripts/upgrade-besu-all-nodes.sh

#!/usr/bin/env bash
# Upgrade all Besu nodes to the latest (or specified) version.
# Requires: SSH to Proxmox host, curl/wget, enough disk in containers.
# Usage:
#   ./scripts/upgrade-besu-all-nodes.sh              # upgrade to latest (25.12.0)
#   ./scripts/upgrade-besu-all-nodes.sh --dry-run    # show what would be done
#   BESU_VERSION=25.11.0 ./scripts/upgrade-besu-all-nodes.sh
# Optional: pre-download to avoid long run (script uses $LOCAL_CACHE/besu-${BESU_VERSION}.tar.gz):
#   curl -sSL -o /tmp/besu-25.12.0.tar.gz https://github.com/hyperledger/besu/releases/download/25.12.0/besu-25.12.0.tar.gz

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

if [ -f "$PROJECT_ROOT/config/ip-addresses.conf" ]; then
    # shellcheck source=../config/ip-addresses.conf
    source "$PROJECT_ROOT/config/ip-addresses.conf"
fi

PROXMOX_HOST="${PROXMOX_HOST:-${PROXMOX_HOST_ML110:-192.168.11.10}}"
# Latest stable as of 2025-12; EIP-7702 requires >= 24.1.0
BESU_VERSION="${BESU_VERSION:-25.12.0}"
BESU_TAR="besu-${BESU_VERSION}.tar.gz"
BESU_DIR="besu-${BESU_VERSION}"
DOWNLOAD_URL="${BESU_DOWNLOAD_URL:-https://github.com/hyperledger/besu/releases/download/${BESU_VERSION}/${BESU_TAR}}"
LOCAL_CACHE="${LOCAL_CACHE:-/tmp}"

DRY_RUN=false
for arg in "$@"; do
    [ "$arg" = "--dry-run" ] && DRY_RUN=true
done

# Same node list and services as restart-all-besu-services.sh
declare -A NODE_SERVICES=(
    ["1000"]="besu-validator"
    ["1001"]="besu-validator"
    ["1002"]="besu-validator"
    ["1003"]="besu-validator"
    ["1004"]="besu-validator"
    ["1500"]="besu-sentry"
    ["1501"]="besu-sentry"
    ["1502"]="besu-sentry"
    ["1503"]="besu-sentry"
    ["2101"]="besu-rpc"
    ["2400"]="besu-rpc"
    ["2401"]="besu-rpc"
    ["2402"]="besu-rpc"
)

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info()    { echo -e "${BLUE}[INFO]${NC} $1"; }
log_ok()      { echo -e "${GREEN}[OK]${NC} $1"; }
log_warn()    { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_err()     { echo -e "${RED}[ERROR]${NC} $1"; }

is_running() {
    local vmid=$1
    ssh -o ConnectTimeout=3 -o StrictHostKeyChecking=accept-new root@"$PROXMOX_HOST" \
        "pct status $vmid 2>/dev/null" | grep -q running
}

# Ensure tarball exists (download to host or use cache)
ensure_tarball() {
    local path="$LOCAL_CACHE/$BESU_TAR"
    if [ -f "$path" ]; then
        log_ok "Using existing $path"
        echo "$path"
        return
    fi
    log_info "Downloading $DOWNLOAD_URL ..."
    if $DRY_RUN; then
        echo ""
        return
    fi
    (cd "$LOCAL_CACHE" && curl -sSfL -o "$BESU_TAR" "$DOWNLOAD_URL") || {
        log_err "Download failed"
        return 1
    }
    log_ok "Downloaded $path"
    echo "$path"
}

upgrade_node() {
    local vmid=$1
    local service="${NODE_SERVICES[$vmid]:-besu-rpc}"
    local tarball_path="$2"

    if ! is_running "$vmid"; then
        log_warn "VMID $vmid not running — skip"
        return 0
    fi

    log_info "VMID $vmid: upgrade to Besu $BESU_VERSION ($service) ..."

    if $DRY_RUN; then
        log_info "  [dry-run] would push $BESU_TAR and extract, switch /opt/besu, restart $service"
        return 0
    fi

    # Copy tarball into container
    if ! ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new root@"$PROXMOX_HOST" \
        "pct push $vmid $tarball_path /tmp/$BESU_TAR" 2>/dev/null; then
        log_err "  Failed to push tarball to $vmid"
        return 1
    fi

    # Extract, switch symlink, fix ownership, restart (each step via pct exec to avoid quoting issues)
    ssh -o ConnectTimeout=60 -o StrictHostKeyChecking=accept-new root@"$PROXMOX_HOST" \
        "pct exec $vmid -- bash -c 'cd /opt && tar -xzf /tmp/$BESU_TAR && rm -f /tmp/$BESU_TAR'" || {
        log_err "  VMID $vmid: extract failed"
        return 1
    }
    ssh -o ConnectTimeout=10 root@"$PROXMOX_HOST" \
        "pct exec $vmid -- bash -c 'cd /opt && rm -f besu && ln -sf $BESU_DIR besu && chown -R besu:besu $BESU_DIR besu 2>/dev/null || true'" || true
    ssh -o ConnectTimeout=15 root@"$PROXMOX_HOST" \
        "pct exec $vmid -- systemctl restart ${service}.service" || {
        log_err "  VMID $vmid: restart failed"
        return 1
    }
    sleep 3
    local active
    active=$(ssh -o ConnectTimeout=5 root@"$PROXMOX_HOST" "pct exec $vmid -- systemctl is-active ${service}.service 2>/dev/null" || echo "unknown")
    if [ "$active" = "active" ]; then
        log_ok "  VMID $vmid upgraded and $service active"
        return 0
    fi
    log_err "  VMID $vmid: service status after restart: $active"
    return 1
}

# --- main ---
log_info "Upgrade Besu on all nodes to $BESU_VERSION (host: $PROXMOX_HOST)"
[ "$DRY_RUN" = true ] && log_warn "DRY RUN — no changes will be made"
echo ""

# Check SSH
if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new root@"$PROXMOX_HOST" "echo OK" &>/dev/null; then
    log_err "Cannot SSH to $PROXMOX_HOST"
    exit 1
fi

tarball_path=""
if ! $DRY_RUN; then
    tarball_path=$(ensure_tarball) || exit 1
    [ -z "$tarball_path" ] && exit 1
fi

PASS=0
FAIL=0
VMIDS_SORTED=$(echo "${!NODE_SERVICES[@]}" | tr ' ' '\n' | sort -n)
for vmid in $VMIDS_SORTED; do
    if upgrade_node "$vmid" "$tarball_path"; then
        ((PASS++)) || true
    else
        ((FAIL++)) || true
    fi
    echo ""
done

echo "────────────────────────────────────────────────────────────"
log_info "Upgrade summary: $PASS succeeded, $FAIL failed"
echo "────────────────────────────────────────────────────────────"

if [ "$FAIL" -gt 0 ]; then
    exit 1
fi
exit 0