#!/usr/bin/env bash # Collect real-time storage data from Proxmox hosts and VMs for growth tracking. # Usage: ./scripts/monitoring/collect-storage-growth-data.sh [--json|--csv|--append] # See: docs/04-configuration/STORAGE_GROWTH_AND_HEALTH.md set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true OUTPUT_MODE="markdown" APPEND_HISTORY=false SSH_OPTS="-o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new" LOG_DIR="${PROJECT_ROOT}/logs/storage-growth" HISTORY_CSV="${LOG_DIR}/history.csv" TIMESTAMP=$(date -Iseconds) for a in "$@"; do case "$a" in --json) OUTPUT_MODE="json" ;; --csv) OUTPUT_MODE="csv" ;; --append) APPEND_HISTORY=true ;; esac done ML110="${PROXMOX_HOST_ML110:-192.168.11.10}" R630_01="${PROXMOX_HOST_R630_01:-192.168.11.11}" R630_02="${PROXMOX_HOST_R630_02:-192.168.11.12}" run_ssh() { ssh $SSH_OPTS "root@$1" "$2" 2>/dev/null || true; } mkdir -p "$LOG_DIR" OUT="${LOG_DIR}/snapshot_$(date +%Y%m%d_%H%M%S).txt" # ---- Host: pvesm, lvs (thin), vgs, df ---- host_data() { local ip="$1" name="$2" echo "=== HOST $name ($ip) ===" run_ssh "$ip" "pvesm status 2>/dev/null" || true echo "" echo "--- LVM thin pools (data% / metadata%) ---" run_ssh "$ip" "lvs --units g -o lv_name,vg_name,lv_size,data_percent,metadata_percent,pool_lv 2>/dev/null | head -20" || true echo "" echo "--- Volume groups (free space) ---" run_ssh "$ip" "vgs --units g -o vg_name,vg_size,vg_free 2>/dev/null" || true echo "" echo "--- Host root df ---" run_ssh "$ip" "df -h / 2>/dev/null" || true echo "" } # ---- Per-VM: df /, /data, /var/log; du /data/besu, /var/log ---- vm_data() { local ip="$1" name="$2" local vmids vmids=$(run_ssh "$ip" "pct list 2>/dev/null | awk 'NR>1 && \$2==\"running\" {print \$1}'" || true) [[ -z "$vmids" ]] && return echo "=== VM/CT on $name (running) ===" for vmid in $vmids; do echo "--- VMID $vmid ---" run_ssh "$ip" "pct exec $vmid -- df -h / /data /var/log 2>/dev/null | grep -E 'Filesystem|^/'" || true run_ssh "$ip" "pct exec $vmid -- sh -c 'du -sh /data/besu 2>/dev/null; du -sh /var/log 2>/dev/null' 2>/dev/null" || true done echo "" } # Collect all { host_data "$ML110" "ml110" host_data "$R630_01" "r630-01" host_data "$R630_02" "r630-02" vm_data "$ML110" "ml110" vm_data "$R630_01" "r630-01" vm_data "$R630_02" "r630-02" } | tee "$OUT" # Compact growth table (from snapshot just written) echo "" echo "=== Growth table (host storage) ===" echo "| Host | Storage | Type | Used% | Used | Avail |" echo "|------|---------|------|-------|------|-------|" for entry in "${ML110}:ml110" "${R630_01}:r630-01" "${R630_02}:r630-02"; do ip="${entry%%:*}" name="${entry##*:}" run_ssh "$ip" "pvesm status 2>/dev/null" | tail -n +2 | while IFS= read -r st_name st_type st_status st_total st_used st_avail st_pct; do [[ -z "$st_name" ]] && continue echo "| $name | $st_name | $st_type | $st_pct | $st_used | $st_avail |" done done echo "" echo "| Host | LV (thin pool) | Data% | Meta% |" echo "|------|-----------------|-------|-------|" for entry in "${ML110}:ml110" "${R630_01}:r630-01" "${R630_02}:r630-02"; do ip="${entry%%:*}" name="${entry##*:}" run_ssh "$ip" "lvs --noheadings -o lv_name,data_percent,metadata_percent,pool_lv 2>/dev/null" | awk '$4!="" {print $1,$2,$3}' | while IFS= read -r lv data meta; do echo "| $name | $lv | $data | $meta |" done done echo "" # Quote CSV field if it contains comma or newline csv_quote() { local s="$1" if [[ "$s" =~ [,\"$'\n'] ]]; then echo "\"${s//\"/\"\"}\"" else echo "$s" fi } # One-line summary for CSV append (detail field quoted when needed) summary_csv() { local ip name for entry in "${ML110}:ml110" "${R630_01}:r630-01" "${R630_02}:r630-02"; do ip="${entry%%:*}" name="${entry##*:}" local pvesm lvs pvesm=$(run_ssh "$ip" "pvesm status 2>/dev/null" || true) lvs=$(run_ssh "$ip" "lvs --noheadings -o lv_name,data_percent,metadata_percent 2>/dev/null" || true) echo "$pvesm" | tail -n +2 | while IFS= read -r line; do [[ -z "$line" ]] && continue echo "${TIMESTAMP},host,${name},${ip},pvesm,$(csv_quote "$line")" done echo "$lvs" | while IFS= read -r lv data meta; do [[ -z "$lv" ]] && continue echo "${TIMESTAMP},host,${name},${ip},thin,${lv},${data},${meta}" done done } if [[ "$OUTPUT_MODE" == "json" ]]; then echo "{ \"timestamp\": \"$TIMESTAMP\", \"csv_rows\": [" first=true while IFS= read -r line; do [[ -z "$line" ]] && continue $first || echo "," first=false escaped="${line//\\/\\\\}"; escaped="${escaped//\"/\\\"}" echo -n " \"$escaped\"" done < <(summary_csv) echo "" echo " ] }" fi if [[ "$OUTPUT_MODE" == "csv" ]]; then echo "timestamp,scope,host,ip,metric,detail" summary_csv fi if $APPEND_HISTORY; then [[ ! -f "$HISTORY_CSV" ]] && echo "timestamp,scope,host,ip,metric,detail" >> "$HISTORY_CSV" summary_csv >> "$HISTORY_CSV" echo "Appended to $HISTORY_CSV" >&2 fi echo "Snapshot: $OUT" >&2