proxmox/scripts/monitoring/collect-storage-growth-data.sh

#!/usr/bin/env bash
# Collect real-time storage data from Proxmox hosts and VMs for growth tracking.
# Usage: ./scripts/monitoring/collect-storage-growth-data.sh [--json|--csv|--append]
# See: docs/04-configuration/STORAGE_GROWTH_AND_HEALTH.md

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true

OUTPUT_MODE="markdown"
APPEND_HISTORY=false
SSH_OPTS="-o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new"
LOG_DIR="${PROJECT_ROOT}/logs/storage-growth"
HISTORY_CSV="${LOG_DIR}/history.csv"
TIMESTAMP=$(date -Iseconds)

for a in "$@"; do
  case "$a" in
    --json)   OUTPUT_MODE="json" ;;
    --csv)    OUTPUT_MODE="csv" ;;
    --append) APPEND_HISTORY=true ;;
  esac
done

ML110="${PROXMOX_HOST_ML110:-192.168.11.10}"
R630_01="${PROXMOX_HOST_R630_01:-192.168.11.11}"
R630_02="${PROXMOX_HOST_R630_02:-192.168.11.12}"
run_ssh() { ssh $SSH_OPTS "root@$1" "$2" 2>/dev/null || true; }

mkdir -p "$LOG_DIR"
OUT="${LOG_DIR}/snapshot_$(date +%Y%m%d_%H%M%S).txt"

# ---- Host: pvesm, lvs (thin), vgs, df ----
host_data() {
  local ip="$1" name="$2"
  echo "=== HOST $name ($ip) ==="
  run_ssh "$ip" "pvesm status 2>/dev/null" || true
  echo ""
  echo "--- LVM thin pools (data% / metadata%) ---"
  run_ssh "$ip" "lvs --units g -o lv_name,vg_name,lv_size,data_percent,metadata_percent,pool_lv 2>/dev/null | head -20" || true
  echo ""
  echo "--- Volume groups (free space) ---"
  run_ssh "$ip" "vgs --units g -o vg_name,vg_size,vg_free 2>/dev/null" || true
  echo ""
  echo "--- Host root df ---"
  run_ssh "$ip" "df -h / 2>/dev/null" || true
  echo ""
}

# ---- Per-VM: df /, /data, /var/log; du /data/besu, /var/log ----
vm_data() {
  local ip="$1" name="$2"
  local vmids
  vmids=$(run_ssh "$ip" "pct list 2>/dev/null | awk 'NR>1 && \$2==\"running\" {print \$1}'" || true)
  [[ -z "$vmids" ]] && return
  echo "=== VM/CT on $name (running) ==="
  for vmid in $vmids; do
    echo "--- VMID $vmid ---"
    run_ssh "$ip" "pct exec $vmid -- df -h / /data /var/log 2>/dev/null | grep -E 'Filesystem|^/'" || true
    run_ssh "$ip" "pct exec $vmid -- sh -c 'du -sh /data/besu 2>/dev/null; du -sh /var/log 2>/dev/null' 2>/dev/null" || true
  done
  echo ""
}

# Collect all
{
  host_data "$ML110" "ml110"
  host_data "$R630_01" "r630-01"
  host_data "$R630_02" "r630-02"
  vm_data "$ML110" "ml110"
  vm_data "$R630_01" "r630-01"
  vm_data "$R630_02" "r630-02"
} | tee "$OUT"

# Compact growth table (from snapshot just written)
echo ""
echo "=== Growth table (host storage) ==="
echo "| Host | Storage | Type | Used% | Used | Avail |"
echo "|------|---------|------|-------|------|-------|"
for entry in "${ML110}:ml110" "${R630_01}:r630-01" "${R630_02}:r630-02"; do
  ip="${entry%%:*}"
  name="${entry##*:}"
  run_ssh "$ip" "pvesm status 2>/dev/null" | tail -n +2 | while IFS= read -r st_name st_type st_status st_total st_used st_avail st_pct; do
    [[ -z "$st_name" ]] && continue
    echo "| $name | $st_name | $st_type | $st_pct | $st_used | $st_avail |"
  done
done
echo ""
echo "| Host | LV (thin pool) | Data% | Meta% |"
echo "|------|-----------------|-------|-------|"
for entry in "${ML110}:ml110" "${R630_01}:r630-01" "${R630_02}:r630-02"; do
  ip="${entry%%:*}"
  name="${entry##*:}"
  run_ssh "$ip" "lvs --noheadings -o lv_name,data_percent,metadata_percent,pool_lv 2>/dev/null" | awk '$4!="" {print $1,$2,$3}' | while IFS= read -r lv data meta; do
    echo "| $name | $lv | $data | $meta |"
  done
done
echo ""

# Quote CSV field if it contains comma or newline
csv_quote() {
  local s="$1"
  if [[ "$s" =~ [,\"$'\n'] ]]; then
    echo "\"${s//\"/\"\"}\""
  else
    echo "$s"
  fi
}

# One-line summary for CSV append (detail field quoted when needed)
summary_csv() {
  local ip name
  for entry in "${ML110}:ml110" "${R630_01}:r630-01" "${R630_02}:r630-02"; do
    ip="${entry%%:*}"
    name="${entry##*:}"
    local pvesm lvs
    pvesm=$(run_ssh "$ip" "pvesm status 2>/dev/null" || true)
    lvs=$(run_ssh "$ip" "lvs --noheadings -o lv_name,data_percent,metadata_percent 2>/dev/null" || true)
    echo "$pvesm" | tail -n +2 | while IFS= read -r line; do
      [[ -z "$line" ]] && continue
      echo "${TIMESTAMP},host,${name},${ip},pvesm,$(csv_quote "$line")"
    done
    echo "$lvs" | while IFS= read -r lv data meta; do
      [[ -z "$lv" ]] && continue
      echo "${TIMESTAMP},host,${name},${ip},thin,${lv},${data},${meta}"
    done
  done
}

if [[ "$OUTPUT_MODE" == "json" ]]; then
  echo "{ \"timestamp\": \"$TIMESTAMP\", \"csv_rows\": ["
  first=true
  while IFS= read -r line; do
    [[ -z "$line" ]] && continue
    $first || echo ","
    first=false
    escaped="${line//\\/\\\\}"; escaped="${escaped//\"/\\\"}"
    echo -n "  \"$escaped\""
  done < <(summary_csv)
  echo ""
  echo " ] }"
fi

if [[ "$OUTPUT_MODE" == "csv" ]]; then
  echo "timestamp,scope,host,ip,metric,detail"
  summary_csv
fi

if $APPEND_HISTORY; then
  [[ ! -f "$HISTORY_CSV" ]] && echo "timestamp,scope,host,ip,metric,detail" >> "$HISTORY_CSV"
  summary_csv >> "$HISTORY_CSV"
  echo "Appended to $HISTORY_CSV" >&2
fi

echo "Snapshot: $OUT" >&2