#!/usr/bin/env bash set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true # Investigate what's using space on thin2 storage pool # Usage: ./investigate-thin2-storage.sh [proxmox-host] set -e PROXMOX_HOST="${1:-192.168.11.12}" STORAGE_POOL="thin2" echo "==========================================" echo "Storage Investigation - thin2 Pool" echo "==========================================" echo "Proxmox Host: $PROXMOX_HOST" echo "Storage Pool: $STORAGE_POOL" echo "==========================================" echo "" # Check if host is reachable if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@$PROXMOX_HOST "echo 'Connected'" 2>/dev/null >/dev/null; then echo "❌ Host $PROXMOX_HOST is not reachable" exit 1 fi HOSTNAME=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "hostname" 2>/dev/null || echo "unknown") echo "Hostname: $HOSTNAME" echo "" # Check storage pool status echo "=== Storage Pool Status ===" STORAGE_INFO=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pvesm status | grep $STORAGE_POOL" 2>/dev/null) echo "$STORAGE_INFO" echo "" # Get all VMs and containers using thin2 echo "=== Containers Using thin2 ===" CONTAINERS=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct list --output-format json" 2>/dev/null || echo "[]") if [ "$CONTAINERS" != "[]" ] && [ -n "$CONTAINERS" ]; then echo "$CONTAINERS" | python3 -c " import sys, json containers = json.load(sys.stdin) thin2_containers = [c for c in containers if '$STORAGE_POOL' in str(c.get('rootfs', ''))] if thin2_containers: print(f\"{'VMID':<6} {'Name':<35} {'Status':<10} {'Size':<15} {'Storage':<30}\") print('-' * 100) for vm in sorted(thin2_containers, key=lambda x: x.get('maxdisk', 0), reverse=True): vmid = str(vm.get('vmid', 'N/A')) name = (vm.get('name', 'N/A') or 'N/A')[:33] status = vm.get('status', 'N/A') maxdisk = vm.get('maxdisk', 0) rootfs = str(vm.get('rootfs', 'N/A'))[:28] if maxdisk: size_gb = maxdisk / (1024**3) size_str = f'{size_gb:.2f}G' else: size_str = 'N/A' print(f\"{vmid:<6} {name:<35} {status:<10} {size_str:<15} {rootfs:<30}\") else: print('No containers found using thin2') " 2>/dev/null || echo "Error parsing containers" fi echo "" # Get all VMs using thin2 echo "=== Virtual Machines Using thin2 ===" VMS=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "qm list --output-format json" 2>/dev/null || echo "[]") if [ "$VMS" != "[]" ] && [ -n "$VMS" ]; then echo "$VMS" | python3 -c " import sys, json vms = json.load(sys.stdin) thin2_vms = [v for v in vms if '$STORAGE_POOL' in str(v.get('disk', ''))] if thin2_vms: print(f\"{'VMID':<6} {'Name':<35} {'Status':<10} {'Size':<15} {'Storage':<30}\") print('-' * 100) for vm in sorted(thin2_vms, key=lambda x: x.get('maxdisk', 0), reverse=True): vmid = str(vm.get('vmid', 'N/A')) name = (vm.get('name', 'N/A') or 'N/A')[:33] status = vm.get('status', 'N/A') maxdisk = vm.get('maxdisk', 0) disk = str(vm.get('disk', 'N/A'))[:28] if maxdisk: size_gb = maxdisk / (1024**3) size_str = f'{size_gb:.2f}G' else: size_str = 'N/A' print(f\"{vmid:<6} {name:<35} {status:<10} {size_str:<15} {disk:<30}\") else: print('No VMs found using thin2') " 2>/dev/null || echo "Error parsing VMs" fi echo "" # Get detailed disk usage for each container on thin2 echo "=== Detailed Disk Usage (Containers on thin2) ===" THIN2_CONTAINERS=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct list --output-format json" 2>/dev/null | \ python3 -c "import sys, json; containers = json.load(sys.stdin); [print(c['vmid']) for c in containers if '$STORAGE_POOL' in str(c.get('rootfs', ''))]" 2>/dev/null || echo "") if [ -n "$THIN2_CONTAINERS" ]; then printf "%-6s | %-35s | %-12s | %-12s | %-12s | %-10s\n" "VMID" "Name" "Used" "Avail" "Total" "Usage %" echo "------------------------------------------------------------------------------------------------------------" TOTAL_USED=0 TOTAL_SIZE=0 for vmid in $THIN2_CONTAINERS; do NAME=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct config $vmid 2>/dev/null | grep '^hostname:' | awk '{print \$2}'" || echo "unknown") STATUS=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct status $vmid 2>/dev/null | awk '{print \$2}'" || echo "unknown") if [ "$STATUS" = "running" ]; then DISK_USAGE=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct exec $vmid -- df -h / 2>/dev/null | tail -1" || echo "") if [ -n "$DISK_USAGE" ]; then USED=$(echo "$DISK_USAGE" | awk '{print $3}') AVAIL=$(echo "$DISK_USAGE" | awk '{print $4}') TOTAL=$(echo "$DISK_USAGE" | awk '{print $2}') PERCENT=$(echo "$DISK_USAGE" | awk '{print $5}') printf "%-6s | %-35s | %-12s | %-12s | %-12s | %-10s\n" "$vmid" "${NAME:0:33}" "$USED" "$AVAIL" "$TOTAL" "$PERCENT" fi else # Get allocated size from Proxmox config SIZE=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct list | grep \"^$vmid\" | awk '{print \$4}'" || echo "N/A") printf "%-6s | %-35s | %-12s | %-12s | %-12s | %-10s\n" "$vmid" "${NAME:0:33}" "N/A (stopped)" "" "$SIZE" "" fi done else echo "No containers found using thin2" fi echo "" # Get actual storage usage from Proxmox echo "=== Storage Pool Disk Usage (from Proxmox) ===" ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pvesm list $STORAGE_POOL --output-format json" 2>/dev/null | \ python3 -c " import sys, json try: data = json.load(sys.stdin) if 'data' in data and len(data['data']) > 0: print(f\"{'Volume':<50} {'Format':<10} {'Size':<15} {'Used':<15} {'Usage %':<10}\") print('-' * 100) total_size = 0 total_used = 0 for item in sorted(data['data'], key=lambda x: x.get('used', 0), reverse=True): volid = item.get('volid', 'N/A')[:48] format_type = item.get('format', 'N/A')[:8] size = item.get('size', 0) used = item.get('used', 0) if size: size_gb = size / (1024**3) size_str = f'{size_gb:.2f}G' else: size_str = 'N/A' if used: used_gb = used / (1024**3) used_str = f'{used_gb:.2f}G' if size: pct = (used / size) * 100 else: pct = 0 else: used_str = 'N/A' pct = 0 total_size += size or 0 total_used += used or 0 print(f\"{volid:<50} {format_type:<10} {size_str:<15} {used_str:<15} {pct:>8.1f}%\") if total_size: total_size_gb = total_size / (1024**3) total_used_gb = total_used / (1024**3) total_pct = (total_used / total_size) * 100 print('-' * 100) print(f\"{'TOTAL':<50} {'':<10} {f'{total_size_gb:.2f}G':<15} {f'{total_used_gb:.2f}G':<15} {total_pct:>8.1f}%\") else: print('No volumes found in storage pool') except Exception as e: print(f'Error parsing storage data: {e}') " 2>/dev/null || ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pvesm list $STORAGE_POOL" 2>/dev/null echo "" echo "=== Largest Directories (if accessible) ===" echo "Checking host storage directories..." ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST " # Find storage directories STORAGE_DIR=\"/var/lib/vz/images\" if [ -d \"\$STORAGE_DIR\" ]; then echo \"Checking \$STORAGE_DIR...\" du -sh \$STORAGE_DIR/*/ 2>/dev/null | sort -hr | head -10 fi # Also check LVM thin pool echo \"\" echo \"Checking LVM thin pool usage...\" lvs -o lv_name,vg_name,data_percent,metadata_percent,size,data_lv,metadata_lv 2>/dev/null | grep -E 'thin|$STORAGE_POOL' | head -10 || echo 'Unable to check LVM pools' " 2>/dev/null || echo "Unable to check host directories" echo "" echo "==========================================" echo "Investigation Complete" echo "=========================================="