212 lines
8.4 KiB
Bash
212 lines
8.4 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
# Load IP configuration
|
||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
|
|
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
|
||
|
|
|
||
|
|
|
||
|
|
# Investigate what's using space on thin2 storage pool
|
||
|
|
# Usage: ./investigate-thin2-storage.sh [proxmox-host]
|
||
|
|
|
||
|
|
set -e
|
||
|
|
|
||
|
|
PROXMOX_HOST="${1:-192.168.11.12}"
|
||
|
|
STORAGE_POOL="thin2"
|
||
|
|
|
||
|
|
echo "=========================================="
|
||
|
|
echo "Storage Investigation - thin2 Pool"
|
||
|
|
echo "=========================================="
|
||
|
|
echo "Proxmox Host: $PROXMOX_HOST"
|
||
|
|
echo "Storage Pool: $STORAGE_POOL"
|
||
|
|
echo "=========================================="
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Check if host is reachable
|
||
|
|
if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@$PROXMOX_HOST "echo 'Connected'" 2>/dev/null >/dev/null; then
|
||
|
|
echo "❌ Host $PROXMOX_HOST is not reachable"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
HOSTNAME=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "hostname" 2>/dev/null || echo "unknown")
|
||
|
|
echo "Hostname: $HOSTNAME"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Check storage pool status
|
||
|
|
echo "=== Storage Pool Status ==="
|
||
|
|
STORAGE_INFO=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pvesm status | grep $STORAGE_POOL" 2>/dev/null)
|
||
|
|
echo "$STORAGE_INFO"
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Get all VMs and containers using thin2
|
||
|
|
echo "=== Containers Using thin2 ==="
|
||
|
|
CONTAINERS=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct list --output-format json" 2>/dev/null || echo "[]")
|
||
|
|
|
||
|
|
if [ "$CONTAINERS" != "[]" ] && [ -n "$CONTAINERS" ]; then
|
||
|
|
echo "$CONTAINERS" | python3 -c "
|
||
|
|
import sys, json
|
||
|
|
containers = json.load(sys.stdin)
|
||
|
|
thin2_containers = [c for c in containers if '$STORAGE_POOL' in str(c.get('rootfs', ''))]
|
||
|
|
if thin2_containers:
|
||
|
|
print(f\"{'VMID':<6} {'Name':<35} {'Status':<10} {'Size':<15} {'Storage':<30}\")
|
||
|
|
print('-' * 100)
|
||
|
|
for vm in sorted(thin2_containers, key=lambda x: x.get('maxdisk', 0), reverse=True):
|
||
|
|
vmid = str(vm.get('vmid', 'N/A'))
|
||
|
|
name = (vm.get('name', 'N/A') or 'N/A')[:33]
|
||
|
|
status = vm.get('status', 'N/A')
|
||
|
|
maxdisk = vm.get('maxdisk', 0)
|
||
|
|
rootfs = str(vm.get('rootfs', 'N/A'))[:28]
|
||
|
|
if maxdisk:
|
||
|
|
size_gb = maxdisk / (1024**3)
|
||
|
|
size_str = f'{size_gb:.2f}G'
|
||
|
|
else:
|
||
|
|
size_str = 'N/A'
|
||
|
|
print(f\"{vmid:<6} {name:<35} {status:<10} {size_str:<15} {rootfs:<30}\")
|
||
|
|
else:
|
||
|
|
print('No containers found using thin2')
|
||
|
|
" 2>/dev/null || echo "Error parsing containers"
|
||
|
|
fi
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Get all VMs using thin2
|
||
|
|
echo "=== Virtual Machines Using thin2 ==="
|
||
|
|
VMS=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "qm list --output-format json" 2>/dev/null || echo "[]")
|
||
|
|
|
||
|
|
if [ "$VMS" != "[]" ] && [ -n "$VMS" ]; then
|
||
|
|
echo "$VMS" | python3 -c "
|
||
|
|
import sys, json
|
||
|
|
vms = json.load(sys.stdin)
|
||
|
|
thin2_vms = [v for v in vms if '$STORAGE_POOL' in str(v.get('disk', ''))]
|
||
|
|
if thin2_vms:
|
||
|
|
print(f\"{'VMID':<6} {'Name':<35} {'Status':<10} {'Size':<15} {'Storage':<30}\")
|
||
|
|
print('-' * 100)
|
||
|
|
for vm in sorted(thin2_vms, key=lambda x: x.get('maxdisk', 0), reverse=True):
|
||
|
|
vmid = str(vm.get('vmid', 'N/A'))
|
||
|
|
name = (vm.get('name', 'N/A') or 'N/A')[:33]
|
||
|
|
status = vm.get('status', 'N/A')
|
||
|
|
maxdisk = vm.get('maxdisk', 0)
|
||
|
|
disk = str(vm.get('disk', 'N/A'))[:28]
|
||
|
|
if maxdisk:
|
||
|
|
size_gb = maxdisk / (1024**3)
|
||
|
|
size_str = f'{size_gb:.2f}G'
|
||
|
|
else:
|
||
|
|
size_str = 'N/A'
|
||
|
|
print(f\"{vmid:<6} {name:<35} {status:<10} {size_str:<15} {disk:<30}\")
|
||
|
|
else:
|
||
|
|
print('No VMs found using thin2')
|
||
|
|
" 2>/dev/null || echo "Error parsing VMs"
|
||
|
|
fi
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Get detailed disk usage for each container on thin2
|
||
|
|
echo "=== Detailed Disk Usage (Containers on thin2) ==="
|
||
|
|
THIN2_CONTAINERS=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct list --output-format json" 2>/dev/null | \
|
||
|
|
python3 -c "import sys, json; containers = json.load(sys.stdin); [print(c['vmid']) for c in containers if '$STORAGE_POOL' in str(c.get('rootfs', ''))]" 2>/dev/null || echo "")
|
||
|
|
|
||
|
|
if [ -n "$THIN2_CONTAINERS" ]; then
|
||
|
|
printf "%-6s | %-35s | %-12s | %-12s | %-12s | %-10s\n" "VMID" "Name" "Used" "Avail" "Total" "Usage %"
|
||
|
|
echo "------------------------------------------------------------------------------------------------------------"
|
||
|
|
|
||
|
|
TOTAL_USED=0
|
||
|
|
TOTAL_SIZE=0
|
||
|
|
|
||
|
|
for vmid in $THIN2_CONTAINERS; do
|
||
|
|
NAME=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct config $vmid 2>/dev/null | grep '^hostname:' | awk '{print \$2}'" || echo "unknown")
|
||
|
|
STATUS=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct status $vmid 2>/dev/null | awk '{print \$2}'" || echo "unknown")
|
||
|
|
|
||
|
|
if [ "$STATUS" = "running" ]; then
|
||
|
|
DISK_USAGE=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct exec $vmid -- df -h / 2>/dev/null | tail -1" || echo "")
|
||
|
|
|
||
|
|
if [ -n "$DISK_USAGE" ]; then
|
||
|
|
USED=$(echo "$DISK_USAGE" | awk '{print $3}')
|
||
|
|
AVAIL=$(echo "$DISK_USAGE" | awk '{print $4}')
|
||
|
|
TOTAL=$(echo "$DISK_USAGE" | awk '{print $2}')
|
||
|
|
PERCENT=$(echo "$DISK_USAGE" | awk '{print $5}')
|
||
|
|
|
||
|
|
printf "%-6s | %-35s | %-12s | %-12s | %-12s | %-10s\n" "$vmid" "${NAME:0:33}" "$USED" "$AVAIL" "$TOTAL" "$PERCENT"
|
||
|
|
fi
|
||
|
|
else
|
||
|
|
# Get allocated size from Proxmox config
|
||
|
|
SIZE=$(ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pct list | grep \"^$vmid\" | awk '{print \$4}'" || echo "N/A")
|
||
|
|
printf "%-6s | %-35s | %-12s | %-12s | %-12s | %-10s\n" "$vmid" "${NAME:0:33}" "N/A (stopped)" "" "$SIZE" ""
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
else
|
||
|
|
echo "No containers found using thin2"
|
||
|
|
fi
|
||
|
|
echo ""
|
||
|
|
|
||
|
|
# Get actual storage usage from Proxmox
|
||
|
|
echo "=== Storage Pool Disk Usage (from Proxmox) ==="
|
||
|
|
ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pvesm list $STORAGE_POOL --output-format json" 2>/dev/null | \
|
||
|
|
python3 -c "
|
||
|
|
import sys, json
|
||
|
|
try:
|
||
|
|
data = json.load(sys.stdin)
|
||
|
|
if 'data' in data and len(data['data']) > 0:
|
||
|
|
print(f\"{'Volume':<50} {'Format':<10} {'Size':<15} {'Used':<15} {'Usage %':<10}\")
|
||
|
|
print('-' * 100)
|
||
|
|
total_size = 0
|
||
|
|
total_used = 0
|
||
|
|
for item in sorted(data['data'], key=lambda x: x.get('used', 0), reverse=True):
|
||
|
|
volid = item.get('volid', 'N/A')[:48]
|
||
|
|
format_type = item.get('format', 'N/A')[:8]
|
||
|
|
size = item.get('size', 0)
|
||
|
|
used = item.get('used', 0)
|
||
|
|
|
||
|
|
if size:
|
||
|
|
size_gb = size / (1024**3)
|
||
|
|
size_str = f'{size_gb:.2f}G'
|
||
|
|
else:
|
||
|
|
size_str = 'N/A'
|
||
|
|
|
||
|
|
if used:
|
||
|
|
used_gb = used / (1024**3)
|
||
|
|
used_str = f'{used_gb:.2f}G'
|
||
|
|
if size:
|
||
|
|
pct = (used / size) * 100
|
||
|
|
else:
|
||
|
|
pct = 0
|
||
|
|
else:
|
||
|
|
used_str = 'N/A'
|
||
|
|
pct = 0
|
||
|
|
|
||
|
|
total_size += size or 0
|
||
|
|
total_used += used or 0
|
||
|
|
|
||
|
|
print(f\"{volid:<50} {format_type:<10} {size_str:<15} {used_str:<15} {pct:>8.1f}%\")
|
||
|
|
|
||
|
|
if total_size:
|
||
|
|
total_size_gb = total_size / (1024**3)
|
||
|
|
total_used_gb = total_used / (1024**3)
|
||
|
|
total_pct = (total_used / total_size) * 100
|
||
|
|
print('-' * 100)
|
||
|
|
print(f\"{'TOTAL':<50} {'':<10} {f'{total_size_gb:.2f}G':<15} {f'{total_used_gb:.2f}G':<15} {total_pct:>8.1f}%\")
|
||
|
|
else:
|
||
|
|
print('No volumes found in storage pool')
|
||
|
|
except Exception as e:
|
||
|
|
print(f'Error parsing storage data: {e}')
|
||
|
|
" 2>/dev/null || ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "pvesm list $STORAGE_POOL" 2>/dev/null
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo "=== Largest Directories (if accessible) ==="
|
||
|
|
echo "Checking host storage directories..."
|
||
|
|
ssh -o StrictHostKeyChecking=no root@$PROXMOX_HOST "
|
||
|
|
# Find storage directories
|
||
|
|
STORAGE_DIR=\"/var/lib/vz/images\"
|
||
|
|
if [ -d \"\$STORAGE_DIR\" ]; then
|
||
|
|
echo \"Checking \$STORAGE_DIR...\"
|
||
|
|
du -sh \$STORAGE_DIR/*/ 2>/dev/null | sort -hr | head -10
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Also check LVM thin pool
|
||
|
|
echo \"\"
|
||
|
|
echo \"Checking LVM thin pool usage...\"
|
||
|
|
lvs -o lv_name,vg_name,data_percent,metadata_percent,size,data_lv,metadata_lv 2>/dev/null | grep -E 'thin|$STORAGE_POOL' | head -10 || echo 'Unable to check LVM pools'
|
||
|
|
" 2>/dev/null || echo "Unable to check host directories"
|
||
|
|
|
||
|
|
echo ""
|
||
|
|
echo "=========================================="
|
||
|
|
echo "Investigation Complete"
|
||
|
|
echo "=========================================="
|