proxmox/scripts/restore-container-filesystems.sh

#!/usr/bin/env bash
# Restore container filesystems from templates or recreate containers
# Usage: ./scripts/restore-container-filesystems.sh

set -uo pipefail

NODE_IP="${PROXMOX_HOST_R630_01}"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

echo ""
log_info "Restoring container filesystems..."
echo ""

# Check for templates (multiple locations/formats)
TEMPLATE=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
    "find /var/lib/vz -name '*ubuntu*22.04*standard*.tar*' -type f 2>/dev/null | head -1" || echo "")

if [[ -z "$TEMPLATE" ]]; then
    log_error "No Ubuntu template found. Please download a template first:"
    log_info "  pveam download local ubuntu-22.04-standard_22.04-1_amd64.tar.zst"
    exit 1
fi

log_info "Using template: $TEMPLATE"
TEMPLATE_FORMAT=$(echo "$TEMPLATE" | grep -oE '\.(tar\.gz|tar\.zst|tar)$' || echo ".tar.gz")

# Containers that need filesystem restoration
CONTAINERS=(3000 3001 3002 3003 3500 3501 5200 6000 6400 10000 10001 10020 10030 10040 10050 10060 10070 10080 10090 10091 10092 10100 10101 10120 10130 10150 10151 10200 10201 10202 10210 10230 10232)

FIXED=0
FAILED=0

for vmid in "${CONTAINERS[@]}"; do
    log_info "Processing CT $vmid..."
    
    # Check if running
    status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct list 2>/dev/null | awk '\$1 == $vmid {print \$2}'" || echo "notfound")
    
    if [[ "$status" == "running" ]]; then
        log_success "  ✓ Already running"
        ((FIXED++))
        continue
    fi
    
    # Get rootfs
    rootfs=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct config $vmid 2>/dev/null | grep '^rootfs:'" || echo "")
    
    if [[ -z "$rootfs" ]]; then
        log_warn "  ⚠️  Config missing, skipping..."
        continue
    fi
    
    # Extract volume info
    volume_name=$(echo "$rootfs" | sed 's/^rootfs: //' | cut -d':' -f2 | cut -d',' -f1)
    mapper_name=$(echo "$volume_name" | sed 's/-/--/g')
    device="/dev/mapper/pve-${mapper_name}"
    
    log_info "  Volume: $volume_name"
    
    # Check if volume exists and is mounted
    mounted=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "mount | grep -q \"$device\" && echo 'mounted' || echo 'unmounted'" || echo "error")
    
    # Mount point
    mount_point="/tmp/ct-${vmid}-mount"
    
    if [[ "$mounted" == "unmounted" ]]; then
        log_info "  Mounting volume..."
        ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
            "mkdir -p $mount_point && mount $device $mount_point" 2>&1 >/dev/null || {
            log_warn "  ⚠️  Mount failed, checking if formatted..."
            # Check if formatted
            fs_check=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
                "blkid $device 2>/dev/null | grep -oP 'TYPE=\"\\K[^\"]+' || echo 'unformatted'" || echo "error")
            
            if [[ "$fs_check" == "unformatted" ]]; then
                log_info "  Formatting volume..."
                ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
                    "mkfs.ext4 -F $device" 2>&1 >/dev/null
                ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
                    "mount $device $mount_point" 2>&1 >/dev/null || {
                log_error "  ❌ Failed to mount after format"
                ((FAILED++))
                continue
            }
            else
                log_error "  ❌ Cannot mount volume"
                ((FAILED++))
                continue
            fi
        }
    else
        log_info "  Volume already mounted"
        mount_point=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
            "mount | grep \"$device\" | awk '{print \$3}'" || echo "")
    fi
    
    # Check if filesystem is empty
    file_count=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "ls -A $mount_point 2>/dev/null | wc -l" || echo "0")
    
    if [[ "$file_count" == "0" ]] || [[ "$file_count" -lt 10 ]]; then
        log_info "  Filesystem appears empty, extracting template..."
        # Handle different archive formats
        extract_success=false
        if echo "$TEMPLATE" | grep -q "\.tar\.zst$"; then
            if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
                "cd $mount_point && zstd -dc $TEMPLATE | tar -x --strip-components=1" 2>&1 >/dev/null; then
                extract_success=true
            fi
        elif echo "$TEMPLATE" | grep -q "\.tar\.gz$"; then
            if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
                "cd $mount_point && tar -xzf $TEMPLATE --strip-components=1" 2>&1 >/dev/null; then
                extract_success=true
            fi
        else
            if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
                "cd $mount_point && tar -xf $TEMPLATE --strip-components=1" 2>&1 >/dev/null; then
                extract_success=true
            fi
        fi
        
        if [[ "$extract_success" == "true" ]]; then
            log_success "  ✓ Template extracted"
        else
            log_error "  ❌ Template extraction failed"
            ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
                "umount $mount_point 2>/dev/null; rmdir $mount_point 2>/dev/null" || true
            ((FAILED++))
            echo ""
            continue
        fi
    else
        log_info "  Filesystem already has content ($file_count items)"
    fi
    
    # Unmount
    ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "umount $mount_point 2>/dev/null; rmdir $mount_point 2>/dev/null" || true
    
    # Try to start
    log_info "  Starting container..."
    if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
        "pct start $vmid" 2>&1 >/dev/null; then
        log_success "  ✓ Started"
        ((FIXED++))
        sleep 1
    else
        error=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${NODE_IP} \
            "pct start $vmid 2>&1" || true)
        if echo "$error" | grep -q "already running"; then
            log_success "  ✓ Already running"
            ((FIXED++))
        else
            log_warn "  ⚠️  Start failed:"
            echo "$error" | sed 's/^/    /' | head -2
            ((FAILED++))
        fi
    fi
    
    echo ""
done

echo ""
log_info "Summary: Fixed: $FIXED, Failed: $FAILED"
log_success "Done!"