fbda1b4beb
Some checks failed
Deploy to Phoenix / deploy (push) Has been cancelled
- ADD_CHAIN138_TO_LEDGER_LIVE: Ledger form done; public code review repo bis-innovations/LedgerLive; init/push commands - CONTRACT_DEPLOYMENT_RUNBOOK: Chain 138 gas price 1 gwei, 36-addr check, TransactionMirror workaround - CONTRACT_*: AddressMapper, MirrorManager deployed 2026-02-12; 36-address on-chain check - NEXT_STEPS_FOR_YOU: Ledger done; steps completable now (no LAN); run-completable-tasks-from-anywhere - MASTER_INDEX, OPERATOR_OPTIONAL, SMART_CONTRACTS_INVENTORY_SIMPLE: updates - LEDGER_BLOCKCHAIN_INTEGRATION_COMPLETE: bis-innovations/LedgerLive reference Co-authored-by: Cursor <cursoragent@cursor.com>
367 lines
13 KiB
Bash
Executable File
367 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Migrate containers VMIDs 3000-10151 from ML110 to R630-01
|
|
# Uses thin1 storage on R630-01 for optimal performance
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
LOG_DIR="${PROJECT_ROOT}/logs/migrations"
|
|
LOG_FILE="${LOG_DIR}/migrate-ml110-to-r630-01_$(date +%Y%m%d_%H%M%S).log"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
|
|
log_info() { echo -e "${BLUE}[INFO]${NC} $1" | tee -a "$LOG_FILE"; }
|
|
log_success() { echo -e "${GREEN}[✓]${NC} $1" | tee -a "$LOG_FILE"; }
|
|
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1" | tee -a "$LOG_FILE"; }
|
|
log_error() { echo -e "${RED}[✗]${NC} $1" | tee -a "$LOG_FILE"; }
|
|
log_header() { echo -e "${CYAN}=== $1 ===${NC}" | tee -a "$LOG_FILE"; }
|
|
|
|
# Create log directory
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Node configuration
|
|
SOURCE_NODE="ml110"
|
|
SOURCE_NODE_IP="192.168.11.10"
|
|
SOURCE_NODE_PASS="L@kers2010"
|
|
|
|
TARGET_NODE="r630-01"
|
|
TARGET_NODE_IP="192.168.11.11"
|
|
TARGET_NODE_PASS="password"
|
|
|
|
# Target storage (using thin1 for best performance)
|
|
TARGET_STORAGE="thin1"
|
|
|
|
# VMID range
|
|
VMID_START=3000
|
|
VMID_END=10151
|
|
|
|
# SSH helpers
|
|
ssh_ml110() {
|
|
sshpass -p "$SOURCE_NODE_PASS" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@"$SOURCE_NODE_IP" "$@" 2>&1
|
|
}
|
|
|
|
ssh_r630_01() {
|
|
sshpass -p "$TARGET_NODE_PASS" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@"$TARGET_NODE_IP" "$@" 2>&1
|
|
}
|
|
|
|
# Check if container exists on source node
|
|
container_exists() {
|
|
local vmid="$1"
|
|
ssh_ml110 "pct list 2>/dev/null | grep -q '^$vmid'" && return 0 || return 1
|
|
}
|
|
|
|
# Get container status
|
|
get_container_status() {
|
|
local vmid="$1"
|
|
ssh_ml110 "pct status $vmid 2>/dev/null | awk '{print \$2}'" || echo "unknown"
|
|
}
|
|
|
|
# Get container name
|
|
get_container_name() {
|
|
local vmid="$1"
|
|
ssh_ml110 "pct list 2>/dev/null | grep '^$vmid' | awk '{for(i=4;i<=NF;i++) printf \"%s \", \$i; print \"\"}' | sed 's/[[:space:]]*$//'" || echo "unknown"
|
|
}
|
|
|
|
# Check if container is already on target node
|
|
is_on_target_node() {
|
|
local vmid="$1"
|
|
ssh_r630_01 "pct list 2>/dev/null | grep -q '^$vmid'" 2>/dev/null && return 0 || return 1
|
|
}
|
|
|
|
# Migrate a single container
|
|
migrate_container() {
|
|
local vmid="$1"
|
|
local container_name="$2"
|
|
|
|
log_header "Migrating Container $vmid ($container_name)"
|
|
|
|
# Check if already on target
|
|
if is_on_target_node "$vmid"; then
|
|
log_info "Container $vmid is already on $TARGET_NODE, skipping..."
|
|
return 0
|
|
fi
|
|
|
|
# Get current status
|
|
local status=$(get_container_status "$vmid")
|
|
log_info "Current status: $status"
|
|
|
|
# Stop container if running
|
|
local was_running=false
|
|
if [ "$status" == "running" ]; then
|
|
was_running=true
|
|
log_info "Stopping container $vmid..."
|
|
ssh_ml110 "pct stop $vmid" || {
|
|
log_warn "pct stop failed, trying graceful shutdown..."
|
|
ssh_ml110 "pvesh create /nodes/$SOURCE_NODE/lxc/$vmid/status/shutdown --timeout 30" || true
|
|
}
|
|
sleep 5
|
|
|
|
# Verify stopped
|
|
status=$(get_container_status "$vmid")
|
|
if [ "$status" != "stopped" ]; then
|
|
log_warn "Container may still be running, waiting..."
|
|
sleep 10
|
|
fi
|
|
fi
|
|
|
|
# Clear any stale lock files before migration
|
|
log_info "Checking for lock files..."
|
|
ssh_ml110 "rm -f /run/lock/lxc/pve-config-$vmid.lock /var/lock/pve-manager/pve-migrate-$vmid 2>/dev/null" || true
|
|
|
|
# Kill any stale migration processes
|
|
local stale_pid=$(ssh_ml110 "ps aux | grep 'vzmigrate:$vmid' | grep -v grep | awk '{print \$2}' | head -1" || echo "")
|
|
if [ -n "$stale_pid" ]; then
|
|
log_warn "Found stale migration process (PID: $stale_pid), killing it..."
|
|
ssh_ml110 "kill -9 $stale_pid 2>/dev/null" || true
|
|
sleep 2
|
|
ssh_ml110 "rm -f /run/lock/lxc/pve-config-$vmid.lock /var/lock/pve-manager/pve-migrate-$vmid 2>/dev/null" || true
|
|
fi
|
|
|
|
# Unlock container if locked
|
|
log_info "Checking for container locks..."
|
|
ssh_ml110 "pvesh delete /nodes/$SOURCE_NODE/lxc/$vmid/lock 2>/dev/null" || true
|
|
sleep 1
|
|
|
|
# Perform migration
|
|
log_info "Migrating container $vmid from $SOURCE_NODE to $TARGET_NODE..."
|
|
log_info "Target storage: $TARGET_STORAGE"
|
|
|
|
# Use pct migrate (storage will be determined by target node default or moved after)
|
|
# Note: pct migrate doesn't support --storage, so we migrate first then move volume
|
|
if ssh_ml110 "pct migrate $vmid $TARGET_NODE" >> "$LOG_FILE" 2>&1; then
|
|
log_success "Migration command completed for container $vmid"
|
|
|
|
# Wait and verify migration
|
|
log_info "Waiting for migration to complete..."
|
|
local migrated=false
|
|
for i in {1..30}; do
|
|
sleep 5
|
|
if is_on_target_node "$vmid"; then
|
|
log_success "Container $vmid is now on $TARGET_NODE"
|
|
migrated=true
|
|
break
|
|
fi
|
|
if [ $i -lt 30 ]; then
|
|
log_info "Still migrating... (attempt $i/30)"
|
|
fi
|
|
done
|
|
|
|
if [ "$migrated" = false ]; then
|
|
log_warn "Migration may still be in progress or failed"
|
|
log_info "Check migration status manually: ssh root@$TARGET_NODE_IP 'pct list | grep $vmid'"
|
|
return 1
|
|
fi
|
|
|
|
# Move volume to target storage if not already there
|
|
log_info "Checking storage location..."
|
|
local current_storage=$(ssh_r630_01 "pct config $vmid 2>/dev/null | grep '^rootfs:' | awk '{print \$2}' | cut -d: -f1" || echo "")
|
|
if [ -n "$current_storage" ] && [ "$current_storage" != "$TARGET_STORAGE" ]; then
|
|
log_info "Moving container $vmid from $current_storage to $TARGET_STORAGE..."
|
|
if ssh_r630_01 "pct move-volume $vmid rootfs $TARGET_STORAGE" >> "$LOG_FILE" 2>&1; then
|
|
log_success "Volume moved to $TARGET_STORAGE"
|
|
else
|
|
log_warn "Failed to move volume to $TARGET_STORAGE, container is on $current_storage"
|
|
fi
|
|
else
|
|
log_info "Container is already on $TARGET_STORAGE"
|
|
fi
|
|
|
|
# Start container if it was running before
|
|
if [ "$was_running" = true ]; then
|
|
log_info "Starting container $vmid on $TARGET_NODE..."
|
|
sleep 2
|
|
ssh_r630_01 "pct start $vmid" || log_warn "Failed to start container $vmid (may need manual start)"
|
|
fi
|
|
|
|
return 0
|
|
else
|
|
log_error "Migration failed for container $vmid"
|
|
# Try to restart if it was running
|
|
if [ "$was_running" = true ]; then
|
|
log_info "Attempting to restart container $vmid on source node..."
|
|
ssh_ml110 "pct start $vmid" || true
|
|
fi
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
log_header "Migration: ML110 to R630-01"
|
|
echo ""
|
|
|
|
log_info "Source Node: $SOURCE_NODE ($SOURCE_NODE_IP)"
|
|
log_info "Target Node: $TARGET_NODE ($TARGET_NODE_IP)"
|
|
log_info "Target Storage: $TARGET_STORAGE"
|
|
log_info "VMID Range: $VMID_START - $VMID_END"
|
|
echo ""
|
|
|
|
# Check connectivity
|
|
log_info "Checking connectivity..."
|
|
if ! sshpass -p "$SOURCE_NODE_PASS" ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@"$SOURCE_NODE_IP" "echo 'connected'" >/dev/null 2>&1; then
|
|
log_error "Cannot connect to $SOURCE_NODE"
|
|
exit 1
|
|
fi
|
|
log_success "Connected to $SOURCE_NODE"
|
|
|
|
if ! sshpass -p "$TARGET_NODE_PASS" ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@"$TARGET_NODE_IP" "echo 'connected'" >/dev/null 2>&1; then
|
|
log_error "Cannot connect to $TARGET_NODE"
|
|
exit 1
|
|
fi
|
|
log_success "Connected to $TARGET_NODE"
|
|
echo ""
|
|
|
|
# Check target storage availability
|
|
log_info "Checking target storage availability..."
|
|
local all_storage=$(ssh_r630_01 "pvesm status 2>/dev/null" || echo "")
|
|
if echo "$all_storage" | grep -q "^$TARGET_STORAGE"; then
|
|
local storage_info=$(echo "$all_storage" | grep "^$TARGET_STORAGE")
|
|
log_success "Target storage $TARGET_STORAGE is available"
|
|
log_info "Storage info: $storage_info"
|
|
else
|
|
log_error "Target storage $TARGET_STORAGE not found on $TARGET_NODE"
|
|
log_info "Available storage on $TARGET_NODE:"
|
|
echo "$all_storage" | tee -a "$LOG_FILE"
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
|
|
# Find containers in range
|
|
log_info "Finding containers in VMID range $VMID_START-$VMID_END..."
|
|
local containers=$(ssh_ml110 "pct list 2>/dev/null | awk '\$1 >= $VMID_START && \$1 <= $VMID_END {print \$1}' | sort -n")
|
|
|
|
if [ -z "$containers" ]; then
|
|
log_warn "No containers found in VMID range $VMID_START-$VMID_END"
|
|
exit 0
|
|
fi
|
|
|
|
local container_list=($containers)
|
|
log_info "Found ${#container_list[@]} containers to migrate"
|
|
|
|
# Get container names
|
|
declare -A container_names
|
|
for vmid in "${container_list[@]}"; do
|
|
local name=$(get_container_name "$vmid")
|
|
container_names[$vmid]="$name"
|
|
log_info " - VMID $vmid: $name"
|
|
done
|
|
echo ""
|
|
|
|
# Check for auto-confirm mode
|
|
local auto_confirm=false
|
|
if [[ "${1:-}" == "--yes" ]] || [[ "${1:-}" == "-y" ]]; then
|
|
auto_confirm=true
|
|
log_info "Auto-confirm mode enabled"
|
|
fi
|
|
|
|
if [ "$auto_confirm" = false ]; then
|
|
log_warn "This will migrate ${#container_list[@]} containers from $SOURCE_NODE to $TARGET_NODE"
|
|
read -p "Continue with migration? (yes/no): " confirm
|
|
if [ "$confirm" != "yes" ]; then
|
|
log_info "Migration cancelled by user"
|
|
exit 0
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Perform migrations
|
|
local success_count=0
|
|
local fail_count=0
|
|
local skip_count=0
|
|
|
|
# Containers to skip (e.g., locked or problematic containers)
|
|
local skip_list=(6000)
|
|
|
|
for vmid in "${container_list[@]}"; do
|
|
local container_name="${container_names[$vmid]}"
|
|
|
|
# Check if container is in skip list
|
|
if [[ " ${skip_list[@]} " =~ " ${vmid} " ]]; then
|
|
log_warn "Container $vmid ($container_name) is in skip list, skipping..."
|
|
((skip_count++))
|
|
echo ""
|
|
continue
|
|
fi
|
|
|
|
# Check if already on target node (suppress errors to avoid script exit)
|
|
set +e
|
|
if is_on_target_node "$vmid" 2>/dev/null; then
|
|
set -e
|
|
log_info "Container $vmid is already on $TARGET_NODE, skipping..."
|
|
((skip_count++))
|
|
echo ""
|
|
continue
|
|
fi
|
|
set -e
|
|
|
|
# Try migration with retry for lock issues
|
|
local retry_count=0
|
|
local max_retries=2
|
|
local migration_success=false
|
|
|
|
while [ $retry_count -lt $max_retries ] && [ "$migration_success" = false ]; do
|
|
if migrate_container "$vmid" "$container_name"; then
|
|
((success_count++))
|
|
migration_success=true
|
|
else
|
|
# Check if it's a lock issue and we haven't exceeded retries
|
|
local error_msg=$(tail -10 "$LOG_FILE" | grep -i "locked\|lock" || echo "")
|
|
if [ -n "$error_msg" ] && [ $retry_count -lt $((max_retries - 1)) ]; then
|
|
log_warn "Container $vmid is locked, cleaning up and retrying (attempt $((retry_count + 2))/$max_retries)..."
|
|
ssh_ml110 "rm -f /run/lock/lxc/pve-config-$vmid.lock /var/lock/pve-manager/pve-migrate-$vmid 2>/dev/null" || true
|
|
ssh_ml110 "ps aux | grep 'vzmigrate:$vmid' | grep -v grep | awk '{print \$2}' | xargs kill -9 2>/dev/null" || true
|
|
ssh_ml110 "pvesh delete /nodes/$SOURCE_NODE/lxc/$vmid/lock 2>/dev/null" || true
|
|
sleep 10
|
|
((retry_count++))
|
|
else
|
|
# Max retries reached or non-lock error
|
|
log_warn "Migration failed for container $vmid after $((retry_count + 1)) attempts, skipping and continuing with next container..."
|
|
((fail_count++))
|
|
migration_success=true # Exit loop to continue with next container
|
|
break
|
|
fi
|
|
fi
|
|
done
|
|
echo ""
|
|
done
|
|
|
|
# Final summary
|
|
log_header "Migration Summary"
|
|
log_info "Total containers found: ${#container_list[@]}"
|
|
log_success "Successfully migrated: $success_count"
|
|
log_info "Already on target (skipped): $skip_count"
|
|
if [ $fail_count -gt 0 ]; then
|
|
log_error "Failed migrations: $fail_count"
|
|
fi
|
|
|
|
# Final verification
|
|
echo ""
|
|
log_info "Final container locations:"
|
|
for vmid in "${container_list[@]}"; do
|
|
if is_on_target_node "$vmid"; then
|
|
log_success "Container $vmid: $TARGET_NODE"
|
|
else
|
|
log_warn "Container $vmid: Still on $SOURCE_NODE"
|
|
fi
|
|
done
|
|
|
|
echo ""
|
|
log_success "Migration complete! Log saved to: $LOG_FILE"
|
|
|
|
if [ $fail_count -eq 0 ]; then
|
|
exit 0
|
|
else
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Run main function
|
|
main "$@"
|