proxmox/scripts/archive/consolidated/deploy/deploy-phoenix-vault-cluster.sh

#!/bin/bash
# Deploy Sankofa Phoenix Vault Cluster with Full Redundancy
# Creates 3-node HA Vault cluster on VLAN 160

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }

# Configuration
PROXMOX_HOST_1="${PROXMOX_HOST_1:-192.168.11.11}"  # r630-01
PROXMOX_HOST_2="${PROXMOX_HOST_2:-192.168.11.12}"  # r630-02
VAULT_NODE_1_VMID=8640
VAULT_NODE_2_VMID=8641
VAULT_NODE_3_VMID=8642
VAULT_NODE_1_IP="10.160.0.40"
VAULT_NODE_2_IP="10.160.0.41"
VAULT_NODE_3_IP="10.160.0.42"
VLAN_ID=160
DRY_RUN="${DRY_RUN:-false}"

echo "═══════════════════════════════════════════════════════════"
echo "  Sankofa Phoenix Vault Cluster Deployment"
echo "═══════════════════════════════════════════════════════════"
echo ""

log_info "Mode: $([ "$DRY_RUN" = "true" ] && echo "DRY RUN" || echo "LIVE")"
log_info "Cluster: 3-node HA Vault cluster"
log_info "Network: VLAN $VLAN_ID (10.160.0.0/22)"
echo ""

# Function to create container
create_vault_node() {
    local vmid=$1
    local hostname=$2
    local ip=$3
    local proxmox_host=$4
    local storage="${5:-local-lvm}"  # Default storage, can be overridden
    
    log_info "Creating Vault node: $hostname (VMID $vmid)"
    
    if [ "$DRY_RUN" = "true" ]; then
        log_info "  Would create container on $proxmox_host"
        log_info "  IP: $ip"
        log_info "  Hostname: $hostname"
        log_info "  Storage: $storage"
        return 0
    fi
    
    # Check if container already exists
    if ssh root@"$proxmox_host" "pct list | grep -q '^$vmid'"; then
        log_warn "Container $vmid already exists on $proxmox_host"
        log_info "Stopping and removing existing container..."
        ssh root@"$proxmox_host" "pct stop $vmid 2>/dev/null || true"
        ssh root@"$proxmox_host" "pct destroy $vmid 2>/dev/null || true"
        sleep 2
    fi
    
    ssh root@"$proxmox_host" "pct create $vmid local:vztmpl/ubuntu-22.04-standard_22.04-1_amd64.tar.zst \
      --hostname $hostname \
      --cores 2 --memory 4096 --swap 2048 \
      --storage $storage --rootfs $storage:50 \
      --net0 name=eth0,bridge=vmbr0,tag=$VLAN_ID,ip=$ip/22,gw=10.160.0.1 \
      --onboot 1 --unprivileged 0 \
      --features nesting=1" || {
        log_error "Failed to create container $vmid"
        return 1
    }
    
    log_success "Container $vmid created"
    
    # Start container
    log_info "Starting container $vmid..."
    ssh root@"$proxmox_host" "pct start $vmid" || {
        log_error "Failed to start container $vmid"
        return 1
    }
    
    # Wait for container to be ready
    sleep 5
    
    log_success "Container $vmid started"
    return 0
}

# Function to install Vault
install_vault() {
    local vmid=$1
    local proxmox_host=$2
    
    log_info "Installing Vault on VMID $vmid..."
    
    if [ "$DRY_RUN" = "true" ]; then
        log_info "  Would install Vault via apt"
        return 0
    fi
    
    ssh root@"$proxmox_host" "pct exec $vmid -- bash" << 'INSTALL_EOF'
set -e
export DEBIAN_FRONTEND=noninteractive

apt-get update
apt-get install -y curl unzip wget gnupg software-properties-common jq lsb-release

# Add HashiCorp GPG key
curl -fsSL https://apt.releases.hashicorp.com/gpg | apt-key add -

# Add HashiCorp repository
apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main"

# Install Vault
apt-get update
apt-get install -y vault

# Verify installation
vault version
INSTALL_EOF
    
    log_success "Vault installed on VMID $vmid"
}

# Function to configure Vault node
configure_vault_node() {
    local vmid=$1
    local node_id=$2
    local ip=$3
    local proxmox_host=$4
    
    log_info "Configuring Vault node $node_id (VMID $vmid)..."
    
    if [ "$DRY_RUN" = "true" ]; then
        log_info "  Would create vault.hcl configuration"
        return 0
    fi
    
    ssh root@"$proxmox_host" "pct exec $vmid -- bash" << CONFIG_EOF
set -e

# Create vault user
useradd --system --home /opt/vault --shell /bin/false vault 2>/dev/null || true

# Create directories
mkdir -p /opt/vault/data
mkdir -p /etc/vault.d
mkdir -p /var/log/vault
chown -R vault:vault /opt/vault
chown -R vault:vault /var/log/vault

# Create vault.hcl
cat > /etc/vault.d/vault.hcl << VAULT_CONFIG
ui = true
disable_mlock = true

listener "tcp" {
  address          = "0.0.0.0:8200"
  cluster_address  = "$ip:8201"
  tls_disable      = 1
}

storage "raft" {
  path    = "/opt/vault/data"
  node_id = "$node_id"
  
  retry_join {
    leader_api_addr = "http://10.160.0.40:8200"
  }
  retry_join {
    leader_api_addr = "http://10.160.0.41:8200"
  }
  retry_join {
    leader_api_addr = "http://10.160.0.42:8200"
  }
}

api_addr = "http://$ip:8200"
cluster_addr = "http://$ip:8201"

log_level = "INFO"
log_file = "/var/log/vault/vault.log"
log_rotate_duration = "24h"
log_rotate_max_files = 30
VAULT_CONFIG

# Create systemd service
cat > /etc/systemd/system/vault.service << 'SERVICE_EOF'
[Unit]
Description=HashiCorp Vault - A tool for managing secrets
Documentation=https://www.vaultproject.io/docs/
After=network-online.target
Wants=network-online.target
ConditionFileNotEmpty=/etc/vault.d/vault.hcl

[Service]
Type=notify
User=vault
Group=vault
ProtectSystem=full
ProtectHome=read-only
PrivateTmp=yes
PrivateDevices=yes
SecureBits=keep-caps
AmbientCapabilities=CAP_IPC_LOCK
CapabilityBoundingSet=CAP_SYSLOG CAP_IPC_LOCK
NoNewPrivileges=yes
ExecStart=/usr/bin/vault server -config=/etc/vault.d/vault.hcl
ExecReload=/bin/kill --signal HUP \$MAINPID
KillMode=process
Restart=on-failure
RestartSec=5
TimeoutStopSec=30
StartLimitInterval=200
StartLimitBurst=5
LimitNOFILE=65536
LimitMEMLOCK=infinity

[Install]
WantedBy=multi-user.target
SERVICE_EOF

# Enable service
systemctl daemon-reload
systemctl enable vault

CONFIG_EOF
    
    log_success "Vault configured on VMID $vmid"
}

# Phase 1: Create containers
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Phase 1: Creating Containers"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Use appropriate storage for each host
# r630-01: local-lvm or thin1
# r630-02: thin3, thin4, thin5, or thin6 (empty pools)
create_vault_node $VAULT_NODE_1_VMID "vault-phoenix-1" $VAULT_NODE_1_IP $PROXMOX_HOST_1 "local-lvm"
create_vault_node $VAULT_NODE_2_VMID "vault-phoenix-2" $VAULT_NODE_2_IP $PROXMOX_HOST_2 "thin3"
create_vault_node $VAULT_NODE_3_VMID "vault-phoenix-3" $VAULT_NODE_3_IP $PROXMOX_HOST_1 "local-lvm"

echo ""

# Phase 2: Install Vault
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Phase 2: Installing Vault"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

install_vault $VAULT_NODE_1_VMID $PROXMOX_HOST_1
install_vault $VAULT_NODE_2_VMID $PROXMOX_HOST_2
install_vault $VAULT_NODE_3_VMID $PROXMOX_HOST_1

echo ""

# Phase 3: Configure Vault
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Phase 3: Configuring Vault Nodes"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

configure_vault_node $VAULT_NODE_1_VMID "vault-phoenix-1" $VAULT_NODE_1_IP $PROXMOX_HOST_1
configure_vault_node $VAULT_NODE_2_VMID "vault-phoenix-2" $VAULT_NODE_2_IP $PROXMOX_HOST_2
configure_vault_node $VAULT_NODE_3_VMID "vault-phoenix-3" $VAULT_NODE_3_IP $PROXMOX_HOST_1

echo ""

# Phase 4: Initialize Cluster
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Phase 4: Cluster Initialization"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

if [ "$DRY_RUN" = "true" ]; then
    log_warn "DRY RUN - Skipping cluster initialization"
    log_info "To initialize cluster, run manually:"
    log_info "  1. Start Vault on Node 1: ssh root@$PROXMOX_HOST_1 'pct exec $VAULT_NODE_1_VMID -- systemctl start vault'"
    log_info "  2. Initialize: ssh root@$PROXMOX_HOST_1 'pct exec $VAULT_NODE_1_VMID -- vault operator init'"
    log_info "  3. Unseal Node 1 with 3 unseal keys"
    log_info "  4. Start Vault on Nodes 2 and 3"
    log_info "  5. Verify cluster: vault operator raft list-peers"
else
    log_info "Starting Vault on Node 1..."
    ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_1_VMID -- systemctl start vault" || {
        log_error "Failed to start Vault on Node 1"
        exit 1
    }
    
    # Wait for Vault to be ready
    log_info "Waiting for Vault to be ready..."
    sleep 10
    
    # Check if already initialized
    INIT_STATUS=$(ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_1_VMID -- vault status -format=json 2>/dev/null | jq -r '.initialized' 2>/dev/null || echo 'false'")
    
    if [ "$INIT_STATUS" = "true" ]; then
        log_warn "Vault cluster already initialized"
        log_info "To reinitialize, you must first remove existing data"
        log_info "Skipping initialization. Use existing unseal keys."
    else
        log_info "Initializing Vault cluster..."
        log_warn "This will generate unseal keys and root token"
        log_warn "Save these securely!"
        
        INIT_OUTPUT=$(ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_1_VMID -- vault operator init -key-shares=5 -key-threshold=3 -recovery-shares=5 -recovery-threshold=3 -format=json 2>&1")
        
        if echo "$INIT_OUTPUT" | grep -q "error"; then
            log_error "Failed to initialize Vault:"
            echo "$INIT_OUTPUT"
            exit 1
        fi
        
        echo "$INIT_OUTPUT" > /tmp/vault-phoenix-init.json
        log_success "Initialization complete. Keys saved to /tmp/vault-phoenix-init.json"
        
        log_info "Extracting unseal keys..."
        UNSEAL_KEYS=$(echo "$INIT_OUTPUT" | jq -r '.unseal_keys_b64[]' 2>/dev/null || echo "")
        ROOT_TOKEN=$(echo "$INIT_OUTPUT" | jq -r '.root_token' 2>/dev/null || echo "")
        
        if [ -z "$UNSEAL_KEYS" ]; then
            log_error "Failed to extract unseal keys from initialization output"
            log_info "Initialization output:"
            echo "$INIT_OUTPUT"
            exit 1
        fi
        
        log_warn "═══════════════════════════════════════════════════════════"
        log_warn "  UNSEAL KEYS (SAVE SECURELY):"
        log_warn "═══════════════════════════════════════════════════════════"
        echo "$UNSEAL_KEYS" | nl -v1
        echo ""
        log_warn "═══════════════════════════════════════════════════════════"
        log_warn "  ROOT TOKEN (SAVE SECURELY):"
        log_warn "═══════════════════════════════════════════════════════════"
        echo "$ROOT_TOKEN"
        echo ""
        
        log_info "Unsealing Node 1 (requires 3 keys)..."
        KEY1=$(echo "$UNSEAL_KEYS" | sed -n '1p')
        KEY2=$(echo "$UNSEAL_KEYS" | sed -n '2p')
        KEY3=$(echo "$UNSEAL_KEYS" | sed -n '3p')
        
        ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_1_VMID -- vault operator unseal $KEY1" || log_warn "Unseal key 1 may have failed"
        ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_1_VMID -- vault operator unseal $KEY2" || log_warn "Unseal key 2 may have failed"
        ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_1_VMID -- vault operator unseal $KEY3" || log_warn "Unseal key 3 may have failed"
        
        # Verify unsealed
        sleep 2
        SEALED=$(ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_1_VMID -- vault status -format=json 2>/dev/null | jq -r '.sealed' 2>/dev/null || echo 'true'")
        if [ "$SEALED" = "false" ]; then
            log_success "Node 1 unsealed successfully"
        else
            log_warn "Node 1 may still be sealed. Check manually: vault status"
        fi
    fi
    
    log_info "Starting Vault on Nodes 2 and 3..."
    ssh root@"$PROXMOX_HOST_2" "pct exec $VAULT_NODE_2_VMID -- systemctl start vault" || log_warn "Failed to start Vault on Node 2"
    ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_3_VMID -- systemctl start vault" || log_warn "Failed to start Vault on Node 3"
    
    sleep 10
    
    log_info "Verifying cluster..."
    ssh root@"$PROXMOX_HOST_1" "pct exec $VAULT_NODE_1_VMID -- vault operator raft list-peers 2>/dev/null || echo 'Cluster not ready yet'"
fi

echo ""
echo "═══════════════════════════════════════════════════════════"
echo "  Deployment Summary"
echo "═══════════════════════════════════════════════════════════"
echo ""

log_success "Vault Cluster Nodes:"
log_info "  Node 1: VMID $VAULT_NODE_1_VMID - $VAULT_NODE_1_IP (vault-phoenix-1)"
log_info "  Node 2: VMID $VAULT_NODE_2_VMID - $VAULT_NODE_2_IP (vault-phoenix-2)"
log_info "  Node 3: VMID $VAULT_NODE_3_VMID - $VAULT_NODE_3_IP (vault-phoenix-3)"

echo ""
log_info "Next Steps:"
log_info "  1. Verify cluster health: vault status"
log_info "  2. List cluster peers: vault operator raft list-peers"
log_info "  3. Configure authentication methods"
log_info "  4. Create policies and secret paths"
log_info "  5. Update Phoenix services to use new Vault cluster"

if [ "$DRY_RUN" = "false" ] && [ -f "/tmp/vault-phoenix-init.json" ]; then
    log_warn "IMPORTANT: Save unseal keys and root token securely!"
    log_info "Keys saved to: /tmp/vault-phoenix-init.json"
    log_info "Move this file to secure location and delete from /tmp"
fi

echo ""