fbda1b4beb
Some checks failed
Deploy to Phoenix / deploy (push) Has been cancelled
- ADD_CHAIN138_TO_LEDGER_LIVE: Ledger form done; public code review repo bis-innovations/LedgerLive; init/push commands - CONTRACT_DEPLOYMENT_RUNBOOK: Chain 138 gas price 1 gwei, 36-addr check, TransactionMirror workaround - CONTRACT_*: AddressMapper, MirrorManager deployed 2026-02-12; 36-address on-chain check - NEXT_STEPS_FOR_YOU: Ledger done; steps completable now (no LAN); run-completable-tasks-from-anywhere - MASTER_INDEX, OPERATOR_OPTIONAL, SMART_CONTRACTS_INVENTORY_SIMPLE: updates - LEDGER_BLOCKCHAIN_INTEGRATION_COMPLETE: bis-innovations/LedgerLive reference Co-authored-by: Cursor <cursoragent@cursor.com>
188 lines
7.5 KiB
Bash
Executable File
188 lines
7.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# Complete HA test suite
|
|
|
|
set -euo pipefail
|
|
|
|
# Load IP configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
|
|
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
if [ -f "$PROJECT_ROOT/.env" ]; then
|
|
set +euo pipefail
|
|
source "$PROJECT_ROOT/.env" 2>/dev/null || true
|
|
set -euo pipefail
|
|
fi
|
|
|
|
PRIMARY_HOST="${PRIMARY_HOST:-192.168.11.11}"
|
|
SECONDARY_HOST="${SECONDARY_HOST:-192.168.11.12}"
|
|
PRIMARY_VMID="${PRIMARY_VMID:-10233}"
|
|
SECONDARY_VMID="${SECONDARY_VMID:-10234}"
|
|
VIP="${VIP:-192.168.11.166}"
|
|
|
|
# Colors
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
RED='\033[0;31m'
|
|
NC='\033[0m'
|
|
|
|
log_info() { echo -e "${BLUE}[TEST]${NC} $1"; }
|
|
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[✗]${NC} $1"; }
|
|
|
|
PASSED=0
|
|
FAILED=0
|
|
|
|
test_check() {
|
|
local name=$1
|
|
shift
|
|
if "$@" >/dev/null 2>&1; then
|
|
log_success "$name"
|
|
((PASSED++))
|
|
return 0
|
|
else
|
|
log_error "$name"
|
|
((FAILED++))
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
echo ""
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "🧪 Complete HA Test Suite"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo ""
|
|
|
|
# Test 1: Container Status
|
|
log_info "Testing container status..."
|
|
test_check "Primary container running" \
|
|
ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" "pct status $PRIMARY_VMID | grep -q running"
|
|
test_check "Secondary container running" \
|
|
ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" "pct status $SECONDARY_VMID | grep -q running"
|
|
|
|
# Test 2: NPMplus Containers
|
|
log_info "Testing NPMplus containers..."
|
|
test_check "Primary NPMplus running" \
|
|
ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" "pct exec $PRIMARY_VMID -- docker ps --filter 'name=npmplus' --format '{{.Status}}' | grep -qE 'Up|healthy'"
|
|
test_check "Secondary NPMplus running" \
|
|
ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" "pct exec $SECONDARY_VMID -- docker ps --filter 'name=npmplus' --format '{{.Status}}' | grep -qE 'Up|healthy'"
|
|
|
|
# Test 3: Keepalived
|
|
log_info "Testing Keepalived..."
|
|
test_check "Primary Keepalived active" \
|
|
ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" "systemctl is-active keepalived | grep -q active"
|
|
test_check "Secondary Keepalived active" \
|
|
ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" "systemctl is-active keepalived | grep -q active"
|
|
|
|
# Test 4: VIP Ownership
|
|
log_info "Testing VIP ownership..."
|
|
test_check "VIP owned by primary" \
|
|
ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" "ip addr show vmbr0 | grep -q $VIP"
|
|
|
|
# Test 5: Network Connectivity
|
|
log_info "Testing network connectivity..."
|
|
test_check "Primary NPMplus accessible" \
|
|
curl -k -s -o /dev/null -w "%{http_code}" --max-time 5 "https://${IP_NPMPLUS_ETH0:-192.168.11.166}:81" | grep -qE "200|301|302"
|
|
test_check "Secondary NPMplus accessible" \
|
|
curl -k -s -o /dev/null -w "%{http_code}" --max-time 5 "https://${IP_NPMPLUS}:81" | grep -qE "200|301|302"
|
|
|
|
# Test 6: Certificate Sync
|
|
log_info "Testing certificate synchronization..."
|
|
PRIMARY_CERTS=$(ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" \
|
|
"pct exec $PRIMARY_VMID -- docker exec npmplus find /data -name 'fullchain.pem' -type f 2>/dev/null | wc -l" || echo "0")
|
|
SECONDARY_CERTS=$(ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" \
|
|
"pct exec $SECONDARY_VMID -- docker exec npmplus find /data -name 'fullchain.pem' -type f 2>/dev/null | wc -l" || echo "0")
|
|
|
|
if [ "$PRIMARY_CERTS" -gt 0 ] && [ "$SECONDARY_CERTS" -gt 0 ]; then
|
|
if [ "$PRIMARY_CERTS" = "$SECONDARY_CERTS" ]; then
|
|
log_success "Certificates synced ($PRIMARY_CERTS certificates)"
|
|
((PASSED++))
|
|
else
|
|
log_warn "Certificate count mismatch: Primary=$PRIMARY_CERTS, Secondary=$SECONDARY_CERTS"
|
|
((FAILED++))
|
|
fi
|
|
else
|
|
if [ "$PRIMARY_CERTS" -eq 0 ]; then
|
|
log_warn "No certificates found in primary"
|
|
fi
|
|
if [ "$SECONDARY_CERTS" -eq 0 ]; then
|
|
log_warn "No certificates found in secondary"
|
|
fi
|
|
fi
|
|
|
|
# Test 7: Configuration Sync
|
|
log_info "Testing configuration synchronization..."
|
|
# Try API method first, fallback to file size check
|
|
PRIMARY_PROXIES=$(ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" \
|
|
"pct exec $PRIMARY_VMID -- docker exec npmplus sqlite3 /data/database.sqlite 'SELECT COUNT(*) FROM proxy_host;' 2>/dev/null" || echo "0")
|
|
SECONDARY_PROXIES=$(ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" \
|
|
"pct exec $SECONDARY_VMID -- docker exec npmplus sqlite3 /data/database.sqlite 'SELECT COUNT(*) FROM proxy_host;' 2>/dev/null" || echo "0")
|
|
|
|
# If sqlite3 not available, check database file size as proxy
|
|
if [ "$PRIMARY_PROXIES" = "0" ] || echo "$PRIMARY_PROXIES" | grep -q "executable file not found"; then
|
|
PRIMARY_DB_SIZE=$(ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" \
|
|
"pct exec $PRIMARY_VMID -- docker exec npmplus stat -c%s /data/database.sqlite 2>/dev/null" || echo "0")
|
|
SECONDARY_DB_SIZE=$(ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" \
|
|
"pct exec $SECONDARY_VMID -- docker exec npmplus stat -c%s /data/database.sqlite 2>/dev/null" || echo "0")
|
|
if [ "$PRIMARY_DB_SIZE" -gt 0 ] && [ "$SECONDARY_DB_SIZE" -gt 0 ]; then
|
|
PRIMARY_PROXIES="db_exists"
|
|
SECONDARY_PROXIES="db_exists"
|
|
fi
|
|
fi
|
|
|
|
if [ "$PRIMARY_PROXIES" -gt 0 ] && [ "$SECONDARY_PROXIES" -gt 0 ]; then
|
|
if [ "$PRIMARY_PROXIES" = "$SECONDARY_PROXIES" ]; then
|
|
log_success "Configuration synced ($PRIMARY_PROXIES proxy hosts)"
|
|
((PASSED++))
|
|
else
|
|
log_warn "Proxy host count mismatch: Primary=$PRIMARY_PROXIES, Secondary=$SECONDARY_PROXIES"
|
|
((FAILED++))
|
|
fi
|
|
else
|
|
if [ "$PRIMARY_PROXIES" -eq 0 ]; then
|
|
log_warn "No proxy hosts found in primary database"
|
|
fi
|
|
if [ "$SECONDARY_PROXIES" -eq 0 ]; then
|
|
log_warn "No proxy hosts found in secondary database"
|
|
fi
|
|
fi
|
|
|
|
# Test 8: Failover Test (non-destructive check)
|
|
log_info "Testing failover readiness..."
|
|
VIP_ON_PRIMARY=$(ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" \
|
|
"ip addr show vmbr0 2>/dev/null | grep -q $VIP && echo 'yes' || echo 'no'")
|
|
VIP_ON_SECONDARY=$(ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" \
|
|
"ip addr show vmbr0 2>/dev/null | grep -q $VIP && echo 'yes' || echo 'no'")
|
|
|
|
if [ "$VIP_ON_PRIMARY" = "yes" ] || [ "$VIP_ON_SECONDARY" = "yes" ]; then
|
|
log_success "VIP is active (on $([ "$VIP_ON_PRIMARY" = "yes" ] && echo "primary" || echo "secondary"))"
|
|
((PASSED++))
|
|
else
|
|
log_error "VIP not found on either host"
|
|
((FAILED++))
|
|
fi
|
|
|
|
# Summary
|
|
echo ""
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "📊 Test Results"
|
|
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
|
echo "Passed: $PASSED"
|
|
echo "Failed: $FAILED"
|
|
echo "Total: $((PASSED + FAILED))"
|
|
echo ""
|
|
|
|
if [ $FAILED -eq 0 ]; then
|
|
log_success "All tests passed! ✅"
|
|
exit 0
|
|
else
|
|
log_warn "Some tests failed or need attention. Review output above."
|
|
exit 0 # Don't fail the script, just report
|
|
fi
|