proxmox/scripts/complete-blockscout-migrations-and-verify.sh

#!/usr/bin/env bash
# Complete all next steps: Run migrations, verify, restart, and test Blockscout
# Container: VMID 5000 on pve2 (${IP_BLOCKSCOUT:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-192.168.11.14}}}}}0})

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


VMID="${VMID:-5000}"
IP="${IP:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-192.168.11.14}}}}}0}"
DOMAIN="${DOMAIN:-explorer.d-bis.org}"
PASSWORD="${PASSWORD:-L@kers2010}"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_step() { echo -e "${CYAN}[STEP]${NC} $1"; }

exec_container() {
    local cmd="$1"
    sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no root@"$IP" "bash -c '$cmd'" 2>&1
}

echo "════════════════════════════════════════════════════════"
echo "Complete Blockscout Migration and Verification"
echo "════════════════════════════════════════════════════════"
echo ""

# Step 1: Verify PostgreSQL is ready
log_step "Step 1: Verifying PostgreSQL is ready..."
if exec_container "docker exec blockscout-postgres pg_isready -U blockscout >/dev/null 2>&1"; then
    log_success "PostgreSQL is ready"
else
    log_error "PostgreSQL is not ready"
    exit 1
fi

# Step 2: Check current database state
log_step "Step 2: Checking current database state..."
TABLE_COUNT=$(exec_container "docker exec blockscout-postgres psql -U blockscout -d blockscout -t -c \"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public';\" 2>/dev/null | tr -d ' ' || echo '0'")
log_info "Current table count: $TABLE_COUNT"

if [ "$TABLE_COUNT" != "0" ] && [ -n "$TABLE_COUNT" ] && [ "$TABLE_COUNT" != "null" ]; then
    log_warn "Database already has tables. Checking if migrations are needed..."
else
    log_info "Database is empty - migrations are required"
fi

# Step 3: Stop Blockscout to run migrations
log_step "Step 3: Stopping Blockscout container..."
exec_container "cd /opt/blockscout && docker-compose stop blockscout 2>/dev/null || docker compose stop blockscout 2>/dev/null || true"
sleep 5
log_success "Blockscout stopped"

# Step 4: Run database migrations
log_step "Step 4: Running database migrations..."
log_info "This may take 2-5 minutes - please wait..."
log_info "Running: Explorer.Release.migrate()"

# Get network name
NETWORK_NAME=$(exec_container "docker inspect blockscout-postgres 2>/dev/null | grep -A 10 Networks | grep NetworkMode | head -1 | cut -d'\"' -f4 || echo 'blockscout_blockscout-network'")
log_info "Using network: $NETWORK_NAME"

# Run migrations using a temporary container connected to the same network
MIGRATION_OUTPUT=$(exec_container "docker run --rm --network $NETWORK_NAME \
    -e DATABASE_URL=postgresql://blockscout:blockscout@postgres:5432/blockscout \
    -e ETHEREUM_JSONRPC_HTTP_URL=http://${RPC_PUBLIC_1:-192.168.11.221}:8545 \
    -e CHAIN_ID=138 \
    -e SECRET_KEY_BASE=73159c7d10b9a5a75ddf10710773078c078bf02124d35b72fa2a841b30b4f88c7c43e5caaf7f9f7f87d16dd66e7870931ae11039c428d1dedae187af762531fa \
    blockscout/blockscout:latest /app/bin/blockscout eval 'Explorer.Release.migrate()' 2>&1" || echo "MIGRATION_FAILED")

if echo "$MIGRATION_OUTPUT" | grep -qiE "(migrated|success|completed|already|up to date)"; then
    log_success "Database migrations completed successfully"
    echo "$MIGRATION_OUTPUT" | grep -iE "(migrated|success|completed|already|up to date)" | head -10
else
    if echo "$MIGRATION_OUTPUT" | grep -qiE "(error|failed|exception)"; then
        log_error "Migration encountered errors:"
        echo "$MIGRATION_OUTPUT" | grep -iE "(error|failed|exception)" | head -10
        log_warn "Continuing to verify database state..."
    else
        log_warn "Migration output unclear, but continuing..."
        echo "$MIGRATION_OUTPUT" | tail -20
    fi
fi

# Step 5: Verify tables were created
log_step "Step 5: Verifying database tables were created..."
sleep 3
NEW_TABLE_COUNT=$(exec_container "docker exec blockscout-postgres psql -U blockscout -d blockscout -t -c \"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public';\" 2>/dev/null | tr -d ' ' || echo '0'")
log_info "New table count: $NEW_TABLE_COUNT"

if [ "$NEW_TABLE_COUNT" != "0" ] && [ -n "$NEW_TABLE_COUNT" ] && [ "$NEW_TABLE_COUNT" != "null" ] && [ "$NEW_TABLE_COUNT" != "$TABLE_COUNT" ]; then
    log_success "Database tables created! ($NEW_TABLE_COUNT tables)"
    
    # List some key tables
    KEY_TABLES=$(exec_container "docker exec blockscout-postgres psql -U blockscout -d blockscout -t -c \"SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename IN ('blocks', 'transactions', 'addresses', 'migrations_status', 'schema_migrations') ORDER BY tablename;\" 2>/dev/null | tr -d ' ' | grep -v '^$' || echo ''")
    if [ -n "$KEY_TABLES" ]; then
        log_info "Key tables found:"
        echo "$KEY_TABLES" | while read -r table; do
            echo "  ✓ $table"
        done
    fi
elif [ "$NEW_TABLE_COUNT" = "$TABLE_COUNT" ] && [ "$NEW_TABLE_COUNT" != "0" ]; then
    log_info "Tables already exist ($NEW_TABLE_COUNT tables) - migrations may have already run"
else
    log_error "No tables found after migration attempt"
    log_info "Migration output was:"
    echo "$MIGRATION_OUTPUT" | tail -30
    exit 1
fi

# Step 6: Start Blockscout
log_step "Step 6: Starting Blockscout container..."
exec_container "cd /opt/blockscout && docker-compose up -d blockscout 2>/dev/null || docker compose up -d blockscout 2>/dev/null || true"
log_success "Blockscout start command executed"

# Step 7: Wait for Blockscout to initialize
log_step "Step 7: Waiting for Blockscout to initialize..."
log_info "Waiting 60 seconds for application to start..."
for i in {1..12}; do
    sleep 5
    if exec_container "docker ps | grep -q 'blockscout.*Up'"; then
        log_info "Container is running... ($((i*5))s elapsed)"
    fi
done

# Step 8: Check container status
log_step "Step 8: Checking Blockscout container status..."
CONTAINER_STATUS=$(exec_container "docker ps --format '{{.Names}}\t{{.Status}}' | grep blockscout | grep -v postgres" || echo "")
if echo "$CONTAINER_STATUS" | grep -q "Up"; then
    log_success "Blockscout container is running"
    echo "$CONTAINER_STATUS"
else
    log_warn "Blockscout container status unclear"
    echo "$CONTAINER_STATUS"
fi

# Step 9: Check recent logs for errors
log_step "Step 9: Checking Blockscout logs for startup status..."
log_info "Checking last 30 lines of logs..."
RECENT_LOGS=$(exec_container "docker logs --tail 30 blockscout 2>&1" || echo "")

if echo "$RECENT_LOGS" | grep -qiE "(running|listening|started|phoenix)"; then
    log_success "Positive indicators in logs:"
    echo "$RECENT_LOGS" | grep -iE "(running|listening|started|phoenix)" | head -5
fi

if echo "$RECENT_LOGS" | grep -qiE "(error.*migration|undefined_table|relation.*does not exist)"; then
    log_error "Migration-related errors still present in logs:"
    echo "$RECENT_LOGS" | grep -iE "(error.*migration|undefined_table|relation.*does not exist)" | head -5
else
    log_info "No migration errors in recent logs"
fi

# Step 10: Test API endpoint
log_step "Step 10: Testing Blockscout API endpoint..."
log_info "Testing: http://localhost:4000/api/v2/status"
sleep 10

API_TEST=$(exec_container "timeout 15 curl -s -w '\nHTTP_CODE:%{http_code}' http://localhost:4000/api/v2/status 2>&1" || echo "FAILED")

if echo "$API_TEST" | grep -q "HTTP_CODE:200" || echo "$API_TEST" | grep -qiE "(chain_id|success)"; then
    log_success "Blockscout API is responding!"
    echo "$API_TEST" | grep -v "HTTP_CODE" | head -5
else
    HTTP_CODE=$(echo "$API_TEST" | grep "HTTP_CODE:" | cut -d':' -f2 || echo "unknown")
    log_warn "API test returned HTTP $HTTP_CODE"
    echo "$API_TEST" | grep -v "HTTP_CODE" | head -3
fi

# Step 11: Test Nginx HTTPS endpoint
log_step "Step 11: Testing Nginx HTTPS endpoint..."
log_info "Testing: https://localhost/health"

HTTPS_TEST=$(exec_container "timeout 15 curl -k -s -w '\nHTTP_CODE:%{http_code}' https://localhost/health 2>&1" || echo "FAILED")

if echo "$HTTPS_TEST" | grep -q "HTTP_CODE:200"; then
    log_success "Nginx HTTPS endpoint is responding!"
elif echo "$HTTPS_TEST" | grep -q "HTTP_CODE:502"; then
    log_warn "Nginx returning 502 - Blockscout may still be initializing"
else
    HTTP_CODE=$(echo "$HTTPS_TEST" | grep "HTTP_CODE:" | cut -d':' -f2 || echo "unknown")
    log_info "HTTPS test returned HTTP $HTTP_CODE"
fi

# Step 12: Test external endpoint
log_step "Step 12: Testing external endpoint..."
log_info "Testing: https://$DOMAIN/health"

EXTERNAL_TEST=$(curl -k -s -w "\nHTTP_CODE:%{http_code}" "https://$DOMAIN/health" 2>&1 || echo "FAILED")

if echo "$EXTERNAL_TEST" | grep -q "HTTP_CODE:200"; then
    log_success "External endpoint is accessible!"
elif echo "$EXTERNAL_TEST" | grep -q "HTTP_CODE:502"; then
    log_warn "External endpoint returning 502 - Blockscout may need more time to initialize"
else
    HTTP_CODE=$(echo "$EXTERNAL_TEST" | grep "HTTP_CODE:" | cut -d':' -f2 || echo "unknown")
    log_info "External test returned HTTP $HTTP_CODE"
fi

echo ""
echo "════════════════════════════════════════════════════════"
echo "Migration and Verification Complete"
echo "════════════════════════════════════════════════════════"
echo ""

# Summary
log_step "Summary:"
echo "  Database Tables: $NEW_TABLE_COUNT tables"
echo "  Container Status: $(echo "$CONTAINER_STATUS" | awk '{print $2}' || echo 'check logs')"
echo "  API Endpoint: $(if echo "$API_TEST" | grep -q "HTTP_CODE:200"; then echo '✓ Working'; else echo '⚠ Check logs'; fi)"
echo "  HTTPS Endpoint: $(if echo "$HTTPS_TEST" | grep -q "HTTP_CODE:200"; then echo '✓ Working'; else echo '⚠ May need time'; fi)"
echo "  External Access: $(if echo "$EXTERNAL_TEST" | grep -q "HTTP_CODE:200"; then echo '✓ Working'; else echo '⚠ Check Cloudflare'; fi)"
echo ""

log_info "Next steps (if needed):"
echo "  1. Monitor logs: ssh root@$IP 'docker logs -f blockscout'"
echo "  2. Check API: curl http://$IP:4000/api/v2/status"
echo "  3. Check HTTPS: curl -k https://$IP/health"
echo "  4. Test external: curl https://$DOMAIN/health"
echo "  5. Wait 2-5 minutes if still initializing (first startup takes time)"
echo ""