proxmox/scripts/diagnose-explorer-502-error.sh

#!/usr/bin/env bash
set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


# Diagnose 502 Bad Gateway error for explorer.d-bis.org
# Checks Blockscout service, nginx, and connectivity on VMID 5000
# Usage: ./diagnose-explorer-502-error.sh

VMID=5000
PROXMOX_HOST="${1:-pve2}"
BLOCKSCOUT_IP="${IP_BLOCKSCOUT}"
BLOCKSCOUT_PORT=4000

echo "=========================================="
echo "Explorer 502 Bad Gateway Diagnostic"
echo "=========================================="
echo "VMID: $VMID ($BLOCKSCOUT_IP)"
echo "Blockscout Expected Port: $BLOCKSCOUT_PORT"
echo "Public URL: https://explorer.d-bis.org"
echo "=========================================="
echo ""

# Check if we're on Proxmox host
if ! command -v pct &>/dev/null; then
    echo "⚠️  pct command not available"
    echo "   This script should be run on Proxmox host or with SSH access"
    echo "   Using SSH to $PROXMOX_HOST..."
    EXEC_PREFIX="ssh root@$PROXMOX_HOST"
else
    EXEC_PREFIX=""
fi

# 1. Check container status
echo "=== 1. Checking Container Status ==="
if [ -n "$EXEC_PREFIX" ]; then
    CONTAINER_STATUS=$($EXEC_PREFIX "pct status $VMID 2>/dev/null" || echo "not found")
else
    CONTAINER_STATUS=$(pct status $VMID 2>/dev/null || echo "not found")
fi

if echo "$CONTAINER_STATUS" | grep -q "running"; then
    echo "   ✅ Container VMID $VMID is running"
    CONTAINER_RUNNING=true
else
    echo "   ❌ Container VMID $VMID is NOT running"
    echo "   Status: $CONTAINER_STATUS"
    echo "   💡 Start with: pct start $VMID"
    CONTAINER_RUNNING=false
fi
echo ""

if [ "$CONTAINER_RUNNING" = false ]; then
    echo "=========================================="
    echo "SUMMARY: Container is not running"
    echo "=========================================="
    echo "Fix: pct start $VMID"
    exit 1
fi

# 2. Check nginx service
echo "=== 2. Checking Nginx Service ==="
if [ -n "$EXEC_PREFIX" ]; then
    NGINX_STATUS=$($EXEC_PREFIX "pct exec $VMID -- systemctl is-active nginx 2>/dev/null" || echo "inactive")
    NGINX_ENABLED=$($EXEC_PREFIX "pct exec $VMID -- systemctl is-enabled nginx 2>/dev/null" || echo "disabled")
else
    NGINX_STATUS=$(pct exec $VMID -- systemctl is-active nginx 2>/dev/null || echo "inactive")
    NGINX_ENABLED=$(pct exec $VMID -- systemctl is-enabled nginx 2>/dev/null || echo "disabled")
fi

if [ "$NGINX_STATUS" = "active" ]; then
    echo "   ✅ Nginx is running"
    NGINX_RUNNING=true
else
    echo "   ❌ Nginx is NOT running (status: $NGINX_STATUS)"
    echo "   💡 Start with: pct exec $VMID -- systemctl start nginx"
    NGINX_RUNNING=false
fi

if [ "$NGINX_ENABLED" = "enabled" ]; then
    echo "   ✅ Nginx is enabled (auto-start on boot)"
else
    echo "   ⚠️  Nginx is not enabled for auto-start"
    echo "   💡 Enable with: pct exec $VMID -- systemctl enable nginx"
fi
echo ""

# 3. Check Blockscout service
echo "=== 3. Checking Blockscout Service ==="
if [ -n "$EXEC_PREFIX" ]; then
    BLOCKSCOUT_SERVICE_STATUS=$($EXEC_PREFIX "pct exec $VMID -- systemctl is-active blockscout.service 2>/dev/null" || echo "inactive")
    BLOCKSCOUT_SERVICE_ENABLED=$($EXEC_PREFIX "pct exec $VMID -- systemctl is-enabled blockscout.service 2>/dev/null" || echo "disabled")
else
    BLOCKSCOUT_SERVICE_STATUS=$(pct exec $VMID -- systemctl is-active blockscout.service 2>/dev/null || echo "inactive")
    BLOCKSCOUT_SERVICE_ENABLED=$(pct exec $VMID -- systemctl is-enabled blockscout.service 2>/dev/null || echo "disabled")
fi

if [ "$BLOCKSCOUT_SERVICE_STATUS" = "active" ]; then
    echo "   ✅ Blockscout service is running"
    BLOCKSCOUT_SERVICE_RUNNING=true
else
    echo "   ❌ Blockscout service is NOT running (status: $BLOCKSCOUT_SERVICE_STATUS)"
    echo "   💡 Start with: pct exec $VMID -- systemctl start blockscout.service"
    BLOCKSCOUT_SERVICE_RUNNING=false
fi

if [ "$BLOCKSCOUT_SERVICE_ENABLED" = "enabled" ]; then
    echo "   ✅ Blockscout service is enabled (auto-start on boot)"
else
    echo "   ⚠️  Blockscout service is not enabled for auto-start"
    echo "   💡 Enable with: pct exec $VMID -- systemctl enable blockscout.service"
fi
echo ""

# 4. Check if port 4000 is listening
echo "=== 4. Checking Port $BLOCKSCOUT_PORT (Blockscout) ==="
if [ -n "$EXEC_PREFIX" ]; then
    PORT_4000_CHECK=$($EXEC_PREFIX "pct exec $VMID -- ss -tlnp 2>/dev/null | grep :$BLOCKSCOUT_PORT || echo 'not listening'")
    PORT_4000_DETAILS=$($EXEC_PREFIX "pct exec $VMID -- netstat -tlnp 2>/dev/null | grep :$BLOCKSCOUT_PORT || echo 'not listening'")
else
    PORT_4000_CHECK=$(pct exec $VMID -- ss -tlnp 2>/dev/null | grep :$BLOCKSCOUT_PORT || echo "not listening")
    PORT_4000_DETAILS=$(pct exec $VMID -- netstat -tlnp 2>/dev/null | grep :$BLOCKSCOUT_PORT || echo "not listening")
fi

if echo "$PORT_4000_CHECK" | grep -q ":$BLOCKSCOUT_PORT"; then
    echo "   ✅ Port $BLOCKSCOUT_PORT is listening"
    echo "   Details: $PORT_4000_CHECK"
    PORT_4000_LISTENING=true
else
    echo "   ❌ Port $BLOCKSCOUT_PORT is NOT listening"
    echo "   💡 This is the likely cause of the 502 error!"
    echo "   💡 Blockscout service may not be started or is not listening on port $BLOCKSCOUT_PORT"
    PORT_4000_LISTENING=false
fi
echo ""

# 5. Test internal Blockscout API
echo "=== 5. Testing Internal Blockscout API ==="
if [ -n "$EXEC_PREFIX" ]; then
    API_TEST=$($EXEC_PREFIX "pct exec $VMID -- curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 http://127.0.0.1:$BLOCKSCOUT_PORT/api/v2/stats 2>/dev/null" || echo "000")
else
    API_TEST=$(pct exec $VMID -- curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 http://127.0.0.1:$BLOCKSCOUT_PORT/api/v2/stats 2>/dev/null || echo "000")
fi

if [ "$API_TEST" = "200" ]; then
    echo "   ✅ Blockscout API is responding (HTTP $API_TEST)"
    API_RESPONDING=true
elif [ "$API_TEST" = "000" ]; then
    echo "   ❌ Blockscout API is not responding (connection failed)"
    echo "   💡 Service is likely not running or not listening on port $BLOCKSCOUT_PORT"
    API_RESPONDING=false
else
    echo "   ⚠️  Blockscout API returned HTTP $API_TEST"
    API_RESPONDING=false
fi
echo ""

# 6. Check nginx configuration
echo "=== 6. Checking Nginx Configuration ==="
if [ -n "$EXEC_PREFIX" ]; then
    NGINX_TEST=$($EXEC_PREFIX "pct exec $VMID -- nginx -t 2>&1" || echo "failed")
else
    NGINX_TEST=$(pct exec $VMID -- nginx -t 2>&1 || echo "failed")
fi

if echo "$NGINX_TEST" | grep -q "test is successful"; then
    echo "   ✅ Nginx configuration is valid"
    NGINX_CONFIG_VALID=true
else
    echo "   ❌ Nginx configuration has errors"
    echo "   Details:"
    echo "$NGINX_TEST" | sed 's/^/      /'
    NGINX_CONFIG_VALID=false
fi
echo ""

# 7. Check nginx error logs
echo "=== 7. Checking Nginx Error Logs (last 10 lines) ==="
if [ -n "$EXEC_PREFIX" ]; then
    NGINX_ERRORS=$($EXEC_PREFIX "pct exec $VMID -- tail -10 /var/log/nginx/error.log 2>/dev/null || echo 'log file not found'" || echo "")
else
    NGINX_ERRORS=$(pct exec $VMID -- tail -10 /var/log/nginx/error.log 2>/dev/null || echo "log file not found")
fi

if [ -n "$NGINX_ERRORS" ] && [ "$NGINX_ERRORS" != "log file not found" ]; then
    echo "   Recent errors:"
    echo "$NGINX_ERRORS" | sed 's/^/      /'
else
    echo "   ℹ️  No recent errors found (or log file not accessible)"
fi
echo ""

# 8. Test external API
echo "=== 8. Testing External API (via domain) ==="
EXTERNAL_API_TEST=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 "https://explorer.d-bis.org/api/v2/stats" 2>/dev/null || echo "000")

if [ "$EXTERNAL_API_TEST" = "200" ]; then
    echo "   ✅ External API is accessible (HTTP $EXTERNAL_API_TEST)"
    EXTERNAL_API_WORKING=true
elif [ "$EXTERNAL_API_TEST" = "502" ]; then
    echo "   ❌ External API returns 502 Bad Gateway"
    echo "   💡 This matches the error you're seeing!"
    echo "   💡 Root cause: Blockscout service is not accessible on port $BLOCKSCOUT_PORT"
    EXTERNAL_API_WORKING=false
elif [ "$EXTERNAL_API_TEST" = "000" ]; then
    echo "   ❌ External API is not accessible (connection failed)"
    EXTERNAL_API_WORKING=false
else
    echo "   ⚠️  External API returned HTTP $EXTERNAL_API_TEST"
    EXTERNAL_API_WORKING=false
fi
echo ""

# Summary
echo "=========================================="
echo "DIAGNOSTIC SUMMARY"
echo "=========================================="
echo "Container Status: $([ "$CONTAINER_RUNNING" = true ] && echo "✅ Running" || echo "❌ Not Running")"
echo "Nginx Status: $([ "$NGINX_RUNNING" = true ] && echo "✅ Running" || echo "❌ Not Running")"
echo "Blockscout Service: $([ "$BLOCKSCOUT_SERVICE_RUNNING" = true ] && echo "✅ Running" || echo "❌ Not Running")"
echo "Port $BLOCKSCOUT_PORT Listening: $([ "$PORT_4000_LISTENING" = true ] && echo "✅ Yes" || echo "❌ No")"
echo "Internal API Test: $([ "$API_RESPONDING" = true ] && echo "✅ Responding" || echo "❌ Not Responding")"
echo "External API Test: $([ "$EXTERNAL_API_WORKING" = true ] && echo "✅ Working" || echo "❌ Not Working ($EXTERNAL_API_TEST)")"
echo "Nginx Config: $([ "$NGINX_CONFIG_VALID" = true ] && echo "✅ Valid" || echo "❌ Invalid")"
echo ""

# Recommendations
echo "=========================================="
echo "RECOMMENDATIONS"
echo "=========================================="

if [ "$BLOCKSCOUT_SERVICE_RUNNING" = false ]; then
    echo "🔴 PRIORITY 1: Start Blockscout Service"
    echo "   pct exec $VMID -- systemctl start blockscout.service"
    echo "   pct exec $VMID -- systemctl enable blockscout.service"
    echo ""
fi

if [ "$PORT_4000_LISTENING" = false ]; then
    echo "🔴 PRIORITY 2: Blockscout is not listening on port $BLOCKSCOUT_PORT"
    echo "   Check Blockscout logs:"
    echo "   pct exec $VMID -- journalctl -u blockscout.service -n 50"
    echo ""
    echo "   Check Docker containers:"
    echo "   pct exec $VMID -- docker ps"
    echo ""
fi

if [ "$NGINX_RUNNING" = false ]; then
    echo "🟠 PRIORITY 3: Start Nginx"
    echo "   pct exec $VMID -- systemctl start nginx"
    echo "   pct exec $VMID -- systemctl enable nginx"
    echo ""
fi

if [ "$NGINX_CONFIG_VALID" = false ]; then
    echo "🟠 PRIORITY 4: Fix Nginx Configuration"
    echo "   pct exec $VMID -- nginx -t"
    echo "   Check config: pct exec $VMID -- cat /etc/nginx/sites-enabled/blockscout"
    echo ""
fi

echo "=========================================="
echo "Quick Fix Commands"
echo "=========================================="
echo "# Start Blockscout service:"
echo "pct exec $VMID -- systemctl start blockscout.service"
echo ""
echo "# Check Blockscout status:"
echo "pct exec $VMID -- systemctl status blockscout.service"
echo ""
echo "# Check Blockscout logs:"
echo "pct exec $VMID -- journalctl -u blockscout.service -n 50"
echo ""
echo "# Restart nginx:"
echo "pct exec $VMID -- systemctl restart nginx"
echo ""
echo "=========================================="