proxmox/scripts/complete-blockscout-firewall-fix.sh

#!/usr/bin/env bash
# Complete Blockscout Firewall Fix - Run all checks and tests
# This script performs comprehensive checks and provides final status

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"
VMID_CLOUDFLARED=102
VMID_BLOCKSCOUT=5000
BLOCKSCOUT_IP="${IP_BLOCKSCOUT}"
BLOCKSCOUT_PORT=80
BLOCKSCOUT_URL="https://explorer.d-bis.org"

# Function to find which node a container is on
find_container_node() {
    local vmid=$1
    # Try to find the container across all nodes
    ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh get /nodes --output-format json" 2>/dev/null | \
        grep -o '"[^"]*"' | tr -d '"' | while read node; do
            if ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pct status $vmid" 2>/dev/null | grep -q "running\|stopped"; then
                echo "$node"
                return
            fi
        done
    # Fallback: check common nodes
    for node in ml110 r630-01 r630-02; do
        if ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh get /nodes/$node/lxc/$vmid/status/current" 2>/dev/null | grep -q "status"; then
            echo "$node"
            return
        fi
    done
    echo "ml110"  # Default fallback
}

echo "════════════════════════════════════════════════════════"
echo "Complete Blockscout Firewall Fix - Comprehensive Check"
echo "════════════════════════════════════════════════════════"
echo ""

# Function to print status
print_status() {
    local status=$1
    local message=$2
    if [ "$status" = "OK" ]; then
        echo -e "${GREEN}✓${NC} $message"
    elif [ "$status" = "FAIL" ]; then
        echo -e "${RED}✗${NC} $message"
    elif [ "$status" = "WARN" ]; then
        echo -e "${YELLOW}⚠${NC} $message"
    else
        echo -e "${BLUE}ℹ${NC} $message"
    fi
}

# Step 1: Check container status
echo "Step 1: Checking container status..."
echo "────────────────────────────────────────────────────────"

# Find container nodes
CLOUDFLARED_NODE=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "for node in \$(pvesh get /nodes --output-format json-pretty 2>/dev/null | grep -o '\"node\":\"[^\"]*\"' | cut -d'\"' -f4 | sort -u); do if pvesh get /nodes/\$node/lxc/$VMID_CLOUDFLARED/status/current 2>/dev/null | grep -q status; then echo \$node; break; fi; done" 2>/dev/null || echo "r630-02")
BLOCKSCOUT_NODE=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "for node in \$(pvesh get /nodes --output-format json-pretty 2>/dev/null | grep -o '\"node\":\"[^\"]*\"' | cut -d'\"' -f4 | sort -u); do if pvesh get /nodes/\$node/lxc/$VMID_BLOCKSCOUT/status/current 2>/dev/null | grep -q status; then echo \$node; break; fi; done" 2>/dev/null || echo "r630-02")

# Check cloudflared container
CLOUDFLARED_STATUS=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh get /nodes/$CLOUDFLARED_NODE/lxc/$VMID_CLOUDFLARED/status/current --output-format json-pretty 2>/dev/null | grep -o '\"status\":\"[^\"]*\"' | cut -d'\"' -f4" || echo "unknown")
if [ "$CLOUDFLARED_STATUS" = "running" ]; then
    print_status "OK" "cloudflared container (VMID $VMID_CLOUDFLARED) is running on $CLOUDFLARED_NODE"
else
    print_status "WARN" "cloudflared container (VMID $VMID_CLOUDFLARED) status: $CLOUDFLARED_STATUS on $CLOUDFLARED_NODE"
    echo "  Attempting to start container..."
    ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh create /nodes/$CLOUDFLARED_NODE/lxc/$VMID_CLOUDFLARED/status/start" 2>/dev/null || true
    sleep 5
    CLOUDFLARED_STATUS=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh get /nodes/$CLOUDFLARED_NODE/lxc/$VMID_CLOUDFLARED/status/current --output-format json-pretty 2>/dev/null | grep -o '\"status\":\"[^\"]*\"' | cut -d'\"' -f4" || echo "unknown")
    if [ "$CLOUDFLARED_STATUS" = "running" ]; then
        print_status "OK" "cloudflared container started successfully"
    fi
fi

# Check Blockscout container
BLOCKSCOUT_STATUS=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh get /nodes/$BLOCKSCOUT_NODE/lxc/$VMID_BLOCKSCOUT/status/current --output-format json-pretty 2>/dev/null | grep -o '\"status\":\"[^\"]*\"' | cut -d'\"' -f4" || echo "unknown")
if [ "$BLOCKSCOUT_STATUS" = "running" ]; then
    print_status "OK" "Blockscout container (VMID $VMID_BLOCKSCOUT) is running on $BLOCKSCOUT_NODE"
else
    print_status "WARN" "Blockscout container (VMID $VMID_BLOCKSCOUT) status: $BLOCKSCOUT_STATUS on $BLOCKSCOUT_NODE"
    echo "  Attempting to start container..."
    ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh create /nodes/$BLOCKSCOUT_NODE/lxc/$VMID_BLOCKSCOUT/status/start" 2>/dev/null || true
    sleep 10
    BLOCKSCOUT_STATUS=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh get /nodes/$BLOCKSCOUT_NODE/lxc/$VMID_BLOCKSCOUT/status/current --output-format json-pretty 2>/dev/null | grep -o '\"status\":\"[^\"]*\"' | cut -d'\"' -f4" || echo "unknown")
    if [ "$BLOCKSCOUT_STATUS" = "running" ]; then
        print_status "OK" "Blockscout container started successfully"
    fi
fi

echo ""

# Step 2: Test internal connectivity (cloudflared → Blockscout)
echo "Step 2: Testing internal connectivity (cloudflared → Blockscout)..."
echo "────────────────────────────────────────────────────────"

CONNECTIVITY_TEST=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pvesh create /nodes/$CLOUDFLARED_NODE/lxc/$VMID_CLOUDFLARED/exec --command 'curl' --arg curl,-s,-o,/dev/null,-w,'%{http_code}',--connect-timeout,5,http://$BLOCKSCOUT_IP:$BLOCKSCOUT_PORT/health 2>&1" || echo "000")

if [ "$CONNECTIVITY_TEST" = "200" ] || [ "$CONNECTIVITY_TEST" = "302" ] || [ "$CONNECTIVITY_TEST" = "301" ]; then
    print_status "OK" "Internal connectivity successful (HTTP $CONNECTIVITY_TEST)"
    INTERNAL_CONNECTIVITY_OK=true
elif echo "$CONNECTIVITY_TEST" | grep -q "No route to host\|Connection refused\|Connection timed out"; then
    print_status "FAIL" "Internal connectivity failed: No route to host"
    echo "  This indicates a firewall rule is blocking traffic"
    INTERNAL_CONNECTIVITY_OK=false
else
    print_status "WARN" "Internal connectivity test returned: HTTP $CONNECTIVITY_TEST"
    echo "  Response: $CONNECTIVITY_TEST"
    INTERNAL_CONNECTIVITY_OK=false
fi

echo ""

# Step 3: Test Blockscout service directly
echo "Step 3: Testing Blockscout service status..."
echo "────────────────────────────────────────────────────────"

# Check if Blockscout service is running inside container
BLOCKSCOUT_SERVICE=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "ssh $BLOCKSCOUT_NODE 'pct exec $VMID_BLOCKSCOUT -- systemctl is-active blockscout 2>/dev/null || pct exec $VMID_BLOCKSCOUT -- docker ps 2>/dev/null | grep -q blockscout && echo running || echo not-running'" 2>/dev/null || echo "unknown")

if [ "$BLOCKSCOUT_SERVICE" = "active" ] || [ "$BLOCKSCOUT_SERVICE" = "running" ]; then
    print_status "OK" "Blockscout service is running"
else
    print_status "WARN" "Blockscout service status: $BLOCKSCOUT_SERVICE"
    echo "  Attempting to start Blockscout service..."
    ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pct exec $VMID_BLOCKSCOUT -- systemctl start blockscout 2>/dev/null || pct exec $VMID_BLOCKSCOUT -- cd /root/blockscout && docker-compose up -d 2>/dev/null || true" 2>/dev/null || true
    sleep 5
fi

# Check Nginx status  
NGINX_STATUS=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "ssh $BLOCKSCOUT_NODE 'pct exec $VMID_BLOCKSCOUT -- systemctl is-active nginx 2>/dev/null'" || echo "inactive")
if [ "$NGINX_STATUS" = "active" ]; then
    print_status "OK" "Nginx is running"
else
    print_status "WARN" "Nginx status: $NGINX_STATUS"
fi

echo ""

# Step 4: Test external connectivity (via Cloudflare Tunnel)
echo "Step 4: Testing external connectivity (via Cloudflare Tunnel)..."
echo "────────────────────────────────────────────────────────"

EXTERNAL_TEST=$(curl -s -o /dev/null -w '%{http_code}' --connect-timeout 10 "$BLOCKSCOUT_URL/health" 2>&1 || echo "000")

if [ "$EXTERNAL_TEST" = "200" ]; then
    print_status "OK" "External connectivity successful (HTTP 200)"
    EXTERNAL_CONNECTIVITY_OK=true
elif [ "$EXTERNAL_TEST" = "502" ]; then
    print_status "FAIL" "External connectivity failed: HTTP 502 Bad Gateway"
    echo "  This indicates Cloudflare Tunnel can't reach Blockscout (likely firewall issue)"
    EXTERNAL_CONNECTIVITY_OK=false
elif [ "$EXTERNAL_TEST" = "522" ]; then
    print_status "FAIL" "External connectivity failed: HTTP 522 Connection Timed Out"
    echo "  This indicates Cloudflare can't connect to the origin server"
    EXTERNAL_CONNECTIVITY_OK=false
else
    print_status "WARN" "External connectivity test returned: HTTP $EXTERNAL_TEST"
    EXTERNAL_CONNECTIVITY_OK=false
fi

echo ""

# Step 5: Check Omada firewall configuration (via API if possible)
echo "Step 5: Checking Omada firewall configuration..."
echo "────────────────────────────────────────────────────────"

if command -v node &> /dev/null; then
    if [ -f "scripts/query-omada-firewall-blockscout-direct.js" ]; then
        print_status "INFO" "Attempting to query Omada Controller API..."
        if node scripts/query-omada-firewall-blockscout-direct.js 2>&1 | grep -q "Found.*firewall rules"; then
            print_status "OK" "Omada Controller API accessible"
            node scripts/query-omada-firewall-blockscout-direct.js 2>&1 | tail -20
        else
            print_status "WARN" "Omada Controller API doesn't expose firewall rules"
            echo "  Firewall rules must be configured via web interface"
        fi
    else
        print_status "WARN" "Omada query script not found"
    fi
else
    print_status "WARN" "Node.js not available - skipping API check"
fi

echo ""

# Step 6: Final Summary and Recommendations
echo "════════════════════════════════════════════════════════"
echo "Final Summary and Recommendations"
echo "════════════════════════════════════════════════════════"
echo ""

if [ "${INTERNAL_CONNECTIVITY_OK:-false}" = "true" ] && [ "${EXTERNAL_CONNECTIVITY_OK:-false}" = "true" ]; then
    print_status "OK" "All connectivity tests passed!"
    echo ""
    echo "✅ Blockscout is accessible both internally and externally"
    echo "✅ No firewall configuration changes needed"
    echo ""
    echo "You can access Blockscout at: $BLOCKSCOUT_URL"
elif [ "${INTERNAL_CONNECTIVITY_OK:-false}" = "false" ]; then
    print_status "FAIL" "Internal connectivity issue detected"
    echo ""
    echo "🔧 Action Required: Configure Omada Firewall Rules"
    echo ""
    echo "The firewall is blocking traffic from cloudflared ($VMID_CLOUDFLARED) to Blockscout ($BLOCKSCOUT_IP:$BLOCKSCOUT_PORT)"
    echo ""
    echo "Steps to fix:"
    echo "  1. Access Omada Cloud Controller: https://omada.tplinkcloud.com"
    echo "     Or run: bash scripts/access-omada-cloud-controller.sh"
    echo ""
    echo "  2. Navigate to: Settings → Firewall → Firewall Rules"
    echo ""
    echo "  3. Create an allow rule:"
    echo "     Name: Allow Internal to Blockscout HTTP"
    echo "     Enable: Yes"
    echo "     Action: Allow"
    echo "     Direction: Forward"
    echo "     Protocol: TCP"
    echo "     Source IP: ${NETWORK_192_168_11_0:-192.168.11.0}/24"
    echo "     Destination IP: $BLOCKSCOUT_IP"
    echo "     Destination Port: $BLOCKSCOUT_PORT"
    echo "     Priority: High (above deny rules)"
    echo ""
    echo "  4. Save and apply the configuration"
    echo ""
    echo "  5. Re-run this script to verify: bash scripts/complete-blockscout-firewall-fix.sh"
    echo ""
elif [ "${EXTERNAL_CONNECTIVITY_OK:-false}" = "false" ] && [ "${INTERNAL_CONNECTIVITY_OK:-false}" = "true" ]; then
    print_status "WARN" "Internal connectivity OK but external connectivity failed"
    echo ""
    echo "This suggests a Cloudflare Tunnel configuration issue rather than firewall."
    echo "Check Cloudflare Tunnel route configuration."
    echo ""
else
    print_status "FAIL" "Multiple connectivity issues detected"
    echo ""
    echo "Both internal and external connectivity tests failed."
    echo "Check both firewall rules and service status."
    echo ""
fi

echo "════════════════════════════════════════════════════════"
echo "Documentation:"
echo "  - docs/OMADA_CLOUD_ACCESS_SUMMARY.md"
echo "  - docs/OMADA_CLOUD_CONTROLLER_FIREWALL_GUIDE.md"
echo "  - scripts/access-omada-cloud-controller.sh"
echo "════════════════════════════════════════════════════════"