2026-03-02 11:37:34 -08:00
#!/usr/bin/env bash
2026-02-12 15:46:57 -08:00
# Comprehensive network configuration review and testing
set -uo pipefail
NODE_IP = " ${ PROXMOX_HOST_R630_01 } "
GATEWAY = " ${ NETWORK_GATEWAY :- 192 .168.11.1 } "
# All containers to test
declare -a all_containers = ( 3000 3001 3002 3003 3500 3501 5200 6000 6400 10000 10001 10020 10030 10040 10050 10060 10070 10080 10090 10091 10092 10100 10101 10120 10130 10150 10151 10200 10201 10202 10210 10230 10232)
echo "═══════════════════════════════════════════════════════════"
echo "Comprehensive Network Configuration Review"
echo "═══════════════════════════════════════════════════════════"
echo " Date: $( date) "
echo " Node: $NODE_IP "
echo ""
# Section 1: Check Proxmox Network Configurations
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "1. PROXMOX NETWORK CONFIGURATIONS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
total = 0
configured = 0
missing = 0
issues = 0
for vmid in " ${ all_containers [@] } " ; do
hostname = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct config $vmid 2>/dev/null | grep '^hostname:' | sed 's/^hostname: //' " || echo "unknown" )
net0 = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct config $vmid 2>/dev/null | grep '^net0:' " || echo "" )
onboot = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct config $vmid 2>/dev/null | grep '^onboot:' | sed 's/^onboot: //' " || echo "0" )
hookscript = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct config $vmid 2>/dev/null | grep '^hookscript:' | sed 's/^hookscript: //' " || echo "none" )
status = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct status $vmid 2>/dev/null | awk '{print \$2}' " || echo "unknown" )
( ( total++) )
if [ -z " $net0 " ] ; then
printf "❌ CT %-6s %-30s Status: %-8s Config: MISSING\\n" " $vmid " " $hostname " " $status "
( ( missing++) )
( ( issues++) )
else
ip = $( echo " $net0 " | grep -oP 'ip=\\K[^,]+' | cut -d'/' -f1 || echo "N/A" )
gw = $( echo " $net0 " | grep -oP 'gw=\\K[^,]+' || echo "N/A" )
bridge = $( echo " $net0 " | grep -oP 'bridge=\\K[^,]+' || echo "N/A" )
printf "✅ CT %-6s %-30s Status: %-8s IP: %-15s\\n" " $vmid " " $hostname " " $status " " $ip "
( ( configured++) )
# Check for issues
if [ " $status " != "running" ] ; then
( ( issues++) )
fi
if [ " $onboot " != "1" ] && [ -n " $net0 " ] ; then
printf " ⚠️ onboot not set\\n"
( ( issues++) )
fi
if [ " $hookscript " = "none" ] && [ [ " $vmid " = ~ ^( 10000| 10001| 10020| 10030| 10040| 10050| 10060| 10070| 10080| 10090| 10091| 10092| 10200| 10201| 10202| 10210| 10230| 10232) $ ] ] ; then
printf " ⚠️ hookscript not set (should have for reassigned containers)\\n"
( ( issues++) )
fi
fi
done
echo ""
echo "Summary:"
echo " Total containers: $total "
echo " Configured: $configured "
echo " Missing config: $missing "
echo " Issues found: $issues "
echo ""
# Section 2: Check Network Interfaces Inside Containers
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "2. NETWORK INTERFACES INSIDE CONTAINERS"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
up = 0
down = 0
no_ip = 0
for vmid in " ${ all_containers [@] } " ; do
status = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct status $vmid 2>/dev/null | awk '{print \$2}' " || echo "unknown" )
if [ " $status " != "running" ] ; then
continue
fi
hostname = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct config $vmid 2>/dev/null | grep '^hostname:' | sed 's/^hostname: //' " || echo "unknown" )
# Check interface status
iface_status = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct exec $vmid -- ip link show eth0 2>/dev/null | grep -oP 'state \\K[^ ]+' || echo 'N/A' " || echo "N/A" )
# Check IP address
ip = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct exec $vmid -- ip addr show eth0 2>/dev/null | grep 'inet ' | awk '{print \$2}' | cut -d'/' -f1 " || echo "N/A" )
# Check default route
route = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct exec $vmid -- ip route | grep default | awk '{print \$3}' " || echo "N/A" )
if [ " $iface_status " = "UP" ] && [ " $ip " != "N/A" ] && [ -n " $ip " ] ; then
printf "✅ CT %-6s %-30s Interface: UP, IP: %-15s, Gateway: %s\\n" " $vmid " " $hostname " " $ip " " $route "
( ( up++) )
elif [ " $iface_status " = "DOWN" ] ; then
printf "❌ CT %-6s %-30s Interface: DOWN\\n" " $vmid " " $hostname "
( ( down++) )
( ( issues++) )
elif [ " $ip " = "N/A" ] || [ -z " $ip " ] ; then
printf "⚠️ CT %-6s %-30s Interface: %-4s, IP: NOT CONFIGURED\\n" " $vmid " " $hostname " " $iface_status "
( ( no_ip++) )
( ( issues++) )
fi
done
echo ""
echo "Summary:"
echo " Interfaces UP with IP: $up "
echo " Interfaces DOWN: $down "
echo " No IP configured: $no_ip "
echo ""
# Section 3: Gateway Connectivity Test
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "3. GATEWAY CONNECTIVITY TEST"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo " Testing connectivity to gateway: $GATEWAY "
echo ""
gateway_success = 0
gateway_failed = 0
for vmid in " ${ all_containers [@] } " ; do
status = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct status $vmid 2>/dev/null | awk '{print \$2}' " || echo "unknown" )
if [ " $status " != "running" ] ; then
continue
fi
hostname = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct config $vmid 2>/dev/null | grep '^hostname:' | sed 's/^hostname: //' " || echo "unknown" )
result = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" timeout 3 pct exec $vmid -- ping -c 1 $GATEWAY 2>&1 " | grep -E '1 received|1 packets received' || echo "failed" )
if echo " $result " | grep -q "1 received\|1 packets received" ; then
printf "✅ CT %-6s %-30s Gateway: REACHABLE\\n" " $vmid " " $hostname "
( ( gateway_success++) )
else
printf "❌ CT %-6s %-30s Gateway: UNREACHABLE\\n" " $vmid " " $hostname "
( ( gateway_failed++) )
( ( issues++) )
fi
done
echo ""
echo "Summary:"
echo " Gateway reachable: $gateway_success "
echo " Gateway unreachable: $gateway_failed "
echo ""
# Section 4: Inter-Container Connectivity Test
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "4. INTER-CONTAINER CONNECTIVITY TEST"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Testing connectivity between key containers"
echo ""
# Test matrix: from_container -> to_container
test_pairs = (
" 10100: ${ PROXMOX_HOST_ML110 } 5:10000: ${ ORDER_POSTGRES_PRIMARY :- ${ ORDER_POSTGRES_PRIMARY :- 192 .168.11.44 } } :DBIS PostgreSQL:Order PostgreSQL "
" 10100: ${ PROXMOX_HOST_ML110 } 5:10120: ${ PROXMOX_HOST_R630_02 } 0:DBIS PostgreSQL:DBIS Redis "
" 10000: ${ ORDER_POSTGRES_PRIMARY :- ${ ORDER_POSTGRES_PRIMARY :- 192 .168.11.44 } } :10001: ${ ORDER_POSTGRES_REPLICA :- ${ ORDER_POSTGRES_REPLICA :- 192 .168.11.45 } } :Order PostgreSQL Primary:Order PostgreSQL Replica "
" 10000: ${ ORDER_POSTGRES_PRIMARY :- ${ ORDER_POSTGRES_PRIMARY :- 192 .168.11.44 } } :10020: ${ ORDER_REDIS_IP :- 192 .168.11.38 } :Order PostgreSQL:Order Redis "
" 10130: ${ IP_DBIS_FRONTEND :- ${ IP_SERVICE_13 :- ${ IP_SERVICE_13 :- ${ IP_SERVICE_13 :- ${ IP_SERVICE_13 :- ${ IP_SERVICE_13 :- 192 .168.11.13 } } } } } 0 } :10150: ${ IP_DBIS_API :- ${ IP_DBIS_API :- 192 .168.11.155 } } :DBIS Frontend:DBIS API "
" 10130: ${ IP_DBIS_FRONTEND :- ${ IP_SERVICE_13 :- ${ IP_SERVICE_13 :- ${ IP_SERVICE_13 :- ${ IP_SERVICE_13 :- ${ IP_SERVICE_13 :- 192 .168.11.13 } } } } } 0 } :10090: ${ IP_SERVICE_36 :- ${ IP_SERVICE_36 :- ${ IP_SERVICE_36 :- ${ IP_SERVICE_36 :- ${ IP_SERVICE_36 :- ${ IP_SERVICE_36 :- 192 .168.11.36 } } } } } } :DBIS Frontend:Order Portal "
)
inter_container_success = 0
inter_container_failed = 0
for pair in " ${ test_pairs [@] } " ; do
IFS = ':' read -r from_vmid from_ip to_vmid to_ip from_name to_name <<< " $pair "
from_status = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct status $from_vmid 2>/dev/null | awk '{print \$2}' " || echo "unknown" )
to_status = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct status $to_vmid 2>/dev/null | awk '{print \$2}' " || echo "unknown" )
if [ " $from_status " != "running" ] || [ " $to_status " != "running" ] ; then
printf "⏸️ CT %-6s → CT %-6s (%s → %s): SKIPPED (not running)\\n" " $from_vmid " " $to_vmid " " $from_name " " $to_name "
continue
fi
result = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" timeout 3 pct exec $from_vmid -- ping -c 1 $to_ip 2>&1 " | grep -E '1 received|1 packets received' || echo "failed" )
if echo " $result " | grep -q "1 received\|1 packets received" ; then
printf "✅ CT %-6s → CT %-6s (%s → %s): REACHABLE\\n" " $from_vmid " " $to_vmid " " $from_name " " $to_name "
( ( inter_container_success++) )
else
printf "❌ CT %-6s → CT %-6s (%s → %s): UNREACHABLE\\n" " $from_vmid " " $to_vmid " " $from_name " " $to_name "
( ( inter_container_failed++) )
( ( issues++) )
fi
done
echo ""
echo "Summary:"
echo " Inter-container reachable: $inter_container_success "
echo " Inter-container unreachable: $inter_container_failed "
echo ""
# Section 5: DNS Resolution Test
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "5. DNS RESOLUTION TEST"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Testing DNS resolution to 8.8.8.8"
echo ""
dns_success = 0
dns_failed = 0
# Test on a few representative containers
test_containers = ( 10000 10100 10130 10200)
for vmid in " ${ test_containers [@] } " ; do
status = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct status $vmid 2>/dev/null | awk '{print \$2}' " || echo "unknown" )
if [ " $status " != "running" ] ; then
continue
fi
hostname = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" pct config $vmid 2>/dev/null | grep '^hostname:' | sed 's/^hostname: //' " || echo "unknown" )
result = $( ssh -o ConnectTimeout = 5 -o StrictHostKeyChecking = no root@${ NODE_IP } \
" timeout 3 pct exec $vmid -- ping -c 1 8.8.8.8 2>&1 " | grep -E '1 received|1 packets received' || echo "failed" )
if echo " $result " | grep -q "1 received\|1 packets received" ; then
printf "✅ CT %-6s %-30s DNS server (8.8.8.8): REACHABLE\\n" " $vmid " " $hostname "
( ( dns_success++) )
else
printf "❌ CT %-6s %-30s DNS server (8.8.8.8): UNREACHABLE\\n" " $vmid " " $hostname "
( ( dns_failed++) )
( ( issues++) )
fi
done
echo ""
echo "Summary:"
echo " DNS reachable: $dns_success "
echo " DNS unreachable: $dns_failed "
echo ""
# Final Summary
echo "═══════════════════════════════════════════════════════════"
echo "FINAL SUMMARY"
echo "═══════════════════════════════════════════════════════════"
echo " Total containers reviewed: $total "
echo " Network configurations: $configured / $total "
echo " Interfaces UP with IP: $up "
echo " Gateway connectivity: $gateway_success successful "
echo " Inter-container connectivity: $inter_container_success successful "
echo " DNS connectivity: $dns_success successful "
echo ""
echo " Total issues found: $issues "
echo ""
if [ $issues -eq 0 ] ; then
echo "✅ ALL NETWORK CONFIGURATIONS ARE HEALTHY"
else
echo "⚠️ SOME ISSUES FOUND - REVIEW OUTPUT ABOVE"
fi
echo "═══════════════════════════════════════════════════════════"