proxmox/scripts/check-ccip-monitor.sh

#!/usr/bin/env bash
# Check CCIP Monitor Service Status
# Usage: ./check-ccip-monitor.sh [VMID]

set -euo pipefail

VMID="${1:-3501}"
PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_section() { echo -e "${CYAN}════════════════════════════════════════${NC}"; }

# Function to run command in container
exec_container() {
    ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pct exec $VMID -- $1" 2>/dev/null || echo ""
}

# Function to check if container exists
check_container_exists() {
    if ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pct list | grep -q '^$VMID'"; then
        return 0
    else
        return 1
    fi
}

log_section
log_info "CCIP Monitor Service Status Check"
log_info "VMID: $VMID"
log_section
echo ""

# Check if container exists
log_info "1. Checking container existence..."
if check_container_exists; then
    log_success "Container $VMID exists"
else
    log_error "Container $VMID not found"
    exit 1
fi
echo ""

# Check container status
log_info "2. Checking container status..."
CONTAINER_STATUS_RAW=$(ssh -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "pct status $VMID 2>/dev/null | head -1 | awk '{print \$2}'" || echo "unknown")
CONTAINER_STATUS=$(echo "$CONTAINER_STATUS_RAW" | tr -d '\r\n' | xargs)
if [ "$CONTAINER_STATUS" = "running" ]; then
    log_success "Container is running"
elif [ "$CONTAINER_STATUS" = "stopped" ]; then
    log_warn "Container is stopped"
    log_info "To start: ssh root@$PROXMOX_HOST 'pct start $VMID'"
elif [ -n "$CONTAINER_STATUS" ] && [ "$CONTAINER_STATUS" != "unknown" ]; then
    log_warn "Container status: $CONTAINER_STATUS"
else
    log_warn "Could not determine container status"
    CONTAINER_STATUS="unknown"
fi
echo ""

# Check systemd service status
log_info "3. Checking systemd service status..."
SERVICE_STATUS=$(exec_container "systemctl is-active ccip-monitor 2>/dev/null || echo 'inactive'")
if [ "$SERVICE_STATUS" = "active" ]; then
    log_success "CCIP Monitor service is active"
    
    # Check if service is enabled
    if exec_container "systemctl is-enabled ccip-monitor 2>/dev/null | grep -q enabled"; then
        log_success "Service is enabled (will start on boot)"
    else
        log_warn "Service is not enabled (won't start on boot)"
    fi
else
    log_warn "CCIP Monitor service is not active (status: $SERVICE_STATUS)"
    log_info "To start: ssh root@$PROXMOX_HOST 'pct exec $VMID -- systemctl start ccip-monitor'"
fi
echo ""

# Check configuration file
log_info "4. Checking configuration file..."
CONFIG_FILE="/opt/ccip-monitor/.env"
if exec_container "[ -f $CONFIG_FILE ]"; then
    log_success "Configuration file exists: $CONFIG_FILE"
    
    # Check key configuration variables
    log_info "   Checking configuration variables..."
    
    RPC_URL=$(exec_container "grep '^RPC_URL' $CONFIG_FILE | cut -d'=' -f2 | tr -d '\"'" || echo "")
    CCIP_ROUTER=$(exec_container "grep '^CCIP_ROUTER_ADDRESS' $CONFIG_FILE | cut -d'=' -f2 | tr -d '\"'" || echo "")
    CCIP_SENDER=$(exec_container "grep '^CCIP_SENDER_ADDRESS' $CONFIG_FILE | cut -d'=' -f2 | tr -d '\"'" || echo "")
    LINK_TOKEN=$(exec_container "grep '^LINK_TOKEN_ADDRESS' $CONFIG_FILE | cut -d'=' -f2 | tr -d '\"'" || echo "")
    METRICS_PORT=$(exec_container "grep '^METRICS_PORT' $CONFIG_FILE | cut -d'=' -f2 | tr -d '\"'" || echo "8000")
    CHECK_INTERVAL=$(exec_container "grep '^CHECK_INTERVAL' $CONFIG_FILE | cut -d'=' -f2 | tr -d '\"'" || echo "60")
    
    if [ -n "$RPC_URL" ]; then
        log_success "   RPC_URL: $RPC_URL"
    else
        log_warn "   RPC_URL: Not configured"
    fi
    
    if [ -n "$CCIP_ROUTER" ] && [ "$CCIP_ROUTER" != "" ]; then
        log_success "   CCIP_ROUTER_ADDRESS: $CCIP_ROUTER"
    else
        log_warn "   CCIP_ROUTER_ADDRESS: Not configured"
    fi
    
    if [ -n "$CCIP_SENDER" ] && [ "$CCIP_SENDER" != "" ]; then
        log_success "   CCIP_SENDER_ADDRESS: $CCIP_SENDER"
    else
        log_warn "   CCIP_SENDER_ADDRESS: Not configured"
    fi
    
    if [ -n "$LINK_TOKEN" ] && [ "$LINK_TOKEN" != "" ]; then
        log_success "   LINK_TOKEN_ADDRESS: $LINK_TOKEN"
    else
        log_warn "   LINK_TOKEN_ADDRESS: Not configured (may use native ETH)"
    fi
    
    log_info "   METRICS_PORT: ${METRICS_PORT:-8000}"
    log_info "   CHECK_INTERVAL: ${CHECK_INTERVAL:-60} seconds"
else
    log_error "Configuration file not found: $CONFIG_FILE"
    log_info "To create: ssh root@$PROXMOX_HOST 'pct exec $VMID -- bash -c \"cat > $CONFIG_FILE <<EOF"
    log_info "RPC_URL_138=http://192.168.11.250:8545"
    log_info "CCIP_ROUTER_ADDRESS="
    log_info "CCIP_SENDER_ADDRESS="
    log_info "LINK_TOKEN_ADDRESS="
    log_info "METRICS_PORT=8000"
    log_info "CHECK_INTERVAL=60"
    log_info "EOF\""
fi
echo ""

# Check if Python script exists
log_info "5. Checking CCIP Monitor script..."
MONITOR_SCRIPT="/opt/ccip-monitor/ccip_monitor.py"
if exec_container "[ -f $MONITOR_SCRIPT ]"; then
    log_success "Monitor script exists: $MONITOR_SCRIPT"
else
    log_warn "Monitor script not found: $MONITOR_SCRIPT"
    log_info "Script should be installed by the installation script"
fi
echo ""

# Check Python virtual environment
log_info "6. Checking Python environment..."
if exec_container "[ -d /opt/ccip-monitor/venv ]"; then
    log_success "Python virtual environment exists"
    
    # Check if Python dependencies are installed
    if exec_container "/opt/ccip-monitor/venv/bin/python -c 'import web3' 2>/dev/null"; then
        log_success "Python dependencies installed (web3 found)"
    else
        log_warn "Python dependencies may be missing"
    fi
else
    log_warn "Python virtual environment not found"
fi
echo ""

# Check metrics endpoint (if service is running)
if [ "$SERVICE_STATUS" = "active" ]; then
    log_info "7. Checking metrics endpoint..."
    METRICS_PORT="${METRICS_PORT:-8000}"
    HEALTH_CHECK=$(exec_container "curl -s http://localhost:$METRICS_PORT/health 2>/dev/null || curl -s http://localhost:$METRICS_PORT/metrics 2>/dev/null || echo 'unavailable'")
    
    if [ "$HEALTH_CHECK" != "unavailable" ] && [ -n "$HEALTH_CHECK" ]; then
        log_success "Metrics endpoint is accessible on port $METRICS_PORT"
    else
        log_warn "Metrics endpoint not accessible on port $METRICS_PORT"
    fi
    echo ""
fi

# Check service logs (last 10 lines)
if [ "$SERVICE_STATUS" = "active" ]; then
    log_info "8. Recent service logs (last 10 lines)..."
    echo ""
    exec_container "journalctl -u ccip-monitor --no-pager -n 10 2>/dev/null" || log_warn "Could not retrieve logs"
    echo ""
fi

# Check RPC connectivity (if configured and container is running)
if [ -n "$RPC_URL" ] && [ "$CONTAINER_STATUS" = "running" ]; then
    log_info "9. Testing RPC connectivity..."
    RPC_TEST=$(exec_container "curl -s -m 5 -X POST '$RPC_URL' -H 'Content-Type: application/json' -d '{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}' 2>/dev/null" || echo "")
    
    if echo "$RPC_TEST" | grep -q '"result"'; then
        BLOCK_HEX=$(echo "$RPC_TEST" | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
        if [ -n "$BLOCK_HEX" ]; then
            BLOCK=$(printf "%d" "$BLOCK_HEX" 2>/dev/null || echo "unknown")
            log_success "RPC endpoint is accessible (Block: $BLOCK)"
        else
            log_success "RPC endpoint is accessible"
        fi
    else
        log_warn "RPC endpoint test failed (container may need to be running)"
    fi
    echo ""
elif [ -n "$RPC_URL" ]; then
    log_info "9. RPC connectivity test skipped (container not running)"
    echo ""
fi

# Summary
log_section
log_info "Summary"
log_section

ISSUES=0

if [ "$CONTAINER_STATUS" != "running" ]; then
    log_error "Container is not running"
    ISSUES=$((ISSUES + 1))
fi

if [ "$SERVICE_STATUS" != "active" ]; then
    log_error "Service is not active"
    ISSUES=$((ISSUES + 1))
fi

if [ -z "$CCIP_ROUTER" ] || [ "$CCIP_ROUTER" = "" ]; then
    log_warn "CCIP_ROUTER_ADDRESS not configured"
    ISSUES=$((ISSUES + 1))
fi

if [ -z "$CCIP_SENDER" ] || [ "$CCIP_SENDER" = "" ]; then
    log_warn "CCIP_SENDER_ADDRESS not configured"
    ISSUES=$((ISSUES + 1))
fi

if [ $ISSUES -eq 0 ]; then
    log_success "CCIP Monitor service appears to be configured correctly!"
else
    log_warn "Found $ISSUES issue(s) that need attention"
fi

echo ""
log_info "To view full logs:"
log_info "  ssh root@$PROXMOX_HOST 'pct exec $VMID -- journalctl -u ccip-monitor -f'"
echo ""
log_info "To restart service:"
log_info "  ssh root@$PROXMOX_HOST 'pct exec $VMID -- systemctl restart ccip-monitor'"
echo ""