proxmox/scripts/fix-all-firefly-issues.sh

#!/usr/bin/env bash
# Fix all Firefly issues for VMIDs 6200 and 6201
# Ensure both connect to Besu RPC on VMID 2500
# Usage: ./scripts/fix-all-firefly-issues.sh

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# Configuration
R630_02_IP="192.168.11.12"
ML110_IP="192.168.11.10"
RPC_VMID=2500
RPC_IP="192.168.11.250"  # Will verify

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_section() { echo -e "\n${CYAN}=== $1 ===${NC}\n"; }

echo ""
log_info "═══════════════════════════════════════════════════════════"
log_info "  FIXING ALL FIREFLY ISSUES"
log_info "  VMIDs: 6200 (r630-02), 6201 (ml110)"
log_info "  RPC Target: VMID 2500 ($RPC_IP)"
log_info "═══════════════════════════════════════════════════════════"
echo ""

# Verify RPC is running
log_section "Verifying Besu RPC (VMID 2500)"
RPC_STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct status $RPC_VMID 2>/dev/null | awk '{print \$2}'" || echo "unknown")

if [[ "$RPC_STATUS" == "running" ]]; then
    log_success "RPC container is running"
    
    # Verify RPC IP
    ACTUAL_RPC_IP=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
        "pct config $RPC_VMID 2>/dev/null | grep -oP 'ip=\\K[^,]+' | head -1" || echo "")
    
    if [[ -n "$ACTUAL_RPC_IP" ]] && [[ "$ACTUAL_RPC_IP" != "dhcp" ]]; then
        RPC_IP="${ACTUAL_RPC_IP%/*}"
        log_success "RPC IP confirmed: $RPC_IP"
    else
        log_warn "Could not determine RPC IP, using default: $RPC_IP"
    fi
else
    log_error "RPC container is not running (status: $RPC_STATUS)"
    log_error "Cannot proceed - RPC must be running"
    exit 1
fi

# Fix VMID 6200
log_section "Fixing VMID 6200 (firefly-1 on r630-02)"

log_info "Step 1: Fixing port conflict in docker-compose.yml..."
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${R630_02_IP} \
    "pct exec 6200 -- bash" <<'EOF'
cd /opt/firefly

# Backup current config
cp docker-compose.yml docker-compose.yml.backup.$(date +%Y%m%d_%H%M%S)

# Remove port 5001 from firefly-core
sed -i '/firefly-core:/,/^[[:space:]]*[^[:space:]]/ {
    /ports:/,/^[[:space:]]*[^[:space:]]/ {
        /- "5001:5001"/d
    }
}' docker-compose.yml

# Update RPC configuration
sed -i "s|FF_BLOCKCHAIN_RPC=.*|FF_BLOCKCHAIN_RPC=http://192.168.11.250:8545|g" docker-compose.yml
sed -i "s|FF_BLOCKCHAIN_WS=.*|FF_BLOCKCHAIN_WS=ws://192.168.11.250:8546|g" docker-compose.yml

echo "docker-compose.yml updated"
EOF

if [[ $? -eq 0 ]]; then
    log_success "docker-compose.yml fixed"
else
    log_error "Failed to fix docker-compose.yml"
    exit 1
fi

log_info "Step 2: Removing stuck firefly-core container..."
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${R630_02_IP} \
    "pct exec 6200 -- docker rm -f firefly-core 2>/dev/null || true"
log_success "Container removed"

log_info "Step 3: Resetting systemd service..."
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${R630_02_IP} \
    "pct exec 6200 -- systemctl reset-failed firefly.service 2>/dev/null || true"
log_success "Service reset"

log_info "Step 4: Starting Firefly service..."
if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${R630_02_IP} \
    "pct exec 6200 -- systemctl start firefly.service 2>&1"; then
    sleep 5
    
    SERVICE_STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${R630_02_IP} \
        "pct exec 6200 -- systemctl is-active firefly.service 2>/dev/null || echo 'inactive'")
    
    if [[ "$SERVICE_STATUS" == "active" ]]; then
        log_success "✅ Firefly service started successfully"
    else
        log_warn "Service status: $SERVICE_STATUS (checking containers...)"
    fi
else
    log_warn "Service start had issues, checking containers..."
fi

log_info "Step 5: Verifying containers..."
CONTAINERS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${R630_02_IP} \
    "pct exec 6200 -- docker ps --format '{{.Names}}\t{{.Status}}' 2>/dev/null | grep firefly" || echo "")

if [[ -n "$CONTAINERS" ]]; then
    log_success "Firefly containers:"
    echo "$CONTAINERS" | while IFS=$'\t' read -r name status; do
        if echo "$status" | grep -q "Up"; then
            log_success "  ✅ $name: $status"
        else
            log_warn "  ⚠️  $name: $status"
        fi
    done
else
    log_warn "No Firefly containers found"
fi

# Fix VMID 6201
log_section "Fixing VMID 6201 (firefly-ali-1 on ml110)"

log_info "Step 1: Starting container..."
if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct start 6201 2>&1"; then
    sleep 3
    log_success "Container started"
else
    log_error "Failed to start container"
    exit 1
fi

log_info "Step 2: Installing Docker (if needed)..."
DOCKER_INSTALLED=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct exec 6201 -- command -v docker >/dev/null 2>&1 && echo 'yes' || echo 'no'")

if [[ "$DOCKER_INSTALLED" == "no" ]]; then
    log_info "Installing Docker..."
    ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
        "pct exec 6201 -- bash" <<'EOF'
apt-get update -qq
apt-get install -y docker.io docker-compose >/dev/null 2>&1
systemctl enable docker
systemctl start docker
echo "Docker installed"
EOF
    log_success "Docker installed"
else
    log_success "Docker already installed"
fi

log_info "Step 3: Creating Firefly directory structure..."
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct exec 6201 -- bash" <<'EOF'
mkdir -p /opt/firefly
chown -R firefly:firefly /opt/firefly 2>/dev/null || true
echo "Directory created"
EOF
log_success "Directory structure created"

log_info "Step 4: Creating docker-compose.yml..."
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct exec 6201 -- bash" <<EOF
cat > /opt/firefly/docker-compose.yml <<'COMPOSE_EOF'
version: '3.8'

services:
  postgres:
    image: postgres:15-alpine
    container_name: firefly-postgres
    environment:
      POSTGRES_USER: firefly
      POSTGRES_PASSWORD: \${DB_PASSWORD:-firefly}
      POSTGRES_DB: firefly
    volumes:
      - postgres-data:/var/lib/postgresql/data
    restart: unless-stopped
    networks:
      - firefly-network

  ipfs:
    image: ipfs/kubo:latest
    container_name: firefly-ipfs
    ports:
      - "5001:5001"
      - "4001:4001"
    volumes:
      - ipfs-data:/data/ipfs
    restart: unless-stopped
    networks:
      - firefly-network

  firefly-core:
    image: ghcr.io/hyperledger/firefly:latest
    container_name: firefly-core
    depends_on:
      - postgres
      - ipfs
    environment:
      - FF_DATABASE_TYPE=postgres
      - FF_DATABASE_URL=postgres://firefly:\${DB_PASSWORD:-firefly}@postgres:5432/firefly?sslmode=disable
      - FF_BLOCKCHAIN_TYPE=ethereum
      - FF_BLOCKCHAIN_RPC=http://${RPC_IP}:8545
      - FF_BLOCKCHAIN_WS=ws://${RPC_IP}:8546
      - FF_CHAIN_ID=\${CHAIN_ID:-138}
      - FF_NODE_NAME=firefly-node-ali-1
      - FF_API_PUBLICURL=\${PUBLIC_URL:-http://localhost:5000}
      - FF_IPFS_API=http://ipfs:5001
      - FF_IPFS_GATEWAY=http://ipfs:8080
      - FF_LOGGING_LEVEL=info
    ports:
      - "5000:5000"
    volumes:
      - firefly-data:/var/lib/firefly
    restart: unless-stopped
    networks:
      - firefly-network

volumes:
  postgres-data:
  ipfs-data:
  firefly-data:

networks:
  firefly-network:
    driver: bridge
COMPOSE_EOF
chown firefly:firefly /opt/firefly/docker-compose.yml
echo "docker-compose.yml created"
EOF

if [[ $? -eq 0 ]]; then
    log_success "docker-compose.yml created"
else
    log_error "Failed to create docker-compose.yml"
    exit 1
fi

log_info "Step 5: Creating systemd service..."
ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct exec 6201 -- bash" <<'EOF'
cat > /etc/systemd/system/firefly.service <<'SERVICE_EOF'
[Unit]
Description=Hyperledger Firefly
After=docker.service
Requires=docker.service

[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=/opt/firefly
User=firefly
Group=firefly
ExecStart=/usr/bin/docker-compose -f /opt/firefly/docker-compose.yml up -d
ExecStop=/usr/bin/docker-compose -f /opt/firefly/docker-compose.yml down
Restart=on-failure

[Install]
WantedBy=multi-user.target
SERVICE_EOF
systemctl daemon-reload
systemctl enable firefly.service
echo "Systemd service created and enabled"
EOF

if [[ $? -eq 0 ]]; then
    log_success "Systemd service created"
else
    log_error "Failed to create systemd service"
    exit 1
fi

log_info "Step 6: Starting Firefly service..."
if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct exec 6201 -- systemctl start firefly.service 2>&1"; then
    sleep 5
    log_success "Service started"
else
    log_warn "Service start had issues, checking containers..."
fi

log_info "Step 7: Verifying containers..."
CONTAINERS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct exec 6201 -- docker ps --format '{{.Names}}\t{{.Status}}' 2>/dev/null | grep firefly" || echo "")

if [[ -n "$CONTAINERS" ]]; then
    log_success "Firefly containers:"
    echo "$CONTAINERS" | while IFS=$'\t' read -r name status; do
        if echo "$status" | grep -q "Up"; then
            log_success "  ✅ $name: $status"
        else
            log_warn "  ⚠️  $name: $status"
        fi
    done
else
    log_warn "No Firefly containers found"
fi

# Final verification
log_section "Final Verification"

log_info "Verifying RPC connectivity from both nodes..."

# Test from VMID 6200
log_info "VMID 6200 -> RPC connectivity:"
RPC_TEST_6200=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${R630_02_IP} \
    "pct exec 6200 -- curl -s -X POST http://${RPC_IP}:8545 -H 'Content-Type: application/json' -d '{\"jsonrpc\":\"2.0\",\"method\":\"eth_chainId\",\"params\":[],\"id\":1}' 2>/dev/null | grep -o '\"result\":\"[^\"]*\"' || echo 'failed'")

if [[ "$RPC_TEST_6200" == *"0x8a"* ]] || [[ "$RPC_TEST_6200" == *"138"* ]]; then
    log_success "  ✅ VMID 6200 can reach RPC (Chain ID: 138)"
else
    log_warn "  ⚠️  VMID 6200 RPC connectivity: $RPC_TEST_6200"
fi

# Test from VMID 6201
log_info "VMID 6201 -> RPC connectivity:"
RPC_TEST_6201=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@${ML110_IP} \
    "pct exec 6201 -- curl -s -X POST http://${RPC_IP}:8545 -H 'Content-Type: application/json' -d '{\"jsonrpc\":\"2.0\",\"method\":\"eth_chainId\",\"params\":[],\"id\":1}' 2>/dev/null | grep -o '\"result\":\"[^\"]*\"' || echo 'failed'")

if [[ "$RPC_TEST_6201" == *"0x8a"* ]] || [[ "$RPC_TEST_6201" == *"138"* ]]; then
    log_success "  ✅ VMID 6201 can reach RPC (Chain ID: 138)"
else
    log_warn "  ⚠️  VMID 6201 RPC connectivity: $RPC_TEST_6201"
fi

log_success "═══════════════════════════════════════════════════════════"
log_success "  ALL FIREFLY ISSUES FIXED"
log_success "═══════════════════════════════════════════════════════════"
echo ""

log_info "Summary:"
log_info "  ✅ VMID 6200: Port conflict fixed, service started"
log_info "  ✅ VMID 6201: Firefly installed and configured"
log_info "  ✅ Both nodes configured to use RPC: $RPC_IP:8545"
log_info "  ✅ Chain ID: 138"
echo ""