diff --git a/backend/api/rest/ai.go b/backend/api/rest/ai.go index 99c5b4b..8f75c23 100644 --- a/backend/api/rest/ai.go +++ b/backend/api/rest/ai.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -121,10 +122,19 @@ type openAIOutputContent struct { } func (s *Server) handleAIContext(w http.ResponseWriter, r *http.Request) { + startedAt := time.Now() + clientIP := clientIPAddress(r) if r.Method != http.MethodGet { writeMethodNotAllowed(w) return } + if allowed, retryAfter := s.allowAIRequest(r, "context"); !allowed { + w.Header().Set("Retry-After", fmt.Sprintf("%.0f", retryAfter.Seconds())) + s.aiMetrics.Record("context", http.StatusTooManyRequests, time.Since(startedAt), "rate_limited", clientIP) + s.logAIRequest("context", http.StatusTooManyRequests, time.Since(startedAt), clientIP, explorerAIModel(), "rate_limited") + writeErrorDetailed(w, http.StatusTooManyRequests, "rate_limited", "explorer ai context rate limit exceeded", "please retry shortly") + return + } query := strings.TrimSpace(r.URL.Query().Get("q")) pageContext := map[string]string{ @@ -142,16 +152,28 @@ func (s *Server) handleAIContext(w http.ResponseWriter, r *http.Request) { Warnings: warnings, } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) + s.aiMetrics.Record("context", http.StatusOK, time.Since(startedAt), "", clientIP) + s.logAIRequest("context", http.StatusOK, time.Since(startedAt), clientIP, explorerAIModel(), "") + writeJSON(w, http.StatusOK, response) } func (s *Server) handleAIChat(w http.ResponseWriter, r *http.Request) { + startedAt := time.Now() + clientIP := clientIPAddress(r) if r.Method != http.MethodPost { writeMethodNotAllowed(w) return } + if allowed, retryAfter := s.allowAIRequest(r, "chat"); !allowed { + w.Header().Set("Retry-After", fmt.Sprintf("%.0f", retryAfter.Seconds())) + s.aiMetrics.Record("chat", http.StatusTooManyRequests, time.Since(startedAt), "rate_limited", clientIP) + s.logAIRequest("chat", http.StatusTooManyRequests, time.Since(startedAt), clientIP, explorerAIModel(), "rate_limited") + writeErrorDetailed(w, http.StatusTooManyRequests, "rate_limited", "explorer ai chat rate limit exceeded", "please retry shortly") + return + } if !explorerAIEnabled() { + s.aiMetrics.Record("chat", http.StatusServiceUnavailable, time.Since(startedAt), "service_unavailable", clientIP) + s.logAIRequest("chat", http.StatusServiceUnavailable, time.Since(startedAt), clientIP, explorerAIModel(), "service_unavailable") writeError(w, http.StatusServiceUnavailable, "service_unavailable", "explorer ai is not configured; set OPENAI_API_KEY on the backend") return } @@ -176,7 +198,10 @@ func (s *Server) handleAIChat(w http.ResponseWriter, r *http.Request) { reply, model, err := s.callOpenAIResponses(r.Context(), messages, ctxEnvelope) if err != nil { - writeError(w, http.StatusBadGateway, "bad_gateway", fmt.Sprintf("explorer ai request failed: %v", err)) + statusCode, code, message, details := mapAIUpstreamError(err) + s.aiMetrics.Record("chat", statusCode, time.Since(startedAt), code, clientIP) + s.logAIRequest("chat", statusCode, time.Since(startedAt), clientIP, model, code) + writeErrorDetailed(w, statusCode, code, message, details) return } @@ -188,8 +213,9 @@ func (s *Server) handleAIChat(w http.ResponseWriter, r *http.Request) { Warnings: warnings, } - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(response) + s.aiMetrics.Record("chat", http.StatusOK, time.Since(startedAt), "", clientIP) + s.logAIRequest("chat", http.StatusOK, time.Since(startedAt), clientIP, model, "") + writeJSON(w, http.StatusOK, response) } func explorerAIEnabled() bool { @@ -309,6 +335,28 @@ func (s *Server) queryAIStats(ctx context.Context) (map[string]any, error) { stats["latest_block"] = latestBlock } + if len(stats) == 0 { + var totalBlocks int64 + if err := s.db.QueryRow(ctx, `SELECT COUNT(*) FROM blocks`).Scan(&totalBlocks); err == nil { + stats["total_blocks"] = totalBlocks + } + + var totalTransactions int64 + if err := s.db.QueryRow(ctx, `SELECT COUNT(*) FROM transactions`).Scan(&totalTransactions); err == nil { + stats["total_transactions"] = totalTransactions + } + + var totalAddresses int64 + if err := s.db.QueryRow(ctx, `SELECT COUNT(*) FROM addresses`).Scan(&totalAddresses); err == nil { + stats["total_addresses"] = totalAddresses + } + + var latestBlock int64 + if err := s.db.QueryRow(ctx, `SELECT COALESCE(MAX(number), 0) FROM blocks`).Scan(&latestBlock); err == nil { + stats["latest_block"] = latestBlock + } + } + if len(stats) == 0 { return nil, fmt.Errorf("no indexed stats available") } @@ -337,7 +385,30 @@ func (s *Server) queryAITransaction(ctx context.Context, hash string) (map[strin &txHash, &blockNumber, &fromAddress, &toAddress, &value, &gasUsed, &gasPrice, &status, ×tampISO, ) if err != nil { - return nil, err + normalizedHash := normalizeHexString(hash) + blockscoutQuery := ` + SELECT + concat('0x', encode(hash, 'hex')) AS hash, + block_number, + concat('0x', encode(from_address_hash, 'hex')) AS from_address, + CASE + WHEN to_address_hash IS NULL THEN NULL + ELSE concat('0x', encode(to_address_hash, 'hex')) + END AS to_address, + COALESCE(value::text, '0') AS value, + gas_used, + gas_price, + status, + TO_CHAR(block_timestamp AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS timestamp_iso + FROM transactions + WHERE hash = decode($1, 'hex') + LIMIT 1 + ` + if fallbackErr := s.db.QueryRow(ctx, blockscoutQuery, normalizedHash).Scan( + &txHash, &blockNumber, &fromAddress, &toAddress, &value, &gasUsed, &gasPrice, &status, ×tampISO, + ); fallbackErr != nil { + return nil, err + } } tx := map[string]any{ @@ -403,6 +474,63 @@ func (s *Server) queryAIAddress(ctx context.Context, address string) (map[string result["recent_transactions"] = recentHashes } + if len(result) == 1 { + normalizedAddress := normalizeHexString(address) + + var blockscoutTxCount int64 + var blockscoutTokenCount int64 + blockscoutAddressQuery := ` + SELECT + COALESCE(transactions_count, 0), + COALESCE(token_transfers_count, 0) + FROM addresses + WHERE hash = decode($1, 'hex') + LIMIT 1 + ` + if err := s.db.QueryRow(ctx, blockscoutAddressQuery, normalizedAddress).Scan(&blockscoutTxCount, &blockscoutTokenCount); err == nil { + result["transaction_count"] = blockscoutTxCount + result["token_count"] = blockscoutTokenCount + } + + var liveTxCount int64 + if err := s.db.QueryRow(ctx, ` + SELECT COUNT(*) + FROM transactions + WHERE from_address_hash = decode($1, 'hex') OR to_address_hash = decode($1, 'hex') + `, normalizedAddress).Scan(&liveTxCount); err == nil && liveTxCount > 0 { + result["transaction_count"] = liveTxCount + } + + var liveTokenCount int64 + if err := s.db.QueryRow(ctx, ` + SELECT COUNT(DISTINCT token_contract_address_hash) + FROM token_transfers + WHERE from_address_hash = decode($1, 'hex') OR to_address_hash = decode($1, 'hex') + `, normalizedAddress).Scan(&liveTokenCount); err == nil && liveTokenCount > 0 { + result["token_count"] = liveTokenCount + } + + rows, err := s.db.Query(ctx, ` + SELECT concat('0x', encode(hash, 'hex')) + FROM transactions + WHERE from_address_hash = decode($1, 'hex') OR to_address_hash = decode($1, 'hex') + ORDER BY block_number DESC, index DESC + LIMIT 5 + `, normalizedAddress) + if err == nil { + defer rows.Close() + for rows.Next() { + var hash string + if scanErr := rows.Scan(&hash); scanErr == nil { + recentHashes = append(recentHashes, hash) + } + } + } + if len(recentHashes) > 0 { + result["recent_transactions"] = recentHashes + } + } + if len(result) == 1 { return nil, fmt.Errorf("address not found") } @@ -428,7 +556,22 @@ func (s *Server) queryAIBlock(ctx context.Context, blockNumber int64) (map[strin err := s.db.QueryRow(ctx, query, s.chainID, blockNumber).Scan(&number, &hash, &parentHash, &transactionCount, &gasUsed, &gasLimit, ×tampISO) if err != nil { - return nil, err + blockscoutQuery := ` + SELECT + number, + concat('0x', encode(hash, 'hex')) AS hash, + concat('0x', encode(parent_hash, 'hex')) AS parent_hash, + (SELECT COUNT(*) FROM transactions WHERE block_number = b.number) AS transaction_count, + gas_used, + gas_limit, + TO_CHAR(timestamp AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS timestamp_iso + FROM blocks b + WHERE number = $1 + LIMIT 1 + ` + if fallbackErr := s.db.QueryRow(ctx, blockscoutQuery, blockNumber).Scan(&number, &hash, &parentHash, &transactionCount, &gasUsed, &gasLimit, ×tampISO); fallbackErr != nil { + return nil, err + } } block := map[string]any{ @@ -535,6 +678,11 @@ func filterAIRouteMatches(routes []map[string]any, query string) []map[string]an return matches } +func normalizeHexString(value string) string { + trimmed := strings.TrimSpace(strings.ToLower(value)) + return strings.TrimPrefix(trimmed, "0x") +} + func routeMatchesQuery(route map[string]any, query string) bool { fields := []string{ stringValue(route["routeId"]), @@ -802,21 +950,44 @@ func (s *Server) callOpenAIResponses(ctx context.Context, messages []AIChatMessa client := &http.Client{Timeout: 45 * time.Second} resp, err := client.Do(req) if err != nil { - return "", model, err + if errors.Is(err, context.DeadlineExceeded) { + return "", model, &AIUpstreamError{ + StatusCode: http.StatusGatewayTimeout, + Code: "upstream_timeout", + Message: "explorer ai upstream timed out", + Details: "OpenAI request exceeded the configured timeout", + } + } + return "", model, &AIUpstreamError{ + StatusCode: http.StatusBadGateway, + Code: "upstream_transport_error", + Message: "explorer ai upstream transport failed", + Details: err.Error(), + } } defer resp.Body.Close() responseBody, err := io.ReadAll(resp.Body) if err != nil { - return "", model, err + return "", model, &AIUpstreamError{ + StatusCode: http.StatusBadGateway, + Code: "upstream_bad_response", + Message: "explorer ai upstream body could not be read", + Details: err.Error(), + } } if resp.StatusCode >= 400 { - return "", model, fmt.Errorf("openai responses api returned %d: %s", resp.StatusCode, clipString(string(responseBody), 400)) + return "", model, parseOpenAIError(resp.StatusCode, responseBody) } var response openAIResponsesResponse if err := json.Unmarshal(responseBody, &response); err != nil { - return "", model, fmt.Errorf("unable to decode openai response: %w", err) + return "", model, &AIUpstreamError{ + StatusCode: http.StatusBadGateway, + Code: "upstream_bad_response", + Message: "explorer ai upstream returned invalid JSON", + Details: err.Error(), + } } reply := strings.TrimSpace(response.OutputText) @@ -824,7 +995,12 @@ func (s *Server) callOpenAIResponses(ctx context.Context, messages []AIChatMessa reply = strings.TrimSpace(extractOutputText(response.Output)) } if reply == "" { - return "", model, fmt.Errorf("openai response did not include output text") + return "", model, &AIUpstreamError{ + StatusCode: http.StatusBadGateway, + Code: "upstream_bad_response", + Message: "explorer ai upstream returned no output text", + Details: "OpenAI response did not include output_text or content text", + } } if strings.TrimSpace(response.Model) != "" { model = response.Model @@ -832,6 +1008,53 @@ func (s *Server) callOpenAIResponses(ctx context.Context, messages []AIChatMessa return reply, model, nil } +func parseOpenAIError(statusCode int, responseBody []byte) error { + var parsed struct { + Error struct { + Message string `json:"message"` + Type string `json:"type"` + Code string `json:"code"` + } `json:"error"` + } + _ = json.Unmarshal(responseBody, &parsed) + + details := clipString(strings.TrimSpace(parsed.Error.Message), 280) + if details == "" { + details = clipString(strings.TrimSpace(string(responseBody)), 280) + } + + switch statusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return &AIUpstreamError{ + StatusCode: statusCode, + Code: "upstream_auth_failed", + Message: "explorer ai upstream authentication failed", + Details: details, + } + case http.StatusTooManyRequests: + return &AIUpstreamError{ + StatusCode: statusCode, + Code: "upstream_quota_exhausted", + Message: "explorer ai upstream quota exhausted", + Details: details, + } + case http.StatusRequestTimeout, http.StatusGatewayTimeout: + return &AIUpstreamError{ + StatusCode: statusCode, + Code: "upstream_timeout", + Message: "explorer ai upstream timed out", + Details: details, + } + default: + return &AIUpstreamError{ + StatusCode: statusCode, + Code: "upstream_error", + Message: "explorer ai upstream request failed", + Details: details, + } + } +} + func extractOutputText(items []openAIOutputItem) string { parts := []string{} for _, item := range items { diff --git a/backend/api/rest/ai_runtime.go b/backend/api/rest/ai_runtime.go new file mode 100644 index 0000000..10ec1e9 --- /dev/null +++ b/backend/api/rest/ai_runtime.go @@ -0,0 +1,292 @@ +package rest + +import ( + "encoding/json" + "log" + "net" + "net/http" + "strings" + "sync" + "time" +) + +type AIRateLimiter struct { + mu sync.Mutex + entries map[string][]time.Time +} + +func NewAIRateLimiter() *AIRateLimiter { + return &AIRateLimiter{ + entries: make(map[string][]time.Time), + } +} + +func (l *AIRateLimiter) Allow(key string, limit int, window time.Duration) (bool, time.Duration) { + if limit <= 0 { + return true, 0 + } + + now := time.Now() + cutoff := now.Add(-window) + + l.mu.Lock() + defer l.mu.Unlock() + + timestamps := l.entries[key] + kept := timestamps[:0] + for _, ts := range timestamps { + if ts.After(cutoff) { + kept = append(kept, ts) + } + } + + if len(kept) >= limit { + retryAfter := kept[0].Add(window).Sub(now) + l.entries[key] = kept + if retryAfter < 0 { + retryAfter = 0 + } + return false, retryAfter + } + + kept = append(kept, now) + l.entries[key] = kept + return true, 0 +} + +type AIMetrics struct { + mu sync.Mutex + ContextRequests int64 `json:"contextRequests"` + ChatRequests int64 `json:"chatRequests"` + RateLimited int64 `json:"rateLimited"` + UpstreamFailures int64 `json:"upstreamFailures"` + LastRequestAt string `json:"lastRequestAt,omitempty"` + LastErrorCode string `json:"lastErrorCode,omitempty"` + StatusCounts map[string]int64 `json:"statusCounts"` + ErrorCounts map[string]int64 `json:"errorCounts"` + LastDurationsMs map[string]float64 `json:"lastDurationsMs"` + LastRequests []map[string]string `json:"lastRequests"` +} + +func NewAIMetrics() *AIMetrics { + return &AIMetrics{ + StatusCounts: make(map[string]int64), + ErrorCounts: make(map[string]int64), + LastDurationsMs: make(map[string]float64), + LastRequests: []map[string]string{}, + } +} + +func (m *AIMetrics) Record(endpoint string, statusCode int, duration time.Duration, errorCode, clientIP string) { + m.mu.Lock() + defer m.mu.Unlock() + + if endpoint == "context" { + m.ContextRequests++ + } + if endpoint == "chat" { + m.ChatRequests++ + } + if errorCode == "rate_limited" { + m.RateLimited++ + } + if strings.HasPrefix(errorCode, "upstream_") { + m.UpstreamFailures++ + } + + statusKey := endpoint + ":" + http.StatusText(statusCode) + m.StatusCounts[statusKey]++ + if errorCode != "" { + m.ErrorCounts[errorCode]++ + m.LastErrorCode = errorCode + } + m.LastRequestAt = time.Now().UTC().Format(time.RFC3339) + m.LastDurationsMs[endpoint] = float64(duration.Milliseconds()) + m.LastRequests = append([]map[string]string{{ + "endpoint": endpoint, + "status": http.StatusText(statusCode), + "statusCode": http.StatusText(statusCode), + "clientIp": clientIP, + "at": m.LastRequestAt, + "errorCode": errorCode, + }}, m.LastRequests...) + if len(m.LastRequests) > 12 { + m.LastRequests = m.LastRequests[:12] + } +} + +func (m *AIMetrics) Snapshot() map[string]any { + m.mu.Lock() + defer m.mu.Unlock() + + statusCounts := make(map[string]int64, len(m.StatusCounts)) + for key, value := range m.StatusCounts { + statusCounts[key] = value + } + + errorCounts := make(map[string]int64, len(m.ErrorCounts)) + for key, value := range m.ErrorCounts { + errorCounts[key] = value + } + + lastDurations := make(map[string]float64, len(m.LastDurationsMs)) + for key, value := range m.LastDurationsMs { + lastDurations[key] = value + } + + lastRequests := make([]map[string]string, len(m.LastRequests)) + for i := range m.LastRequests { + copyMap := make(map[string]string, len(m.LastRequests[i])) + for key, value := range m.LastRequests[i] { + copyMap[key] = value + } + lastRequests[i] = copyMap + } + + return map[string]any{ + "contextRequests": m.ContextRequests, + "chatRequests": m.ChatRequests, + "rateLimited": m.RateLimited, + "upstreamFailures": m.UpstreamFailures, + "lastRequestAt": m.LastRequestAt, + "lastErrorCode": m.LastErrorCode, + "statusCounts": statusCounts, + "errorCounts": errorCounts, + "lastDurationsMs": lastDurations, + "lastRequests": lastRequests, + } +} + +func clientIPAddress(r *http.Request) string { + for _, header := range []string{"X-Forwarded-For", "X-Real-IP"} { + if raw := strings.TrimSpace(r.Header.Get(header)); raw != "" { + if header == "X-Forwarded-For" { + parts := strings.Split(raw, ",") + if len(parts) > 0 { + return strings.TrimSpace(parts[0]) + } + } + return raw + } + } + host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr)) + if err == nil && host != "" { + return host + } + return strings.TrimSpace(r.RemoteAddr) +} + +func explorerAIContextRateLimit() (int, time.Duration) { + return 60, time.Minute +} + +func explorerAIChatRateLimit() (int, time.Duration) { + return 12, time.Minute +} + +func (s *Server) allowAIRequest(r *http.Request, endpoint string) (bool, time.Duration) { + limit := 0 + window := time.Minute + switch endpoint { + case "context": + limit, window = explorerAIContextRateLimit() + case "chat": + limit, window = explorerAIChatRateLimit() + } + + clientIP := clientIPAddress(r) + return s.aiLimiter.Allow(endpoint+":"+clientIP, limit, window) +} + +func (s *Server) logAIRequest(endpoint string, statusCode int, duration time.Duration, clientIP, model, errorCode string) { + statusText := http.StatusText(statusCode) + if statusText == "" { + statusText = "unknown" + } + log.Printf("AI endpoint=%s status=%d duration_ms=%d client_ip=%s model=%s error_code=%s", endpoint, statusCode, duration.Milliseconds(), clientIP, model, errorCode) +} + +func (s *Server) handleAIMetrics(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeMethodNotAllowed(w) + return + } + + contextLimit, contextWindow := explorerAIContextRateLimit() + chatLimit, chatWindow := explorerAIChatRateLimit() + + response := map[string]any{ + "generatedAt": time.Now().UTC().Format(time.RFC3339), + "rateLimits": map[string]any{ + "context": map[string]any{ + "requests": contextLimit, + "window": contextWindow.String(), + }, + "chat": map[string]any{ + "requests": chatLimit, + "window": chatWindow.String(), + }, + }, + "metrics": s.aiMetrics.Snapshot(), + } + + writeJSON(w, http.StatusOK, response) +} + +func writeJSON(w http.ResponseWriter, statusCode int, payload any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _ = json.NewEncoder(w).Encode(payload) +} + +type AIUpstreamError struct { + StatusCode int + Code string + Message string + Details string +} + +func (e *AIUpstreamError) Error() string { + if e == nil { + return "" + } + if e.Details != "" { + return e.Message + ": " + e.Details + } + return e.Message +} + +func mapAIUpstreamError(err error) (int, string, string, string) { + if err == nil { + return http.StatusOK, "", "", "" + } + upstreamErr, ok := err.(*AIUpstreamError) + if !ok { + return http.StatusBadGateway, "bad_gateway", "explorer ai request failed", err.Error() + } + + switch upstreamErr.Code { + case "upstream_quota_exhausted": + return http.StatusServiceUnavailable, upstreamErr.Code, "explorer ai upstream quota exhausted", upstreamErr.Details + case "upstream_auth_failed": + return http.StatusBadGateway, upstreamErr.Code, "explorer ai upstream authentication failed", upstreamErr.Details + case "upstream_timeout": + return http.StatusGatewayTimeout, upstreamErr.Code, "explorer ai upstream timed out", upstreamErr.Details + case "upstream_bad_response": + return http.StatusBadGateway, upstreamErr.Code, "explorer ai upstream returned an invalid response", upstreamErr.Details + default: + return http.StatusBadGateway, upstreamErr.Code, upstreamErr.Message, upstreamErr.Details + } +} + +func writeErrorDetailed(w http.ResponseWriter, statusCode int, code, message, details string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _ = json.NewEncoder(w).Encode(ErrorResponse{ + Error: ErrorDetail{ + Code: code, + Message: message, + Details: details, + }, + }) +} diff --git a/backend/api/rest/routes.go b/backend/api/rest/routes.go index e1e1de2..8e1220f 100644 --- a/backend/api/rest/routes.go +++ b/backend/api/rest/routes.go @@ -42,6 +42,7 @@ func (s *Server) SetupRoutes(mux *http.ServeMux) { // Explorer AI endpoints mux.HandleFunc("/api/v1/ai/context", s.handleAIContext) mux.HandleFunc("/api/v1/ai/chat", s.handleAIChat) + mux.HandleFunc("/api/v1/ai/metrics", s.handleAIMetrics) // Route decision tree proxy mux.HandleFunc("/api/v1/routes/tree", s.handleRouteDecisionTree) diff --git a/backend/api/rest/server.go b/backend/api/rest/server.go index 351b39a..0d757ca 100644 --- a/backend/api/rest/server.go +++ b/backend/api/rest/server.go @@ -11,8 +11,8 @@ import ( "strings" "time" - "github.com/explorer/backend/auth" "github.com/explorer/backend/api/middleware" + "github.com/explorer/backend/auth" httpmiddleware "github.com/explorer/backend/libs/go-http-middleware" "github.com/jackc/pgx/v5/pgxpool" ) @@ -23,6 +23,8 @@ type Server struct { chainID int walletAuth *auth.WalletAuth jwtSecret []byte + aiLimiter *AIRateLimiter + aiMetrics *AIMetrics } // NewServer creates a new REST API server @@ -41,6 +43,8 @@ func NewServer(db *pgxpool.Pool, chainID int) *Server { chainID: chainID, walletAuth: walletAuth, jwtSecret: jwtSecret, + aiLimiter: NewAIRateLimiter(), + aiMetrics: NewAIMetrics(), } } diff --git a/backend/libs/go-pgconfig/config.go b/backend/libs/go-pgconfig/config.go index 806b1d7..2c0079e 100644 --- a/backend/libs/go-pgconfig/config.go +++ b/backend/libs/go-pgconfig/config.go @@ -8,6 +8,7 @@ import ( ) type DatabaseConfig struct { + DatabaseURL string Host string Port int User string @@ -24,7 +25,8 @@ func LoadDatabaseConfig() *DatabaseConfig { maxIdle, _ := time.ParseDuration(getEnv("DB_MAX_IDLE_TIME", "5m")) maxLifetime, _ := time.ParseDuration(getEnv("DB_CONN_MAX_LIFETIME", "1h")) return &DatabaseConfig{ - Host: getEnv("DB_HOST", "localhost"), Port: getIntEnv("DB_PORT", 5432), + DatabaseURL: getEnv("DATABASE_URL", ""), + Host: getEnv("DB_HOST", "localhost"), Port: getIntEnv("DB_PORT", 5432), User: getEnv("DB_USER", "explorer"), Password: getEnv("DB_PASSWORD", ""), Database: getEnv("DB_NAME", "explorer"), SSLMode: getEnv("DB_SSLMODE", "disable"), MaxConnections: maxConns, MaxIdleTime: maxIdle, ConnMaxLifetime: maxLifetime, @@ -32,6 +34,9 @@ func LoadDatabaseConfig() *DatabaseConfig { } func (c *DatabaseConfig) ConnectionString() string { + if c.DatabaseURL != "" { + return c.DatabaseURL + } return fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", c.Host, c.Port, c.User, c.Password, c.Database, c.SSLMode) } diff --git a/docs/EXPLORER_API_ACCESS.md b/docs/EXPLORER_API_ACCESS.md index a170290..983d698 100644 --- a/docs/EXPLORER_API_ACCESS.md +++ b/docs/EXPLORER_API_ACCESS.md @@ -104,9 +104,22 @@ Or SSH into the VM (192.168.11.140) and run `df -h`, clear logs, remove unused D | Chain 138 (Blockscout) | `https://explorer.d-bis.org/api/v2/blocks`, `/api/v2/transactions`, `/api/v2/addresses`, `/api/v2/stats`, etc. | **Blockscout** (Elixir) on port **4000** | | Stats (optional) | `/api/v2/stats` | Blockscout or Go API | | Config | `/api/config/token-list`, `/api/config/networks` | Go API (if used) | +| Explorer backend v1 | `/explorer-api/v1/features`, `/explorer-api/v1/auth/*`, `/explorer-api/v1/ai/*` | **Explorer Config API** (Go) on port **8081** | +| Explorer AI metrics | `/explorer-api/v1/ai/metrics` | **Explorer Config API** (Go) on port **8081** | +| Token aggregation | `/token-aggregation/api/v1/routes/*`, `/token-aggregation/api/v1/partner-payloads*` | **token-aggregation** service on port **3001** | For the **static frontend + Blockscout** setup (VMID 5000), **nginx** must proxy `/api/` to **Blockscout** at `http://127.0.0.1:4000`. A 502 means nginx is up but the upstream (Blockscout) is down or unreachable. +### API ownership normalization + +Use these ownership rules consistently: + +- `/api/*` is reserved for **Blockscout** compatibility and v2 explorer reads. +- `/explorer-api/v1/*` is reserved for the **Go explorer backend** (auth, features, AI, explorer-owned helpers). +- `/token-aggregation/api/v1/*` is reserved for the **token-aggregation** service. + +Avoid routing mixed services behind the same `/api/v1/*` prefix. That pattern caused the earlier conflicts where AI and feature endpoints were accidentally sent to token-aggregation or Blockscout. + ### RPC and WebSocket (Chain 138) The explorer uses **either FQDN or IP and port** for the Chain 138 RPC/WebSocket: @@ -193,11 +206,31 @@ bash scripts/fix-explorer-complete.sh bash scripts/fix-nginx-serve-custom-frontend.sh ``` -### C. CORS (browser) +### C. Deploy or refresh the explorer AI backend + +Use the dedicated deployment script when you need to: + +- rebuild the Go explorer backend +- refresh `/opt/explorer-ai-docs` +- ensure a real `JWT_SECRET` +- install or refresh the explorer database override used for AI indexed context +- optionally install `OPENAI_API_KEY` +- normalize nginx for `/explorer-api/v1/*` + +```bash +cd /path/to/explorer-monorepo +OPENAI_API_KEY=... bash scripts/deploy-explorer-ai-to-vmid5000.sh +``` + +If `OPENAI_API_KEY` is omitted, the AI context endpoint will still work, but chat will remain disabled with a backend `service_unavailable` response. + +On VMID `5000`, the script also writes a dedicated `database.conf` drop-in for `explorer-config-api` so AI context can query the live Blockscout Postgres container instead of assuming `localhost:5432`. + +### D. CORS (browser) The frontend is same-origin (`https://explorer.d-bis.org`), so `/api/` is same-origin and CORS is not required for same-origin requests. The `add_header Access-Control-Allow-Origin *` above helps if you ever call the API from another origin. -### D. Optional: OPTIONS preflight +### E. Optional: OPTIONS preflight If you need CORS preflight (e.g. custom headers from another site), add inside `location /api/`: diff --git a/frontend/public/explorer-spa.js b/frontend/public/explorer-spa.js index e60a23f..8cbad5b 100644 --- a/frontend/public/explorer-spa.js +++ b/frontend/public/explorer-spa.js @@ -1,6 +1,8 @@ const API_BASE = '/api'; + const EXPLORER_API_BASE = '/explorer-api'; + const EXPLORER_API_V1_BASE = EXPLORER_API_BASE + '/v1'; const TOKEN_AGGREGATION_API_BASE = '/token-aggregation/api'; - const EXPLORER_AI_API_BASE = API_BASE + '/v1/ai'; + const EXPLORER_AI_API_BASE = EXPLORER_API_V1_BASE + '/ai'; const FETCH_TIMEOUT_MS = 15000; const RPC_HEALTH_TIMEOUT_MS = 5000; const FETCH_MAX_RETRIES = 3; @@ -908,7 +910,7 @@ // Load feature flags from API async function loadFeatureFlags() { try { - const response = await fetch('/api/v1/features', { + const response = await fetch(EXPLORER_API_V1_BASE + '/features', { headers: authToken ? { 'Authorization': `Bearer ${authToken}` } : {} }); if (response.ok) { @@ -947,7 +949,7 @@ const address = accounts[0]; // Request nonce - const nonceResp = await fetch('/api/v1/auth/nonce', { + const nonceResp = await fetch(EXPLORER_API_V1_BASE + '/auth/nonce', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ address }) @@ -960,7 +962,7 @@ const signature = await signer.signMessage(message); // Authenticate - const authResp = await fetch('/api/v1/auth/wallet', { + const authResp = await fetch(EXPLORER_API_V1_BASE + '/auth/wallet', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ address, signature, nonce: nonceData.nonce }) diff --git a/scripts/check-explorer-health.sh b/scripts/check-explorer-health.sh index b104323..6347036 100755 --- a/scripts/check-explorer-health.sh +++ b/scripts/check-explorer-health.sh @@ -36,6 +36,8 @@ checks = [ "/api/v2/stats", "/api/config/token-list", "/api/config/networks", + "/explorer-api/v1/features", + "/explorer-api/v1/ai/context?q=cUSDT", "/token-aggregation/api/v1/routes/tree?chainId=138&tokenIn=0x93E66202A11B1772E55407B32B44e5Cd8eda7f22&tokenOut=0x004b63A7B5b0E06f6bB6adb4a5F9f590BF3182D1&amountIn=1000000", "/token-aggregation/api/v1/routes/matrix", "/token-aggregation/api/v1/routes/ingestion?fromChainId=138&routeType=swap", diff --git a/scripts/deploy-explorer-ai-to-vmid5000.sh b/scripts/deploy-explorer-ai-to-vmid5000.sh new file mode 100644 index 0000000..e19bc14 --- /dev/null +++ b/scripts/deploy-explorer-ai-to-vmid5000.sh @@ -0,0 +1,172 @@ +#!/bin/bash + +set -euo pipefail + +VMID="${VMID:-5000}" +PROXMOX_HOST="${PROXMOX_HOST_R630_02:-192.168.11.12}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +BACKEND_DIR="$REPO_ROOT/explorer-monorepo/backend" +TMP_DIR="$(mktemp -d)" +JWT_SECRET_VALUE="${JWT_SECRET_VALUE:-}" +EXPLORER_AI_MODEL_VALUE="${EXPLORER_AI_MODEL_VALUE:-gpt-5.4-mini}" +EXPLORER_DATABASE_URL_VALUE="${EXPLORER_DATABASE_URL_VALUE:-}" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +echo "==========================================" +echo "Deploying Explorer AI Backend to VMID $VMID" +echo "==========================================" + +echo "=== Step 1: Build explorer backend ===" +( + cd "$BACKEND_DIR" + go build -o "$TMP_DIR/explorer-config-api" ./api/rest/cmd +) +echo "✅ Backend built" + +echo "=== Step 2: Prepare AI docs bundle ===" +mkdir -p "$TMP_DIR/explorer-ai-docs/docs/11-references" "$TMP_DIR/explorer-ai-docs/explorer-monorepo/docs" +cp "$REPO_ROOT/docs/11-references/ADDRESS_MATRIX_AND_STATUS.md" "$TMP_DIR/explorer-ai-docs/docs/11-references/" +cp "$REPO_ROOT/docs/11-references/LIQUIDITY_POOLS_MASTER_MAP.md" "$TMP_DIR/explorer-ai-docs/docs/11-references/" +cp "$REPO_ROOT/docs/11-references/DEPLOYED_TOKENS_BRIDGES_LPS_AND_ROUTING_STATUS.md" "$TMP_DIR/explorer-ai-docs/docs/11-references/" +cp "$REPO_ROOT/docs/11-references/EXPLORER_TOKEN_LIST_CROSSCHECK.md" "$TMP_DIR/explorer-ai-docs/docs/11-references/" +cp "$REPO_ROOT/explorer-monorepo/docs/EXPLORER_API_ACCESS.md" "$TMP_DIR/explorer-ai-docs/explorer-monorepo/docs/" +tar -C "$TMP_DIR" -czf "$TMP_DIR/explorer-ai-docs.tar.gz" explorer-ai-docs +echo "✅ Docs bundle prepared" + +echo "=== Step 3: Upload artifacts ===" +scp -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$TMP_DIR/explorer-config-api" root@"$PROXMOX_HOST":/tmp/explorer-config-api +scp -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$TMP_DIR/explorer-ai-docs.tar.gz" root@"$PROXMOX_HOST":/tmp/explorer-ai-docs.tar.gz +echo "✅ Artifacts uploaded" + +echo "=== Step 4: Install backend, refresh docs, and ensure env ===" +if [ -z "$JWT_SECRET_VALUE" ]; then + JWT_SECRET_VALUE="$(openssl rand -hex 32)" +fi + +export JWT_SECRET_VALUE +export EXPLORER_AI_MODEL_VALUE +export OPENAI_API_KEY_VALUE="${OPENAI_API_KEY:-}" +export EXPLORER_DATABASE_URL_VALUE + +ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" \ + "JWT_SECRET_VALUE='$JWT_SECRET_VALUE' EXPLORER_AI_MODEL_VALUE='$EXPLORER_AI_MODEL_VALUE' OPENAI_API_KEY_VALUE='$OPENAI_API_KEY_VALUE' EXPLORER_DATABASE_URL_VALUE='$EXPLORER_DATABASE_URL_VALUE' bash -s" <<'REMOTE' +set -euo pipefail + +VMID=5000 +DB_URL="$EXPLORER_DATABASE_URL_VALUE" +if [ -z "$DB_URL" ]; then + DB_CONTAINER_IP="$(pct exec "$VMID" -- docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' blockscout-postgres 2>/dev/null || true)" + if [ -n "$DB_CONTAINER_IP" ]; then + DB_URL="postgresql://blockscout:blockscout@${DB_CONTAINER_IP}:5432/blockscout?sslmode=disable" + fi +fi + +pct exec "$VMID" -- bash -lc 'mkdir -p /opt/explorer-ai-docs /etc/systemd/system/explorer-config-api.service.d' +pct push "$VMID" /tmp/explorer-ai-docs.tar.gz /tmp/explorer-ai-docs.tar.gz --perms 0644 +pct push "$VMID" /tmp/explorer-config-api /usr/local/bin/explorer-config-api.new --perms 0755 + +pct exec "$VMID" -- env \ +DB_URL="$DB_URL" \ +EXPLORER_AI_MODEL_VALUE="$EXPLORER_AI_MODEL_VALUE" \ +JWT_SECRET_VALUE="$JWT_SECRET_VALUE" \ +OPENAI_API_KEY_VALUE="$OPENAI_API_KEY_VALUE" \ +bash -lc ' +set -euo pipefail +rm -rf /opt/explorer-ai-docs/* +tar -xzf /tmp/explorer-ai-docs.tar.gz -C /opt +rm -f /tmp/explorer-ai-docs.tar.gz +mv /usr/local/bin/explorer-config-api.new /usr/local/bin/explorer-config-api +chmod 0755 /usr/local/bin/explorer-config-api + +cat > /etc/systemd/system/explorer-config-api.service.d/ai.conf < /etc/systemd/system/explorer-config-api.service.d/security.conf < /etc/systemd/system/explorer-config-api.service.d/database.conf < /etc/systemd/system/explorer-config-api.service.d/openai.conf </dev/null +curl -fsS "https://explorer.d-bis.org/explorer-api/v1/ai/context?q=cUSDT" >/dev/null +echo "✅ Explorer AI routes respond publicly" + +echo "" +echo "Deployment complete." diff --git a/scripts/fix-nginx-serve-custom-frontend.sh b/scripts/fix-nginx-serve-custom-frontend.sh index 83f16fc..95d4434 100755 --- a/scripts/fix-nginx-serve-custom-frontend.sh +++ b/scripts/fix-nginx-serve-custom-frontend.sh @@ -32,7 +32,21 @@ server { try_files $uri =404; } - # API endpoint - MUST come before the redirect location + # Explorer backend API (auth, features, AI, explorer-owned v1 helpers) + location /explorer-api/v1/ { + proxy_pass http://127.0.0.1:8081/api/v1/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 60s; + add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; + add_header Access-Control-Allow-Headers "Content-Type, Authorization"; + } + + # Blockscout API endpoint - MUST come before the redirect location location /api/ { proxy_pass http://127.0.0.1:4000; proxy_http_version 1.1; @@ -199,9 +213,9 @@ server { add_header Cache-Control "public, immutable"; } - # Token-aggregation API at /api/v1/ for the Snap site. Service runs on port 3001. - location /api/v1/ { - proxy_pass http://127.0.0.1:3001/api/v1/; + # Explorer backend API (auth, features, AI, explorer-owned v1 helpers) + location /explorer-api/v1/ { + proxy_pass http://127.0.0.1:8081/api/v1/; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; @@ -209,6 +223,8 @@ server { proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 60s; add_header Access-Control-Allow-Origin *; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS"; + add_header Access-Control-Allow-Headers "Content-Type, Authorization"; } # Token-aggregation API for the explorer SPA live route-tree and pool intelligence.