Files
defi-arbitrage/services/monitoring/alert.service.ts
DBIS Core Team 73e8d30190 chore: sync state and push to Gitea
Made-with: Cursor
2026-03-02 13:17:20 -08:00

273 lines
7.7 KiB
TypeScript

// Alert Service
// Sends alerts for critical events and risk violations
import { logger } from '@/infrastructure/monitoring/logger';
import { metricsService } from './metrics.service';
export enum AlertSeverity {
LOW = 'low',
MEDIUM = 'medium',
HIGH = 'high',
CRITICAL = 'critical',
}
export interface Alert {
severity: AlertSeverity;
message: string;
dealId?: string;
violationType?: string;
metadata?: Record<string, any>;
timestamp: Date;
}
export class AlertService {
private alertChannels: AlertChannel[] = [];
constructor() {
// Initialize alert channels based on environment
if (process.env.SLACK_WEBHOOK_URL) {
this.alertChannels.push(new SlackAlertChannel(process.env.SLACK_WEBHOOK_URL));
}
if (process.env.PAGERDUTY_INTEGRATION_KEY) {
this.alertChannels.push(new PagerDutyAlertChannel(process.env.PAGERDUTY_INTEGRATION_KEY));
}
if (process.env.EMAIL_ALERT_RECIPIENTS) {
this.alertChannels.push(new EmailAlertChannel(process.env.EMAIL_ALERT_RECIPIENTS.split(',')));
}
}
/**
* Send alert
*/
async sendAlert(alert: Alert): Promise<void> {
logger.error('Alert triggered', {
severity: alert.severity,
message: alert.message,
dealId: alert.dealId,
violationType: alert.violationType,
});
// Record in metrics
if (alert.violationType) {
metricsService.recordRiskViolation(alert.violationType, alert.severity);
}
// Send to all channels
const promises = this.alertChannels.map(channel =>
channel.send(alert).catch(err => {
logger.error('Failed to send alert via channel', {
channel: channel.constructor.name,
error: err instanceof Error ? err.message : 'Unknown error',
});
})
);
await Promise.allSettled(promises);
}
/**
* Alert on risk violation
*/
async alertRiskViolation(
violationType: string,
message: string,
dealId?: string,
severity: AlertSeverity = AlertSeverity.HIGH
): Promise<void> {
await this.sendAlert({
severity,
message: `Risk Violation: ${message}`,
dealId,
violationType,
timestamp: new Date(),
});
}
/**
* Alert on LTV threshold
*/
async alertLtvThreshold(dealId: string, currentLtv: number, maxLtv: number): Promise<void> {
const percentage = (currentLtv / maxLtv) * 100;
let severity = AlertSeverity.MEDIUM;
if (percentage >= 95) {
severity = AlertSeverity.CRITICAL;
} else if (percentage >= 85) {
severity = AlertSeverity.HIGH;
}
await this.alertRiskViolation(
'ltv_threshold',
`LTV at ${(currentLtv * 100).toFixed(2)}% (${percentage.toFixed(1)}% of max ${(maxLtv * 100).toFixed(2)}%)`,
dealId,
severity
);
}
/**
* Alert on USDTz exposure
*/
async alertUsdtzExposure(dealId: string, exposure: number, maxExposure: number): Promise<void> {
const percentage = (exposure / maxExposure) * 100;
let severity = AlertSeverity.MEDIUM;
if (percentage >= 95) {
severity = AlertSeverity.CRITICAL;
} else if (percentage >= 85) {
severity = AlertSeverity.HIGH;
}
await this.alertRiskViolation(
'usdtz_exposure',
`USDTz exposure at $${exposure.toFixed(2)} (${percentage.toFixed(1)}% of max $${maxExposure.toFixed(2)})`,
dealId,
severity
);
}
/**
* Alert on deal failure
*/
async alertDealFailure(dealId: string, error: string, step?: string): Promise<void> {
await this.sendAlert({
severity: AlertSeverity.HIGH,
message: `Deal execution failed: ${error}`,
dealId,
metadata: { step },
timestamp: new Date(),
});
}
/**
* Alert on system error
*/
async alertSystemError(error: string, metadata?: Record<string, any>): Promise<void> {
await this.sendAlert({
severity: AlertSeverity.CRITICAL,
message: `System error: ${error}`,
metadata,
timestamp: new Date(),
});
}
}
// Alert Channel Interfaces
interface AlertChannel {
send(alert: Alert): Promise<void>;
}
class SlackAlertChannel implements AlertChannel {
constructor(private webhookUrl: string) {}
async send(alert: Alert): Promise<void> {
const color = {
[AlertSeverity.LOW]: '#36a64f',
[AlertSeverity.MEDIUM]: '#ffa500',
[AlertSeverity.HIGH]: '#ff6600',
[AlertSeverity.CRITICAL]: '#ff0000',
}[alert.severity];
const fields: Array<{ title: string; value: string; short: boolean }> = [
...(alert.dealId ? [{ title: 'Deal ID', value: alert.dealId, short: true }] : []),
...(alert.violationType ? [{ title: 'Violation Type', value: alert.violationType, short: true }] : []),
{ title: 'Timestamp', value: alert.timestamp.toISOString(), short: true },
...(alert.metadata ? Object.entries(alert.metadata).map(([k, v]) => ({ title: k, value: String(v), short: true })) : []),
];
const payload = {
attachments: [{ color, title: `Arbitrage Alert: ${alert.severity.toUpperCase()}`, text: alert.message, fields }],
};
try {
const res = await fetch(this.webhookUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload),
});
if (!res.ok) throw new Error(`Slack webhook failed: ${res.status}`);
} catch (err) {
logger.error('Slack alert delivery failed', { error: err instanceof Error ? err.message : err });
}
}
}
class PagerDutyAlertChannel implements AlertChannel {
constructor(private integrationKey: string) {}
async send(alert: Alert): Promise<void> {
const severity = {
[AlertSeverity.LOW]: 'info',
[AlertSeverity.MEDIUM]: 'warning',
[AlertSeverity.HIGH]: 'error',
[AlertSeverity.CRITICAL]: 'critical',
}[alert.severity];
const payload = {
routing_key: this.integrationKey,
event_action: 'trigger',
payload: {
summary: alert.message,
severity,
source: 'arbitrage-service',
custom_details: {
dealId: alert.dealId,
violationType: alert.violationType,
...alert.metadata,
},
},
};
try {
const res = await fetch('https://events.pagerduty.com/v2/enqueue', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(payload),
});
if (!res.ok) {
const text = await res.text();
throw new Error(`PagerDuty API ${res.status}: ${text}`);
}
} catch (err) {
logger.error('PagerDuty alert delivery failed', {
error: err instanceof Error ? err.message : String(err),
});
}
}
}
class EmailAlertChannel implements AlertChannel {
constructor(private recipients: string[]) {}
async send(alert: Alert): Promise<void> {
if (alert.severity !== AlertSeverity.CRITICAL && alert.severity !== AlertSeverity.HIGH) {
return;
}
const emailApiUrl = process.env.EMAIL_ALERT_API_URL;
if (!emailApiUrl) {
logger.warn('Email alert skipped: Set EMAIL_ALERT_API_URL (e.g. SendGrid) to enable');
return;
}
try {
const res = await fetch(emailApiUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
to: this.recipients,
subject: `[${alert.severity.toUpperCase()}] Arbitrage Alert`,
text: alert.message,
html: `<p>${alert.message}</p><p>Deal ID: ${alert.dealId || 'N/A'}</p>`,
}),
});
if (!res.ok) throw new Error(`Email API failed: ${res.status}`);
} catch (err) {
logger.error('Email alert delivery failed', { error: err instanceof Error ? err.message : err });
}
}
}
export const alertService = new AlertService();