""" AWOOOI Alert Chain Metrics =========================== ADR-037 Wave A.5: 告警鏈路 Prometheus 指標 用於監控告警鏈路健康狀態: - Webhook 請求計數與延遲 - 告警處理成功率 - 異常頻率統計 版本: v1.0 建立: 2026-03-29 (台北時區) 建立者: Claude Code (Phase 21 ADR-037) """ from prometheus_client import Counter, Gauge, Histogram # ============================================================================= # Webhook Metrics (告警來源: Alertmanager/Sentry/SignOz) # ============================================================================= WEBHOOK_REQUESTS_TOTAL = Counter( "awoooi_webhook_requests_total", "Total webhook requests received", ["source", "status"], # source: alertmanager/sentry/signoz, status: success/error ) WEBHOOK_LATENCY_HISTOGRAM = Histogram( "awoooi_webhook_latency_seconds", "Webhook processing latency in seconds", ["source"], buckets=[0.1, 0.5, 1, 2, 5, 10, 30], ) # ============================================================================= # Alert Processing Metrics (告警處理) # ============================================================================= ALERT_PROCESSED_TOTAL = Counter( "awoooi_alerts_processed_total", "Total alerts processed", ["source", "severity", "outcome"], # outcome: incident_created/deduped/ignored ) INCIDENT_CREATED_TOTAL = Counter( "awoooi_incidents_created_total", "Total incidents created from alerts", ["source", "severity"], ) TELEGRAM_NOTIFICATIONS_TOTAL = Counter( "awoooi_telegram_notifications_total", "Total Telegram notifications sent", ["source", "status"], # status: success/failed ) # ============================================================================= # Anomaly Counter Metrics (ADR-037) # ============================================================================= ANOMALY_RECORDED_TOTAL = Counter( "awoooi_anomaly_recorded_total", "Total anomalies recorded to counter", ["alert_name", "service"], ) ANOMALY_ESCALATION_TOTAL = Counter( "awoooi_anomaly_escalation_total", "Total anomaly escalations", ["level"], # level: REPEAT/ESCALATE/PERMANENT_FIX ) ANOMALY_FREQUENCY_GAUGE = Gauge( "awoooi_anomaly_frequency_24h", "Current 24h anomaly frequency", ["anomaly_key"], ) # ============================================================================= # Auto Repair Metrics # ============================================================================= AUTO_REPAIR_ATTEMPTS_TOTAL = Counter( "awoooi_auto_repair_attempts_total", "Total auto repair attempts", ["action", "tier", "outcome"], # outcome: success/failed/skipped ) AUTO_REPAIR_SUCCESS_RATE = Gauge( "awoooi_auto_repair_success_rate", "Auto repair success rate by action", ["action"], ) # ============================================================================= # Alert Chain Health Metrics # ============================================================================= ALERT_CHAIN_LAST_SUCCESS = Gauge( "awoooi_alert_chain_last_success_timestamp", "Last successful alert chain completion timestamp", ["source"], ) ALERT_CHAIN_HEALTHY = Gauge( "awoooi_alert_chain_healthy", "Alert chain health status (1=healthy, 0=unhealthy)", ["source"], ) # ============================================================================= # Sentry Comment Metrics # ============================================================================= SENTRY_COMMENT_TOTAL = Counter( "awoooi_sentry_comment_total", "Total Sentry comments posted", ["status"], # status: success/failed/skipped ) # ============================================================================= # Learning Service Metrics (ADR-037 Phase G) # ============================================================================= LEARNING_SKIP_TOTAL = Counter( "awoooi_learning_skip_total", "Actions skipped due to low success rate", ["action"], ) # ============================================================================= # Helper Functions # ============================================================================= def record_webhook_request(source: str, status: str, latency: float) -> None: """記錄 Webhook 請求指標""" WEBHOOK_REQUESTS_TOTAL.labels(source=source, status=status).inc() WEBHOOK_LATENCY_HISTOGRAM.labels(source=source).observe(latency) def record_alert_processed(source: str, severity: str, outcome: str) -> None: """記錄告警處理指標""" ALERT_PROCESSED_TOTAL.labels( source=source, severity=severity, outcome=outcome ).inc() def record_telegram_notification(source: str, success: bool) -> None: """記錄 Telegram 通知指標""" status = "success" if success else "failed" TELEGRAM_NOTIFICATIONS_TOTAL.labels(source=source, status=status).inc() def record_anomaly(alert_name: str, service: str, frequency_24h: int, escalation_level: str | None) -> None: """記錄異常頻率指標""" ANOMALY_RECORDED_TOTAL.labels(alert_name=alert_name, service=service).inc() if escalation_level: ANOMALY_ESCALATION_TOTAL.labels(level=escalation_level).inc() def record_auto_repair(action: str, tier: int, success: bool) -> None: """記錄自動修復指標""" outcome = "success" if success else "failed" AUTO_REPAIR_ATTEMPTS_TOTAL.labels( action=action, tier=str(tier), outcome=outcome ).inc() def record_alert_chain_success(source: str) -> None: """記錄告警鏈路成功完成""" import time ALERT_CHAIN_LAST_SUCCESS.labels(source=source).set(time.time()) ALERT_CHAIN_HEALTHY.labels(source=source).set(1) def record_alert_chain_failure(source: str) -> None: """記錄告警鏈路失敗""" ALERT_CHAIN_HEALTHY.labels(source=source).set(0)