From ac2715e541412e65398909c90f54e4d946d0ed0d Mon Sep 17 00:00:00 2001 From: OG T Date: Sun, 29 Mar 2026 11:17:27 +0800 Subject: [PATCH] =?UTF-8?q?fix(api):=20P1-2=20ApprovalRequestCreate=20?= =?UTF-8?q?=E6=AC=84=E4=BD=8D=E5=B0=8D=E9=BD=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修正 SignOz + GitHub Webhook 的 ApprovalRequestCreate: Before (錯誤欄位): - action_type, target_resource, source - blast_radius=BlastRadius.SINGLE (enum 不存在) - dry_run_check=DryRunCheck.SKIPPED (錯誤格式) - 缺少 action, description, requested_by After (正確欄位): - action, description (必填) - blast_radius=BlastRadius(...) (Pydantic Model) - dry_run_checks=[] (list) - requested_by (必填) - 其他欄位移至 metadata 遵循: ApprovalRequestBase schema (approval.py) Co-Authored-By: Claude Opus 4.5 --- apps/api/src/api/v1/github_webhook.py | 63 +++-- apps/api/src/api/v1/signoz_webhook.py | 362 ++++++++++++++++++++++++++ 2 files changed, 399 insertions(+), 26 deletions(-) create mode 100644 apps/api/src/api/v1/signoz_webhook.py diff --git a/apps/api/src/api/v1/github_webhook.py b/apps/api/src/api/v1/github_webhook.py index 29e4e9675..16c7af96c 100644 --- a/apps/api/src/api/v1/github_webhook.py +++ b/apps/api/src/api/v1/github_webhook.py @@ -1270,23 +1270,26 @@ async def create_ci_failure_approval( } risk_level = risk_map.get(diagnosis.risk_level, RiskLevel.MEDIUM) - # 組裝 Approval 請求 + # P1-2 修正: 欄位對齊 ApprovalRequestBase (2026-03-29) + suggestion = diagnosis.fix_command or "; ".join(diagnosis.suggestions[:2]) approval_request = ApprovalRequestCreate( - source="github", - alert_type="ci_failure_repair", - target_resource=repo, - namespace="github-actions", + action=f"CI Failure Repair: {repo}", + description=f"Root Cause: {diagnosis.root_cause}\nSuggestion: {suggestion}", risk_level=risk_level, - root_cause=diagnosis.root_cause, - suggestion=diagnosis.fix_command or "; ".join(diagnosis.suggestions[:2]), - blast_radius=BlastRadius.NAMESPACE if diagnosis.auto_fixable else BlastRadius.SERVICE, - data_impact=DataImpact.NONE, - dry_run_check=DryRunCheck.SKIPPED, - llm_provider=diagnosis.analyzed_by, - llm_confidence=diagnosis.confidence, + blast_radius=BlastRadius( + affected_pods=1 if diagnosis.auto_fixable else 2, + estimated_downtime="~5min", + related_services=[repo], + data_impact=DataImpact.NONE, + ), + dry_run_checks=[], + requested_by="github-webhook", metadata={ + "source": "github", + "alert_type": "ci_failure_repair", + "target_resource": repo, + "namespace": "github-actions", "ci_diagnosis_id": diagnosis_id, - "repo": repo, "workflow_name": workflow_run.name, "workflow_id": workflow_run.id, "workflow_url": workflow_run.html_url, @@ -1294,6 +1297,8 @@ async def create_ci_failure_approval( "error_type": diagnosis.error_type, "auto_fixable": diagnosis.auto_fixable, "fix_command": diagnosis.fix_command, + "llm_provider": diagnosis.analyzed_by, + "llm_confidence": diagnosis.confidence, }, ) @@ -1471,28 +1476,34 @@ async def create_github_approval( else: risk_level = RiskLevel.MEDIUM - # 組裝 Approval 請求 + # P1-2 修正: 欄位對齊 ApprovalRequestBase (2026-03-29) + root_cause = f"Code review found security concerns in {target}" + suggestion = f"Review {len(analysis.security_concerns)} security concern(s): {', '.join(analysis.security_concerns[:3])}" approval_request = ApprovalRequestCreate( - source="github", - alert_type="code_review_security", - target_resource=repo, - namespace="github", + action=f"Code Review Security: {repo}", + description=f"Root Cause: {root_cause}\nSuggestion: {suggestion}", risk_level=risk_level, - root_cause=f"Code review found security concerns in {target}", - suggestion=f"Review {len(analysis.security_concerns)} security concern(s): {', '.join(analysis.security_concerns[:3])}", - blast_radius=BlastRadius.SERVICE, - data_impact=DataImpact.READ_ONLY, - dry_run_check=DryRunCheck.PASSED, - llm_provider=analysis.analyzed_by, - llm_confidence=analysis.confidence, + blast_radius=BlastRadius( + affected_pods=1, + estimated_downtime="0", + related_services=[repo], + data_impact=DataImpact.READ_ONLY, + ), + dry_run_checks=[], + requested_by="github-webhook", metadata={ + "source": "github", + "alert_type": "code_review_security", + "target_resource": repo, + "namespace": "github", "github_review_id": review_id, - "repo": repo, "target": target, "url": url, "quality_score": analysis.quality_score, "security_concerns": analysis.security_concerns, "issues_count": len(analysis.issues), + "llm_provider": analysis.analyzed_by, + "llm_confidence": analysis.confidence, }, ) diff --git a/apps/api/src/api/v1/signoz_webhook.py b/apps/api/src/api/v1/signoz_webhook.py new file mode 100644 index 000000000..7deb0615f --- /dev/null +++ b/apps/api/src/api/v1/signoz_webhook.py @@ -0,0 +1,362 @@ +""" +AWOOOI API - SignOz Webhook Handler +==================================== +接收 SignOz Alert，建立 Incident 並發送 Telegram 通知 + +整合流程: +1. SignOz Alert → AWOOOI API Webhook +2. 記錄異常頻率 (AnomalyCounter - ADR-037) +3. 建立 Incident (雙層記憶寫入) +4. 建立 Approval 供人工審核 +5. 發送 Telegram 告警 + +🔴 HARD RULE: 時間顯示使用 Asia/Taipei (UTC+8) +""" + +import uuid + +import structlog +from fastapi import APIRouter, BackgroundTasks, HTTPException, Request +from pydantic import BaseModel + +from src.models.approval import ( + ApprovalRequestCreate, + BlastRadius, + DataImpact, + RiskLevel, +) +from src.core.metrics import ( + record_alert_chain_failure, + record_alert_chain_success, + record_alert_processed, + record_anomaly, + record_telegram_notification, + record_webhook_request, +) +from src.services.anomaly_counter import get_anomaly_counter +from src.services.approval_db import get_approval_service +from src.services.incident_service import get_incident_service +from src.services.telegram_gateway import get_telegram_gateway +from src.utils.timezone import now_taipei_iso + +logger = structlog.get_logger(__name__) + +router = APIRouter(prefix="/webhooks/signoz", tags=["SignOz Webhook"]) + + +# SignOz Severity → Risk Level 映射 +SIGNOZ_SEVERITY_TO_RISK = { + "critical": RiskLevel.CRITICAL, + "error": RiskLevel.HIGH, + "warning": RiskLevel.MEDIUM, + "info": RiskLevel.LOW, +} + + +class SignOzAlertPayload(BaseModel): + """SignOz Alert Payload (標準 Prometheus AlertManager 格式)""" + + alertname: str + status: str # firing, resolved + labels: dict = {} + annotations: dict = {} + startsAt: str | None = None + endsAt: str | None = None + generatorURL: str | None = None + + +@router.post("/alert") +async def handle_signoz_alert( + request: Request, + background_tasks: BackgroundTasks, +): + """ + SignOz 告警 Webhook Handler + + 觸發條件: + - Alert status=firing + + 處理流程: + 1. 記錄異常頻率 (AnomalyCounter) + 2. 建立 Incident + 3. 建立 Approval + 4. 發送 Telegram 告警 + """ + try: + payload = await request.json() + logger.info("signoz_alert_received", payload=payload) + + # 解析 payload (支援陣列或單一告警) + alerts = payload.get("alerts", [payload]) if isinstance(payload, dict) else [payload] + + results = [] + for alert in alerts: + # 只處理 firing 狀態 + status = alert.get("status", "firing") + if status != "firing": + results.append({"status": "ignored", "reason": "not firing"}) + continue + + # 提取告警資訊 + alert_name = alert.get("alertname", alert.get("labels", {}).get("alertname", "unknown")) + labels = alert.get("labels", {}) + annotations = alert.get("annotations", {}) + severity = labels.get("severity", "warning") + + # 背景處理 + background_tasks.add_task( + process_signoz_alert, + alert_name=alert_name, + labels=labels, + annotations=annotations, + severity=severity, + starts_at=alert.get("startsAt"), + ) + + results.append({ + "status": "accepted", + "alert_name": alert_name, + }) + + return {"status": "ok", "processed": len(results), "results": results} + + except Exception as e: + logger.exception("signoz_webhook_error", error=str(e)) + raise HTTPException(status_code=500, detail=str(e)) + + +async def process_signoz_alert( + alert_name: str, + labels: dict, + annotations: dict, + severity: str, + starts_at: str | None, +): + """ + 背景處理 SignOz 告警 + + ADR-037 Phase 21: 完整告警處理流程 + 1. 記錄異常頻率 + 2. 建立 Incident + 3. 建立 Approval + 4. 發送 Telegram + """ + try: + # ================================================================= + # Step 1: 記錄異常頻率 (ADR-037) + # ================================================================= + anomaly_counter = get_anomaly_counter() + anomaly_signature = { + "alert_name": alert_name, + "service": labels.get("service_name", labels.get("service", "unknown")), + "namespace": labels.get("namespace", "signoz"), + "error_type": labels.get("error_type", severity), + } + frequency = await anomaly_counter.record_anomaly(anomaly_signature) + anomaly_frequency = frequency.model_dump() if frequency else None + + logger.info( + "signoz_anomaly_recorded", + alert_name=alert_name, + count_24h=frequency.count_24h if frequency else 0, + escalation_level=frequency.escalation_level if frequency else None, + ) + + # Wave A.5: 記錄異常指標 (ADR-037) + if frequency: + record_anomaly( + alert_name=alert_name, + service=labels.get("service_name", labels.get("service", "unknown")), + frequency_24h=frequency.count_24h, + escalation_level=frequency.escalation_level, + ) + + # ================================================================= + # Step 2: 建立 Incident + # ================================================================= + incident_service = get_incident_service() + signal_data = { + "alert_name": alert_name, + "severity": severity, + "source": "signoz", + "target": labels.get("service_name", labels.get("service", "unknown")), + "labels": labels, + "annotations": annotations, + "fingerprint": f"signoz-{alert_name}-{labels.get('service_name', 'unknown')}", + } + # ADR-037: 傳遞頻率統計到 Incident + incident = await incident_service.create_incident_from_signal( + signal_data, frequency_stats=anomaly_frequency + ) + + if not incident: + logger.error("signoz_incident_creation_failed", alert_name=alert_name) + return + + # ================================================================= + # Step 3: 建立 Approval + # ================================================================= + approval_id = await create_signoz_approval( + alert_name=alert_name, + labels=labels, + annotations=annotations, + severity=severity, + incident_id=incident.incident_id, + anomaly_frequency=anomaly_frequency, + ) + + # ================================================================= + # Step 4: 發送 Telegram 告警 + # ================================================================= + await send_signoz_telegram( + approval_id=approval_id, + alert_name=alert_name, + labels=labels, + annotations=annotations, + severity=severity, + anomaly_frequency=anomaly_frequency, + ) + + logger.info( + "signoz_alert_processed", + alert_name=alert_name, + incident_id=incident.incident_id, + approval_id=approval_id, + ) + + # Wave A.5: 記錄告警鏈路成功 (ADR-037) + record_alert_chain_success("signoz") + record_alert_processed( + source="signoz", + severity=severity, + outcome="incident_created", + ) + + except Exception as e: + logger.exception("signoz_process_error", error=str(e), alert_name=alert_name) + # Wave A.5: 記錄告警鏈路失敗 (ADR-037) + record_alert_chain_failure("signoz") + + +async def create_signoz_approval( + alert_name: str, + labels: dict, + annotations: dict, + severity: str, + incident_id: str, + anomaly_frequency: dict | None = None, +) -> str: + """ + 為 SignOz 告警建立 Approval 記錄 + + ADR-037: 含異常頻率統計 + """ + try: + approval_service = get_approval_service() + + # 決定風險等級 (考慮頻率升級) + risk_level = SIGNOZ_SEVERITY_TO_RISK.get(severity, RiskLevel.MEDIUM) + + # ADR-037: 根據頻率升級風險等級 + if anomaly_frequency: + escalation = anomaly_frequency.get("escalation_level") + if escalation == "PERMANENT_FIX": + risk_level = RiskLevel.CRITICAL + elif escalation == "ESCALATE": + risk_level = max(risk_level, RiskLevel.HIGH) + + # 建立 Approval + service_name = labels.get("service_name", labels.get("service", "unknown")) + summary = annotations.get("summary", f"SignOz Alert: {alert_name}") + description = annotations.get("description", summary) + + # P1-2 修正: 欄位對齊 ApprovalRequestBase (2026-03-29) + approval_request = ApprovalRequestCreate( + action=f"SignOz Alert: {alert_name}", + description=description, + risk_level=risk_level, + blast_radius=BlastRadius( + affected_pods=1, + estimated_downtime="0", + related_services=[service_name], + data_impact=DataImpact.READ_ONLY, + ), + dry_run_checks=[], + requested_by="signoz-webhook", + metadata={ + "source": "signoz", + "alert_name": alert_name, + "labels": labels, + "annotations": annotations, + "incident_id": incident_id, + "anomaly_frequency": anomaly_frequency, + }, + ) + + approval = await approval_service.create_approval(approval_request) + return str(approval.id) + + except Exception as e: + logger.exception("signoz_approval_error", error=str(e)) + # 回傳臨時 ID，不中斷流程 + return str(uuid.uuid4()) + + +async def send_signoz_telegram( + approval_id: str, + alert_name: str, + labels: dict, + annotations: dict, + severity: str, + anomaly_frequency: dict | None = None, +): + """ + 發送 SignOz 告警到 Telegram + + ADR-037: 含異常頻率顯示 + """ + try: + telegram = get_telegram_gateway() + await telegram.initialize() + + service_name = labels.get("service_name", labels.get("service", "unknown")) + summary = annotations.get("summary", f"SignOz Alert: {alert_name}") + description = annotations.get("description", "") + + await telegram.send_approval_card( + approval_id=approval_id, + risk_level="critical" if severity == "critical" else ( + "high" if severity == "error" else "medium" + ), + resource_name=service_name, + root_cause=summary, + suggested_action=description or "請檢查 SignOz 儀表板", + primary_responsibility="BE", + confidence=0.7, # SignOz 自動檢測的信心度 + namespace="signoz", + anomaly_frequency=anomaly_frequency, + ) + + logger.info( + "signoz_telegram_sent", + approval_id=approval_id, + alert_name=alert_name, + escalation_level=anomaly_frequency.get("escalation_level") if anomaly_frequency else None, + ) + + except Exception as e: + logger.exception("signoz_telegram_error", error=str(e)) + + +# ============================================================================= +# Health Check (供 SignOz 確認 Webhook 可達) +# ============================================================================= + +@router.get("/health") +async def signoz_webhook_health(): + """SignOz Webhook 健康檢查""" + return { + "status": "ok", + "service": "signoz-webhook", + "timestamp": now_taipei_iso(), + }