diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 2febfd715..9397e6887 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -25,6 +25,7 @@ Endpoints: import asyncio import hashlib import hmac +import time import uuid from typing import Literal @@ -34,7 +35,7 @@ from pydantic import BaseModel, Field from src.core.config import settings from src.core.constants import is_cicd_alertname, is_heartbeat_alertname from src.core.logging import get_logger -from src.core.metrics import record_alert_chain_success +from src.core.metrics import record_alert_chain_success, record_webhook_request from src.models.approval import ( ApprovalRequestCreate, BlastRadius, @@ -2606,6 +2607,27 @@ async def alertmanager_webhook( - 內網 IP (192.168.x.x, 10.x.x.x, 172.x.x.x): 免 HMAC - 外網 IP: 拒絕 """ + metric_started_at = time.monotonic() + metric_recorded = False + + def _record_webhook_metric(metric_status: str) -> None: + nonlocal metric_recorded + if metric_recorded: + return + metric_recorded = True + try: + record_webhook_request( + "alertmanager", + metric_status, + time.monotonic() - metric_started_at, + ) + except Exception as metric_err: + logger.warning( + "alertmanager_webhook_metric_record_failed", + status=metric_status, + error=str(metric_err), + ) + # 取得客戶端 IP client_ip = request.client.host if request.client else "unknown" forwarded_for = request.headers.get("X-Forwarded-For", "").split(",")[0].strip() @@ -2618,6 +2640,7 @@ async def alertmanager_webhook( client_ip=actual_ip, reason="External IP must use /alerts with HMAC", ) + _record_webhook_metric("error") raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail="External sources must use /alerts endpoint with HMAC signature", @@ -2633,6 +2656,7 @@ async def alertmanager_webhook( # 只處理第一個 firing 告警 (避免告警風暴) firing_alerts = [a for a in payload.alerts if a.status == "firing"] if not firing_alerts: + _record_webhook_metric("success") return AlertResponse( success=True, message="No firing alerts to process", @@ -2712,6 +2736,7 @@ async def alertmanager_webhook( ) record_alert_chain_success("alertmanager") + _record_webhook_metric("success") return AlertResponse( success=True, message=f"CI/CD alert processed (simple format): {alertname}", @@ -2721,6 +2746,7 @@ async def alertmanager_webhook( except Exception as e: logger.error("cicd_telegram_failed", error=str(e), alertname=alertname) # CI/CD 通知失敗不阻擋流程 + _record_webhook_metric("success") return AlertResponse( success=True, message=f"CI/CD alert logged (telegram failed): {alertname}", @@ -2866,6 +2892,7 @@ async def alertmanager_webhook( parent_fingerprint=grouping_result.parent_fingerprint, fingerprint=fingerprint, ) + _record_webhook_metric("success") return AlertResponse( success=True, message=( @@ -2940,6 +2967,7 @@ async def alertmanager_webhook( notification_type=notification_type, ) + _record_webhook_metric("success") return AlertResponse( success=True, message=f"🛡️ 告警收斂 (x{updated_approval.hit_count}) - 已排程節流再通知", @@ -3004,6 +3032,7 @@ async def alertmanager_webhook( severity=severity, ) record_alert_chain_success("alertmanager") + _record_webhook_metric("success") return AlertResponse( success=True, message="✅ TYPE-1 純資訊告警已通知 (no LLM)", @@ -3050,6 +3079,7 @@ async def alertmanager_webhook( notification_type=notification_type, recurrence_stage="llm_inflight", ) + _record_webhook_metric("success") return AlertResponse( success=True, message="🛡️ 告警已由同指紋背景 AI 分析處理中，已排程節流再通知", @@ -3096,6 +3126,7 @@ async def alertmanager_webhook( ) record_alert_chain_success("alertmanager") + _record_webhook_metric("success") return AlertResponse( success=True, message="✅ 告警已排入背景分析 (202 Accepted)", @@ -3110,6 +3141,7 @@ async def alertmanager_webhook( fingerprint=fingerprint, error=str(e), ) + _record_webhook_metric("error") return AlertResponse( success=False, message="⚠️ 告警已接收但處理降級，避免 Alertmanager retry storm；已交由背景治理/人工介入追蹤", diff --git a/apps/api/tests/test_alertmanager_webhook_metrics.py b/apps/api/tests/test_alertmanager_webhook_metrics.py new file mode 100644 index 000000000..3da038a96 --- /dev/null +++ b/apps/api/tests/test_alertmanager_webhook_metrics.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import pytest +from fastapi import BackgroundTasks, HTTPException +from prometheus_client import generate_latest +from starlette.requests import Request + +from src.api.v1.webhooks import AlertmanagerPayload, alertmanager_webhook +from src.core.metrics import WEBHOOK_REQUESTS_TOTAL + + +def _request( + *, + client_host: str = "127.0.0.1", + headers: list[tuple[bytes, bytes]] | None = None, +) -> Request: + return Request( + { + "type": "http", + "method": "POST", + "path": "/api/v1/webhooks/alertmanager", + "scheme": "http", + "server": ("testserver", 80), + "client": (client_host, 50000), + "query_string": b"", + "headers": headers or [], + } + ) + + +def _webhook_count(status: str) -> float: + return WEBHOOK_REQUESTS_TOTAL.labels( + source="alertmanager", + status=status, + )._value.get() + + +def _metrics_text() -> str: + return generate_latest().decode("utf-8") + + +@pytest.mark.asyncio +async def test_alertmanager_no_firing_payload_records_success_metric() -> None: + before = _webhook_count("success") + + response = await alertmanager_webhook( + _request(), + AlertmanagerPayload(status="resolved", alerts=[]), + BackgroundTasks(), + ) + + assert response.success is True + assert response.message == "No firing alerts to process" + assert _webhook_count("success") == before + 1 + assert ( + 'awoooi_webhook_requests_total{source="alertmanager",status="success"}' + in _metrics_text() + ) + + +@pytest.mark.asyncio +async def test_alertmanager_external_reject_records_error_metric() -> None: + before = _webhook_count("error") + + with pytest.raises(HTTPException) as exc_info: + await alertmanager_webhook( + _request(client_host="8.8.8.8"), + AlertmanagerPayload(status="resolved", alerts=[]), + BackgroundTasks(), + ) + + assert exc_info.value.status_code == 403 + assert _webhook_count("error") == before + 1 + assert ( + 'awoooi_webhook_requests_total{source="alertmanager",status="error"}' + in _metrics_text() + ) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 81a6e40fd..e62e649b8 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,20 @@ +## 2026-07-02 — 20:42 P0-006 Alertmanager webhook request counter exposure gap + +**完成內容**： +- 補上 `/api/v1/webhooks/alertmanager` route-level metrics：所有主要 return path 現在都會 once-only 呼叫 `record_webhook_request("alertmanager", "success" / "error", latency)`。 +- 覆蓋 success path：no-firing payload、CI/CD simple alert、grouped skip、converged recurrence、TYPE-1 info notification、LLM inflight suppressed、background analysis accepted。 +- 覆蓋 error path：external IP reject 與 degraded accepted response；metric 寫入失敗時只記 warning，不阻斷 webhook 回應，避免 Alertmanager retry storm。 +- 新增 `apps/api/tests/test_alertmanager_webhook_metrics.py`，直接驗證 no-firing success 與 external reject error 都會讓 `awoooi_webhook_requests_total{source="alertmanager",status="..."}` 出現在 `generate_latest()` scrape output。 + +**驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_alertmanager_webhook_metrics.py apps/api/tests/test_alertmanager_project_context.py -q -p no:cacheprovider`：`5 passed`。 +- `python3.11 -m py_compile apps/api/src/api/v1/webhooks.py apps/api/tests/test_alertmanager_webhook_metrics.py`：通過。 +- `python3 ops/runner/guard-gitea-runner-pressure.py --root .`：`GITEA_RUNNER_PRESSURE_GUARD_OK ... auto_branch_events_on_110=0 generic_runner_labels=0`。 +- `git diff --check`：通過。 + +**仍維持**： +- 未讀 Telegram token / secret / `.env` / raw sessions / SQLite / auth；未送 controlled probe；未 workflow_dispatch；未重啟主機 / VM / Docker daemon / Nginx / K3s / DB / firewall；未寫 production DB。 + ## 2026-07-02 — 20:18 CIR-P0-TG-001 Telegram alert learning registry readback **完成內容**：