fix(runtime): preserve lifecycle closure receipts

2026-07-15 04:32:15 +08:00
parent 6c1a12e388
commit 8f741ce100
2 changed files with 97 additions and 24 deletions
--- a/apps/api/src/jobs/incident_lifecycle_reconciler.py
+++ b/apps/api/src/jobs/incident_lifecycle_reconciler.py
@@ -8,9 +8,14 @@ Incident Lifecycle Reconciler
 - auto_repair_executions.success = true
 - approval_records.status = EXECUTION_SUCCESS
 - approval_records.status = EXPIRED
+- Prometheus 已無 firing alertname 的歷史 incident
+- 同一 firing alertname 只保留最新一筆，舊重複案依統一 lifecycle 路徑收旂

 不處理單純 APPROVED / NO_ACTION / manual_required，避免把仍需人工的事件
 誤當作自動修復完成。
+
+所有結案都必須經過 ``IncidentService.resolve_incident`` 並留下
+durable timeline receipt；禁止只改 incidents.status 製造假性終局。
 """

 from __future__ import annotations
@@ -25,7 +30,6 @@ from sqlalchemy import text
 from src.core.config import settings
 from src.core.context import clear_project_context, set_project_context
 from src.db.base import get_db_context
-from src.utils.timezone import now_taipei

 logger = structlog.get_logger(__name__)

@@ -40,7 +44,6 @@ class LifecycleCandidate:
    incident_id: str
    resolution_type: str
    reason: str
-    direct_db_only: bool = False


 async def run_incident_lifecycle_reconciler_loop() -> None:
@@ -96,31 +99,49 @@ async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int]
    if not candidates:
        return 0, 0

+    from src.services.approval_db import get_timeline_service
    from src.services.incident_service import get_incident_service

    incident_service = get_incident_service()
+    timeline_service = get_timeline_service()
    resolved = 0
    errors = 0

    for candidate in candidates:
        try:
-            if candidate.direct_db_only:
-                result = await _resolve_db_only(candidate.incident_id)
-            else:
-                result = await incident_service.resolve_incident(
-                    candidate.incident_id,
-                    resolution_type=candidate.resolution_type,
-                    emit_postmortem=False,
-                )
+            result = await incident_service.resolve_incident(
+                candidate.incident_id,
+                resolution_type=candidate.resolution_type,
+                emit_postmortem=False,
+            )
            if not result:
                continue
+            timeline_receipt = await timeline_service.add_event(
+                event_type="lifecycle_reconciled",
+                status="success",
+                title="Incident lifecycle reconciled from durable source truth",
+                description=(
+                    f"reason={candidate.reason}; "
+                    f"resolution_type={candidate.resolution_type}; "
+                    "postmortem=suppressed_batch_reconcile; "
+                    "telegram=not_sent"
+                ),
+                actor="incident_lifecycle_reconciler",
+                actor_role="ai_agent",
+                risk_level="low",
+                incident_id=candidate.incident_id,
+                project_id=_PROJECT_ID,
+            )
+            if not timeline_receipt.get("id"):
+                raise RuntimeError("lifecycle_timeline_receipt_missing")
            resolved += 1
            logger.info(
                "incident_lifecycle_reconciled",
                incident_id=candidate.incident_id,
                reason=candidate.reason,
                resolution_type=candidate.resolution_type,
-                direct_db_only=candidate.direct_db_only,
+                timeline_receipt_id=str(timeline_receipt["id"]),
+                telegram_sent=False,
            )
        except Exception as exc:
            errors += 1
@@ -161,18 +182,6 @@ async def _fetch_active_alertnames() -> set[str] | None:
    return active_alertnames


-async def _resolve_db_only(incident_id: str) -> bool:
-    from src.repositories.incident_repository import get_incident_repository
-
-    now = now_taipei()
-    return await get_incident_repository().update_status(
-        incident_id=incident_id,
-        status="resolved",
-        updated_at=now,
-        resolved_at=now,
-    )
-
-
 async def _fetch_candidates(limit: int) -> list[LifecycleCandidate]:
    async with get_db_context() as db:
        result = await db.execute(
@@ -297,7 +306,6 @@ async def _fetch_inactive_or_duplicate_alert_candidates(
            incident_id=str(row["incident_id"]),
            resolution_type="timeout",
            reason=str(row["reason"]),
-            direct_db_only=True,
        )
        for row in rows
    ]
--- a/apps/api/tests/test_incident_lifecycle_reconciler.py
+++ b/apps/api/tests/test_incident_lifecycle_reconciler.py
@@ -15,6 +15,9 @@ from src.jobs.incident_lifecycle_reconciler import (
@pytest.mark.asyncio
 async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch):
    service = SimpleNamespace(resolve_incident=AsyncMock(return_value=object()))
+    timeline = SimpleNamespace(
+        add_event=AsyncMock(side_effect=[{"id": "tl-1"}, {"id": "tl-2"}])
+    )

    monkeypatch.setattr(
        "src.jobs.incident_lifecycle_reconciler._fetch_candidates",
@@ -37,6 +40,10 @@ async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch):
        "src.services.incident_service.get_incident_service",
        lambda: service,
    )
+    monkeypatch.setattr(
+        "src.services.approval_db.get_timeline_service",
+        lambda: timeline,
+    )

    resolved, errors = await reconcile_stuck_incidents(limit=2)

@@ -53,6 +60,64 @@ async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch):
        "resolution_type": "timeout",
        "emit_postmortem": False,
    }
+    assert timeline.add_event.await_count == 2
+    assert len(timeline.add_event.await_args_list[0].kwargs["event_type"]) <= 20
+    assert timeline.add_event.await_args_list[0].kwargs == {
+        "event_type": "lifecycle_reconciled",
+        "status": "success",
+        "title": "Incident lifecycle reconciled from durable source truth",
+        "description": (
+            "reason=approval_execution_success; resolution_type=auto_repair; "
+            "postmortem=suppressed_batch_reconcile; telegram=not_sent"
+        ),
+        "actor": "incident_lifecycle_reconciler",
+        "actor_role": "ai_agent",
+        "risk_level": "low",
+        "incident_id": "INC-EXEC-SUCCESS",
+        "project_id": "awoooi",
+    }
+
+
+@pytest.mark.asyncio
+async def test_inactive_alert_uses_service_and_writes_timeline(monkeypatch):
+    service = SimpleNamespace(resolve_incident=AsyncMock(return_value=object()))
+    timeline = SimpleNamespace(add_event=AsyncMock(return_value={"id": "tl-inactive"}))
+    candidate = LifecycleCandidate(
+        incident_id="INC-INACTIVE",
+        resolution_type="timeout",
+        reason="inactive_alert_stale",
+    )
+
+    monkeypatch.setattr(
+        "src.jobs.incident_lifecycle_reconciler._fetch_candidates",
+        AsyncMock(return_value=[]),
+    )
+    monkeypatch.setattr(
+        "src.jobs.incident_lifecycle_reconciler._fetch_active_alertnames",
+        AsyncMock(return_value=set()),
+    )
+    monkeypatch.setattr(
+        "src.jobs.incident_lifecycle_reconciler._fetch_inactive_or_duplicate_alert_candidates",
+        AsyncMock(return_value=[candidate]),
+    )
+    monkeypatch.setattr(
+        "src.services.incident_service.get_incident_service",
+        lambda: service,
+    )
+    monkeypatch.setattr(
+        "src.services.approval_db.get_timeline_service",
+        lambda: timeline,
+    )
+
+    resolved, errors = await reconcile_stuck_incidents(limit=1)
+
+    assert (resolved, errors) == (1, 0)
+    service.resolve_incident.assert_awaited_once_with(
+        "INC-INACTIVE",
+        resolution_type="timeout",
+        emit_postmortem=False,
+    )
+    timeline.add_event.assert_awaited_once()


@pytest.mark.asyncio