From 8f741ce1006c49452cc8be34055c53059bdeadbc Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 04:32:15 +0800 Subject: [PATCH] fix(runtime): preserve lifecycle closure receipts --- .../src/jobs/incident_lifecycle_reconciler.py | 56 +++++++++------- .../test_incident_lifecycle_reconciler.py | 65 +++++++++++++++++++ 2 files changed, 97 insertions(+), 24 deletions(-) diff --git a/apps/api/src/jobs/incident_lifecycle_reconciler.py b/apps/api/src/jobs/incident_lifecycle_reconciler.py index b85a31f59..e1861b9a1 100644 --- a/apps/api/src/jobs/incident_lifecycle_reconciler.py +++ b/apps/api/src/jobs/incident_lifecycle_reconciler.py @@ -8,9 +8,14 @@ Incident Lifecycle Reconciler - auto_repair_executions.success = true - approval_records.status = EXECUTION_SUCCESS - approval_records.status = EXPIRED +- Prometheus 已無 firing alertname 的歷史 incident +- 同一 firing alertname 只保留最新一筆，舊重複案依統一 lifecycle 路徑收旂 不處理單純 APPROVED / NO_ACTION / manual_required，避免把仍需人工的事件 誤當作自動修復完成。 + +所有結案都必須經過 ``IncidentService.resolve_incident`` 並留下 +durable timeline receipt；禁止只改 incidents.status 製造假性終局。 """ from __future__ import annotations @@ -25,7 +30,6 @@ from sqlalchemy import text from src.core.config import settings from src.core.context import clear_project_context, set_project_context from src.db.base import get_db_context -from src.utils.timezone import now_taipei logger = structlog.get_logger(__name__) @@ -40,7 +44,6 @@ class LifecycleCandidate: incident_id: str resolution_type: str reason: str - direct_db_only: bool = False async def run_incident_lifecycle_reconciler_loop() -> None: @@ -96,31 +99,49 @@ async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int] if not candidates: return 0, 0 + from src.services.approval_db import get_timeline_service from src.services.incident_service import get_incident_service incident_service = get_incident_service() + timeline_service = get_timeline_service() resolved = 0 errors = 0 for candidate in candidates: try: - if candidate.direct_db_only: - result = await _resolve_db_only(candidate.incident_id) - else: - result = await incident_service.resolve_incident( - candidate.incident_id, - resolution_type=candidate.resolution_type, - emit_postmortem=False, - ) + result = await incident_service.resolve_incident( + candidate.incident_id, + resolution_type=candidate.resolution_type, + emit_postmortem=False, + ) if not result: continue + timeline_receipt = await timeline_service.add_event( + event_type="lifecycle_reconciled", + status="success", + title="Incident lifecycle reconciled from durable source truth", + description=( + f"reason={candidate.reason}; " + f"resolution_type={candidate.resolution_type}; " + "postmortem=suppressed_batch_reconcile; " + "telegram=not_sent" + ), + actor="incident_lifecycle_reconciler", + actor_role="ai_agent", + risk_level="low", + incident_id=candidate.incident_id, + project_id=_PROJECT_ID, + ) + if not timeline_receipt.get("id"): + raise RuntimeError("lifecycle_timeline_receipt_missing") resolved += 1 logger.info( "incident_lifecycle_reconciled", incident_id=candidate.incident_id, reason=candidate.reason, resolution_type=candidate.resolution_type, - direct_db_only=candidate.direct_db_only, + timeline_receipt_id=str(timeline_receipt["id"]), + telegram_sent=False, ) except Exception as exc: errors += 1 @@ -161,18 +182,6 @@ async def _fetch_active_alertnames() -> set[str] | None: return active_alertnames -async def _resolve_db_only(incident_id: str) -> bool: - from src.repositories.incident_repository import get_incident_repository - - now = now_taipei() - return await get_incident_repository().update_status( - incident_id=incident_id, - status="resolved", - updated_at=now, - resolved_at=now, - ) - - async def _fetch_candidates(limit: int) -> list[LifecycleCandidate]: async with get_db_context() as db: result = await db.execute( @@ -297,7 +306,6 @@ async def _fetch_inactive_or_duplicate_alert_candidates( incident_id=str(row["incident_id"]), resolution_type="timeout", reason=str(row["reason"]), - direct_db_only=True, ) for row in rows ] diff --git a/apps/api/tests/test_incident_lifecycle_reconciler.py b/apps/api/tests/test_incident_lifecycle_reconciler.py index 342d932a7..2b839fb84 100644 --- a/apps/api/tests/test_incident_lifecycle_reconciler.py +++ b/apps/api/tests/test_incident_lifecycle_reconciler.py @@ -15,6 +15,9 @@ from src.jobs.incident_lifecycle_reconciler import ( @pytest.mark.asyncio async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch): service = SimpleNamespace(resolve_incident=AsyncMock(return_value=object())) + timeline = SimpleNamespace( + add_event=AsyncMock(side_effect=[{"id": "tl-1"}, {"id": "tl-2"}]) + ) monkeypatch.setattr( "src.jobs.incident_lifecycle_reconciler._fetch_candidates", @@ -37,6 +40,10 @@ async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch): "src.services.incident_service.get_incident_service", lambda: service, ) + monkeypatch.setattr( + "src.services.approval_db.get_timeline_service", + lambda: timeline, + ) resolved, errors = await reconcile_stuck_incidents(limit=2) @@ -53,6 +60,64 @@ async def test_reconcile_stuck_incidents_resolves_strong_evidence(monkeypatch): "resolution_type": "timeout", "emit_postmortem": False, } + assert timeline.add_event.await_count == 2 + assert len(timeline.add_event.await_args_list[0].kwargs["event_type"]) <= 20 + assert timeline.add_event.await_args_list[0].kwargs == { + "event_type": "lifecycle_reconciled", + "status": "success", + "title": "Incident lifecycle reconciled from durable source truth", + "description": ( + "reason=approval_execution_success; resolution_type=auto_repair; " + "postmortem=suppressed_batch_reconcile; telegram=not_sent" + ), + "actor": "incident_lifecycle_reconciler", + "actor_role": "ai_agent", + "risk_level": "low", + "incident_id": "INC-EXEC-SUCCESS", + "project_id": "awoooi", + } + + +@pytest.mark.asyncio +async def test_inactive_alert_uses_service_and_writes_timeline(monkeypatch): + service = SimpleNamespace(resolve_incident=AsyncMock(return_value=object())) + timeline = SimpleNamespace(add_event=AsyncMock(return_value={"id": "tl-inactive"})) + candidate = LifecycleCandidate( + incident_id="INC-INACTIVE", + resolution_type="timeout", + reason="inactive_alert_stale", + ) + + monkeypatch.setattr( + "src.jobs.incident_lifecycle_reconciler._fetch_candidates", + AsyncMock(return_value=[]), + ) + monkeypatch.setattr( + "src.jobs.incident_lifecycle_reconciler._fetch_active_alertnames", + AsyncMock(return_value=set()), + ) + monkeypatch.setattr( + "src.jobs.incident_lifecycle_reconciler._fetch_inactive_or_duplicate_alert_candidates", + AsyncMock(return_value=[candidate]), + ) + monkeypatch.setattr( + "src.services.incident_service.get_incident_service", + lambda: service, + ) + monkeypatch.setattr( + "src.services.approval_db.get_timeline_service", + lambda: timeline, + ) + + resolved, errors = await reconcile_stuck_incidents(limit=1) + + assert (resolved, errors) == (1, 0) + service.resolve_incident.assert_awaited_once_with( + "INC-INACTIVE", + resolution_type="timeout", + emit_postmortem=False, + ) + timeline.add_event.assert_awaited_once() @pytest.mark.asyncio