fix(runtime): preserve lifecycle closure receipts

This commit is contained in:
ogt
2026-07-15 04:32:15 +08:00
parent 6c1a12e388
commit 8f741ce100
2 changed files with 97 additions and 24 deletions

View File

@@ -8,9 +8,14 @@ Incident Lifecycle Reconciler
- auto_repair_executions.success = true
- approval_records.status = EXECUTION_SUCCESS
- approval_records.status = EXPIRED
- Prometheus 已無 firing alertname 的歷史 incident
- 同一 firing alertname 只保留最新一筆,舊重複案依統一 lifecycle 路徑收旂
不處理單純 APPROVED / NO_ACTION / manual_required避免把仍需人工的事件
誤當作自動修復完成。
所有結案都必須經過 ``IncidentService.resolve_incident`` 並留下
durable timeline receipt禁止只改 incidents.status 製造假性終局。
"""
from __future__ import annotations
@@ -25,7 +30,6 @@ from sqlalchemy import text
from src.core.config import settings
from src.core.context import clear_project_context, set_project_context
from src.db.base import get_db_context
from src.utils.timezone import now_taipei
logger = structlog.get_logger(__name__)
@@ -40,7 +44,6 @@ class LifecycleCandidate:
incident_id: str
resolution_type: str
reason: str
direct_db_only: bool = False
async def run_incident_lifecycle_reconciler_loop() -> None:
@@ -96,31 +99,49 @@ async def reconcile_stuck_incidents(limit: int = BATCH_LIMIT) -> tuple[int, int]
if not candidates:
return 0, 0
from src.services.approval_db import get_timeline_service
from src.services.incident_service import get_incident_service
incident_service = get_incident_service()
timeline_service = get_timeline_service()
resolved = 0
errors = 0
for candidate in candidates:
try:
if candidate.direct_db_only:
result = await _resolve_db_only(candidate.incident_id)
else:
result = await incident_service.resolve_incident(
candidate.incident_id,
resolution_type=candidate.resolution_type,
emit_postmortem=False,
)
result = await incident_service.resolve_incident(
candidate.incident_id,
resolution_type=candidate.resolution_type,
emit_postmortem=False,
)
if not result:
continue
timeline_receipt = await timeline_service.add_event(
event_type="lifecycle_reconciled",
status="success",
title="Incident lifecycle reconciled from durable source truth",
description=(
f"reason={candidate.reason}; "
f"resolution_type={candidate.resolution_type}; "
"postmortem=suppressed_batch_reconcile; "
"telegram=not_sent"
),
actor="incident_lifecycle_reconciler",
actor_role="ai_agent",
risk_level="low",
incident_id=candidate.incident_id,
project_id=_PROJECT_ID,
)
if not timeline_receipt.get("id"):
raise RuntimeError("lifecycle_timeline_receipt_missing")
resolved += 1
logger.info(
"incident_lifecycle_reconciled",
incident_id=candidate.incident_id,
reason=candidate.reason,
resolution_type=candidate.resolution_type,
direct_db_only=candidate.direct_db_only,
timeline_receipt_id=str(timeline_receipt["id"]),
telegram_sent=False,
)
except Exception as exc:
errors += 1
@@ -161,18 +182,6 @@ async def _fetch_active_alertnames() -> set[str] | None:
return active_alertnames
async def _resolve_db_only(incident_id: str) -> bool:
from src.repositories.incident_repository import get_incident_repository
now = now_taipei()
return await get_incident_repository().update_status(
incident_id=incident_id,
status="resolved",
updated_at=now,
resolved_at=now,
)
async def _fetch_candidates(limit: int) -> list[LifecycleCandidate]:
async with get_db_context() as db:
result = await db.execute(
@@ -297,7 +306,6 @@ async def _fetch_inactive_or_duplicate_alert_candidates(
incident_id=str(row["incident_id"]),
resolution_type="timeout",
reason=str(row["reason"]),
direct_db_only=True,
)
for row in rows
]