fix(api): reconcile completed stuck incidents

apps/api/src/main.py

@@ -34,17 +34,22 @@ from sentry_sdk.integrations.starlette import StarletteIntegration
 
 from src.api.v1 import agents as agents_v1  # Phase 9.5: Agent Teams API
 from src.api.v1 import ai as ai_v1
-from src.api.v1 import aider_events as aider_events_v1  # aider-watch v2 ADR-091
-from src.api.v1 import ai_governance as ai_governance_v1  # 2026-05-02: /governance 頁面 3 endpoints
+from src.api.v1 import (
+    ai_governance as ai_governance_v1,  # 2026-05-02: /governance 頁面 3 endpoints
+)
 from src.api.v1 import ai_slo as ai_slo_v1  # Phase 6 ADR-087: AI SLO 自我治理
+from src.api.v1 import aider_events as aider_events_v1  # aider-watch v2 ADR-091
 from src.api.v1 import aiops_kpi as aiops_kpi_v1  # ADR-090 § Phase 7 KPI Dashboard
-from src.api.v1 import aiops_timeline as aiops_timeline_v1  # 2026-04-27 Wave8-X3 B4 timeline endpoint
-from src.api.v1 import approvals as approvals_v1
+from src.api.v1 import (
+    aiops_timeline as aiops_timeline_v1,  # 2026-04-27 Wave8-X3 B4 timeline endpoint
+)
 from src.api.v1 import alert_operation_logs as alert_operation_logs_v1
+from src.api.v1 import approvals as approvals_v1
 from src.api.v1 import audit_logs as audit_logs_v1
 from src.api.v1 import auto_repair as auto_repair_v1  # #8: 自動升級決策
 from src.api.v1 import csrf as csrf_v1  # Phase 20: CSRF Protection
 from src.api.v1 import dashboard as dashboard_v1
+from src.api.v1 import drift as drift_v1  # Phase 25 P2: Config Drift Detection
 from src.api.v1 import errors as errors_v1  # #40: Sentry 錯誤 BFF API
 from src.api.v1 import (
     gitea_webhook as gitea_webhook_v1,  # ADR-059: Gitea → OpenClaw (GitHub → Gitea 遷移)
@@ -56,19 +61,20 @@ from src.api.v1 import incidents as incidents_v1  # Phase 6.4: Decision Proposal
 from src.api.v1 import knowledge as knowledge_v1  # KB Phase 1: Knowledge Base
 from src.api.v1 import learning as learning_v1  # Phase D-G P0: Learning API
 from src.api.v1 import metrics as metrics_v1  # Phase 7: Gold Metrics (真實血脈)
+from src.api.v1 import monitoring as monitoring_v1  # 2026-04-03: 監控工具狀態
+from src.api.v1 import notifications as notifications_v1  # 2026-04-10: 通知頻道狀態
+from src.api.v1 import (
+    platform as platform_v1,  # AwoooP Phase 4: Platform Shell（Shadow Mode）
+)
 from src.api.v1 import playbooks as playbooks_v1  # #7: Playbook 萃取
 from src.api.v1 import proposals as proposals_v1  # Phase 6.4h: Proposals CRUD API
+from src.api.v1 import rag as rag_v1  # Phase 33 ADR-067: RAG 知識庫
 from src.api.v1 import (
     sentry_webhook as sentry_webhook_v1,  # Phase 10.2.1: Sentry → Telegram
 )
 from src.api.v1 import (
     signoz_webhook as signoz_webhook_v1,  # Phase 21: SignOz → Telegram (ADR-037)
 )
-from src.api.v1 import drift as drift_v1  # Phase 25 P2: Config Drift Detection
-from src.api.v1 import platform as platform_v1  # AwoooP Phase 4: Platform Shell（Shadow Mode）
-from src.api.v1 import rag as rag_v1  # Phase 33 ADR-067: RAG 知識庫
-from src.api.v1 import monitoring as monitoring_v1  # 2026-04-03: 監控工具狀態
-from src.api.v1 import notifications as notifications_v1  # 2026-04-10: 通知頻道狀態
 from src.api.v1 import stats as stats_v1  # Phase 6.5: Statistics Analytics
 from src.api.v1 import telegram as telegram_v1  # Phase 5.4: Telegram Gateway
 from src.api.v1 import telegram_webhook as telegram_webhook_v1  # ADR-094: Webhook入口
@@ -81,8 +87,6 @@ from src.core.logging import get_logger, setup_logging
 from src.core.redis_client import close_redis_pool, init_redis_pool
 from src.core.sse import get_publisher
 from src.core.telemetry import setup_telemetry, shutdown_telemetry
-from src.services.adr100_slo_metrics_service import get_adr100_slo_metrics_service
-from src.services.flywheel_stats_service import get_flywheel_stats_service
 
 # CTO-201: Database & Executor
 from src.db.base import close_db, init_db
@@ -92,7 +96,9 @@ from src.routers import proposals as proposals_router
 
 # Legacy route imports (to be migrated)
 from src.routes import agent, notifications, pipelines, plugins
+from src.services.adr100_slo_metrics_service import get_adr100_slo_metrics_service
 from src.services.executor import close_executor
+from src.services.flywheel_stats_service import get_flywheel_stats_service
 
 # Phase 5: OpenClaw AI Engine
 from src.services.openclaw import close_openclaw
@@ -267,16 +273,21 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
     # 2026-04-05 ogt: 重開機後 Redis 清空，從 DB restore 未解決的 incidents
     # 統帥批准: 數據必須長久記錄，重開機後自動恢復 Working Memory
     try:
-        from src.services.incident_service import get_incident_service
+        from sqlalchemy import select
+
         from src.db.base import get_db_context
         from src.db.models import IncidentRecord
-        from sqlalchemy import select
+        from src.models.incident import IncidentStatus
+        from src.services.incident_service import get_incident_service
 
         incident_service = get_incident_service()
         async with get_db_context() as db:
             result = await db.execute(
                 select(IncidentRecord).where(
-                    IncidentRecord.status.in_(["investigating", "mitigating"])
+                    IncidentRecord.status.in_([
+                        IncidentStatus.INVESTIGATING,
+                        IncidentStatus.MITIGATING,
+                    ])
                 )
             )
             records = result.scalars().all()
@@ -284,31 +295,16 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
         restored = 0
         for record in records:
             try:
-                from src.models.incident import Incident
-                incident = Incident(
-                    incident_id=record.incident_id,
-                    status=record.status,
-                    severity=record.severity,
-                    signals=record.signals or [],
-                    affected_services=record.affected_services or [],
-                    decision_chain=record.decision_chain,
-                    proposal_ids=record.proposal_ids or [],
-                    outcome=record.outcome,
-                    created_at=record.created_at,
-                    updated_at=record.updated_at,
-                    resolved_at=record.resolved_at,
-                    closed_at=record.closed_at,
-                    ttl_days=record.ttl_days,
-                    vectorized=record.vectorized,
-                    # ADR-073: 分類欄位必須還原，否則 KM 寫入時全為 "unknown"
-                    notification_type=record.notification_type,
-                    alert_category=record.alert_category,
-                )
+                incident = incident_service._record_to_incident(record)
                 if await incident_service.save_to_working_memory(incident):
                     restored += 1
-            except Exception:
+            except Exception as record_error:
                 # 舊資料 source 值不合法（node-exporter 等）→ 跳過
-                pass
+                logger.warning(
+                    "working_memory_warmup_record_skipped",
+                    incident_id=getattr(record, "incident_id", None),
+                    error=str(record_error),
+                )
 
         logger.info("working_memory_warmed_up", restored=restored, total=len(records))
     except Exception as e:
@@ -351,7 +347,9 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
         logger.warning("playbook_pg_backfill_schedule_failed", error=str(e))
 
     try:
-        from src.services.playbook_embedding_service import ensure_playbook_embeddings_indexed
+        from src.services.playbook_embedding_service import (
+            ensure_playbook_embeddings_indexed,
+        )
         asyncio.create_task(ensure_playbook_embeddings_indexed())
         logger.info("playbook_embedding_indexing_scheduled")
     except Exception as e:
@@ -499,6 +497,24 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
     except Exception as e:
         logger.warning("approval_timeout_resolver_schedule_failed", error=str(e))
 
+    # T73: 已有完成證據但仍卡在 INVESTIGATING 的舊 incident 小批次收斂。
+    # 僅處理 auto-repair success / approval EXECUTION_SUCCESS / approval EXPIRED，
+    # 不自動關閉 manual_required 或單純 APPROVED 事件。
+    try:
+        from src.jobs.incident_lifecycle_reconciler import (
+            INTERVAL_SECONDS as INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
+        )
+        from src.jobs.incident_lifecycle_reconciler import (
+            run_incident_lifecycle_reconciler_loop,
+        )
+        asyncio.create_task(run_incident_lifecycle_reconciler_loop())
+        logger.info(
+            "incident_lifecycle_reconciler_scheduled",
+            interval_sec=INCIDENT_LIFECYCLE_RECONCILER_INTERVAL,
+        )
+    except Exception as e:
+        logger.warning("incident_lifecycle_reconciler_schedule_failed", error=str(e))
+
     # ADR-083 Phase 3: Evolver Agent（每日）— Playbook 自動合併 + 低信任封存
     # 2026-04-15 ogt + Claude Sonnet 4.6（亞太）: Phase 3 初始建立
     try:
@@ -510,7 +526,9 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
 
     # ADR-104 T2: LLM Playbook DRAFT governance（每小時）
     try:
-        from src.jobs.playbook_generation_governance_job import run_playbook_generation_governance_loop
+        from src.jobs.playbook_generation_governance_job import (
+            run_playbook_generation_governance_loop,
+        )
         asyncio.create_task(run_playbook_generation_governance_loop())
         logger.info(
             "playbook_generation_governance_loop_scheduled",
@@ -556,8 +574,9 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
         from src.utils.timezone import now_taipei
 
         async def _run_kb_rot_cleaner_loop() -> None:
-            from src.jobs.kb_rot_cleaner import get_kb_rot_cleaner
             import asyncio as _asyncio
+
+            from src.jobs.kb_rot_cleaner import get_kb_rot_cleaner
             while True:
                 try:
                     now = now_taipei()
@@ -654,8 +673,8 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
     #   recovery service 每 30s 檢查 → 111 連續 3 次 HEALTHY → 自動切回 → clear_cache
     # 順序：先取 singleton → wire callback → 啟動 recovery service（才能接收 callback）
     try:
-        from src.services.ollama_failover_manager import get_ollama_failover_manager
         from src.services.ollama_auto_recovery import get_ollama_auto_recovery_service
+        from src.services.ollama_failover_manager import get_ollama_failover_manager
 
         _failover_mgr = get_ollama_failover_manager()
         _recovery_svc = get_ollama_auto_recovery_service()
@@ -668,8 +687,8 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
         # alerter 還沒注入 Redis → dedup fail-open，告警會送出且無 dedup 保護（重複告警風險）
         # 修法：configure_alerter() 提前到 start() 之前；Redis pool 在 lifespan 早期已就緒
         try:
-            from src.services.failover_alerter import configure_alerter
             from src.core.redis_client import get_redis
+            from src.services.failover_alerter import configure_alerter
             configure_alerter(get_redis())
             logger.info("failover_alerter_configured")
         except Exception as _alerter_err:
@@ -810,6 +829,7 @@ else:
 # 原因: FastAPI 不知道自己在 HTTPS 後面，redirect 回 http://
 # 效果: 有了此中間件，307 Location 會是 https://
 from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware
+
 app.add_middleware(ProxyHeadersMiddleware, trusted_hosts="*")
 
 # CORS - Strict Whitelist (Iron Law #2)