feat(governance): emit adr100 slo metrics

apps/api/src/main.py

@@ -76,13 +76,13 @@ from src.api.v1 import terminal as terminal_v1  # Phase 19.1: Omni-Terminal SSE
 from src.api.v1 import timeline as timeline_v1
 from src.api.v1 import webhooks as webhooks_v1
 from src.core.config import settings
-from src.core.feature_flags import aiops_flags  # ADR-080: AI 自主化飛輪 feature flags 啟動驗證
 from src.core.http_client import close_all_http_clients, init_all_http_clients
 from src.core.logging import get_logger, setup_logging
 from src.core.redis_client import close_redis_pool, init_redis_pool
-from src.services.flywheel_stats_service import get_flywheel_stats_service
 from src.core.sse import get_publisher
 from src.core.telemetry import setup_telemetry, shutdown_telemetry
+from src.services.adr100_slo_metrics_service import get_adr100_slo_metrics_service
+from src.services.flywheel_stats_service import get_flywheel_stats_service
 
 # CTO-201: Database & Executor
 from src.db.base import close_db, init_db
@@ -554,7 +554,6 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
     # 2026-04-27 P3.1-T3 by Claude
     try:
         from src.utils.timezone import now_taipei
-        from datetime import datetime as _dt
 
         async def _run_kb_rot_cleaner_loop() -> None:
             from src.jobs.kb_rot_cleaner import get_kb_rot_cleaner
@@ -1016,6 +1015,13 @@ async def prometheus_metrics() -> Response:
         content += flywheel_metrics.to_prometheus_lines()
     except Exception:
         logger.warning("prometheus_metrics_flywheel_error")
+    # 2026-05-14 Codex — T18 ADR-100 SLO emitter
+    # GovernanceAgent 讀 Prometheus recording rules；若 /metrics 不吐底層 DB totals，
+    # sli:* rules 會全空並每小時重複發 governance_slo_data_gap。
+    try:
+        content += await get_adr100_slo_metrics_service().to_prometheus_lines()
+    except Exception as exc:
+        logger.warning("prometheus_metrics_adr100_slo_error", error=str(exc))
     return Response(content=content, media_type=CONTENT_TYPE_LATEST)
 
 

apps/api/src/services/adr100_slo_metrics_service.py

@@ -0,0 +1,217 @@
+"""
+ADR-100 SLO metrics emitter.
+
+Prometheus recording rules for the AI flywheel SLOs expect a small set of
+counter-like metrics. The source of truth already lives in PostgreSQL, so this
+read-side emitter exposes DB totals on /metrics without changing runtime write
+paths or introducing another state store.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from time import time
+
+from sqlalchemy import text
+
+from src.db.base import get_db_context
+
+
+@dataclass(frozen=True)
+class AutomationOperationSample:
+    outcome: str
+    operation_type: str
+    count: int
+
+
+@dataclass(frozen=True)
+class VerificationSample:
+    outcome: str
+    count: int
+
+
+@dataclass(frozen=True)
+class Adr100SloMetricsSnapshot:
+    automation_operations: list[AutomationOperationSample] = field(default_factory=list)
+    post_execution_verifications: list[VerificationSample] = field(default_factory=list)
+    knowledge_entries_total: int = 0
+    high_confidence_total: int = 0
+    high_confidence_success_total: int = 0
+    emitted_at: float = field(default_factory=time)
+
+
+class Adr100SloMetricsService:
+    """Build ADR-100 Prometheus samples from production DB state."""
+
+    async def to_prometheus_lines(self) -> str:
+        snapshot = await self.fetch_snapshot()
+        return render_adr100_slo_metrics(snapshot)
+
+    async def fetch_snapshot(self) -> Adr100SloMetricsSnapshot:
+        async with get_db_context() as db:
+            automation_rows = (
+                await db.execute(text(_AUTOMATION_OPERATION_SQL))
+            ).fetchall()
+            verification_rows = (
+                await db.execute(text(_POST_EXECUTION_VERIFICATION_SQL))
+            ).fetchall()
+            knowledge_total = int(
+                (await db.execute(text("SELECT count(*) FROM knowledge_entries"))).scalar()
+                or 0
+            )
+            confidence_row = (
+                await db.execute(text(_HIGH_CONFIDENCE_APPROVAL_SQL))
+            ).one()
+
+        return Adr100SloMetricsSnapshot(
+            automation_operations=[
+                AutomationOperationSample(
+                    outcome=str(row.outcome),
+                    operation_type=str(row.operation_type),
+                    count=int(row.count or 0),
+                )
+                for row in automation_rows
+            ],
+            post_execution_verifications=[
+                VerificationSample(
+                    outcome=str(row.outcome),
+                    count=int(row.count or 0),
+                )
+                for row in verification_rows
+            ],
+            knowledge_entries_total=knowledge_total,
+            high_confidence_total=int(confidence_row.high_confidence_total or 0),
+            high_confidence_success_total=int(
+                confidence_row.high_confidence_success_total or 0
+            ),
+        )
+
+
+def render_adr100_slo_metrics(snapshot: Adr100SloMetricsSnapshot) -> str:
+    """Render ADR-100 SLO metrics in Prometheus text exposition format."""
+    lines: list[str] = [
+        "",
+        "# HELP automation_operation_log_total DB-derived AI automation operation count for ADR-100 SLOs",
+        "# TYPE automation_operation_log_total counter",
+    ]
+    if snapshot.automation_operations:
+        for sample in snapshot.automation_operations:
+            lines.append(
+                "automation_operation_log_total"
+                f'{{outcome="{_escape_label(sample.outcome)}",'
+                f'operation_type="{_escape_label(sample.operation_type)}"}} '
+                f"{sample.count}"
+            )
+    else:
+        lines.append(
+            'automation_operation_log_total{outcome="none",operation_type="none"} 0'
+        )
+
+    lines.extend([
+        "# HELP post_execution_verification_total DB-derived post execution verification result count for ADR-100 SLOs",
+        "# TYPE post_execution_verification_total counter",
+    ])
+    if snapshot.post_execution_verifications:
+        for sample in snapshot.post_execution_verifications:
+            lines.append(
+                "post_execution_verification_total"
+                f'{{outcome="{_escape_label(sample.outcome)}"}} {sample.count}'
+            )
+    else:
+        lines.append('post_execution_verification_total{outcome="none"} 0')
+
+    lines.extend([
+        "# HELP knowledge_entries_total DB-derived knowledge entry count for ADR-100 SLOs",
+        "# TYPE knowledge_entries_total counter",
+        f"knowledge_entries_total {snapshot.knowledge_entries_total}",
+        "# HELP approval_records_high_confidence_total DB-derived high confidence approval decisions for ADR-100 SLOs",
+        "# TYPE approval_records_high_confidence_total counter",
+        f"approval_records_high_confidence_total {snapshot.high_confidence_total}",
+        "# HELP approval_records_high_confidence_success_total DB-derived high confidence approval decisions with successful verification for ADR-100 SLOs",
+        "# TYPE approval_records_high_confidence_success_total counter",
+        (
+            "approval_records_high_confidence_success_total "
+            f"{snapshot.high_confidence_success_total}"
+        ),
+        "# HELP adr100_slo_emitter_last_success_timestamp Last successful ADR-100 DB metrics emission timestamp",
+        "# TYPE adr100_slo_emitter_last_success_timestamp gauge",
+        f"adr100_slo_emitter_last_success_timestamp {snapshot.emitted_at:.0f}",
+        "",
+    ])
+    return "\n".join(lines)
+
+
+def _escape_label(value: str) -> str:
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+_AUTOMATION_OPERATION_SQL = """
+    SELECT
+        CASE
+            WHEN status <> 'success' THEN status
+            WHEN actor = 'approval_execution'
+                 AND COALESCE(input->>'requested_by', '') NOT ILIKE 'auto%%'
+                THEN 'human_required'
+            ELSE 'auto_executed'
+        END AS outcome,
+        operation_type,
+        count(*) AS count
+    FROM automation_operation_log
+    GROUP BY outcome, operation_type
+    ORDER BY outcome, operation_type
+"""
+
+
+_POST_EXECUTION_VERIFICATION_SQL = """
+    SELECT verification_result AS outcome, count(*) AS count
+    FROM incident_evidence
+    WHERE verification_result IS NOT NULL
+    GROUP BY verification_result
+    ORDER BY verification_result
+"""
+
+
+_HIGH_CONFIDENCE_APPROVAL_SQL = """
+    WITH approval_confidence AS (
+        SELECT
+            id,
+            incident_id,
+            COALESCE(
+                CASE
+                    WHEN extra_metadata->>'confidence_score' ~ '^[0-9]+(\\.[0-9]+)?$'
+                        THEN (extra_metadata->>'confidence_score')::numeric
+                    ELSE NULL
+                END,
+                CASE
+                    WHEN extra_metadata->>'confidence' ~ '^[0-9]+(\\.[0-9]+)?$'
+                        THEN (extra_metadata->>'confidence')::numeric
+                    ELSE NULL
+                END,
+                composite_score,
+                0
+            ) AS confidence
+        FROM approval_records
+    )
+    SELECT
+        count(*) FILTER (WHERE confidence >= 0.8) AS high_confidence_total,
+        count(*) FILTER (
+            WHERE confidence >= 0.8
+              AND EXISTS (
+                  SELECT 1
+                  FROM incident_evidence ev
+                  WHERE ev.incident_id = approval_confidence.incident_id
+                    AND ev.verification_result = 'success'
+              )
+        ) AS high_confidence_success_total
+    FROM approval_confidence
+"""
+
+
+_adr100_slo_metrics_service: Adr100SloMetricsService | None = None
+
+
+def get_adr100_slo_metrics_service() -> Adr100SloMetricsService:
+    global _adr100_slo_metrics_service
+    if _adr100_slo_metrics_service is None:
+        _adr100_slo_metrics_service = Adr100SloMetricsService()
+    return _adr100_slo_metrics_service

apps/api/src/services/governance_agent.py

@@ -447,13 +447,13 @@ class GovernanceAgent:
                                 "status": "skipped",
                                 "error": "no_data",
                                 "reason": "prometheus_empty_result_metric_not_emitted",
-                                "hint": "ADR-100 emitter 未實作或 PROMETHEUS_MULTIPROC_DIR 未設",
+                                "hint": "ADR-100 emitter 未輸出、Prometheus recording rule 未載入，或 multiprocess 目錄未掛載",
                             }
                             logger.warning(
                                 "governance_slo_no_data",
                                 slo=name,
                                 query=query,
-                                hint="ADR-100 emitter not yet implemented",
+                                hint="ADR-100 metrics, recording rules, or multiprocess mount not ready",
                             )
                             continue
                         value = float(result_list[0]["value"][1])
@@ -655,15 +655,15 @@ class GovernanceAgent:
                         },
                         "remediation": {
                             "items": [
-                                "補齊 ADR-100 SLO emitter（automation_operation_log_total / post_execution_verification_total / km_entries_total）",
-                                "設置 PROMETHEUS_MULTIPROC_DIR 並掛載可寫目錄（如 emptyDir）",
+                                "補齊 ADR-100 SLO emitter（automation_operation_log_total / post_execution_verification_total / knowledge_entries_total）",
+                                "確認 Prometheus recording rules 已載入，且 API Pod multiprocess 目錄可寫",
                             ],
                             "next_action": "run_adr100_slo_emit_playbook",
-                            "hint": "ADR-100 emitter 未實作或 PROMETHEUS_MULTIPROC_DIR 未設",
+                            "hint": "ADR-100 emitter、Prometheus recording rules、或 multiprocess 目錄任一環節未就緒",
                         },
                         "actionable": {
                             "items": [
-                                "先確認所有 API Pod 是否有 PROMETHEUS_MULTIPROC_DIR 掛載",
+                                "先確認 /metrics 是否已輸出 ADR-100 底層指標",
                                 "檢查 Prometheus rule 是否已載入 sli:autonomy_rate:5m 等 4 項告警規則",
                             ],
                         },