feat(governance): surface verification coverage

apps/api/src/services/adr100_slo_status_service.py

@@ -10,13 +10,15 @@ from __future__ import annotations
 
 import math
 from dataclasses import dataclass
-from datetime import UTC, datetime
 from typing import Any
 
 import httpx
 import structlog
+from sqlalchemy import text
 
 from src.core.config import settings
+from src.db.base import get_db_context
+from src.utils.timezone import now_taipei_iso
 
 logger = structlog.get_logger(__name__)
 
@@ -99,17 +101,19 @@ class Adr100SloStatusService:
         evaluable = [metric for metric in metrics if metric.get("evaluable")]
         ok_count = sum(1 for metric in evaluable if metric.get("status") == "ok")
         overall_compliance = (ok_count / len(evaluable)) if evaluable else None
-        overall_status = _overall_status(metrics, evaluable)
+        verification_coverage = await self._fetch_verification_coverage()
+        overall_status = _overall_status(metrics, evaluable, verification_coverage)
 
         return {
             "schema_version": "adr100_slo_status_v1",
-            "source": "prometheus",
-            "evaluated_at": datetime.now(UTC).isoformat(),
+            "source": "prometheus+postgresql",
+            "evaluated_at": now_taipei_iso(),
             "overall_status": overall_status,
             "overall_compliance": overall_compliance,
             "evaluable_count": len(evaluable),
             "metric_count": len(metrics),
             "metrics": metrics,
+            "verification_coverage": verification_coverage,
         }
 
     async def _fetch_metric(
@@ -176,6 +180,116 @@ class Adr100SloStatusService:
             sample_count=sample_count if sample_count is not None else value,
         )
 
+    async def _fetch_verification_coverage(self) -> dict[str, Any]:
+        """Summarize whether recent auto-repair executions have verifier evidence."""
+        try:
+            async with get_db_context() as db:
+                summary_row = (
+                    await db.execute(text(_VERIFICATION_COVERAGE_SQL))
+                ).mappings().one()
+                recent_rows = (
+                    await db.execute(text(_VERIFICATION_COVERAGE_RECENT_SQL))
+                ).mappings().all()
+        except Exception as exc:
+            logger.warning("adr100_verification_coverage_query_error", error=str(exc))
+            return {
+                "schema_version": "adr100_verification_coverage_v1",
+                "source": "postgresql",
+                "window": "24h",
+                "status": "error",
+                "reason": "postgresql_query_error",
+                "evaluable": False,
+                "total_auto": 0,
+                "successful_auto": 0,
+                "verified_auto": 0,
+                "verified_success": 0,
+                "verified_non_success": 0,
+                "unverified_auto": 0,
+                "coverage_rate": None,
+                "verification_success_rate": None,
+                "last_auto_at": None,
+                "last_verified_auto_at": None,
+                "last_verification_evidence_at": None,
+                "latest_auto_age_seconds": None,
+                "last_verified_auto_age_seconds": None,
+                "recent_unverified": [],
+            }
+
+        return _build_verification_coverage_payload(summary_row, recent_rows)
+
+
+_VERIFICATION_COVERAGE_SQL = """
+    WITH recent_auto AS (
+        SELECT id, incident_id, success, created_at
+        FROM auto_repair_executions
+        WHERE created_at >= NOW() - INTERVAL '24 hours'
+    ),
+    per_auto AS (
+        SELECT
+            are.id,
+            are.incident_id,
+            are.success,
+            are.created_at,
+            latest.verification_result,
+            latest.collected_at AS verification_collected_at,
+            latest.self_healing_score
+        FROM recent_auto are
+        LEFT JOIN LATERAL (
+            SELECT ev.verification_result, ev.collected_at, ev.self_healing_score
+            FROM incident_evidence ev
+            WHERE ev.incident_id = are.incident_id
+              AND ev.verification_result IS NOT NULL
+            ORDER BY ev.collected_at DESC
+            LIMIT 1
+        ) latest ON TRUE
+    )
+    SELECT
+        count(*)::int AS total_auto,
+        count(*) FILTER (WHERE success)::int AS successful_auto,
+        count(*) FILTER (WHERE verification_result IS NOT NULL)::int AS verified_auto,
+        count(*) FILTER (WHERE verification_result = 'success')::int AS verified_success,
+        count(*) FILTER (WHERE verification_result IN ('degraded','failed','timeout'))::int AS verified_non_success,
+        count(*) FILTER (WHERE verification_result IS NULL)::int AS unverified_auto,
+        max(created_at) AS last_auto_at,
+        max(created_at) FILTER (WHERE verification_result IS NOT NULL) AS last_verified_auto_at,
+        max(verification_collected_at) AS last_verification_evidence_at,
+        EXTRACT(EPOCH FROM (NOW() - max(created_at)))::int AS latest_auto_age_seconds,
+        EXTRACT(EPOCH FROM (NOW() - (max(created_at) FILTER (WHERE verification_result IS NOT NULL))))::int
+            AS last_verified_auto_age_seconds
+    FROM per_auto
+"""
+
+
+_VERIFICATION_COVERAGE_RECENT_SQL = """
+    WITH recent_auto AS (
+        SELECT id, incident_id, success, created_at
+        FROM auto_repair_executions
+        WHERE created_at >= NOW() - INTERVAL '24 hours'
+    ),
+    per_auto AS (
+        SELECT
+            are.id,
+            are.incident_id,
+            are.success,
+            are.created_at,
+            latest.verification_result
+        FROM recent_auto are
+        LEFT JOIN LATERAL (
+            SELECT ev.verification_result
+            FROM incident_evidence ev
+            WHERE ev.incident_id = are.incident_id
+              AND ev.verification_result IS NOT NULL
+            ORDER BY ev.collected_at DESC
+            LIMIT 1
+        ) latest ON TRUE
+    )
+    SELECT id, incident_id, success, created_at
+    FROM per_auto
+    WHERE verification_result IS NULL
+    ORDER BY created_at DESC
+    LIMIT 5
+"""
+
 
 async def _query_prometheus_value(
     client: httpx.AsyncClient,
@@ -254,9 +368,86 @@ def _classify_status(value: float, definition: Adr100SloDefinition) -> str:
     return "ok"
 
 
-def _overall_status(metrics: list[dict[str, Any]], evaluable: list[dict[str, Any]]) -> str:
+def _build_verification_coverage_payload(
+    summary_row: Any,
+    recent_unverified_rows: Any,
+) -> dict[str, Any]:
+    row = dict(summary_row)
+    total_auto = int(row.get("total_auto") or 0)
+    verified_auto = int(row.get("verified_auto") or 0)
+    verified_success = int(row.get("verified_success") or 0)
+    verified_non_success = int(row.get("verified_non_success") or 0)
+    unverified_auto = int(row.get("unverified_auto") or 0)
+
+    if total_auto == 0:
+        status = "skipped_low_volume"
+        reason = "no_auto_repair_executions_24h"
+        evaluable = False
+    elif unverified_auto > 0:
+        status = "warning"
+        reason = "verification_backlog_present"
+        evaluable = True
+    elif verified_non_success > 0:
+        status = "warning"
+        reason = "non_success_verification_present"
+        evaluable = True
+    else:
+        status = "ok"
+        reason = None
+        evaluable = True
+
+    coverage_rate = (verified_auto / total_auto) if total_auto else None
+    verification_success_rate = (verified_success / verified_auto) if verified_auto else None
+
+    return {
+        "schema_version": "adr100_verification_coverage_v1",
+        "source": "postgresql",
+        "window": "24h",
+        "status": status,
+        "reason": reason,
+        "evaluable": evaluable,
+        "total_auto": total_auto,
+        "successful_auto": int(row.get("successful_auto") or 0),
+        "verified_auto": verified_auto,
+        "verified_success": verified_success,
+        "verified_non_success": verified_non_success,
+        "unverified_auto": unverified_auto,
+        "coverage_rate": coverage_rate,
+        "verification_success_rate": verification_success_rate,
+        "last_auto_at": _iso(row.get("last_auto_at")),
+        "last_verified_auto_at": _iso(row.get("last_verified_auto_at")),
+        "last_verification_evidence_at": _iso(row.get("last_verification_evidence_at")),
+        "latest_auto_age_seconds": _int_or_none(row.get("latest_auto_age_seconds")),
+        "last_verified_auto_age_seconds": _int_or_none(row.get("last_verified_auto_age_seconds")),
+        "recent_unverified": [
+            {
+                "id": str(item.get("id")),
+                "incident_id": str(item.get("incident_id")),
+                "success": bool(item.get("success")),
+                "created_at": _iso(item.get("created_at")),
+            }
+            for item in (dict(raw) for raw in recent_unverified_rows)
+        ],
+    }
+
+
+def _iso(value: Any) -> str | None:
+    return value.isoformat() if hasattr(value, "isoformat") else None
+
+
+def _int_or_none(value: Any) -> int | None:
+    return int(value) if value is not None else None
+
+
+def _overall_status(
+    metrics: list[dict[str, Any]],
+    evaluable: list[dict[str, Any]],
+    verification_coverage: dict[str, Any] | None = None,
+) -> str:
     if any(metric.get("status") == "violated" for metric in metrics):
         return "violated"
+    if verification_coverage and verification_coverage.get("status") in {"violated", "warning"}:
+        return str(verification_coverage["status"])
     if any(metric.get("status") == "warning" for metric in metrics):
         return "warning"
     if evaluable and any(metric.get("status") == "skipped_low_volume" for metric in metrics):