feat(governance): explain verifier failures

apps/api/src/services/adr100_slo_status_service.py

@@ -190,6 +190,9 @@ class Adr100SloStatusService:
                 recent_rows = (
                     await db.execute(text(_VERIFICATION_COVERAGE_RECENT_SQL))
                 ).mappings().all()
+                recent_non_success_rows = (
+                    await db.execute(text(_VERIFICATION_COVERAGE_NON_SUCCESS_SQL))
+                ).mappings().all()
         except Exception as exc:
             logger.warning("adr100_verification_coverage_query_error", error=str(exc))
             return {
@@ -213,9 +216,18 @@ class Adr100SloStatusService:
                 "latest_auto_age_seconds": None,
                 "last_verified_auto_age_seconds": None,
                 "recent_unverified": [],
+                "recent_non_success": [],
+                "non_success_breakdown": {
+                    "by_verification_result": [],
+                    "by_failure_class": [],
+                },
             }
 
-        return _build_verification_coverage_payload(summary_row, recent_rows)
+        return _build_verification_coverage_payload(
+            summary_row,
+            recent_rows,
+            recent_non_success_rows,
+        )
 
 
 _VERIFICATION_COVERAGE_SQL = """
@@ -291,6 +303,65 @@ _VERIFICATION_COVERAGE_RECENT_SQL = """
 """
 
 
+_VERIFICATION_COVERAGE_NON_SUCCESS_SQL = """
+    WITH recent_auto AS (
+        SELECT
+            id,
+            incident_id,
+            success,
+            playbook_id,
+            playbook_name,
+            triggered_by,
+            risk_level,
+            error_message,
+            created_at
+        FROM auto_repair_executions
+        WHERE created_at >= NOW() - INTERVAL '24 hours'
+    ),
+    per_auto AS (
+        SELECT
+            are.id AS auto_repair_id,
+            are.incident_id,
+            are.success AS auto_success,
+            are.playbook_id,
+            are.playbook_name,
+            are.triggered_by,
+            are.risk_level,
+            left(coalesce(are.error_message, ''), 240) AS auto_error,
+            are.created_at AS auto_created_at,
+            latest.verification_result,
+            latest.collected_at AS verification_collected_at,
+            left(coalesce(latest.post_execution_state::text, ''), 700) AS post_state_text,
+            left(coalesce(latest.evidence_summary, ''), 300) AS evidence_summary
+        FROM recent_auto are
+        LEFT JOIN LATERAL (
+            SELECT
+                ev.verification_result,
+                ev.collected_at,
+                ev.post_execution_state,
+                ev.evidence_summary
+            FROM incident_evidence ev
+            WHERE ev.incident_id = are.incident_id
+              AND ev.verification_result IS NOT NULL
+            ORDER BY ev.collected_at DESC
+            LIMIT 1
+        ) latest ON TRUE
+    )
+    SELECT
+        p.*,
+        i.status::text AS incident_status,
+        i.severity::text AS incident_severity,
+        i.alert_category,
+        i.alertname
+    FROM per_auto p
+    LEFT JOIN incidents i ON i.incident_id = p.incident_id
+    WHERE p.verification_result IS NOT NULL
+      AND p.verification_result <> 'success'
+    ORDER BY p.auto_created_at DESC
+    LIMIT 8
+"""
+
+
 async def _query_prometheus_value(
     client: httpx.AsyncClient,
     prom_url: str,
@@ -371,6 +442,7 @@ def _classify_status(value: float, definition: Adr100SloDefinition) -> str:
 def _build_verification_coverage_payload(
     summary_row: Any,
     recent_unverified_rows: Any,
+    recent_non_success_rows: Any = (),
 ) -> dict[str, Any]:
     row = dict(summary_row)
     total_auto = int(row.get("total_auto") or 0)
@@ -398,6 +470,10 @@ def _build_verification_coverage_payload(
 
     coverage_rate = (verified_auto / total_auto) if total_auto else None
     verification_success_rate = (verified_success / verified_auto) if verified_auto else None
+    recent_non_success = [
+        _non_success_finding_payload(dict(raw))
+        for raw in recent_non_success_rows
+    ]
 
     return {
         "schema_version": "adr100_verification_coverage_v1",
@@ -428,9 +504,96 @@ def _build_verification_coverage_payload(
             }
             for item in (dict(raw) for raw in recent_unverified_rows)
         ],
+        "recent_non_success": recent_non_success,
+        "non_success_breakdown": {
+            "by_verification_result": _count_breakdown(
+                item["verification_result"] for item in recent_non_success
+            ),
+            "by_failure_class": _count_breakdown(
+                item["failure_class"] for item in recent_non_success
+            ),
+        },
     }
 
 
+def _non_success_finding_payload(row: dict[str, Any]) -> dict[str, Any]:
+    failure_class = _classify_non_success_failure(row)
+    return {
+        "auto_repair_id": str(row.get("auto_repair_id")),
+        "incident_id": str(row.get("incident_id")),
+        "incident_status": str(row.get("incident_status") or "unknown"),
+        "incident_severity": str(row.get("incident_severity") or "unknown"),
+        "alert_category": row.get("alert_category"),
+        "alertname": row.get("alertname"),
+        "auto_success": bool(row.get("auto_success")),
+        "playbook_id": row.get("playbook_id"),
+        "playbook_name": row.get("playbook_name"),
+        "triggered_by": row.get("triggered_by"),
+        "risk_level": row.get("risk_level"),
+        "verification_result": str(row.get("verification_result") or "unknown"),
+        "failure_class": failure_class,
+        "next_step": _next_step_for_failure_class(failure_class),
+        "auto_error_excerpt": _short_text(row.get("auto_error"), 180),
+        "evidence_excerpt": _short_text(row.get("evidence_summary"), 180),
+        "auto_created_at": _iso(row.get("auto_created_at")),
+        "verification_collected_at": _iso(row.get("verification_collected_at")),
+    }
+
+
+def _classify_non_success_failure(row: dict[str, Any]) -> str:
+    combined = " ".join(
+        str(row.get(key) or "")
+        for key in ("auto_error", "post_state_text", "evidence_summary")
+    ).lower()
+    if "unsupported scheme" in combined:
+        return "unsupported_action_scheme"
+    if "missing_query_parameter" in combined:
+        return "verifier_missing_promql"
+    if "empty_pod_name" in combined:
+        return "verifier_target_missing_pod"
+    if not bool(row.get("auto_success")):
+        return "auto_repair_execution_failed"
+
+    result = str(row.get("verification_result") or "").lower()
+    if result in {"failed", "timeout"}:
+        return f"verification_{result}"
+    return "verification_degraded"
+
+
+def _next_step_for_failure_class(failure_class: str) -> str:
+    if failure_class == "unsupported_action_scheme":
+        return "normalize_playbook_executor"
+    if failure_class == "verifier_missing_promql":
+        return "add_verifier_query_template"
+    if failure_class == "verifier_target_missing_pod":
+        return "map_verifier_target"
+    if failure_class == "auto_repair_execution_failed":
+        return "review_auto_repair_execution"
+    if failure_class in {"verification_failed", "verification_timeout"}:
+        return "escalate_verification_failure"
+    return "review_degraded_verification"
+
+
+def _count_breakdown(values: Any) -> list[dict[str, Any]]:
+    counts: dict[str, int] = {}
+    for value in values:
+        key = str(value or "unknown")
+        counts[key] = counts.get(key, 0) + 1
+    return [
+        {"name": name, "count": count}
+        for name, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+    ]
+
+
+def _short_text(value: Any, limit: int) -> str | None:
+    if value is None:
+        return None
+    text = " ".join(str(value).split())
+    if not text:
+        return None
+    return text[:limit]
+
+
 def _iso(value: Any) -> str | None:
     return value.isoformat() if hasattr(value, "isoformat") else None