feat(awooop): summarize automation quality

apps/api/src/services/awooop_truth_chain_service.py

@@ -8,7 +8,7 @@ Telegram cards can be audited without guessing which subsystem owns the truth.
 from __future__ import annotations
 
 import json
-from datetime import date, datetime
+from datetime import UTC, date, datetime, timedelta
 from decimal import Decimal
 from typing import Any
 from uuid import UUID
@@ -489,6 +489,130 @@ def build_automation_quality(
     }
 
 
+def _automation_quality_score_bucket(score: int) -> str:
+    if score >= 85:
+        return "green"
+    if score >= 60:
+        return "yellow"
+    return "red"
+
+
+def summarize_automation_quality_records(
+    *,
+    project_id: str,
+    window_hours: int,
+    records: list[dict[str, Any]],
+    limit: int,
+) -> dict[str, Any]:
+    """Aggregate per-incident automation quality into an operator summary."""
+    verdicts: dict[str, dict[str, Any]] = {}
+    gate_failures: dict[str, dict[str, Any]] = {}
+    score_buckets: dict[str, int] = {"green": 0, "yellow": 0, "red": 0}
+    examples: list[dict[str, Any]] = []
+    total_score = 0
+    evaluated_total = 0
+    verified_total = 0
+
+    for record in records:
+        incident = record.get("incident") if isinstance(record.get("incident"), dict) else {}
+        truth_status = record.get("truth_status") if isinstance(record.get("truth_status"), dict) else {}
+        quality = record.get("automation_quality") if isinstance(record.get("automation_quality"), dict) else {}
+        if quality.get("applicable") is not True:
+            continue
+
+        evaluated_total += 1
+        score = int(quality.get("score") or 0)
+        total_score += score
+        bucket = _automation_quality_score_bucket(score)
+        score_buckets[bucket] += 1
+
+        verdict = str(quality.get("verdict") or "unknown")
+        if verdict == "auto_repaired_verified":
+            verified_total += 1
+        verdict_row = verdicts.setdefault(
+            verdict,
+            {
+                "verdict": verdict,
+                "total": 0,
+                "score_sum": 0,
+                "min_score": score,
+                "max_score": score,
+                "needs_human": False,
+            },
+        )
+        verdict_row["total"] += 1
+        verdict_row["score_sum"] += score
+        verdict_row["min_score"] = min(int(verdict_row["min_score"]), score)
+        verdict_row["max_score"] = max(int(verdict_row["max_score"]), score)
+        verdict_row["needs_human"] = bool(
+            verdict_row["needs_human"] or truth_status.get("needs_human")
+        )
+
+        for gate in quality.get("gates") or []:
+            if not isinstance(gate, dict):
+                continue
+            gate_status = str(gate.get("status") or "")
+            if gate_status not in {"failed", "missing"}:
+                continue
+            gate_name = str(gate.get("name") or "unknown")
+            gate_row = gate_failures.setdefault(
+                gate_name,
+                {"gate": gate_name, "total": 0, "statuses": {}},
+            )
+            gate_row["total"] += 1
+            gate_row["statuses"][gate_status] = int(gate_row["statuses"].get(gate_status, 0)) + 1
+
+        examples.append({
+            "incident_id": incident.get("incident_id"),
+            "alertname": incident.get("alertname"),
+            "severity": incident.get("severity"),
+            "status": incident.get("status"),
+            "created_at": incident.get("created_at"),
+            "truth_stage": truth_status.get("current_stage"),
+            "truth_stage_status": truth_status.get("stage_status"),
+            "needs_human": bool(truth_status.get("needs_human")),
+            "verdict": verdict,
+            "score": score,
+            "score_bucket": bucket,
+            "blockers": list(quality.get("blockers") or [])[:8],
+        })
+
+    by_verdict = []
+    for row in verdicts.values():
+        total = int(row["total"])
+        row["avg_score"] = round(float(row.pop("score_sum")) / total, 1) if total else 0.0
+        by_verdict.append(row)
+    by_verdict.sort(key=lambda row: (-int(row["total"]), str(row["verdict"])))
+
+    failing_gates = sorted(
+        gate_failures.values(),
+        key=lambda row: (-int(row["total"]), str(row["gate"])),
+    )
+
+    return {
+        "schema_version": "automation_quality_summary_v1",
+        "project_id": project_id,
+        "window_hours": window_hours,
+        "limit": limit,
+        "incident_total": len(records),
+        "evaluated_total": evaluated_total,
+        "verified_auto_repair_total": verified_total,
+        "average_score": round(total_score / evaluated_total, 1) if evaluated_total else 0.0,
+        "score_buckets": score_buckets,
+        "by_verdict": by_verdict,
+        "gate_failures": failing_gates,
+        "examples": examples[:25],
+        "production_claim": {
+            "can_claim_full_auto_repair": evaluated_total > 0 and verified_total == evaluated_total,
+            "reason": (
+                "all_evaluated_incidents_auto_repaired_verified"
+                if evaluated_total > 0 and verified_total == evaluated_total
+                else "some_incidents_are_not_auto_repaired_verified"
+            ),
+        },
+    }
+
+
 def _summarize_mcp(rows: list[dict[str, Any]]) -> dict[str, Any]:
     by_tool: dict[str, dict[str, Any]] = {}
     success_count = 0
@@ -1108,3 +1232,72 @@ async def fetch_truth_chain(source_id: str, project_id: str = "awoooi") -> dict[
         current_stage=truth_status["current_stage"],
     )
     return result
+
+
+async def fetch_automation_quality_summary(
+    *,
+    project_id: str = "awoooi",
+    hours: int = 24,
+    limit: int = 200,
+) -> dict[str, Any]:
+    """Return a recent incident-level quality summary for the automation flywheel."""
+    bounded_hours = max(1, min(int(hours), 168))
+    bounded_limit = max(1, min(int(limit), 500))
+    cutoff = datetime.now(UTC) - timedelta(hours=bounded_hours)
+
+    async with get_db_context(project_id) as db:
+        incidents = await _fetch_all(
+            db,
+            """
+            SELECT
+                incident_id,
+                project_id,
+                status::text AS status,
+                severity::text AS severity,
+                alertname,
+                alert_category,
+                notification_type,
+                created_at,
+                updated_at,
+                resolved_at,
+                verification_result
+            FROM incidents
+            WHERE (project_id = :project_id OR project_id IS NULL)
+              AND created_at >= :cutoff
+            ORDER BY created_at DESC
+            LIMIT :limit
+            """,
+            {
+                "project_id": project_id,
+                "cutoff": cutoff,
+                "limit": bounded_limit,
+            },
+        )
+
+    records: list[dict[str, Any]] = []
+    for incident in incidents:
+        incident_id = str(incident.get("incident_id") or "")
+        if not incident_id:
+            continue
+        truth_chain = await fetch_truth_chain(source_id=incident_id, project_id=project_id)
+        records.append({
+            "incident": truth_chain.get("incident") or incident,
+            "truth_status": truth_chain.get("truth_status") or {},
+            "automation_quality": truth_chain.get("automation_quality") or {},
+        })
+
+    summary = summarize_automation_quality_records(
+        project_id=project_id,
+        window_hours=bounded_hours,
+        records=records,
+        limit=bounded_limit,
+    )
+    logger.info(
+        "awooop_automation_quality_summary_fetched",
+        project_id=project_id,
+        window_hours=bounded_hours,
+        incident_total=summary["incident_total"],
+        evaluated_total=summary["evaluated_total"],
+        can_claim_full_auto_repair=summary["production_claim"]["can_claim_full_auto_repair"],
+    )
+    return summary