feat(awooop): summarize automation quality
This commit is contained in:
@@ -8,11 +8,13 @@ from src.services.awooop_ansible_audit_service import (
|
||||
build_ansible_truth,
|
||||
)
|
||||
from src.services.awooop_truth_chain_service import (
|
||||
_automation_quality_score_bucket,
|
||||
build_automation_quality,
|
||||
build_incident_reconciliation,
|
||||
_clean_row,
|
||||
_summarize_gateway_mcp,
|
||||
_truth_status,
|
||||
summarize_automation_quality_records,
|
||||
)
|
||||
from src.services.drift_repeat_state import (
|
||||
build_drift_fingerprint,
|
||||
@@ -280,6 +282,89 @@ def test_automation_quality_marks_verified_auto_repair() -> None:
|
||||
assert quality["blockers"] == []
|
||||
|
||||
|
||||
def test_automation_quality_score_buckets_are_stable() -> None:
|
||||
assert _automation_quality_score_bucket(100) == "green"
|
||||
assert _automation_quality_score_bucket(85) == "green"
|
||||
assert _automation_quality_score_bucket(84) == "yellow"
|
||||
assert _automation_quality_score_bucket(60) == "yellow"
|
||||
assert _automation_quality_score_bucket(59) == "red"
|
||||
|
||||
|
||||
def test_automation_quality_summary_denies_full_claim_when_unverified() -> None:
|
||||
summary = summarize_automation_quality_records(
|
||||
project_id="awoooi",
|
||||
window_hours=24,
|
||||
limit=200,
|
||||
records=[
|
||||
{
|
||||
"incident": {
|
||||
"incident_id": "INC-OK",
|
||||
"alertname": "container recovered",
|
||||
"severity": "P4",
|
||||
"status": "RESOLVED",
|
||||
"created_at": "2026-05-13T01:00:00+00:00",
|
||||
},
|
||||
"truth_status": {
|
||||
"current_stage": "execution_succeeded",
|
||||
"stage_status": "success",
|
||||
"needs_human": False,
|
||||
},
|
||||
"automation_quality": {
|
||||
"applicable": True,
|
||||
"verdict": "auto_repaired_verified",
|
||||
"score": 100,
|
||||
"gates": [
|
||||
{"name": "verification_recorded", "status": "passed"},
|
||||
],
|
||||
"blockers": [],
|
||||
},
|
||||
},
|
||||
{
|
||||
"incident": {
|
||||
"incident_id": "INC-GAP",
|
||||
"alertname": "low risk action",
|
||||
"severity": "P4",
|
||||
"status": "INVESTIGATING",
|
||||
"created_at": "2026-05-13T02:00:00+00:00",
|
||||
},
|
||||
"truth_status": {
|
||||
"current_stage": "execution_succeeded",
|
||||
"stage_status": "success",
|
||||
"needs_human": False,
|
||||
},
|
||||
"automation_quality": {
|
||||
"applicable": True,
|
||||
"verdict": "execution_unverified",
|
||||
"score": 65,
|
||||
"gates": [
|
||||
{"name": "verification_recorded", "status": "missing"},
|
||||
{"name": "learning_recorded", "status": "missing"},
|
||||
],
|
||||
"blockers": ["verification_recorded", "learning_recorded"],
|
||||
},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
assert summary["schema_version"] == "automation_quality_summary_v1"
|
||||
assert summary["incident_total"] == 2
|
||||
assert summary["evaluated_total"] == 2
|
||||
assert summary["verified_auto_repair_total"] == 1
|
||||
assert summary["score_buckets"] == {"green": 1, "yellow": 1, "red": 0}
|
||||
assert summary["production_claim"]["can_claim_full_auto_repair"] is False
|
||||
assert summary["production_claim"]["reason"] == "some_incidents_are_not_auto_repaired_verified"
|
||||
assert {row["verdict"]: row["total"] for row in summary["by_verdict"]} == {
|
||||
"auto_repaired_verified": 1,
|
||||
"execution_unverified": 1,
|
||||
}
|
||||
assert {row["gate"]: row["total"] for row in summary["gate_failures"]} == {
|
||||
"learning_recorded": 1,
|
||||
"verification_recorded": 1,
|
||||
}
|
||||
assert summary["examples"][1]["incident_id"] == "INC-GAP"
|
||||
assert summary["examples"][1]["score_bucket"] == "yellow"
|
||||
|
||||
|
||||
def test_reconciliation_marks_consistent_resolved_execution() -> None:
|
||||
reconciliation = build_incident_reconciliation(
|
||||
incident={"incident_id": "INC-2", "status": "RESOLVED"},
|
||||
|
||||
@@ -35,3 +35,15 @@ def test_truth_chain_route_is_registered() -> None:
|
||||
]
|
||||
|
||||
assert "/truth-chain/{source_id}" in paths
|
||||
|
||||
|
||||
def test_truth_chain_quality_summary_route_is_registered_before_dynamic_source_id() -> None:
|
||||
paths = [
|
||||
route.path
|
||||
for route in router.routes
|
||||
if "GET" in getattr(route, "methods", set())
|
||||
]
|
||||
|
||||
assert "/truth-chain/quality/summary" in paths
|
||||
assert "/truth-chain/{source_id}" in paths
|
||||
assert paths.index("/truth-chain/quality/summary") < paths.index("/truth-chain/{source_id}")
|
||||
|
||||
Reference in New Issue
Block a user