feat(awooop): surface ai route repair evidence
All checks were successful
CD Pipeline / tests (push) Successful in 1m24s
Code Review / ai-code-review (push) Successful in 12s
CD Pipeline / build-and-deploy (push) Successful in 3m53s
CD Pipeline / post-deploy-checks (push) Successful in 1m48s

This commit is contained in:
Your Name
2026-05-25 14:21:25 +08:00
parent e570d9f6a9
commit 67296746c0
8 changed files with 642 additions and 7 deletions

View File

@@ -21,6 +21,7 @@ from src.services.platform_operator_service import (
_ai_route_health_map,
_ai_route_lane_state,
_ai_route_policy_order,
_ai_route_repair_evidence_item,
_build_awooop_status_chain,
_callback_reply_event_item,
_callback_reply_summary_matches_status,
@@ -1562,6 +1563,12 @@ def test_ai_route_status_response_preserves_route_fields() -> None:
"action": "monitor",
"reason": "primary_lane_active",
},
"repair_evidence": {
"provider": "ai_route_repair",
"stage": "repair_diagnosis",
"target_resource": "ollama_gcp_a",
"access_blockers": ["gcloud_compute_instances_get_missing"],
},
"checked_at": datetime(2026, 5, 19, 12, 0, 0),
})
@@ -1569,6 +1576,79 @@ def test_ai_route_status_response_preserves_route_fields() -> None:
assert dumped["policy_order"][-1]["provider_name"] == "gemini"
assert dumped["selected_provider"] == "ollama_gcp_a"
assert dumped["lane_mode"] == "primary"
assert dumped["repair_evidence"]["target_resource"] == "ollama_gcp_a"
def test_ai_route_repair_evidence_item_summarizes_operator_safe_fields() -> None:
row = {
"event_id": UUID("dff309f0-f159-4537-8f58-47714ce94dca"),
"run_id": UUID("ca67ebcc-a24f-53e7-9505-2db15d855ecc"),
"provider_event_id": (
"ai_route_repair:repair_diagnosis:"
"gcp-a-primary-lane-down-20260525T060415Z"
),
"provider_ts": datetime(2026, 5, 25, 6, 4, 15),
"received_at": datetime(2026, 5, 25, 6, 5, 3),
"source_envelope": {
"provider": "ai_route_repair",
"stage": "repair_diagnosis",
"log_correlation": {
"alertname": "GcpAPrimaryLaneDown",
"severity": "warning",
"target_resource": "ollama_gcp_a",
"fingerprint": "ai-route-gcp-a-primary-down",
},
"source_refs": {
"alert_ids": ["gcp-a-primary-lane-down"],
"signoz_alerts": ["signoz:gcp-a"],
"sentry_issue_ids": [],
"fingerprints": ["ai-route-gcp-a-primary-down"],
"run_ids": ["ca67ebcc-a24f-53e7-9505-2db15d855ecc"],
},
"extra": {
"payload": {
"schema_version": "ai_route_repair_diagnosis_v1",
"observed_state": {
"target_resource": "ollama_gcp_a",
"lane_mode": "degraded_failover",
},
"live_probe": {
"gcp_a_direct_11434": "connection_refused",
"gcp_b_direct_11434": "http_200",
},
"access_blockers": [
"gcloud_compute_instances_get_missing",
"gcp_a_ollama_11434_refused",
],
"side_effects": {
"incident_created": False,
"telegram_sent": False,
"approval_created": False,
"runtime_route_changed": False,
},
},
},
},
}
item = _ai_route_repair_evidence_item(row)
assert item["provider"] == "ai_route_repair"
assert item["stage"] == "repair_diagnosis"
assert item["target_resource"] == "ollama_gcp_a"
assert item["run_id"] == "ca67ebcc-a24f-53e7-9505-2db15d855ecc"
assert item["source_ref_count"] == 4
assert item["access_blockers"] == [
"gcloud_compute_instances_get_missing",
"gcp_a_ollama_11434_refused",
]
assert item["live_probe"]["gcp_a_direct_11434"] == "connection_refused"
assert item["side_effects"] == {
"incident_created": False,
"telegram_sent": False,
"approval_created": False,
"runtime_route_changed": False,
}
def test_ai_route_lane_state_marks_degraded_failover() -> None:
@@ -1669,6 +1749,15 @@ async def test_ai_route_status_times_out_before_slow_provider_checks(monkeypatch
fake_connectivity,
)
async def no_repair_evidence(**_kwargs):
return None
monkeypatch.setattr(
platform_operator_service,
"_latest_ai_route_repair_evidence",
no_repair_evidence,
)
response = await platform_operator_service.get_ai_route_status("deep_rca")
assert response["route_reason"] == (
@@ -1722,6 +1811,15 @@ async def test_ai_route_status_lightweight_fallback_keeps_gemini_policy_only(
fake_offline_connectivity,
)
async def no_repair_evidence(**_kwargs):
return None
monkeypatch.setattr(
platform_operator_service,
"_latest_ai_route_repair_evidence",
no_repair_evidence,
)
response = await platform_operator_service.get_ai_route_status("deep_rca")
assert response["selected_provider"] == "gemini"