feat(awooop): surface degraded ai route lanes
All checks were successful
CD Pipeline / tests (push) Successful in 1m25s
Code Review / ai-code-review (push) Successful in 12s
CD Pipeline / build-and-deploy (push) Successful in 3m37s
CD Pipeline / post-deploy-checks (push) Successful in 1m44s

This commit is contained in:
Your Name
2026-05-25 13:24:53 +08:00
parent 19d306c720
commit ed3e658578
8 changed files with 455 additions and 17 deletions

View File

@@ -619,7 +619,8 @@ async def get_ai_route_status(
route_error=str(exc),
)
return {
health = _ai_route_health_map(route)
response = {
"schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
"workload_type": workload,
"policy_order": policy_order,
@@ -633,9 +634,15 @@ async def get_ai_route_status(
"route_reason": route.routing_reason,
"route_source": "ollama_failover_manager",
"route_error": None,
"health": _ai_route_health_map(route),
"health": health,
"checked_at": checked_at,
}
response.update(_ai_route_lane_state(
policy_order=policy_order,
selected_provider=route.primary.provider_name,
health=health,
))
return response
def _validate_ai_route_workload(workload_type: str | None) -> OllamaWorkloadType:
@@ -712,7 +719,7 @@ async def _ai_route_lightweight_status_from_policy(
)
if selected_index is None:
return {
response = {
"schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
"workload_type": workload,
"policy_order": policy_order,
@@ -729,6 +736,12 @@ async def _ai_route_lightweight_status_from_policy(
"health": health_by_provider,
"checked_at": checked_at,
}
response.update(_ai_route_lane_state(
policy_order=policy_order,
selected_provider="gemini",
health=health_by_provider,
))
return response
selected = endpoints[selected_index]
model = get_settings().OLLAMA_HEALTH_CHECK_MODEL
@@ -748,7 +761,7 @@ async def _ai_route_lightweight_status_from_policy(
"runtime": "cloud",
})
return {
response = {
"schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
"workload_type": workload,
"policy_order": policy_order,
@@ -765,6 +778,12 @@ async def _ai_route_lightweight_status_from_policy(
"health": health_by_provider,
"checked_at": checked_at,
}
response.update(_ai_route_lane_state(
policy_order=policy_order,
selected_provider=selected.provider_name,
health=health_by_provider,
))
return response
async def _ai_route_probe_connectivity(
@@ -832,7 +851,7 @@ def _ai_route_unavailable_status(
route_error: str,
route_source: str,
) -> dict[str, Any]:
return {
response = {
"schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
"workload_type": workload,
"policy_order": policy_order,
@@ -846,6 +865,101 @@ def _ai_route_unavailable_status(
"health": {},
"checked_at": checked_at,
}
response.update(_ai_route_lane_state(
policy_order=policy_order,
selected_provider=None,
health={},
))
return response
def _ai_route_lane_state(
*,
policy_order: list[dict[str, Any]],
selected_provider: str | None,
health: dict[str, dict[str, Any]],
) -> dict[str, Any]:
"""Expose failover lane state separately from policy labels."""
selected_index = next(
(
index
for index, item in enumerate(policy_order)
if item.get("provider_name") == selected_provider
),
None,
)
active_item = (
policy_order[selected_index]
if selected_index is not None
else None
)
skipped_items = policy_order[:selected_index] if selected_index is not None else []
skipped_lanes = [
_ai_route_lane_item(item, health.get(str(item.get("provider_name"))))
for item in skipped_items
if item.get("runtime") == "ollama"
]
if not selected_provider or active_item is None:
lane_mode = "unavailable"
operator_action = {
"human_required": True,
"action": "inspect_ai_router",
"reason": "no_active_provider",
}
elif active_item.get("runtime") == "cloud":
lane_mode = "cloud_fallback"
operator_action = {
"human_required": True,
"action": "restore_ollama_lanes",
"reason": "all_ollama_lanes_unavailable",
}
elif skipped_lanes:
lane_mode = "degraded_failover"
operator_action = {
"human_required": True,
"action": "repair_skipped_primary_lane",
"reason": "fallback_lane_active",
}
else:
lane_mode = "primary"
operator_action = {
"human_required": False,
"action": "monitor",
"reason": "primary_lane_active",
}
return {
"lane_mode": lane_mode,
"active_lane": (
_ai_route_lane_item(active_item, health.get(str(active_item.get("provider_name"))))
if active_item
else None
),
"skipped_lanes": skipped_lanes,
"operator_action": operator_action,
}
def _ai_route_lane_item(
item: dict[str, Any],
health_item: dict[str, Any] | None,
) -> dict[str, Any]:
return {
"priority": item.get("priority"),
"provider_name": item.get("provider_name"),
"role": item.get("role"),
"runtime": item.get("runtime"),
"url": item.get("url"),
"health_status": (health_item or {}).get("status", "not_checked"),
"reason": (health_item or {}).get("reason") or item.get("reason"),
"action_required": (health_item or {}).get("status") not in {
"healthy",
"not_checked",
None,
},
}
def _ai_route_policy_endpoint_item(