feat(awooop): surface degraded ai route lanes
This commit is contained in:
@@ -619,7 +619,8 @@ async def get_ai_route_status(
|
||||
route_error=str(exc),
|
||||
)
|
||||
|
||||
return {
|
||||
health = _ai_route_health_map(route)
|
||||
response = {
|
||||
"schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
|
||||
"workload_type": workload,
|
||||
"policy_order": policy_order,
|
||||
@@ -633,9 +634,15 @@ async def get_ai_route_status(
|
||||
"route_reason": route.routing_reason,
|
||||
"route_source": "ollama_failover_manager",
|
||||
"route_error": None,
|
||||
"health": _ai_route_health_map(route),
|
||||
"health": health,
|
||||
"checked_at": checked_at,
|
||||
}
|
||||
response.update(_ai_route_lane_state(
|
||||
policy_order=policy_order,
|
||||
selected_provider=route.primary.provider_name,
|
||||
health=health,
|
||||
))
|
||||
return response
|
||||
|
||||
|
||||
def _validate_ai_route_workload(workload_type: str | None) -> OllamaWorkloadType:
|
||||
@@ -712,7 +719,7 @@ async def _ai_route_lightweight_status_from_policy(
|
||||
)
|
||||
|
||||
if selected_index is None:
|
||||
return {
|
||||
response = {
|
||||
"schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
|
||||
"workload_type": workload,
|
||||
"policy_order": policy_order,
|
||||
@@ -729,6 +736,12 @@ async def _ai_route_lightweight_status_from_policy(
|
||||
"health": health_by_provider,
|
||||
"checked_at": checked_at,
|
||||
}
|
||||
response.update(_ai_route_lane_state(
|
||||
policy_order=policy_order,
|
||||
selected_provider="gemini",
|
||||
health=health_by_provider,
|
||||
))
|
||||
return response
|
||||
|
||||
selected = endpoints[selected_index]
|
||||
model = get_settings().OLLAMA_HEALTH_CHECK_MODEL
|
||||
@@ -748,7 +761,7 @@ async def _ai_route_lightweight_status_from_policy(
|
||||
"runtime": "cloud",
|
||||
})
|
||||
|
||||
return {
|
||||
response = {
|
||||
"schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
|
||||
"workload_type": workload,
|
||||
"policy_order": policy_order,
|
||||
@@ -765,6 +778,12 @@ async def _ai_route_lightweight_status_from_policy(
|
||||
"health": health_by_provider,
|
||||
"checked_at": checked_at,
|
||||
}
|
||||
response.update(_ai_route_lane_state(
|
||||
policy_order=policy_order,
|
||||
selected_provider=selected.provider_name,
|
||||
health=health_by_provider,
|
||||
))
|
||||
return response
|
||||
|
||||
|
||||
async def _ai_route_probe_connectivity(
|
||||
@@ -832,7 +851,7 @@ def _ai_route_unavailable_status(
|
||||
route_error: str,
|
||||
route_source: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
response = {
|
||||
"schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
|
||||
"workload_type": workload,
|
||||
"policy_order": policy_order,
|
||||
@@ -846,6 +865,101 @@ def _ai_route_unavailable_status(
|
||||
"health": {},
|
||||
"checked_at": checked_at,
|
||||
}
|
||||
response.update(_ai_route_lane_state(
|
||||
policy_order=policy_order,
|
||||
selected_provider=None,
|
||||
health={},
|
||||
))
|
||||
return response
|
||||
|
||||
|
||||
def _ai_route_lane_state(
|
||||
*,
|
||||
policy_order: list[dict[str, Any]],
|
||||
selected_provider: str | None,
|
||||
health: dict[str, dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
"""Expose failover lane state separately from policy labels."""
|
||||
selected_index = next(
|
||||
(
|
||||
index
|
||||
for index, item in enumerate(policy_order)
|
||||
if item.get("provider_name") == selected_provider
|
||||
),
|
||||
None,
|
||||
)
|
||||
active_item = (
|
||||
policy_order[selected_index]
|
||||
if selected_index is not None
|
||||
else None
|
||||
)
|
||||
skipped_items = policy_order[:selected_index] if selected_index is not None else []
|
||||
|
||||
skipped_lanes = [
|
||||
_ai_route_lane_item(item, health.get(str(item.get("provider_name"))))
|
||||
for item in skipped_items
|
||||
if item.get("runtime") == "ollama"
|
||||
]
|
||||
|
||||
if not selected_provider or active_item is None:
|
||||
lane_mode = "unavailable"
|
||||
operator_action = {
|
||||
"human_required": True,
|
||||
"action": "inspect_ai_router",
|
||||
"reason": "no_active_provider",
|
||||
}
|
||||
elif active_item.get("runtime") == "cloud":
|
||||
lane_mode = "cloud_fallback"
|
||||
operator_action = {
|
||||
"human_required": True,
|
||||
"action": "restore_ollama_lanes",
|
||||
"reason": "all_ollama_lanes_unavailable",
|
||||
}
|
||||
elif skipped_lanes:
|
||||
lane_mode = "degraded_failover"
|
||||
operator_action = {
|
||||
"human_required": True,
|
||||
"action": "repair_skipped_primary_lane",
|
||||
"reason": "fallback_lane_active",
|
||||
}
|
||||
else:
|
||||
lane_mode = "primary"
|
||||
operator_action = {
|
||||
"human_required": False,
|
||||
"action": "monitor",
|
||||
"reason": "primary_lane_active",
|
||||
}
|
||||
|
||||
return {
|
||||
"lane_mode": lane_mode,
|
||||
"active_lane": (
|
||||
_ai_route_lane_item(active_item, health.get(str(active_item.get("provider_name"))))
|
||||
if active_item
|
||||
else None
|
||||
),
|
||||
"skipped_lanes": skipped_lanes,
|
||||
"operator_action": operator_action,
|
||||
}
|
||||
|
||||
|
||||
def _ai_route_lane_item(
|
||||
item: dict[str, Any],
|
||||
health_item: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"priority": item.get("priority"),
|
||||
"provider_name": item.get("provider_name"),
|
||||
"role": item.get("role"),
|
||||
"runtime": item.get("runtime"),
|
||||
"url": item.get("url"),
|
||||
"health_status": (health_item or {}).get("status", "not_checked"),
|
||||
"reason": (health_item or {}).get("reason") or item.get("reason"),
|
||||
"action_required": (health_item or {}).get("status") not in {
|
||||
"healthy",
|
||||
"not_checked",
|
||||
None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _ai_route_policy_endpoint_item(
|
||||
|
||||
Reference in New Issue
Block a user