feat(awooop): surface degraded ai route lanes

apps/api/src/api/v1/platform/operator_runs.py

@@ -147,6 +147,10 @@ class AiRouteStatusResponse(BaseModel):
     route_source: str
     route_error: str | None = None
     health: dict[str, dict[str, Any]]
+    lane_mode: str | None = None
+    active_lane: dict[str, Any] | None = None
+    skipped_lanes: list[dict[str, Any]] = Field(default_factory=list)
+    operator_action: dict[str, Any] | None = None
     checked_at: datetime
 
 

apps/api/src/services/platform_operator_service.py

@@ -619,7 +619,8 @@ async def get_ai_route_status(
             route_error=str(exc),
         )
 
-    return {
+    health = _ai_route_health_map(route)
+    response = {
         "schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
         "workload_type": workload,
         "policy_order": policy_order,
@@ -633,9 +634,15 @@ async def get_ai_route_status(
         "route_reason": route.routing_reason,
         "route_source": "ollama_failover_manager",
         "route_error": None,
-        "health": _ai_route_health_map(route),
+        "health": health,
         "checked_at": checked_at,
     }
+    response.update(_ai_route_lane_state(
+        policy_order=policy_order,
+        selected_provider=route.primary.provider_name,
+        health=health,
+    ))
+    return response
 
 
 def _validate_ai_route_workload(workload_type: str | None) -> OllamaWorkloadType:
@@ -712,7 +719,7 @@ async def _ai_route_lightweight_status_from_policy(
     )
 
     if selected_index is None:
-        return {
+        response = {
             "schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
             "workload_type": workload,
             "policy_order": policy_order,
@@ -729,6 +736,12 @@ async def _ai_route_lightweight_status_from_policy(
             "health": health_by_provider,
             "checked_at": checked_at,
         }
+        response.update(_ai_route_lane_state(
+            policy_order=policy_order,
+            selected_provider="gemini",
+            health=health_by_provider,
+        ))
+        return response
 
     selected = endpoints[selected_index]
     model = get_settings().OLLAMA_HEALTH_CHECK_MODEL
@@ -748,7 +761,7 @@ async def _ai_route_lightweight_status_from_policy(
         "runtime": "cloud",
     })
 
-    return {
+    response = {
         "schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
         "workload_type": workload,
         "policy_order": policy_order,
@@ -765,6 +778,12 @@ async def _ai_route_lightweight_status_from_policy(
         "health": health_by_provider,
         "checked_at": checked_at,
     }
+    response.update(_ai_route_lane_state(
+        policy_order=policy_order,
+        selected_provider=selected.provider_name,
+        health=health_by_provider,
+    ))
+    return response
 
 
 async def _ai_route_probe_connectivity(
@@ -832,7 +851,7 @@ def _ai_route_unavailable_status(
     route_error: str,
     route_source: str,
 ) -> dict[str, Any]:
-    return {
+    response = {
         "schema_version": _AI_ROUTE_STATUS_SCHEMA_VERSION,
         "workload_type": workload,
         "policy_order": policy_order,
@@ -846,6 +865,101 @@ def _ai_route_unavailable_status(
         "health": {},
         "checked_at": checked_at,
     }
+    response.update(_ai_route_lane_state(
+        policy_order=policy_order,
+        selected_provider=None,
+        health={},
+    ))
+    return response
+
+
+def _ai_route_lane_state(
+    *,
+    policy_order: list[dict[str, Any]],
+    selected_provider: str | None,
+    health: dict[str, dict[str, Any]],
+) -> dict[str, Any]:
+    """Expose failover lane state separately from policy labels."""
+    selected_index = next(
+        (
+            index
+            for index, item in enumerate(policy_order)
+            if item.get("provider_name") == selected_provider
+        ),
+        None,
+    )
+    active_item = (
+        policy_order[selected_index]
+        if selected_index is not None
+        else None
+    )
+    skipped_items = policy_order[:selected_index] if selected_index is not None else []
+
+    skipped_lanes = [
+        _ai_route_lane_item(item, health.get(str(item.get("provider_name"))))
+        for item in skipped_items
+        if item.get("runtime") == "ollama"
+    ]
+
+    if not selected_provider or active_item is None:
+        lane_mode = "unavailable"
+        operator_action = {
+            "human_required": True,
+            "action": "inspect_ai_router",
+            "reason": "no_active_provider",
+        }
+    elif active_item.get("runtime") == "cloud":
+        lane_mode = "cloud_fallback"
+        operator_action = {
+            "human_required": True,
+            "action": "restore_ollama_lanes",
+            "reason": "all_ollama_lanes_unavailable",
+        }
+    elif skipped_lanes:
+        lane_mode = "degraded_failover"
+        operator_action = {
+            "human_required": True,
+            "action": "repair_skipped_primary_lane",
+            "reason": "fallback_lane_active",
+        }
+    else:
+        lane_mode = "primary"
+        operator_action = {
+            "human_required": False,
+            "action": "monitor",
+            "reason": "primary_lane_active",
+        }
+
+    return {
+        "lane_mode": lane_mode,
+        "active_lane": (
+            _ai_route_lane_item(active_item, health.get(str(active_item.get("provider_name"))))
+            if active_item
+            else None
+        ),
+        "skipped_lanes": skipped_lanes,
+        "operator_action": operator_action,
+    }
+
+
+def _ai_route_lane_item(
+    item: dict[str, Any],
+    health_item: dict[str, Any] | None,
+) -> dict[str, Any]:
+    return {
+        "priority": item.get("priority"),
+        "provider_name": item.get("provider_name"),
+        "role": item.get("role"),
+        "runtime": item.get("runtime"),
+        "url": item.get("url"),
+        "health_status": (health_item or {}).get("status", "not_checked"),
+        "reason": (health_item or {}).get("reason") or item.get("reason"),
+        "action_required": (health_item or {}).get("status") not in {
+            "healthy",
+            "not_checked",
+            None,
+        },
+    }
 
 
 def _ai_route_policy_endpoint_item(

apps/api/tests/test_awooop_operator_timeline_labels.py

@@ -19,6 +19,7 @@ from src.services.ollama_failover_manager import OllamaEndpoint, OllamaRoutingRe
 from src.services.ollama_health_monitor import HealthReport, HealthStatus
 from src.services.platform_operator_service import (
     _ai_route_health_map,
+    _ai_route_lane_state,
     _ai_route_policy_order,
     _build_awooop_status_chain,
     _callback_reply_event_item,
@@ -1549,12 +1550,87 @@ def test_ai_route_status_response_preserves_route_fields() -> None:
                 "checked": True,
             },
         },
+        "lane_mode": "primary",
+        "active_lane": {
+            "provider_name": "ollama_gcp_a",
+            "health_status": "healthy",
+            "action_required": False,
+        },
+        "skipped_lanes": [],
+        "operator_action": {
+            "human_required": False,
+            "action": "monitor",
+            "reason": "primary_lane_active",
+        },
         "checked_at": datetime(2026, 5, 19, 12, 0, 0),
     })
 
     dumped = response.model_dump(mode="json")
     assert dumped["policy_order"][-1]["provider_name"] == "gemini"
     assert dumped["selected_provider"] == "ollama_gcp_a"
+    assert dumped["lane_mode"] == "primary"
+
+
+def test_ai_route_lane_state_marks_degraded_failover() -> None:
+    policy = _ai_route_policy_order("deep_rca")
+    health = {
+        "ollama_gcp_a": {
+            "status": "offline",
+            "reason": "recent_endpoint_failure_cooldown:25s",
+        },
+        "ollama_gcp_b": {
+            "status": "healthy",
+            "reason": "",
+        },
+        "ollama_local": {
+            "status": "healthy",
+            "reason": "",
+        },
+    }
+
+    state = _ai_route_lane_state(
+        policy_order=policy,
+        selected_provider="ollama_gcp_b",
+        health=health,
+    )
+
+    assert state["lane_mode"] == "degraded_failover"
+    assert state["active_lane"]["provider_name"] == "ollama_gcp_b"
+    assert len(state["skipped_lanes"]) == 1
+    assert state["skipped_lanes"][0]["provider_name"] == "ollama_gcp_a"
+    assert state["skipped_lanes"][0]["role"] == "primary"
+    assert state["skipped_lanes"][0]["health_status"] == "offline"
+    assert state["skipped_lanes"][0]["reason"] == "recent_endpoint_failure_cooldown:25s"
+    assert state["skipped_lanes"][0]["action_required"] is True
+    assert state["operator_action"] == {
+        "human_required": True,
+        "action": "repair_skipped_primary_lane",
+        "reason": "fallback_lane_active",
+    }
+
+
+def test_ai_route_lane_state_marks_cloud_fallback() -> None:
+    policy = _ai_route_policy_order("deep_rca")
+    health = {
+        "ollama_gcp_a": {"status": "offline", "reason": "timeout"},
+        "ollama_gcp_b": {"status": "offline", "reason": "timeout"},
+        "ollama_local": {"status": "offline", "reason": "timeout"},
+    }
+
+    state = _ai_route_lane_state(
+        policy_order=policy,
+        selected_provider="gemini",
+        health=health,
+    )
+
+    assert state["lane_mode"] == "cloud_fallback"
+    assert state["active_lane"]["provider_name"] == "gemini"
+    assert [lane["provider_name"] for lane in state["skipped_lanes"]] == [
+        "ollama_gcp_a",
+        "ollama_gcp_b",
+        "ollama_local",
+    ]
+    assert state["operator_action"]["action"] == "restore_ollama_lanes"
 
 
 @pytest.mark.asyncio