fix(health): surface ollama endpoint diagnosis
This commit is contained in:
@@ -47,6 +47,11 @@ class ComponentHealth(BaseModel):
|
||||
status: Literal["up", "down", "degraded"]
|
||||
latency_ms: float | None = None
|
||||
error: str | None = None
|
||||
provider_name: str | None = None
|
||||
diagnosis_code: str | None = None
|
||||
retry_after_seconds: float | None = None
|
||||
cooldown_remaining_seconds: float | None = None
|
||||
is_cooldown: bool = False
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
@@ -194,16 +199,47 @@ async def _ollama_endpoint_health_check(name: str, url: str) -> ComponentHealth:
|
||||
return ComponentHealth(
|
||||
status="down",
|
||||
error=f"recent endpoint failure cooldown: {cooldown_remaining:.0f}s",
|
||||
provider_name=name,
|
||||
diagnosis_code="endpoint_cooldown",
|
||||
retry_after_seconds=round(cooldown_remaining, 1),
|
||||
cooldown_remaining_seconds=round(cooldown_remaining, 1),
|
||||
is_cooldown=True,
|
||||
)
|
||||
|
||||
result = await _http_health_check(name, url, "/api/tags")
|
||||
result.provider_name = name
|
||||
if result.status == "up":
|
||||
result.diagnosis_code = "endpoint_reachable"
|
||||
record_ollama_endpoint_success(url)
|
||||
else:
|
||||
result.diagnosis_code = _classify_ollama_endpoint_failure(name, result.error)
|
||||
record_ollama_endpoint_failure(url)
|
||||
return result
|
||||
|
||||
|
||||
def _classify_ollama_endpoint_failure(
|
||||
provider_name: str,
|
||||
error: str | None,
|
||||
) -> str:
|
||||
"""Return a stable diagnosis code for UI/alert rendering."""
|
||||
normalized_error = (error or "").lower()
|
||||
if "cooldown" in normalized_error:
|
||||
return "endpoint_cooldown"
|
||||
if "502" in normalized_error or "bad gateway" in normalized_error:
|
||||
return (
|
||||
"local_proxy_upstream_unreachable"
|
||||
if provider_name == "ollama_local"
|
||||
else "proxy_upstream_unreachable"
|
||||
)
|
||||
if "timeout" in normalized_error:
|
||||
return "endpoint_timeout"
|
||||
if "connection refused" in normalized_error:
|
||||
return "endpoint_connection_refused"
|
||||
if "no route to host" in normalized_error or "network is unreachable" in normalized_error:
|
||||
return "endpoint_network_unreachable"
|
||||
return "endpoint_unreachable"
|
||||
|
||||
|
||||
async def check_openclaw() -> ComponentHealth:
|
||||
"""Async OpenClaw health check via /health"""
|
||||
return await _http_health_check("openclaw", settings.OPENCLAW_URL, "/health")
|
||||
|
||||
@@ -43,7 +43,10 @@ async def test_ollama_provider_chain_reports_fallback_when_primary_down(monkeypa
|
||||
assert aggregate.latency_ms == 42.0
|
||||
assert aggregate.error == "primary unavailable; fallback active: ollama_gcp_b"
|
||||
assert details["ollama_gcp_a"].status == "down"
|
||||
assert details["ollama_gcp_a"].provider_name == "ollama_gcp_a"
|
||||
assert details["ollama_gcp_a"].diagnosis_code == "endpoint_timeout"
|
||||
assert details["ollama_gcp_b"].status == "up"
|
||||
assert details["ollama_gcp_b"].diagnosis_code == "endpoint_reachable"
|
||||
assert details["ollama_local"].status == "up"
|
||||
|
||||
|
||||
@@ -108,6 +111,9 @@ async def test_ollama_provider_chain_uses_cooldown_after_failure(
|
||||
assert aggregate.status == "degraded"
|
||||
assert details["ollama_gcp_a"].status == "down"
|
||||
assert "cooldown" in (details["ollama_gcp_a"].error or "")
|
||||
assert details["ollama_gcp_a"].diagnosis_code == "endpoint_cooldown"
|
||||
assert details["ollama_gcp_a"].is_cooldown is True
|
||||
assert details["ollama_gcp_a"].retry_after_seconds is not None
|
||||
assert "ollama_gcp_a" not in calls
|
||||
assert {"ollama_gcp_b", "ollama_local"} == set(calls)
|
||||
|
||||
@@ -132,3 +138,23 @@ async def test_ollama_provider_chain_success_clears_cooldown(
|
||||
await health.check_ollama_provider_chain()
|
||||
|
||||
assert not is_ollama_endpoint_blocked("http://gcp-a:11434")
|
||||
|
||||
|
||||
def test_ollama_failure_classifier_marks_local_proxy_502() -> None:
|
||||
assert (
|
||||
health._classify_ollama_endpoint_failure(
|
||||
"ollama_local",
|
||||
"Server error '502 Bad Gateway' for url",
|
||||
)
|
||||
== "local_proxy_upstream_unreachable"
|
||||
)
|
||||
|
||||
|
||||
def test_ollama_failure_classifier_marks_network_unreachable() -> None:
|
||||
assert (
|
||||
health._classify_ollama_endpoint_failure(
|
||||
"ollama_local",
|
||||
"No route to host",
|
||||
)
|
||||
== "endpoint_network_unreachable"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user