fix(health): surface ollama endpoint diagnosis
Some checks failed
CD Pipeline / tests (push) Successful in 1m30s
Code Review / ai-code-review (push) Successful in 13s
CD Pipeline / build-and-deploy (push) Successful in 3m44s
CD Pipeline / post-deploy-checks (push) Has been cancelled

This commit is contained in:
Your Name
2026-05-31 18:38:23 +08:00
parent e6a433da22
commit 7987da7f3f
7 changed files with 163 additions and 3 deletions

View File

@@ -47,6 +47,11 @@ class ComponentHealth(BaseModel):
status: Literal["up", "down", "degraded"]
latency_ms: float | None = None
error: str | None = None
provider_name: str | None = None
diagnosis_code: str | None = None
retry_after_seconds: float | None = None
cooldown_remaining_seconds: float | None = None
is_cooldown: bool = False
class HealthResponse(BaseModel):
@@ -194,16 +199,47 @@ async def _ollama_endpoint_health_check(name: str, url: str) -> ComponentHealth:
return ComponentHealth(
status="down",
error=f"recent endpoint failure cooldown: {cooldown_remaining:.0f}s",
provider_name=name,
diagnosis_code="endpoint_cooldown",
retry_after_seconds=round(cooldown_remaining, 1),
cooldown_remaining_seconds=round(cooldown_remaining, 1),
is_cooldown=True,
)
result = await _http_health_check(name, url, "/api/tags")
result.provider_name = name
if result.status == "up":
result.diagnosis_code = "endpoint_reachable"
record_ollama_endpoint_success(url)
else:
result.diagnosis_code = _classify_ollama_endpoint_failure(name, result.error)
record_ollama_endpoint_failure(url)
return result
def _classify_ollama_endpoint_failure(
provider_name: str,
error: str | None,
) -> str:
"""Return a stable diagnosis code for UI/alert rendering."""
normalized_error = (error or "").lower()
if "cooldown" in normalized_error:
return "endpoint_cooldown"
if "502" in normalized_error or "bad gateway" in normalized_error:
return (
"local_proxy_upstream_unreachable"
if provider_name == "ollama_local"
else "proxy_upstream_unreachable"
)
if "timeout" in normalized_error:
return "endpoint_timeout"
if "connection refused" in normalized_error:
return "endpoint_connection_refused"
if "no route to host" in normalized_error or "network is unreachable" in normalized_error:
return "endpoint_network_unreachable"
return "endpoint_unreachable"
async def check_openclaw() -> ComponentHealth:
"""Async OpenClaw health check via /health"""
return await _http_health_check("openclaw", settings.OPENCLAW_URL, "/health")

View File

@@ -43,7 +43,10 @@ async def test_ollama_provider_chain_reports_fallback_when_primary_down(monkeypa
assert aggregate.latency_ms == 42.0
assert aggregate.error == "primary unavailable; fallback active: ollama_gcp_b"
assert details["ollama_gcp_a"].status == "down"
assert details["ollama_gcp_a"].provider_name == "ollama_gcp_a"
assert details["ollama_gcp_a"].diagnosis_code == "endpoint_timeout"
assert details["ollama_gcp_b"].status == "up"
assert details["ollama_gcp_b"].diagnosis_code == "endpoint_reachable"
assert details["ollama_local"].status == "up"
@@ -108,6 +111,9 @@ async def test_ollama_provider_chain_uses_cooldown_after_failure(
assert aggregate.status == "degraded"
assert details["ollama_gcp_a"].status == "down"
assert "cooldown" in (details["ollama_gcp_a"].error or "")
assert details["ollama_gcp_a"].diagnosis_code == "endpoint_cooldown"
assert details["ollama_gcp_a"].is_cooldown is True
assert details["ollama_gcp_a"].retry_after_seconds is not None
assert "ollama_gcp_a" not in calls
assert {"ollama_gcp_b", "ollama_local"} == set(calls)
@@ -132,3 +138,23 @@ async def test_ollama_provider_chain_success_clears_cooldown(
await health.check_ollama_provider_chain()
assert not is_ollama_endpoint_blocked("http://gcp-a:11434")
def test_ollama_failure_classifier_marks_local_proxy_502() -> None:
assert (
health._classify_ollama_endpoint_failure(
"ollama_local",
"Server error '502 Bad Gateway' for url",
)
== "local_proxy_upstream_unreachable"
)
def test_ollama_failure_classifier_marks_network_unreachable() -> None:
assert (
health._classify_ollama_endpoint_failure(
"ollama_local",
"No route to host",
)
== "endpoint_network_unreachable"
)