fix(ollama): cooldown provider health probes

apps/api/tests/test_health_ollama_provider_chain.py

@@ -3,6 +3,19 @@ from __future__ import annotations
 import pytest
 
 from src.api.v1 import health
+from src.services.ollama_endpoint_circuit_breaker import (
+    is_ollama_endpoint_blocked,
+    record_ollama_endpoint_failure,
+    reset_ollama_endpoint_cooldown_for_tests,
+)
+
+
+def setup_function() -> None:
+    reset_ollama_endpoint_cooldown_for_tests()
+
+
+def teardown_function() -> None:
+    reset_ollama_endpoint_cooldown_for_tests()
 
 
 def _set_ollama_settings(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -66,3 +79,56 @@ def test_overall_status_uses_aggregate_ollama_not_endpoint_details() -> None:
     }
 
     assert health._determine_overall_status(components) == "degraded"
+
+
+@pytest.mark.asyncio
+async def test_ollama_provider_chain_uses_cooldown_after_failure(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    _set_ollama_settings(monkeypatch)
+    calls: list[str] = []
+
+    async def fake_http_check(name: str, _url: str, _path: str) -> health.ComponentHealth:
+        calls.append(name)
+        if name == "ollama_gcp_a":
+            return health.ComponentHealth(status="down", error="http 502")
+        return health.ComponentHealth(status="up", latency_ms=9.0)
+
+    monkeypatch.setattr(health, "_http_health_check", fake_http_check)
+
+    aggregate, details = await health.check_ollama_provider_chain()
+
+    assert aggregate.status == "degraded"
+    assert details["ollama_gcp_a"].status == "down"
+    assert is_ollama_endpoint_blocked("http://gcp-a:11434")
+
+    calls.clear()
+    aggregate, details = await health.check_ollama_provider_chain()
+
+    assert aggregate.status == "degraded"
+    assert details["ollama_gcp_a"].status == "down"
+    assert "cooldown" in (details["ollama_gcp_a"].error or "")
+    assert "ollama_gcp_a" not in calls
+    assert {"ollama_gcp_b", "ollama_local"} == set(calls)
+
+
+@pytest.mark.asyncio
+async def test_ollama_provider_chain_success_clears_cooldown(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    _set_ollama_settings(monkeypatch)
+    record_ollama_endpoint_failure("http://gcp-a:11434")
+    monkeypatch.setattr(
+        health,
+        "get_ollama_endpoint_cooldown_remaining_seconds",
+        lambda _url: 0.0,
+    )
+
+    async def fake_http_check(name: str, _url: str, _path: str) -> health.ComponentHealth:
+        return health.ComponentHealth(status="up", latency_ms=8.0)
+
+    monkeypatch.setattr(health, "_http_health_check", fake_http_check)
+
+    await health.check_ollama_provider_chain()
+
+    assert not is_ollama_endpoint_blocked("http://gcp-a:11434")

apps/api/tests/test_ollama_endpoint_circuit_breaker.py

@@ -4,6 +4,7 @@ from types import SimpleNamespace
 
 from src.services.ollama_endpoint_circuit_breaker import (
     filter_ollama_urls_with_cooldown,
+    get_ollama_endpoint_cooldown_remaining_seconds,
     is_ollama_endpoint_blocked,
     record_ollama_endpoint_failure,
     record_ollama_endpoint_success,
@@ -41,6 +42,26 @@ def test_cooldown_expires_and_success_clears_block() -> None:
     assert not is_ollama_endpoint_blocked("http://gcp-a:11434", now=201.0)
 
 
+def test_cooldown_remaining_seconds_expires_cleanly() -> None:
+    record_ollama_endpoint_failure("http://gcp-a:11434", cooldown_seconds=10.0, now=100.0)
+
+    assert (
+        get_ollama_endpoint_cooldown_remaining_seconds(
+            "http://gcp-a:11434",
+            now=103.0,
+        )
+        == 7.0
+    )
+    assert (
+        get_ollama_endpoint_cooldown_remaining_seconds(
+            "http://gcp-a:11434",
+            now=111.0,
+        )
+        == 0.0
+    )
+    assert not is_ollama_endpoint_blocked("http://gcp-a:11434", now=112.0)
+
+
 def test_all_blocked_returns_full_order_for_recovery_probe() -> None:
     urls = ("http://gcp-a:11434", "http://gcp-b:11434")
 

apps/api/tests/test_ollama_health_monitor.py

@@ -19,16 +19,17 @@ OllamaHealthMonitor 單元測試 - P1.1c
 from __future__ import annotations
 
 import asyncio
-import json
-import time
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import httpx
 import pytest
 
+from src.services.ollama_endpoint_circuit_breaker import (
+    is_ollama_endpoint_blocked,
+    record_ollama_endpoint_failure,
+    reset_ollama_endpoint_cooldown_for_tests,
+)
 from src.services.ollama_health_monitor import (
-    LATENCY_HEALTHY_THRESHOLD_MS,
-    LATENCY_SLOW_THRESHOLD_MS,
     HealthReport,
     HealthStatus,
     OllamaHealthMonitor,
@@ -48,8 +49,10 @@ HOST_LOCAL = "http://192.168.0.111:11434"  # Local fallback（已移出 188 主
 @pytest.fixture(autouse=True)
 def reset_singleton():
     """每個測試後重置 singleton"""
+    reset_ollama_endpoint_cooldown_for_tests()
     yield
     reset_ollama_health_monitor()
+    reset_ollama_endpoint_cooldown_for_tests()
 
 
 @pytest.fixture
@@ -110,6 +113,29 @@ class TestConnectivity:
             result = await monitor._check_connectivity(HOST)
         assert result is False
 
+    @pytest.mark.asyncio
+    async def test_run_checks_records_connectivity_failure_cooldown(self, monitor, monkeypatch):
+        """連通性失敗會建立短暫 cooldown，避免重複撞同一個 upstream。"""
+        monkeypatch.setattr(monitor, "_check_connectivity", AsyncMock(return_value=False))
+
+        report = await monitor._run_checks(HOST)
+
+        assert report.status == HealthStatus.OFFLINE
+        assert is_ollama_endpoint_blocked(HOST)
+
+    @pytest.mark.asyncio
+    async def test_run_checks_respects_existing_failure_cooldown(self, monitor, monkeypatch):
+        """cooldown 中直接回報 OFFLINE，不再打 /api/tags。"""
+        record_ollama_endpoint_failure(HOST)
+        connectivity = AsyncMock(return_value=True)
+        monkeypatch.setattr(monitor, "_check_connectivity", connectivity)
+
+        report = await monitor._run_checks(HOST)
+
+        assert report.status == HealthStatus.OFFLINE
+        assert "cooldown" in report.reason
+        connectivity.assert_not_awaited()
+
     @pytest.mark.asyncio
     async def test_connectivity_timeout(self, monitor):
         """連線 timeout → 返回 False（不 raise）"""
@@ -160,7 +186,6 @@ class TestInference:
 
         # 模擬 0.5s 延遲（< 10s threshold）
         call_count = [0]
-        original_perf_counter = time.perf_counter
 
         def _fake_perf_counter():
             call_count[0] += 1