fix(ollama): cooldown noisy failed endpoints

apps/api/src/services/embedding_service.py

@@ -22,6 +22,11 @@ import httpx
 import structlog
 
 from src.services.model_registry import get_model as _get_model
+from src.services.ollama_endpoint_circuit_breaker import (
+    filter_ollama_urls_with_cooldown,
+    record_ollama_endpoint_failure,
+    record_ollama_endpoint_success,
+)
 from src.services.ollama_endpoint_resolver import resolve_ollama_order
 
 logger = structlog.get_logger(__name__)
@@ -135,6 +140,17 @@ class OllamaEmbeddingService:
             )
         return self._client
 
+    def _active_ollama_endpoints(self) -> tuple[tuple[str, str], ...]:
+        active_urls = filter_ollama_urls_with_cooldown(
+            url for url, _provider_name in self._ollama_endpoints
+        )
+        active_url_set = set(active_urls)
+        return tuple(
+            (url, provider_name)
+            for url, provider_name in self._ollama_endpoints
+            if url in active_url_set
+        )
+
     async def embed_text(self, text: str) -> list[float]:
         """
         將單一文本轉換為向量
@@ -151,7 +167,7 @@ class OllamaEmbeddingService:
         client = await self._get_client()
 
         last_error: Exception | None = None
-        for endpoint_url, provider_name in self._ollama_endpoints:
+        for endpoint_url, provider_name in self._active_ollama_endpoints():
             try:
                 response = await client.post(
                     f"{endpoint_url}/api/embeddings",
@@ -161,6 +177,7 @@ class OllamaEmbeddingService:
                     },
                 )
                 response.raise_for_status()
+                record_ollama_endpoint_success(endpoint_url)
 
                 data = response.json()
                 embedding = data.get("embedding", [])
@@ -179,6 +196,7 @@ class OllamaEmbeddingService:
 
             except httpx.TimeoutException as e:
                 last_error = e
+                record_ollama_endpoint_failure(endpoint_url)
                 logger.error(
                     "embedding_timeout",
                     model=self._model,
@@ -187,6 +205,8 @@ class OllamaEmbeddingService:
                 )
             except httpx.HTTPStatusError as e:
                 last_error = e
+                if e.response.status_code >= 500:
+                    record_ollama_endpoint_failure(endpoint_url)
                 logger.error(
                     "embedding_http_error",
                     status=e.response.status_code,
@@ -195,6 +215,8 @@ class OllamaEmbeddingService:
                 )
             except Exception as e:
                 last_error = e
+                if isinstance(e, httpx.TransportError):
+                    record_ollama_endpoint_failure(endpoint_url)
                 logger.error(
                     "embedding_error",
                     error=str(e),

apps/api/src/services/knowledge_rag_service.py

@@ -22,7 +22,11 @@ import structlog
 
 import src.repositories.rag_chunk_repository as rag_repo
 from src.core.config import settings
-from src.services.ollama_endpoint_resolver import resolve_ollama_order
+from src.services.ollama_endpoint_circuit_breaker import (
+    record_ollama_endpoint_failure,
+    record_ollama_endpoint_success,
+    resolve_ollama_order_with_cooldown,
+)
 
 logger = structlog.get_logger(__name__)
 
@@ -129,7 +133,7 @@ class KnowledgeRAGService:
 
     async def _embed(self, text: str) -> list[float] | None:
         http = await self._get_http()
-        for endpoint in resolve_ollama_order("embedding"):
+        for endpoint in resolve_ollama_order_with_cooldown("embedding"):
             if not endpoint.url:
                 continue
             try:
@@ -141,17 +145,22 @@ class KnowledgeRAGService:
                     },
                 )
                 if resp.status_code == 200:
+                    record_ollama_endpoint_success(endpoint.url)
                     logger.debug(
                         "rag_embed_success",
                         provider=endpoint.provider_name,
                     )
                     return resp.json().get("embedding")
+                if resp.status_code >= 500:
+                    record_ollama_endpoint_failure(endpoint.url)
                 logger.warning(
                     "rag_embed_http_error",
                     provider=endpoint.provider_name,
                     status=resp.status_code,
                 )
             except Exception as e:
+                if isinstance(e, (httpx.TimeoutException, httpx.TransportError)):
+                    record_ollama_endpoint_failure(endpoint.url)
                 logger.warning(
                     "rag_embed_failed",
                     provider=endpoint.provider_name,
@@ -167,7 +176,7 @@ class KnowledgeRAGService:
             f"=== 問題 ===\n{question}"
         )
         http = await self._get_http()
-        for endpoint in resolve_ollama_order("rag"):
+        for endpoint in resolve_ollama_order_with_cooldown("rag"):
             if not endpoint.url:
                 continue
             try:
@@ -182,17 +191,22 @@ class KnowledgeRAGService:
                     timeout=httpx.Timeout(90.0, connect=10.0),
                 )
                 if resp.status_code == 200:
+                    record_ollama_endpoint_success(endpoint.url)
                     logger.debug(
                         "rag_generate_success",
                         provider=endpoint.provider_name,
                     )
                     return resp.json().get("response", "").strip()
+                if resp.status_code >= 500:
+                    record_ollama_endpoint_failure(endpoint.url)
                 logger.warning(
                     "rag_generate_http_error",
                     provider=endpoint.provider_name,
                     status=resp.status_code,
                 )
             except Exception as e:
+                if isinstance(e, (httpx.TimeoutException, httpx.TransportError)):
+                    record_ollama_endpoint_failure(endpoint.url)
                 logger.error(
                     "rag_generate_failed",
                     provider=endpoint.provider_name,

apps/api/src/services/ollama_endpoint_circuit_breaker.py

@@ -0,0 +1,90 @@
+"""
+Lightweight in-process cooldown for noisy Ollama endpoint failures.
+
+This does not change ADR-110 policy order. It only suppresses endpoints that
+just failed for short-lived high-volume callers such as embedding/RAG, while
+leaving health checks and failover status free to probe the full topology.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Iterable
+
+from src.services.ollama_endpoint_resolver import (
+    OllamaEndpointSelection,
+    OllamaWorkloadType,
+    resolve_ollama_order,
+)
+
+DEFAULT_OLLAMA_ENDPOINT_COOLDOWN_SECONDS = 60.0
+
+_blocked_until_by_url: dict[str, float] = {}
+
+
+def _normalize_url(url: str) -> str:
+    return url.rstrip("/")
+
+
+def record_ollama_endpoint_failure(
+    url: str,
+    *,
+    cooldown_seconds: float = DEFAULT_OLLAMA_ENDPOINT_COOLDOWN_SECONDS,
+    now: float | None = None,
+) -> None:
+    """Temporarily suppress an endpoint after network/5xx failure."""
+    if not url:
+        return
+    current_time = time.monotonic() if now is None else now
+    _blocked_until_by_url[_normalize_url(url)] = current_time + cooldown_seconds
+
+
+def record_ollama_endpoint_success(url: str) -> None:
+    """Clear cooldown when an endpoint succeeds again."""
+    if not url:
+        return
+    _blocked_until_by_url.pop(_normalize_url(url), None)
+
+
+def is_ollama_endpoint_blocked(url: str, *, now: float | None = None) -> bool:
+    if not url:
+        return False
+    current_time = time.monotonic() if now is None else now
+    normalized = _normalize_url(url)
+    blocked_until = _blocked_until_by_url.get(normalized)
+    if blocked_until is None:
+        return False
+    if blocked_until <= current_time:
+        _blocked_until_by_url.pop(normalized, None)
+        return False
+    return True
+
+
+def filter_ollama_urls_with_cooldown(
+    urls: Iterable[str],
+    *,
+    now: float | None = None,
+) -> tuple[str, ...]:
+    """Return non-cooldown URLs, preserving original order and all-blocked recovery."""
+    ordered_urls = tuple(url for url in urls if url)
+    available = tuple(url for url in ordered_urls if not is_ollama_endpoint_blocked(url, now=now))
+    return available or ordered_urls
+
+
+def resolve_ollama_order_with_cooldown(
+    workload_type: OllamaWorkloadType = "interactive",
+    *,
+    now: float | None = None,
+) -> tuple[OllamaEndpointSelection, ...]:
+    """Return resolver order with short-lived noisy-failure cooldown applied."""
+    order = resolve_ollama_order(workload_type)
+    available = tuple(
+        endpoint
+        for endpoint in order
+        if endpoint.url and not is_ollama_endpoint_blocked(endpoint.url, now=now)
+    )
+    return available or order
+
+
+def reset_ollama_endpoint_cooldown_for_tests() -> None:
+    _blocked_until_by_url.clear()

apps/api/src/services/playbook_rag.py

@@ -32,6 +32,11 @@ import structlog
 from src.core.config import settings
 from src.models.playbook import Playbook, SymptomPattern
 from src.repositories.interfaces import IEmbeddingCacheRepository
+from src.services.ollama_endpoint_circuit_breaker import (
+    filter_ollama_urls_with_cooldown,
+    record_ollama_endpoint_failure,
+    record_ollama_endpoint_success,
+)
 from src.services.ollama_endpoint_resolver import resolve_ollama_endpoint
 
 logger = structlog.get_logger(__name__)
@@ -204,7 +209,7 @@ class PlaybookRAGService:
         try:
             client = await self._get_http_client()
             last_error = ""
-            for endpoint_url in self.ollama_urls:
+            for endpoint_url in filter_ollama_urls_with_cooldown(self.ollama_urls):
                 try:
                     response = await client.post(
                         f"{endpoint_url}/api/embeddings",
@@ -217,6 +222,8 @@ class PlaybookRAGService:
 
                     if response.status_code != 200:
                         last_error = f"http_{response.status_code}"
+                        if response.status_code >= 500:
+                            record_ollama_endpoint_failure(endpoint_url)
                         logger.warning(
                             "ollama_embedding_failed",
                             endpoint=endpoint_url,
@@ -237,10 +244,16 @@ class PlaybookRAGService:
                         )
                         continue
 
+                    record_ollama_endpoint_success(endpoint_url)
                     logger.info("ollama_embedding_success", endpoint=endpoint_url)
                     return normalize_vector(embedding)
                 except Exception as endpoint_error:
                     last_error = str(endpoint_error)
+                    if isinstance(
+                        endpoint_error,
+                        (httpx.TimeoutException, httpx.TransportError),
+                    ):
+                        record_ollama_endpoint_failure(endpoint_url)
                     logger.warning(
                         "ollama_embedding_endpoint_error",
                         endpoint=endpoint_url,