feat(p1): Ollama 多層容災系統 — P1.1 健康檢測 + P1.2 ai_router 整合 + P1.5 容災告警

ADR-092 P1 飛輪閉環的 Ollama 失敗轉移子系統，全部 Engineer-A2/C/C2 補上。新服務 (1581 行)： - ollama_health_monitor.py (356)：3 層健康檢測（TCP/HTTP/推理） - ollama_failover_manager.py (571)：111→188 自動切換 + Redis 持久化 + recovery callback - ollama_auto_recovery.py (436)：30s 背景監控 + 連續 3 次 HEALTHY → 切回 + clear_cache - failover_alerter.py (218)：P1.5 Telegram 容災告警服務整合： - ai_router.py: AIProviderEnum.OLLAMA_188 + 120s budget + failover fallback chain - main.py lifespan: 啟動時 wire callback + start recovery，關閉時優雅 stop - config.py: OLLAMA_FALLBACK_URL / OLLAMA_HEALTH_CHECK_MODEL / GEMINI_DAILY_QUOTA（帳單熔斷） K8s 配置： - 04-configmap.yaml.patch-188-fallback：注入 OLLAMA_FALLBACK_URL=http://192.168.0.188:11434 測試 (2082 行)： - test_ollama_health_monitor.py (402) - test_ollama_failover_manager.py (707) - test_ollama_auto_recovery.py (580) - test_ai_router_failover_integration.py (257) - test_lifespan_failover_wiring.py (136) 依賴鏈：service 三件套 + ai_router + main.py 一起 commit，缺一就 ImportError。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-26 20:18:33 +08:00
parent d3a4fb4d15
commit 55c6b4e2d9
13 changed files with 3798 additions and 2 deletions
--- a/apps/api/src/services/ollama_failover_manager.py
+++ b/apps/api/src/services/ollama_failover_manager.py
@@ -0,0 +1,571 @@
+"""
+Ollama 自動容災管理 - P1.1b
+============================
+依 OllamaHealthMonitor 健康狀態決定 Ollama 路由方案。
+
+路由邏輯（2026-04-25 統帥指令：Gemini 優先，188 最後備援）：
+  111 HEALTHY  → 主 111，fallback [Gemini, 188, Nemotron]
+  111 SLOW     → 主 Gemini，fallback [111, 188]
+  111 DEGRADED → 主 Gemini，fallback [188, Nemotron, Claude]
+  111 OFFLINE  → 主 Gemini，fallback [188, Nemotron, Claude]
+  111 OFFLINE + 188 OFFLINE → 主 Gemini，fallback [Nemotron, Claude]
+
+設計說明：
+- 不直接依賴 AIProviderEnum（P1.2 Engineer-A 整合時再對齊）
+- 返回輕量 OllamaRoutingResult，含主 endpoint + fallback 清單
+- 並行檢查 111 + 188（asyncio.gather）
+- 切換觸發時寫 audit_logs service="ollama_failover"
+- clear_cache() 方法供 OllamaAutoRecoveryService 切回後清空路由快取
+
+版本: v2.0
+建立: 2026-04-25 (台北時區)
+建立者: Claude Engineer-C (P1.1b)
+# Created 2026-04-25 P1.1 by Claude Engineer-C
+# 2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+from dataclasses import dataclass, field
+# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
+# 用標準庫 timezone(timedelta(hours=8)) 取代 zoneinfo，保證一定有 +8 時區
+# 原 zoneinfo.ZoneInfo("Asia/Taipei") 失敗時 = None → datetime.now(None) 為 UTC
+from datetime import timezone, timedelta
+
+import structlog
+
+from src.core.config import get_settings
+
+# 台北時區 +8（標準庫保險絲，100% 可用）
+# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
+TAIPEI_TZ = timezone(timedelta(hours=8))
+from src.services.ollama_health_monitor import (
+    HealthReport,
+    HealthStatus,
+    OllamaHealthMonitor,
+    get_ollama_health_monitor,
+)
+
+logger = structlog.get_logger(__name__)
+
+
+# =============================================================================
+# 路由結果模型（輕量，P1.2 整合時轉換為 RoutingDecision）
+# =============================================================================
+
+
+@dataclass
+class OllamaEndpoint:
+    """Ollama 端點描述"""
+
+    url: str
+    provider_name: str  # 給 AIRouterExecutor 用的 provider 名稱
+    model: str
+
+    def to_dict(self) -> dict:
+        return {"url": self.url, "provider_name": self.provider_name, "model": self.model}
+
+
+@dataclass
+class OllamaRoutingResult:
+    """
+    Ollama 容災路由結果
+
+    P1.2 Engineer-A 整合時，將此結果轉換為 ai_router.RoutingDecision：
+      - selected_provider = AIProviderEnum[result.primary.provider_name.upper()] or 新的 OLLAMA_188
+      - selected_model = result.primary.model
+      - fallback_chain = [(AIProviderEnum[p.provider_name.upper()], p.model) for p in result.fallback_chain]
+    """
+
+    primary: OllamaEndpoint
+    fallback_chain: list[OllamaEndpoint]
+    routing_reason: str
+    health_111: HealthReport
+    health_188: HealthReport | None = None
+
+    def all_endpoints_in_order(self) -> list[OllamaEndpoint]:
+        """返回完整的優先序端點列表（primary 在前）"""
+        return [self.primary, *self.fallback_chain]
+
+    def to_dict(self) -> dict:
+        return {
+            "primary": {
+                "url": self.primary.url,
+                "provider": self.primary.provider_name,
+                "model": self.primary.model,
+            },
+            "fallback_chain": [
+                {"url": e.url, "provider": e.provider_name, "model": e.model}
+                for e in self.fallback_chain
+            ],
+            "routing_reason": self.routing_reason,
+            "health_111": self.health_111.to_dict(),
+            "health_188": self.health_188.to_dict() if self.health_188 else None,
+        }
+
+
+# =============================================================================
+# 已知 Fallback 端點定義（Nemotron / Gemini / Claude）
+# =============================================================================
+
+# 以 provider_name 對應 ai_router.AIProviderEnum 的 value
+_NEMOTRON_ENDPOINT = OllamaEndpoint(
+    url="",  # Nemotron 不是 HTTP URL，由 AIRouterExecutor 從 Registry 取得
+    provider_name="nemotron",
+    model="nvidia/nemotron-mini-4b-instruct",
+)
+_GEMINI_ENDPOINT = OllamaEndpoint(
+    url="",
+    provider_name="gemini",
+    model="gemini-1.5-flash",
+)
+_CLAUDE_ENDPOINT = OllamaEndpoint(
+    url="",
+    provider_name="claude",
+    model="claude-3-5-haiku-20241022",
+)
+
+
+# =============================================================================
+# OllamaFailoverManager
+# =============================================================================
+
+
+class OllamaFailoverManager:
+    """
+    Ollama 自動容災管理器
+
+    並行檢查 111 + 188，依健康狀態選擇最佳路由。
+
+    使用方式：
+        manager = OllamaFailoverManager()
+        result = await manager.select_provider()
+        # result.primary.url → 使用的 Ollama URL
+        # result.fallback_chain → 依序 fallback
+
+    2026-04-25 Claude Engineer-C (P1.1b)
+    """
+
+    def __init__(
+        self,
+        health_monitor: OllamaHealthMonitor | None = None,
+        recovery_callback=None,
+    ) -> None:
+        self._monitor = health_monitor or get_ollama_health_monitor()
+        self._settings = get_settings()
+        # 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
+        # recovery_callback: async callable(provider_name: str) → None
+        # OllamaAutoRecoveryService.set_current_primary 在 failover 時被通知，
+        # 避免重啟後 _current_primary 停留在 "ollama" 而永不啟動恢復監控
+        self._recovery_callback = recovery_callback
+
+    # -------------------------------------------------------------------------
+    # Public API
+    # -------------------------------------------------------------------------
+
+    async def select_provider(
+        self,
+        task_type: str = "",
+        context: dict | None = None,
+    ) -> OllamaRoutingResult:
+        """
+        並行檢查 111 + 188，返回路由結果
+
+        Args:
+            task_type: 任務類型（預留，目前未影響路由邏輯）
+            context: 額外上下文（預留）
+
+        Returns:
+            OllamaRoutingResult
+        """
+        url_111 = self._settings.OLLAMA_URL
+        url_188 = self._settings.OLLAMA_FALLBACK_URL or ""
+
+        # 並行檢查
+        # 2026-04-25 critic-fix Part2 H4 by Claude Engineer-C2
+        # return_exceptions=True 防止任一 check 例外導致整個 select_provider 炸
+        if url_188:
+            results = await asyncio.gather(
+                self._monitor.check(url_111),
+                self._monitor.check(url_188),
+                return_exceptions=True,
+            )
+            # 處理 exception — 任一失敗視為 OFFLINE
+            health_111_raw, health_188_raw = results
+            health_111: HealthReport = (
+                HealthReport(status=HealthStatus.OFFLINE, reason=f"check error: {health_111_raw}")
+                if isinstance(health_111_raw, Exception)
+                else health_111_raw
+            )
+            health_188: HealthReport | None = (
+                HealthReport(status=HealthStatus.OFFLINE, reason=f"check error: {health_188_raw}")
+                if isinstance(health_188_raw, Exception)
+                else health_188_raw
+            )
+        else:
+            health_111 = await self._monitor.check(url_111)
+            health_188 = None
+
+        result = self._decide_route(
+            health_111=health_111,
+            health_188=health_188,
+            url_111=url_111,
+            url_188=url_188,
+        )
+
+        # Gemini 帳單熔斷（quota gate）
+        # 2026-04-25 critic-fix Part2 H7 by Claude Engineer-C2
+        if result.primary.provider_name == "gemini":
+            quota_ok = await self._check_gemini_quota()
+            if not quota_ok:
+                quota = getattr(self._settings, "GEMINI_DAILY_QUOTA", 1000)
+                logger.warning(
+                    "gemini_quota_exceeded_falling_to_188",
+                    quota=quota,
+                    health_111=health_111.status.value,
+                )
+                result = self._build_quota_exceeded_route(
+                    health_111=health_111,
+                    health_188=health_188,
+                    url_111=url_111,
+                    url_188=url_188,
+                )
+
+        # 寫入 audit_log（best-effort）
+        await self._write_failover_audit(result)
+
+        logger.info(
+            "ollama_failover_decision",
+            primary=result.primary.provider_name,
+            primary_url=result.primary.url,
+            reason=result.routing_reason,
+            fallback_count=len(result.fallback_chain),
+            health_111=health_111.status.value,
+            health_188=health_188.status.value if health_188 else "not_configured",
+        )
+
+        # 通知 recovery service 當前 primary（跨重啟持久化）
+        # 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
+        if self._recovery_callback is not None:
+            try:
+                await self._recovery_callback(result.primary.provider_name)
+            except Exception as e:
+                logger.warning(
+                    "ollama_failover_recovery_callback_failed",
+                    error=str(e),
+                )
+
+        return result
+
+    # -------------------------------------------------------------------------
+    # 路由決策邏輯
+    # -------------------------------------------------------------------------
+
+    def _decide_route(
+        self,
+        health_111: HealthReport,
+        health_188: HealthReport | None,
+        url_111: str,
+        url_188: str,
+    ) -> OllamaRoutingResult:
+        """
+        決策矩陣（2026-04-25 統帥指令：Gemini 優先，188 最後備援）：
+
+          111 HEALTHY  → primary=111, fallback=[Gemini, 188, Nemotron]
+          111 SLOW     → primary=Gemini, fallback=[111, 188]
+          111 DEGRADED → primary=Gemini, fallback=[188, Nemotron, Claude]
+          111 OFFLINE  → primary=Gemini, fallback=[188, Nemotron, Claude]
+          111 OFFLINE + 188 OFFLINE → primary=Gemini, fallback=[Nemotron, Claude]
+
+        關鍵原則：
+        - 111 非 HEALTHY 時，primary 必為 Gemini（快速雲端，不等 188 慢推理）
+        - 188 永遠在 fallback chain，作為 Gemini 額度耗盡的最後備援
+        - degradation_reason 記錄切換原因 + 時間戳
+
+        2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
+        """
+        model_111 = self._settings.OLLAMA_HEALTH_CHECK_MODEL
+        model_188 = "qwen2.5:7b-instruct"  # 188 CPU-only 備援推薦模型（plan 方案 C）
+
+        ep_111 = OllamaEndpoint(url=url_111, provider_name="ollama", model=model_111)
+        ep_188 = (
+            OllamaEndpoint(url=url_188, provider_name="ollama_188", model=model_188)
+            if url_188
+            else None
+        )
+
+        # 188 可用性判斷（僅供 fallback 使用）
+        has_188 = ep_188 is not None and (
+            health_188 is not None and health_188.status != HealthStatus.OFFLINE
+        )
+
+        # 切換時間戳（台北時區 +8，標準庫保證）
+        # 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
+        now_ts = datetime.datetime.now(TAIPEI_TZ).isoformat()
+
+        # ==========================================================
+        # 111 HEALTHY → 主 111，Gemini 作為第一 fallback（快速雲端）
+        # ==========================================================
+        if health_111.status == HealthStatus.HEALTHY:
+            fallback: list[OllamaEndpoint] = [_GEMINI_ENDPOINT]
+            if has_188 and ep_188:
+                fallback.append(ep_188)
+            fallback.append(_NEMOTRON_ENDPOINT)
+            return OllamaRoutingResult(
+                primary=ep_111,
+                fallback_chain=fallback,
+                routing_reason="111 HEALTHY → 主 111",
+                health_111=health_111,
+                health_188=health_188,
+            )
+
+        # ==========================================================
+        # 111 SLOW → primary=Gemini，fallback=[111, 188]
+        # 111 實測 eval rate 0.09 token/s，~111s 推理，Gemini 更快
+        # ==========================================================
+        if health_111.status == HealthStatus.SLOW:
+            fallback_slow: list[OllamaEndpoint] = [ep_111]
+            if has_188 and ep_188:
+                fallback_slow.append(ep_188)
+            degradation_reason = (
+                f"111 SLOW（eval ~0.09 token/s, ~111s）→ 切 Gemini at {now_ts}"
+            )
+            return OllamaRoutingResult(
+                primary=_GEMINI_ENDPOINT,
+                fallback_chain=fallback_slow,
+                routing_reason=degradation_reason,
+                health_111=health_111,
+                health_188=health_188,
+            )
+
+        # ==========================================================
+        # 111 DEGRADED 或 OFFLINE → primary=Gemini，188 在 fallback
+        # ==========================================================
+        status_label = health_111.status.value  # "degraded" / "offline"
+        degradation_reason = f"111 {status_label} → 切 Gemini at {now_ts}"
+        if has_188 and ep_188:
+            return OllamaRoutingResult(
+                primary=_GEMINI_ENDPOINT,
+                fallback_chain=[ep_188, _NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
+                routing_reason=degradation_reason,
+                health_111=health_111,
+                health_188=health_188,
+            )
+
+        # 188 也不可用 → Gemini 主力，最後備援 Nemotron / Claude
+        degradation_reason = f"111 {status_label} + 188 不可用 → 切 Gemini at {now_ts}"
+        return OllamaRoutingResult(
+            primary=_GEMINI_ENDPOINT,
+            fallback_chain=[_NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
+            routing_reason=degradation_reason,
+            health_111=health_111,
+            health_188=health_188,
+        )
+
+    # -------------------------------------------------------------------------
+    # Gemini 帳單熔斷（quota gate）
+    # 2026-04-25 critic-fix Part2 H7 by Claude Engineer-C2
+    # -------------------------------------------------------------------------
+
+    async def _check_gemini_quota(self) -> bool:
+        """
+        檢查每日 Gemini call 配額，超過上限則禁用。
+
+        Redis key: ollama:gemini_daily_count:{YYYY-MM-DD}，TTL 86400s
+        計數 atomic（incr）。
+
+        Returns:
+            True  → 仍在配額內，可使用 Gemini
+            False → 已超配額，應切到 188+Nemotron
+
+        fail-open：Redis 不可用時允許走 Gemini（不阻擋服務）
+        """
+        try:
+            from src.core.redis_client import get_redis
+            redis = get_redis()
+            if redis is None:
+                return True  # fail-open
+            quota = getattr(self._settings, "GEMINI_DAILY_QUOTA", 1000)
+            key = f"ollama:gemini_daily_count:{datetime.date.today().isoformat()}"
+            count_raw = await redis.get(key)
+            count = int(count_raw or 0)
+            if count >= quota:
+                return False
+            # atomic incr + 設定 TTL（確保跨日自動重置）
+            await redis.incr(key)
+            await redis.expire(key, 86400)
+            return True
+        except Exception as e:
+            logger.warning("gemini_quota_check_failed", error=str(e))
+            return True  # fail-open
+
+    def _build_quota_exceeded_route(
+        self,
+        health_111: HealthReport,
+        health_188: HealthReport | None,
+        url_111: str,  # noqa: ARG002 — 保留供 OllamaRoutingResult 結構完整性（health_111 對應）
+        url_188: str,
+    ) -> OllamaRoutingResult:
+        """
+        Gemini 配額耗盡時的備援路由：primary=OLLAMA_188，fallback=[Nemotron, Claude]
+        若 188 也不可用，則 primary=Nemotron。
+        """
+        model_188 = "qwen2.5:7b-instruct"
+        ep_188 = (
+            OllamaEndpoint(url=url_188, provider_name="ollama_188", model=model_188)
+            if url_188
+            else None
+        )
+        has_188 = ep_188 is not None and (
+            health_188 is not None and health_188.status != HealthStatus.OFFLINE
+        )
+
+        if has_188 and ep_188:
+            return OllamaRoutingResult(
+                primary=ep_188,
+                fallback_chain=[_NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
+                routing_reason="Gemini quota exceeded → 188 CPU-only 備援",
+                health_111=health_111,
+                health_188=health_188,
+            )
+
+        # 188 也不可用
+        return OllamaRoutingResult(
+            primary=_NEMOTRON_ENDPOINT,
+            fallback_chain=[_CLAUDE_ENDPOINT],
+            routing_reason="Gemini quota exceeded + 188 不可用 → Nemotron 備援",
+            health_111=health_111,
+            health_188=health_188,
+        )
+
+    # -------------------------------------------------------------------------
+    # Recovery API（供 OllamaAutoRecoveryService 呼叫）
+    # -------------------------------------------------------------------------
+
+    def set_recovery_callback(self, callback) -> None:
+        """
+        設定 recovery callback（供 lifespan wiring 使用）。
+        callback signature: async (provider_name: str) -> None
+
+        # 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan
+        """
+        self._recovery_callback = callback
+
+    async def clear_cache(self) -> None:
+        """
+        清空路由決策快取，讓下次 select_provider 重新評估健康狀態。
+        OllamaAutoRecoveryService 在偵測 111 恢復後呼叫此方法。
+
+        2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
+        # 2026-04-25 P1.2 by Claude Engineer-A2 — 改用 make_cache_key 動態組 key，消除硬編碼 IP
+        """
+        try:
+            from src.core.redis_client import get_redis
+            from src.services.ollama_health_monitor import make_cache_key
+            redis = get_redis()
+            if redis is None:
+                return
+            # 動態由 settings URL 組 cache key，避免硬編碼 IP
+            keys = [
+                make_cache_key(self._settings.OLLAMA_URL),
+                make_cache_key(self._settings.OLLAMA_FALLBACK_URL or ""),
+            ]
+            for k in keys:
+                if k and k != "ollama_health:":  # 空 URL 會產生無意義的 key，跳過
+                    await redis.delete(k)
+            logger.info(
+                "ollama_failover_cache_cleared",
+                service="ollama_failover",
+                reason="recovery_triggered",
+            )
+        except Exception as e:
+            logger.debug("ollama_failover_clear_cache_failed", error=str(e))
+
+    def notify_recovery(self, provider: str) -> None:
+        """
+        預留：P1.5 Engineer 接入 Telegram alerter 時使用。
+        目前僅寫 structlog audit。
+
+        2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
+        """
+        logger.info(
+            "ollama_recovery_notified",
+            service="ollama_failover",
+            provider=provider,
+            action="recovery_received",
+        )
+
+    # -------------------------------------------------------------------------
+    # Audit Log
+    # -------------------------------------------------------------------------
+
+    async def _write_failover_audit(self, result: OllamaRoutingResult) -> None:
+        """
+        切換觸發時寫 structlog audit（best-effort）+ Telegram 告警
+
+        # 2026-04-25 critic-fix Part2 B1 by Claude Engineer-C2
+        # 原 AuditLog DB 寫入使用不存在的欄位（service/action/target/status/metadata）
+        # → SQLAlchemy crash → except 吃掉 → 零稽核
+        # 修法：刪除 DB 寫入路徑，改用 structlog only（audit 不依賴 DB schema）
+
+        # 2026-04-25 P1.5 by Claude Engineer-D — 新增 Telegram 告警（dedup 10min）
+
+        service="ollama_failover"（per 任務規格）
+        僅在 primary 非 111 時記錄（真正發生切換）
+        """
+        if result.primary.provider_name == "ollama":
+            # 111 正常，無切換事件
+            return
+
+        logger.info(
+            "ollama_failover_triggered",
+            service="ollama_failover",
+            action="failover_triggered",
+            from_provider="ollama",
+            to_provider=result.primary.provider_name,
+            reason=result.routing_reason,
+            primary_url=result.primary.url or result.primary.provider_name,
+            health_111=result.health_111.status.value,
+            health_188=result.health_188.status.value if result.health_188 else "not_configured",
+        )
+
+        # Telegram 告警（首次切換才通知，dedup 10min 內建）
+        # 2026-04-25 P1.5 by Claude Engineer-D — 告警失敗不阻斷主路由邏輯
+        try:
+            from src.services.failover_alerter import get_failover_alerter
+            fallback_chain_str = " → ".join(
+                p.provider_name for p in result.fallback_chain
+            )
+            alerter = get_failover_alerter()
+            await alerter.alert_failover({
+                "to_provider": result.primary.provider_name,
+                "model": result.primary.model,
+                "reason": result.routing_reason,
+                "timestamp": datetime.datetime.now(TAIPEI_TZ).isoformat(),
+                "fallback_chain_str": fallback_chain_str,
+            })
+        except Exception as e:
+            logger.warning("failover_alert_failed", error=str(e))
+
+
+# =============================================================================
+# Singleton
+# =============================================================================
+
+_failover_manager: OllamaFailoverManager | None = None
+
+
+def get_ollama_failover_manager() -> OllamaFailoverManager:
+    """取得 OllamaFailoverManager singleton"""
+    global _failover_manager
+    if _failover_manager is None:
+        _failover_manager = OllamaFailoverManager()
+    return _failover_manager
+
+
+def reset_ollama_failover_manager() -> None:
+    """重置 singleton（測試用）"""
+    global _failover_manager
+    _failover_manager = None