feat(ollama): ADR-110 GCP 三層容災架構（GCP-A → GCP-B → Local → Gemini）

## 變更摘要 - Primary: http://34.143.170.20:11434 (GCP-A SSD, 9x 載速 + 2x 推理) - Secondary: http://34.21.145.224:11434 (GCP-B SSD) - Fallback: http://192.168.0.111:11434 (M1 Pro Local HDD，最後防線) - 廢止 ADR-105「111 唯一鐵律」，新建 ADR-110 ## 核心改動 - config.py: 新增 OLLAMA_SECONDARY_URL；validator 加 GCP IP 白名單（34.143.170.20, 34.21.145.224） - ollama_failover_manager.py: 三層 Ollama 決策矩陣；並行健康檢查三台；health_111 → health_gcp_a - ollama_health_monitor.py: host label 萃取改為通用版（支援 GCP 公網 IP） - failover_alerter.py: 故障/恢復主機動態顯示，不再硬編碼「Ollama 111 (GPU)」 - ollama_auto_recovery.py: notify_recovery 改為 ollama_gcp_a；recovered_host 動態 - k8s/awoooi-prod: configmap + deployment + network-policy 同步更新（egress 加 GCP /32） - 服務層: 10 個服務檔案硬編碼 192.168.0.111 改為讀 settings.OLLAMA_URL - 測試: URL 常數更新，新增三層容災場景，GCP IP 白名單驗證測試 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-03 22:49:23 +08:00
parent e45b055e0e
commit b1ef05fa8c
28 changed files with 816 additions and 355 deletions
--- a/apps/api/src/services/chat_manager.py
+++ b/apps/api/src/services/chat_manager.py
@@ -166,8 +166,10 @@ class ChatManager:

        import httpx
        import re
+        from src.core.config import get_settings as _get_settings

-        OLLAMA_URL = "http://192.168.0.111:11434"
+        # 2026-05-03 ogt: ADR-110 GCP-A Primary — 改從 settings 讀取，不再硬編碼 111
+        OLLAMA_URL = _get_settings().OLLAMA_URL
        MODEL = "deepseek-r1:14b"

        try:
--- a/apps/api/src/services/decision_fusion.py
+++ b/apps/api/src/services/decision_fusion.py
@@ -188,7 +188,7 @@ class DecisionFusionEngine:

    @property
    def _ollama_url(self) -> str:
-        return getattr(self._settings, "OLLAMA_URL", "http://192.168.0.111:11434")
+        return getattr(self._settings, "OLLAMA_URL", "http://34.143.170.20:11434")  # 2026-05-03 ogt: ADR-110 GCP-A Primary

    # =========================================================================
    # Public API
--- a/apps/api/src/services/decision_fusion_adapter.py
+++ b/apps/api/src/services/decision_fusion_adapter.py
@@ -254,7 +254,7 @@ class DecisionFusionAdapter:
            "只輸出 CONFIDENCE 和 ACTION 兩行，不要其他解釋。"
        )

-        ollama_url = getattr(self._settings, "OLLAMA_URL", "http://192.168.0.111:11434")
+        ollama_url = getattr(self._settings, "OLLAMA_URL", "http://34.143.170.20:11434")  # 2026-05-03 ogt: ADR-110 GCP-A Primary

        try:
            async with httpx.AsyncClient(
--- a/apps/api/src/services/decision_manager.py
+++ b/apps/api/src/services/decision_manager.py
@@ -616,7 +616,7 @@ async def _nemoclaw_second_opinion(incident: "Incident", primary_result: dict) -
        import httpx as _httpx

        from src.services.model_registry import get_model as _get_model
-        ollama_url = getattr(settings, "OLLAMA_URL", "http://192.168.0.111:11434")
+        ollama_url = getattr(settings, "OLLAMA_URL", "http://34.143.170.20:11434")  # 2026-05-03 ogt: ADR-110 GCP-A Primary
        # D1 集中化 2026-04-11: 從 models.json providers.ollama.models.nemoclaw 讀取
        model = _get_model("ollama", "nemoclaw")

@@ -709,7 +709,7 @@ async def _generate_playbook_draft_if_new(incident: "Incident") -> None:
        )

        from src.services.model_registry import get_model as _get_model
-        ollama_url = getattr(settings, "OLLAMA_URL", "http://192.168.0.111:11434")
+        ollama_url = getattr(settings, "OLLAMA_URL", "http://34.143.170.20:11434")  # 2026-05-03 ogt: ADR-110 GCP-A Primary
        # D1 集中化 2026-04-11: 從 models.json providers.ollama.models.playbook_draft 讀取
        _pb_model = _get_model("ollama", "playbook_draft")
        async with _httpx.AsyncClient(timeout=45.0) as client:
--- a/apps/api/src/services/drift_narrator_service.py
+++ b/apps/api/src/services/drift_narrator_service.py
@@ -33,7 +33,10 @@ logger = structlog.get_logger(__name__)
 # ============================================================
 # 設定
 # ============================================================
-OLLAMA_URL = "http://192.168.0.111:11434"
+# 2026-05-03 ogt: ADR-110 GCP-A Primary — 改從 settings 讀取，不再硬編碼 111
+def _get_ollama_url() -> str:
+    from src.core.config import get_settings
+    return get_settings().OLLAMA_URL
 # D1 集中化 2026-04-11: 從 models.json providers.ollama.models.drift_summary 讀取
 NARRATOR_MODEL = get_model("ollama", "drift_summary")
 NARRATOR_TIMEOUT = 90.0  # seconds
--- a/apps/api/src/services/failover_alerter.py
+++ b/apps/api/src/services/failover_alerter.py
@@ -40,7 +40,9 @@ class FailoverAlerter:
        self._memory_dedup_max_size = 1000

    async def alert_failover(self, event: dict[str, Any]) -> None:
-        """111 故障切換到 Gemini/188 — 10min dedup"""
+        """Ollama 故障切換告警 — 10min dedup
+        # 2026-05-03 ogt: ADR-110 三層容災，故障主機從 event["failed_host"] 動態讀取
+        """
        to_provider = event.get("to_provider", "unknown")
        dedup_key = f"alert:failover:{to_provider}"
        if not await self._check_dedup(dedup_key, ttl=DEDUP_TTL_SEC):
@@ -51,10 +53,12 @@ class FailoverAlerter:
        model = event.get("model", "?")
        timestamp = event.get("timestamp", datetime.now(TAIPEI_TZ).isoformat())
        fallback_chain_str = event.get("fallback_chain_str", "?")
+        # 2026-05-03 ogt: ADR-110 — 故障主機動態，不再硬編碼 111
+        failed_host = event.get("failed_host", "Ollama")

        msg = (
            f"*Ollama 容災激活*\n\n"
-            f"故障主機：Ollama 111 \\(GPU\\)\n"
+            f"故障主機：{_escape_md(failed_host)}\n"
            f"故障狀態：{_escape_md(reason)}\n"
            f"切換目標：{_escape_md(to_provider)} \\(model: {_escape_md(model)}\\)\n"
            f"切換時間：{_escape_md(timestamp)}\n\n"
@@ -65,9 +69,8 @@ class FailoverAlerter:
        logger.info("failover_alert_sent", to_provider=to_provider)

    async def alert_recovery(self, event: dict[str, Any]) -> None:
-        """111 恢復後自動切回 — 10min dedup
-
-        2026-04-25 P1.5 by Claude Engineer-D — 用戶明確要求要通知
+        """Ollama 自動恢復告警 — 10min dedup
+        # 2026-05-03 ogt: ADR-110 三層容災，恢復主機從 event["recovered_host"] 動態讀取
        """
        dedup_key = "alert:recovery"
        if not await self._check_dedup(dedup_key, ttl=DEDUP_TTL_SEC):
@@ -77,16 +80,18 @@ class FailoverAlerter:
        stable_count = event.get("stable_count", event.get("to", "?"))
        # 相容 auto_recovery 傳入的 {"from": ..., "to": ...} 格式
        from_provider = event.get("from_provider", event.get("from", "?"))
-        to_provider = event.get("to_provider", event.get("to", "ollama_111"))
+        to_provider = event.get("to_provider", event.get("to", "ollama"))
        recovery_time = event.get("recovery_time", datetime.now(TAIPEI_TZ).isoformat())
+        # 2026-05-03 ogt: ADR-110 — 恢復主機動態，不再硬編碼 111
+        recovered_host = event.get("recovered_host", to_provider)

        msg = (
            f"*Ollama 自動恢復*\n\n"
-            f"恢復主機：Ollama 111 \\(GPU\\)\n"
+            f"恢復主機：{_escape_md(str(recovered_host))}\n"
            f"穩定計數：連續 {stable_count} 次 HEALTHY\n"
            f"切回時間：{_escape_md(str(recovery_time))}\n"
            f"切換路徑：{_escape_md(str(from_provider))} → {_escape_md(str(to_provider))}\n\n"
-            f"自動化飛輪已恢復至 GPU 推理模式"
+            f"自動化飛輪已恢復至高效能推理模式"
        )
        await self._send(msg)
        logger.info("recovery_alert_sent", from_provider=from_provider)
--- a/apps/api/src/services/knowledge_extractor_service.py
+++ b/apps/api/src/services/knowledge_extractor_service.py
@@ -15,7 +15,10 @@ import structlog

 logger = structlog.get_logger(__name__)

-_OLLAMA_BASE = "http://192.168.0.111:11434"
+# 2026-05-03 ogt: ADR-110 GCP-A Primary — 改從 settings 讀取，不再硬編碼 111
+def _get_ollama_base() -> str:
+    from src.core.config import get_settings
+    return get_settings().OLLAMA_URL
 _EXTRACT_MODEL = "llama3.2:3b"
 _EXTRACT_TIMEOUT = 30.0  # 秒，容忍慢速

@@ -159,7 +162,7 @@ class KnowledgeExtractorService:
        try:
            async with httpx.AsyncClient(timeout=_EXTRACT_TIMEOUT) as client:
                r = await client.post(
-                    f"{_OLLAMA_BASE}/api/generate",
+                    f"{_get_ollama_base()}/api/generate",
                    json={
                        "model": _EXTRACT_MODEL,
                        "prompt": prompt,
@@ -179,7 +182,7 @@ class KnowledgeExtractorService:
            logger.exception(
                "kb_ollama_call_failed",
                model=_EXTRACT_MODEL,
-                base=_OLLAMA_BASE,
+                base=_get_ollama_base(),
            )
            return None

--- a/apps/api/src/services/log_summary_service.py
+++ b/apps/api/src/services/log_summary_service.py
@@ -33,7 +33,10 @@ logger = structlog.get_logger(__name__)
 # ============================================================
 # 設定
 # ============================================================
-OLLAMA_URL = "http://192.168.0.111:11434"
+# 2026-05-03 ogt: ADR-110 GCP-A Primary — 改從 settings 讀取，不再硬編碼 111
+def _get_ollama_url() -> str:
+    from src.core.config import get_settings
+    return get_settings().OLLAMA_URL
 # D1 集中化 2026-04-11: 從 models.json providers.ollama.models.log_anomaly 讀取
 SUMMARY_MODEL = get_model("ollama", "log_anomaly")
 LLM_TIMEOUT = 180.0       # deepseek-r1 硬超時
@@ -208,7 +211,7 @@ class LogSummaryService:
        try:
            async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
                resp = await client.post(
-                    f"{OLLAMA_URL}/api/generate",
+                    f"{_get_ollama_url()}/api/generate",
                    json={
                        "model": SUMMARY_MODEL,
                        "prompt": prompt,
--- a/apps/api/src/services/model_version_probe.py
+++ b/apps/api/src/services/model_version_probe.py
@@ -4,7 +4,7 @@ AI Provider 版本探測 — 為每個 Provider 提供 get_version()
 每個 probe 函數獨立運作，失敗只影響該 provider，不 crash 整批。

 Provider:
-  - ollama        : 192.168.0.111 Ollama (primary)
+  - ollama        : 34.143.170.20 GCP-A Ollama (primary) — 2026-05-03 ogt: ADR-110 GCP-A Primary
  - ollama_188    : 192.168.0.188 Ollama (fallback)
  - gemini        : Google Gemini API (版本 = model name)
  - claude        : Anthropic Claude (版本 = model name)
@@ -43,14 +43,14 @@ class ProviderVersionInfo:
 # =============================================================================

 async def probe_ollama_version(url: str, model: str) -> ProviderVersionInfo:
-    """探測 Ollama（111 或 188）：GET /api/tags 取 model digest + modified_at
+    """探測 Ollama（GCP-A 或 188）：GET /api/tags 取 model digest + modified_at

    Args:
-        url:   Ollama base URL，例如 "http://192.168.0.111:11434"
+        url:   Ollama base URL，例如 "http://34.143.170.20:11434"（GCP-A Primary）
        model: model name，例如 "qwen2.5:7b-instruct"

    Returns:
-        ProviderVersionInfo — provider 依 URL 自動判斷（111=ollama, 否則=ollama_188）
+        ProviderVersionInfo — provider 依 URL 自動判斷（GCP-A=ollama, 本地111=ollama_local, 否則=ollama_remote）

    Raises:
        ValueError: model 不在清單
@@ -58,7 +58,15 @@ async def probe_ollama_version(url: str, model: str) -> ProviderVersionInfo:
    """
    import httpx

-    provider_name = "ollama" if "192.168.0.111" in url else "ollama_188"
+    # 2026-05-03 ogt: ADR-110 GCP-A Primary — 擴展 provider 判斷邏輯支援 GCP 三層容災
+    _GCP_OLLAMA_IPS = {"34.143.170.20", "34.21.145.224"}
+    _LOCAL_OLLAMA_IPS = {"192.168.0.111"}
+    if any(ip in url for ip in _GCP_OLLAMA_IPS):
+        provider_name = "ollama"
+    elif any(ip in url for ip in _LOCAL_OLLAMA_IPS):
+        provider_name = "ollama_local"
+    else:
+        provider_name = "ollama_remote"

    async with httpx.AsyncClient(timeout=5.0) as client:
        resp = await client.get(f"{url}/api/tags")
--- a/apps/api/src/services/ollama_auto_recovery.py
+++ b/apps/api/src/services/ollama_auto_recovery.py
@@ -376,7 +376,8 @@ class OllamaAutoRecoveryService:

        # 通知 failover_manager
        try:
-            self._failover_manager.notify_recovery("ollama_111")
+            # 2026-05-03 ogt: ADR-110 — provider name 改為 "ollama_gcp_a"（GCP-A Primary）
+            self._failover_manager.notify_recovery("ollama_gcp_a")
        except Exception:
            logger.exception(
                "ollama_auto_recovery_notify_error",
@@ -391,9 +392,12 @@ class OllamaAutoRecoveryService:
            if alerter is None:
                from src.services.failover_alerter import get_failover_alerter
                alerter = get_failover_alerter()
+            # 2026-05-03 ogt: ADR-110 — recovered_host 動態，顯示恢復的實際主機 URL
+            _recovered_url = getattr(self._settings, "OLLAMA_URL", "Ollama Primary")
            await alerter.alert_recovery({
                "from": from_provider,
-                "to": "ollama_111",
+                "to": "ollama",
+                "recovered_host": f"GCP-A {_recovered_url}",
                "stable_count": stable_count,
                "recovery_time": recovery_time.isoformat(),
            })
@@ -420,7 +424,7 @@ class OllamaAutoRecoveryService:
            service="ollama_auto_recovery",
            action="switched_back",
            from_provider=from_provider,
-            to_provider="ollama_111",
+            to_provider="ollama_gcp_a",  # 2026-05-03 ogt: ADR-110
            stable_count=stable_count,
            recovery_time=recovery_time.isoformat(),
        )
--- a/apps/api/src/services/ollama_failover_manager.py
+++ b/apps/api/src/services/ollama_failover_manager.py
@@ -3,25 +3,30 @@ Ollama 自動容災管理 - P1.1b
 ============================
 依 OllamaHealthMonitor 健康狀態決定 Ollama 路由方案。

-路由邏輯（2026-04-26 統帥鐵律：111 = 唯一 Ollama，備援只用 Gemini）：
-  111 HEALTHY  → 主 111，fallback [Gemini]
-  111 SLOW/DEGRADED/OFFLINE → 主 Gemini，fallback [Nemotron, Claude]
-  Gemini quota 超過 → 主 Nemotron，fallback [Claude]
+路由邏輯（2026-05-03 統帥新令：GCP 三層容災，ADR-110）：
+  GCP-A HEALTHY  → primary=GCP-A, fallback=[GCP-B, Local]
+  GCP-A 不健康 + GCP-B HEALTHY → primary=GCP-B, fallback=[Local]
+  GCP-A + GCP-B 都不健康 + Local HEALTHY → primary=Local, fallback=[Gemini]
+  全部 Ollama 不健康 → primary=Gemini, fallback=[Nemotron, Claude]
+  Gemini quota 超過 → primary=Nemotron, fallback=[Claude]

 設計說明：
- 188 CPU-only 禁止用於即時回應（0.45 tok/s），完全移出 routing chain
- 唯一 Ollama 主機：192.168.0.111（M1 Pro, Metal 加速）
+- GCP-A 主機：34.143.170.20（SSD，9x 載速 + 2x 推理）
+- GCP-B 備援：34.21.145.224（SSD，9x 載速 + 2x 推理）
+- Local 最後防線：192.168.0.111（M1 Pro, Metal 加速，HDD）
 - 不直接依賴 AIProviderEnum（P1.2 Engineer-A 整合時再對齊）
 - 返回輕量 OllamaRoutingResult，含主 endpoint + fallback 清單
- 只檢查 111（不再並行檢查 188）
+- 並行檢查三台 Ollama 主機健康狀態
 - 切換觸發時寫 audit_logs service="ollama_failover"
 - clear_cache() 方法供 OllamaAutoRecoveryService 切回後清空路由快取

-版本: v2.0
+版本: v3.0
 建立: 2026-04-25 (台北時區)
 建立者: Claude Engineer-C (P1.1b)
+更新: 2026-05-03 ogt — GCP 三層容災（ADR-110），GCP-A → GCP-B → Local → Gemini
 # Created 2026-04-25 P1.1 by Claude Engineer-C
 # 2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
+# 2026-05-03 ogt: GCP 三層容災（ADR-110），GCP-A → GCP-B → Local → Gemini
 """

 from __future__ import annotations
@@ -71,10 +76,10 @@ class OllamaEndpoint:
@dataclass
 class OllamaRoutingResult:
    """
-    Ollama 容災路由結果
+    Ollama 容災路由結果（2026-05-03 ogt: 更新為三層 GCP 容災，ADR-110）

    P1.2 Engineer-A 整合時，將此結果轉換為 ai_router.RoutingDecision：
-      - selected_provider = AIProviderEnum[result.primary.provider_name.upper()] or 新的 OLLAMA_188
+      - selected_provider = AIProviderEnum[result.primary.provider_name.upper()]
      - selected_model = result.primary.model
      - fallback_chain = [(AIProviderEnum[p.provider_name.upper()], p.model) for p in result.fallback_chain]
    """
@@ -82,8 +87,14 @@ class OllamaRoutingResult:
    primary: OllamaEndpoint
    fallback_chain: list[OllamaEndpoint]
    routing_reason: str
-    health_111: HealthReport
-    health_188: HealthReport | None = None  # optional backward-compat（188 不在路由鏈時為 None）
+    health_gcp_a: HealthReport           # GCP-A 健康狀態（原 health_111）
+    health_gcp_b: HealthReport | None = None   # GCP-B 健康狀態
+    health_local: HealthReport | None = None   # Local(111) 健康狀態
+
+    @property
+    def health_111(self) -> HealthReport:
+        """向後相容屬性（舊測試 / log 使用）"""
+        return self.health_gcp_a

    def all_endpoints_in_order(self) -> list[OllamaEndpoint]:
        """返回完整的優先序端點列表（primary 在前）"""
@@ -101,8 +112,9 @@ class OllamaRoutingResult:
                for e in self.fallback_chain
            ],
            "routing_reason": self.routing_reason,
-            "health_111": self.health_111.to_dict(),
-            "health_188": self.health_188.to_dict() if self.health_188 else None,
+            "health_gcp_a": self.health_gcp_a.to_dict(),
+            "health_gcp_b": self.health_gcp_b.to_dict() if self.health_gcp_b else None,
+            "health_local": self.health_local.to_dict() if self.health_local else None,
        }


@@ -171,10 +183,8 @@ class OllamaFailoverManager:
        context: dict | None = None,
    ) -> OllamaRoutingResult:
        """
-        檢查 111 健康狀態，返回路由結果。
-
-        2026-04-26 統帥鐵律：唯一 Ollama = 111，188 禁止用於即時回應。
-        只檢查 111，不再並行檢查 188。
+        三層 Ollama 容災路由（2026-05-03 統帥新令，ADR-110）：
+        GCP-A → GCP-B → Local(111) → Gemini → Nemotron → Claude

        Args:
            task_type: 任務類型（預留，目前未影響路由邏輯）
@@ -183,17 +193,35 @@ class OllamaFailoverManager:
        Returns:
            OllamaRoutingResult
        """
-        url_111 = self._settings.OLLAMA_URL
+        # 2026-05-03 ogt: GCP 三層容災（ADR-110），GCP-A → GCP-B → Local → Gemini
+        url_gcp_a = self._settings.OLLAMA_URL            # 34.143.170.20
+        url_gcp_b = self._settings.OLLAMA_SECONDARY_URL  # 34.21.145.224
+        url_local = self._settings.OLLAMA_FALLBACK_URL   # 192.168.0.111

-        # 只檢查 111（188 移出 routing chain）
-        try:
-            health_111 = await self._monitor.check(url_111)
-        except Exception as e:
-            health_111 = HealthReport(status=HealthStatus.OFFLINE, reason=f"check error: {e}")
+        # 並行檢查三台 Ollama 主機（asyncio.gather 提升效率）
+        results_raw = await asyncio.gather(
+            self._monitor.check(url_gcp_a),
+            self._monitor.check(url_gcp_b),
+            self._monitor.check(url_local),
+            return_exceptions=True,
+        )
+
+        def _to_health(r, label: str) -> HealthReport:
+            if isinstance(r, Exception):
+                return HealthReport(status=HealthStatus.OFFLINE, reason=f"{label} check error: {r}")
+            return r
+
+        health_gcp_a = _to_health(results_raw[0], "GCP-A")
+        health_gcp_b = _to_health(results_raw[1], "GCP-B")
+        health_local = _to_health(results_raw[2], "Local")

        result = self._decide_route(
-            health_111=health_111,
-            url_111=url_111,
+            health_gcp_a=health_gcp_a,
+            health_gcp_b=health_gcp_b,
+            health_local=health_local,
+            url_gcp_a=url_gcp_a,
+            url_gcp_b=url_gcp_b,
+            url_local=url_local,
        )

        # Gemini 帳單熔斷（quota gate）
@@ -205,10 +233,9 @@ class OllamaFailoverManager:
                logger.warning(
                    "gemini_quota_exceeded_fallback_to_nemotron",
                    quota=quota,
-                    health_111=health_111.status.value,
+                    health_gcp_a=health_gcp_a.status.value,
                )
-                # 2026-04-26 統帥鐵律：188 移出，quota 超過 → Nemotron → Claude
-                result = self._build_quota_exceeded_route(health_111=health_111)
+                result = self._build_quota_exceeded_route(health_gcp_a=health_gcp_a)
                # Quota 耗盡 Telegram 告警（24h dedup）
                try:
                    from src.services.failover_alerter import get_failover_alerter
@@ -241,7 +268,9 @@ class OllamaFailoverManager:
            primary_url=result.primary.url,
            reason=result.routing_reason,
            fallback_count=len(result.fallback_chain),
-            health_111=health_111.status.value,
+            health_gcp_a=health_gcp_a.status.value,
+            health_gcp_b=health_gcp_b.status.value,
+            health_local=health_local.status.value,
        )

        # 通知 recovery service 當前 primary（跨重啟持久化）
@@ -263,48 +292,78 @@ class OllamaFailoverManager:

    def _decide_route(
        self,
-        health_111: HealthReport,
-        url_111: str,
+        health_gcp_a: HealthReport,
+        health_gcp_b: HealthReport,
+        health_local: HealthReport,
+        url_gcp_a: str,
+        url_gcp_b: str,
+        url_local: str,
    ) -> OllamaRoutingResult:
        """
-        決策矩陣（2026-04-26 統帥鐵律：唯一 Ollama=111，備援只用 Gemini）：
+        三層 Ollama 決策矩陣（2026-05-03 ogt，ADR-110）：

-          111 HEALTHY  → primary=111, fallback=[Gemini]
-          111 SLOW     → primary=Gemini, fallback=[111, Nemotron, Claude]
-          111 DEGRADED → primary=Gemini, fallback=[Nemotron, Claude]
-          111 OFFLINE  → primary=Gemini, fallback=[Nemotron, Claude]
+          GCP-A HEALTHY → primary=GCP-A, fallback=[GCP-B, Local]
+          GCP-A 不健康 + GCP-B HEALTHY → primary=GCP-B, fallback=[Local]
+          GCP-A + GCP-B 不健康 + Local HEALTHY → primary=Local, fallback=[Gemini]
+          全部 Ollama 不健康 → primary=Gemini, fallback=[Nemotron, Claude]

-        188 完全移出 routing chain（CPU-only 0.45 tok/s，禁止即時回應）。
        Gemini quota 超過由 _build_quota_exceeded_route() 接管。
        """
-        model_111 = self._settings.OLLAMA_HEALTH_CHECK_MODEL
-        ep_111 = OllamaEndpoint(url=url_111, provider_name="ollama", model=model_111)
+        model = self._settings.OLLAMA_HEALTH_CHECK_MODEL
+        ep_gcp_a = OllamaEndpoint(url=url_gcp_a, provider_name="ollama_gcp_a", model=model)
+        ep_gcp_b = OllamaEndpoint(url=url_gcp_b, provider_name="ollama_gcp_b", model=model)
+        ep_local = OllamaEndpoint(url=url_local, provider_name="ollama_local", model=model)

        now_ts = datetime.datetime.now(TAIPEI_TZ).isoformat()

-        if health_111.status == HealthStatus.HEALTHY:
+        # GCP-A 健康 → 主 GCP-A
+        if health_gcp_a.status == HealthStatus.HEALTHY:
            return OllamaRoutingResult(
-                primary=ep_111,
+                primary=ep_gcp_a,
+                fallback_chain=[ep_gcp_b, ep_local],
+                routing_reason="GCP-A HEALTHY → primary GCP-A",
+                health_gcp_a=health_gcp_a,
+                health_gcp_b=health_gcp_b,
+                health_local=health_local,
+            )
+
+        # GCP-A 不健康，GCP-B 健康 → 切 GCP-B
+        if health_gcp_b.status == HealthStatus.HEALTHY:
+            return OllamaRoutingResult(
+                primary=ep_gcp_b,
+                fallback_chain=[ep_local],
+                routing_reason=f"GCP-A {health_gcp_a.status.value} → 切 GCP-B at {now_ts}",
+                health_gcp_a=health_gcp_a,
+                health_gcp_b=health_gcp_b,
+                health_local=health_local,
+            )
+
+        # GCP-A + GCP-B 都不健康，Local 健康 → 切 Local(111)
+        if health_local.status == HealthStatus.HEALTHY:
+            return OllamaRoutingResult(
+                primary=ep_local,
                fallback_chain=[_GEMINI_ENDPOINT],
-                routing_reason="111 HEALTHY → 主 111",
-                health_111=health_111,
+                routing_reason=(
+                    f"GCP-A {health_gcp_a.status.value} + GCP-B {health_gcp_b.status.value}"
+                    f" → 切 Local(111) at {now_ts}"
+                ),
+                health_gcp_a=health_gcp_a,
+                health_gcp_b=health_gcp_b,
+                health_local=health_local,
            )

-        if health_111.status == HealthStatus.SLOW:
-            return OllamaRoutingResult(
-                primary=_GEMINI_ENDPOINT,
-                fallback_chain=[ep_111, _NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
-                routing_reason=f"111 SLOW → 切 Gemini at {now_ts}",
-                health_111=health_111,
-            )
-
-        # DEGRADED / OFFLINE
-        status_label = health_111.status.value
+        # 全部 Ollama 不健康 → Gemini
        return OllamaRoutingResult(
            primary=_GEMINI_ENDPOINT,
            fallback_chain=[_NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
-            routing_reason=f"111 {status_label} → 切 Gemini at {now_ts}",
-            health_111=health_111,
+            routing_reason=(
+                f"所有 Ollama 不健康（GCP-A {health_gcp_a.status.value}，"
+                f"GCP-B {health_gcp_b.status.value}，"
+                f"Local {health_local.status.value}）→ 切 Gemini at {now_ts}"
+            ),
+            health_gcp_a=health_gcp_a,
+            health_gcp_b=health_gcp_b,
+            health_local=health_local,
        )

    # -------------------------------------------------------------------------
@@ -382,17 +441,17 @@ class OllamaFailoverManager:

    def _build_quota_exceeded_route(
        self,
-        health_111: HealthReport,
+        health_gcp_a: HealthReport,
    ) -> OllamaRoutingResult:
        """
        Gemini 配額耗盡時的備援路由：primary=Nemotron, fallback=[Claude]
-        2026-04-26 統帥鐵律：188 移出，quota 超過直接走 Nemotron → Claude。
+        2026-05-03 ogt: 更新參數名 health_111 → health_gcp_a（ADR-110）
        """
        return OllamaRoutingResult(
            primary=_NEMOTRON_ENDPOINT,
            fallback_chain=[_CLAUDE_ENDPOINT],
            routing_reason="Gemini quota exceeded → Nemotron 備援",
-            health_111=health_111,
+            health_gcp_a=health_gcp_a,
        )

    # -------------------------------------------------------------------------
@@ -423,8 +482,10 @@ class OllamaFailoverManager:
            if redis is None:
                return
            # 動態由 settings URL 組 cache key，避免硬編碼 IP
+            # 2026-05-03 ogt: 新增 OLLAMA_SECONDARY_URL（ADR-110 GCP-B）
            keys = [
                make_cache_key(self._settings.OLLAMA_URL),
+                make_cache_key(self._settings.OLLAMA_SECONDARY_URL or ""),
                make_cache_key(self._settings.OLLAMA_FALLBACK_URL or ""),
            ]
            for k in keys:
@@ -470,8 +531,8 @@ class OllamaFailoverManager:
        service="ollama_failover"（per 任務規格）
        僅在 primary 非 111 時記錄（真正發生切換）
        """
-        if result.primary.provider_name == "ollama":
-            # 111 正常，無切換事件
+        # 2026-05-03 ogt: GCP 三層容災下，三個 ollama_* provider 都是正常狀態，無需告警
+        if result.primary.provider_name in ("ollama", "ollama_gcp_a", "ollama_gcp_b", "ollama_local"):
            return

        # 2026-04-26 P2.3 by Claude Sonnet 4.6 (tool-expert) — 記錄 failover Prometheus metric
@@ -496,17 +557,29 @@ class OllamaFailoverManager:
            to_provider=result.primary.provider_name,
            reason=result.routing_reason,
            primary_url=result.primary.url or result.primary.provider_name,
-            health_111=result.health_111.status.value,
-            health_188=result.health_188.status.value if result.health_188 else "not_configured",
+            health_gcp_a=result.health_gcp_a.status.value,
+            health_gcp_b=result.health_gcp_b.status.value if result.health_gcp_b else "not_configured",
+            health_local=result.health_local.status.value if result.health_local else "not_configured",
        )

        # Telegram 告警（首次切換才通知，dedup 10min 內建）
        # 2026-04-25 P1.5 by Claude Engineer-D — 告警失敗不阻斷主路由邏輯
+        # 2026-05-03 ogt: ADR-110 — failed_host 動態計算，顯示哪台 GCP/Local 故障
        try:
            from src.services.failover_alerter import get_failover_alerter
+            from src.services.ollama_health_monitor import HealthStatus
            fallback_chain_str = " → ".join(
                p.provider_name for p in result.fallback_chain
            )
+            # 計算故障主機描述（哪層 Ollama 不健康）
+            _failed = []
+            if result.health_gcp_a.status != HealthStatus.HEALTHY:
+                _failed.append(f"GCP-A {self._settings.OLLAMA_URL}")
+            if result.health_gcp_b and result.health_gcp_b.status != HealthStatus.HEALTHY:
+                _failed.append(f"GCP-B {self._settings.OLLAMA_SECONDARY_URL}")
+            if result.health_local and result.health_local.status != HealthStatus.HEALTHY:
+                _failed.append(f"Local {self._settings.OLLAMA_FALLBACK_URL}")
+            failed_host = " + ".join(_failed) if _failed else "Ollama"
            alerter = get_failover_alerter()
            await alerter.alert_failover({
                "to_provider": result.primary.provider_name,
@@ -514,6 +587,7 @@ class OllamaFailoverManager:
                "reason": result.routing_reason,
                "timestamp": datetime.datetime.now(TAIPEI_TZ).isoformat(),
                "fallback_chain_str": fallback_chain_str,
+                "failed_host": failed_host,
            })
        except Exception as e:
            logger.warning("failover_alert_failed", error=str(e))
--- a/apps/api/src/services/ollama_health_monitor.py
+++ b/apps/api/src/services/ollama_health_monitor.py
@@ -177,8 +177,15 @@ class OllamaHealthMonitor:
            from src.core.metrics import OLLAMA_HEALTH_STATUS
            from urllib.parse import urlparse as _urlparse
            _netloc = _urlparse(host).hostname or host
-            # 192.168.0.111 → "111"，192.168.0.188 → "188"
-            _host_label = _netloc.split(".")[-1] if "." in _netloc else _netloc
+            # 2026-05-03 ogt: GCP 三層容災（ADR-110）— 通用 host label 萃取
+            # 私網 IP（192.168.x.y）→ 取最後一段（"111"/"188"）維持短標識
+            # GCP 公網 IP（34.x.y.z）→ 取前 8 字元避免 label 過長（"34.143.1"）
+            if _netloc.startswith("192.168."):
+                _host_label = _netloc.split(".")[-1]
+            elif "." in _netloc:
+                _host_label = _netloc.replace(".", "_")[:12]  # e.g. "34_143_170_2"
+            else:
+                _host_label = _netloc
            _is_healthy = 1 if report.status == HealthStatus.HEALTHY else 0
            OLLAMA_HEALTH_STATUS.labels(host=_host_label).set(_is_healthy)
        except Exception as _metric_err: