fix(alerts): let GCP Ollama finish before cloud fallback

apps/api/src/core/config.py

@@ -529,6 +529,14 @@ class Settings(BaseSettings):
             "GCP-B, and 111 fail."
         ),
     )
+    INCIDENT_LLM_TIMEOUT_SECONDS: int = Field(
+        default=240,
+        description=(
+            "Outer timeout for incident OpenClaw proposal generation. This must "
+            "be long enough for the GCP-A/GCP-B/111 Ollama lane to complete "
+            "before Gemini backup is considered useful."
+        ),
+    )
     # 2026-03-29 ogt: ADR-036 Nemotron Tool Calling 整合
     NVIDIA_API_KEY: str = Field(
         default="",

apps/api/src/services/agent_orchestrator.py

@@ -27,7 +27,7 @@ from __future__ import annotations
 
 import asyncio
 import dataclasses
-import json
+import os
 import time
 import uuid
 from datetime import UTC, datetime
@@ -63,11 +63,25 @@ if TYPE_CHECKING:
 
 logger = structlog.get_logger(__name__)
 
+def _agent_debate_global_timeout_seconds() -> float:
+    """Return the full Phase 2 debate timeout.
+
+    GCP Ollama incident analysis can legitimately take longer than the old
+    90s guard. Keep a hard ceiling, but make it an explicit deployment knob.
+    """
+
+    raw = os.environ.get("AGENT_DEBATE_GLOBAL_TIMEOUT_SEC", "260.0")
+    try:
+        timeout = float(raw)
+    except (TypeError, ValueError):
+        timeout = 260.0
+    return max(timeout, 90.0)
+
+
 # 全局超時（所有 Agent 加起來）
-# 2026-04-16 Claude Sonnet 4.6: deepseek-r1:14b 實測 2.2-27.3s avg 10.6s
-# 原 30s 對 3 個序列 Agent 每個只剩 10s → 頻繁 timeout → confidence=20%
-# 調整: 每 Agent 25s, 3個序列+1組並行 = 最差 75s + buffer = 90s
-GLOBAL_TIMEOUT_SEC = 90.0
+# 2026-05-06 Codex: configurable for GCP-A/GCP-B/111 Ollama-first incident
+# diagnosis. The old 90s guard was cutting off valid deep diagnosis runs.
+GLOBAL_TIMEOUT_SEC = _agent_debate_global_timeout_seconds()
 
 # 2026-04-16 ogt + Claude Sonnet 4.6: 移除 _PER_AGENT_TIMEOUT_SEC
 # LLM 必須等到完整回應，不得人工截斷。降級只在真正異常（連線失敗、模型崩潰）觸發。

apps/api/src/services/decision_manager.py

@@ -89,6 +89,22 @@ def _phase2_fallback_reason(package: Any) -> str | None:
     return None
 
 
+def _incident_llm_timeout_seconds() -> float:
+    """Return the outer timeout for incident LLM proposals.
+
+    The provider layer already has per-provider timeouts. This outer guard must
+    not be shorter than the GCP Ollama lane, or alert diagnosis will be cut off
+    before the free/local-first route can answer.
+    """
+
+    configured = getattr(settings, "INCIDENT_LLM_TIMEOUT_SECONDS", None)
+    try:
+        timeout = float(configured)
+    except (TypeError, ValueError):
+        timeout = 240.0
+    return max(timeout, float(getattr(settings, "OPENCLAW_TIMEOUT", 30)))
+
+
 def _should_escalate_auto_approve_rejection(reason: Any) -> bool:
     """Return True for manual gates that mean the automation path went blind."""
 
@@ -841,7 +857,6 @@ async def _resolve_target_from_k8s(incident: "Incident", namespace: str) -> str
                 reason="alertname 有對應但 keywords=[]，走 fallback 取第一個非 infra pod",
             )
 
-        import re as _re
         for line in pod_lines:
             pod = line.removeprefix("pod/").strip()
             if not pod:
@@ -2708,9 +2723,10 @@ class DecisionManager:
             if context_parts:
                 llm_expert_context["diagnosis_context"] = "\n\n".join(context_parts)
 
-            # GAP-B4 (2026-04-14 Claude Sonnet 4.6): LLM 25s hard timeout，
-            # 比外層 decide() 30s wait_for 更嚴格，留 5s 給 YAML risk override + NemoClaw second opinion
-            # Timeout → 明確 llm_timeout_fallback 日誌，返回 expert_result 而非等外層觸發
+            # 2026-05-06 Codex: The alert goal is resolution quality, not a
+            # fast-but-paid card. The outer guard is configurable and must allow
+            # the GCP-A → GCP-B → 111 Ollama lane to finish before cloud backup.
+            llm_timeout_seconds = _incident_llm_timeout_seconds()
             llm_result, provider, success = await asyncio.wait_for(
                 self._openclaw.generate_incident_proposal_with_tools(
                     incident_id=incident.incident_id,
@@ -2719,7 +2735,7 @@ class DecisionManager:
                     affected_services=incident.affected_services,
                     expert_context=llm_expert_context if llm_expert_context else None,
                 ),
-                timeout=25.0,
+                timeout=llm_timeout_seconds,
             )
 
             if success and llm_result:
@@ -2786,7 +2802,7 @@ class DecisionManager:
             logger.warning(
                 "llm_timeout_fallback",
                 incident_id=incident.incident_id,
-                timeout_sec=25.0,
+                timeout_sec=llm_timeout_seconds,
                 action="降級 Expert System",
             )
         except Exception as e:

apps/api/src/services/ollama_endpoint_resolver.py

@@ -35,9 +35,6 @@ _GCP_A_PREFERRED_WORKLOADS = {
     "interactive",
     "healthcheck",
     "alert_fast",
-}
-
-_LOCAL_PREFERRED_WORKLOADS = {
     "batch",
     "embedding",
     "rag",
@@ -47,6 +44,9 @@ _LOCAL_PREFERRED_WORKLOADS = {
     "deep_rca",
     "image_analysis",
     "hermes",
+}
+
+_LOCAL_PREFERRED_WORKLOADS = {
     "local_required",
     "privacy_sensitive",
     "dr",
@@ -102,11 +102,27 @@ def resolve_ollama_selection(
             reason="gcp_b_default_non_alert_lane",
         )
 
+    if primary:
+        return OllamaEndpointSelection(
+            url=primary,
+            provider_name="ollama_gcp_a",
+            workload_type=workload_type,
+            reason="primary_interactive_lane",
+        )
+
+    if secondary:
+        return OllamaEndpointSelection(
+            url=secondary,
+            provider_name="ollama_gcp_b",
+            workload_type=workload_type,
+            reason="primary_missing_gcp_b_fallback",
+        )
+
     return OllamaEndpointSelection(
-        url=primary,
-        provider_name="ollama_gcp_a",
+        url=fallback,
+        provider_name="ollama_local",
         workload_type=workload_type,
-        reason="primary_interactive_lane",
+        reason="gcp_missing_local_fallback",
     )
 
 

apps/api/src/services/playbook_rag.py

@@ -29,6 +29,7 @@ from typing import Any
 import httpx
 import structlog
 
+from src.core.config import settings
 from src.models.playbook import Playbook, SymptomPattern
 from src.repositories.interfaces import IEmbeddingCacheRepository
 from src.services.ollama_endpoint_resolver import resolve_ollama_endpoint
@@ -45,6 +46,20 @@ EMBEDDING_MODEL = "nomic-embed-text"
 EMBEDDING_DIM = 768  # nomic-embed-text 向量維度
 
 
+def _dedupe_urls(urls: list[str]) -> list[str]:
+    """Return configured Ollama URLs in order without blanks or duplicates."""
+
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = (url or "").rstrip("/")
+        if not normalized or normalized in seen:
+            continue
+        deduped.append(normalized)
+        seen.add(normalized)
+    return deduped
+
+
 # =============================================================================
 # Data Models
 # =============================================================================
@@ -147,6 +162,14 @@ class PlaybookRAGService:
         self._http_client = http_client
         self._embedding_cache = embedding_cache
         self.ollama_url = resolve_ollama_endpoint("embedding")
+        self.ollama_urls = _dedupe_urls(
+            [
+                self.ollama_url,
+                getattr(settings, "OLLAMA_URL", ""),
+                getattr(settings, "OLLAMA_SECONDARY_URL", ""),
+                getattr(settings, "OLLAMA_FALLBACK_URL", ""),
+            ]
+        )
         self.embedding_model = EMBEDDING_MODEL
 
     # =========================================================================
@@ -180,31 +203,57 @@ class PlaybookRAGService:
         """
         try:
             client = await self._get_http_client()
-            response = await client.post(
-                f"{self.ollama_url}/api/embeddings",
-                json={
-                    "model": self.embedding_model,
-                    "prompt": text,
-                },
-                timeout=30.0,  # 單次請求 timeout
+            last_error = ""
+            for endpoint_url in self.ollama_urls:
+                try:
+                    response = await client.post(
+                        f"{endpoint_url}/api/embeddings",
+                        json={
+                            "model": self.embedding_model,
+                            "prompt": text,
+                        },
+                        timeout=30.0,  # 單次請求 timeout
+                    )
+
+                    if response.status_code != 200:
+                        last_error = f"http_{response.status_code}"
+                        logger.warning(
+                            "ollama_embedding_failed",
+                            endpoint=endpoint_url,
+                            status_code=response.status_code,
+                            text_preview=text[:50],
+                        )
+                        continue
+
+                    result = response.json()
+                    embedding = result.get("embedding", [])
+
+                    if not embedding:
+                        last_error = "empty_embedding"
+                        logger.warning(
+                            "ollama_embedding_empty",
+                            endpoint=endpoint_url,
+                            text_preview=text[:50],
+                        )
+                        continue
+
+                    logger.info("ollama_embedding_success", endpoint=endpoint_url)
+                    return normalize_vector(embedding)
+                except Exception as endpoint_error:
+                    last_error = str(endpoint_error)
+                    logger.warning(
+                        "ollama_embedding_endpoint_error",
+                        endpoint=endpoint_url,
+                        error=last_error,
+                        text_preview=text[:50],
+                    )
+
+            logger.warning(
+                "ollama_embedding_error",
+                error=last_error or "all endpoints failed",
+                text_preview=text[:50],
             )
-
-            if response.status_code != 200:
-                logger.warning(
-                    "ollama_embedding_failed",
-                    status_code=response.status_code,
-                    text_preview=text[:50],
-                )
-                return None
-
-            result = response.json()
-            embedding = result.get("embedding", [])
-
-            if not embedding:
-                logger.warning("ollama_embedding_empty", text_preview=text[:50])
-                return None
-
-            return normalize_vector(embedding)
+            return None
 
         except Exception as e:
             logger.warning(

apps/api/tests/test_agent_step_timeouts.py

@@ -17,7 +17,6 @@ from __future__ import annotations
 import asyncio
 import importlib
 import sys
-import time
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -72,6 +71,17 @@ class TestTimeoutDefaults:
             f"Critic default timeout 期望 15.0，實際 {mod.AGENT_CRITIC_TIMEOUT_SEC}"
         )
 
+    def test_agent_debate_global_timeout_default_is_260(self, monkeypatch):
+        """Agent debate global timeout defaults to the GCP Ollama-first budget."""
+        monkeypatch.delenv("AGENT_DEBATE_GLOBAL_TIMEOUT_SEC", raising=False)
+
+        if "src.services.agent_orchestrator" in sys.modules:
+            del sys.modules["src.services.agent_orchestrator"]
+        import src.services.agent_orchestrator as mod
+        importlib.reload(mod)
+
+        assert mod.GLOBAL_TIMEOUT_SEC == 260.0
+
     def test_deprecated_alias_matches_new_constant_diagnostician(self, monkeypatch):
         """PHASE2_STEP_TIMEOUT_SEC alias 應等於 AGENT_DIAGNOSTICIAN_TIMEOUT_SEC（相容性保證）"""
         monkeypatch.delenv("AGENT_DIAGNOSTICIAN_TIMEOUT_SEC", raising=False)
@@ -172,6 +182,17 @@ class TestEnvOverride:
         assert mod.PHASE2_STEP_TIMEOUT_SEC == 8.0
         assert mod.PHASE2_STEP_TIMEOUT_SEC == mod.AGENT_CRITIC_TIMEOUT_SEC
 
+    def test_agent_debate_global_timeout_env_override(self, monkeypatch):
+        """AGENT_DEBATE_GLOBAL_TIMEOUT_SEC=300 覆蓋 default 260.0"""
+        monkeypatch.setenv("AGENT_DEBATE_GLOBAL_TIMEOUT_SEC", "300")
+
+        if "src.services.agent_orchestrator" in sys.modules:
+            del sys.modules["src.services.agent_orchestrator"]
+        import src.services.agent_orchestrator as mod
+        importlib.reload(mod)
+
+        assert mod.GLOBAL_TIMEOUT_SEC == 300.0
+
 
 # =============================================================================
 # Section 3: Metric Histogram observe 驗證

apps/api/tests/test_ollama_endpoint_resolver.py

@@ -20,7 +20,7 @@ def _settings(
     )
 
 
-def test_heavy_workloads_prefer_local_lane() -> None:
+def test_non_sensitive_workloads_prefer_gcp_a_lane() -> None:
     cfg = _settings()
 
     for workload in (
@@ -35,9 +35,9 @@ def test_heavy_workloads_prefer_local_lane() -> None:
         "hermes",
     ):
         selection = resolve_ollama_selection(workload, config=cfg)
-        assert selection.url == "http://192.168.0.110:11437"
-        assert selection.provider_name == "ollama_local"
-        assert selection.reason == "local_heavy_or_privacy_lane"
+        assert selection.url == "http://192.168.0.110:11435"
+        assert selection.provider_name == "ollama_gcp_a"
+        assert selection.reason == "primary_interactive_lane"
 
 
 def test_interactive_workloads_stay_on_gcp_a() -> None:
@@ -58,10 +58,10 @@ def test_local_required_workloads_use_local_lane() -> None:
         assert selection.provider_name == "ollama_local"
 
 
-def test_heavy_workloads_fall_back_to_gcp_b_when_local_missing() -> None:
-    cfg = _settings(fallback="")
+def test_non_sensitive_workloads_fall_back_to_gcp_b_when_primary_missing() -> None:
+    cfg = _settings(primary="")
 
     selection = resolve_ollama_selection("embedding", config=cfg)
     assert selection.url == "http://192.168.0.110:11436"
     assert selection.provider_name == "ollama_gcp_b"
-    assert selection.reason == "local_missing_gcp_b_fallback"
+    assert selection.reason == "primary_missing_gcp_b_fallback"

apps/api/tests/test_phase2_fallback.py

@@ -1,7 +1,11 @@
 from types import SimpleNamespace
 
 from src.agents.protocol import AgentSessionStatus
-from src.services.decision_manager import _phase2_fallback_reason
+from src.services import decision_manager as decision_manager_module
+from src.services.decision_manager import (
+    _incident_llm_timeout_seconds,
+    _phase2_fallback_reason,
+)
 
 
 def _package(**kwargs):
@@ -35,3 +39,17 @@ def test_phase2_actionable_package_stays_primary() -> None:
     pkg = _package()
 
     assert _phase2_fallback_reason(pkg) is None
+
+
+def test_incident_llm_timeout_uses_configured_value(monkeypatch) -> None:
+    monkeypatch.setattr(decision_manager_module.settings, "INCIDENT_LLM_TIMEOUT_SECONDS", 240)
+    monkeypatch.setattr(decision_manager_module.settings, "OPENCLAW_TIMEOUT", 120)
+
+    assert _incident_llm_timeout_seconds() == 240.0
+
+
+def test_incident_llm_timeout_never_below_openclaw_timeout(monkeypatch) -> None:
+    monkeypatch.setattr(decision_manager_module.settings, "INCIDENT_LLM_TIMEOUT_SECONDS", 60)
+    monkeypatch.setattr(decision_manager_module.settings, "OPENCLAW_TIMEOUT", 120)
+
+    assert _incident_llm_timeout_seconds() == 120.0

docs/LOGBOOK.md

@@ -6,6 +6,21 @@
 
 ---
 
+## 2026-05-06 | Incident Ollama-first path stops timing out before GCP answers
+
+**背景**：production log 顯示告警 provider order 已是 `ollama_gcp_a -> ollama_gcp_b -> ollama_local -> gemini`，且 GCP-A 可用 `qwen3:14b` 成功回應（52s/75s），但 DecisionManager 仍用 25s 外層 timeout、Phase 2 debate 仍用 90s 全局 timeout，導致合法的 GCP Ollama 深度診斷被提前截斷；同時 RAG/embedding resolver 仍優先打目前不可達的 111，造成大量 `ollama_embedding_error`。
+
+**本次修補**：
+- 新增 `INCIDENT_LLM_TIMEOUT_SECONDS`，production 設為 240s；Incident LLM 外層 guard 不再硬編 25s，且不得低於 `OPENCLAW_TIMEOUT`。
+- 新增 `AGENT_DEBATE_GLOBAL_TIMEOUT_SEC`，production 設為 260s；Phase 2 debate 不再被 90s 固定值卡死。
+- `ollama_endpoint_resolver` 改為非敏感工作（embedding/RAG/deep_rca/Hermes/code_review 等）GCP-A 優先、GCP-B 備援、111 兜底；只有 `local_required` / `privacy_sensitive` / `dr` 維持 local-first。
+- `PlaybookRAGService.embed_text()` 改為依序嘗試配置的 Ollama endpoints，單一 endpoint 失敗不再直接放棄 RAG。
+
+**驗證**：
+- `py_compile` touched backend files OK；ruff `E9,F401,F821,F841` OK。
+- 相關測試：timeout/resolver 32 passed（1 個既有 coroutine warning）、OpenClaw Ollama route 13 passed、action/parser/learning guard 74 passed、Ollama failover/recovery 73 passed。
+- 現場確認 GCP-A/GCP-B 均可列出 `qwen3:14b`、`qwen2.5:7b-instruct`、`bge-m3`、`gemma3:4b`；111 `/api/tags` 目前 timeout，仍需後續修 111 連通性，但 Gemini 已回到 GCP-A/GCP-B/111 之後的最後備援角色。
+
 ## 2026-05-06 | Decision Telegram dedup no longer reads missing Incident.title
 
 **背景**：新 Ollama-first 部署後，production log 顯示 alert diagnosis 已走 `ollama_gcp_a -> ollama_gcp_b -> ollama_local -> gemini` 且 `phase24_ai_router_used` provider=`ollama`，但 DecisionManager 推送 Telegram decision card 時出現 `telegram_decision_push_failed: 'Incident' object has no attribute 'title'`。

k8s/awoooi-prod/04-configmap.yaml

@@ -60,6 +60,16 @@ data:
   # 回滾: kubectl set env deployment/awoooi-api -n awoooi-prod USE_AI_ROUTER=false
   # ============================================================================
   USE_AI_ROUTER: "true"
+  ALERT_AI_ALLOW_CLOUD_FALLBACK: "true"
+  ALERT_AI_ENFORCE_OLLAMA_FIRST: "true"
+  ALERT_OLLAMA_MODEL: "qwen3:14b"
+  OLLAMA_HEALTH_CHECK_MODEL: "gemma3:4b"
+  OPENCLAW_DEFAULT_MODEL: "qwen2.5:7b-instruct"
+  OPENCLAW_TIMEOUT: "120"
+  INCIDENT_LLM_TIMEOUT_SECONDS: "240"
+  AGENT_DEBATE_GLOBAL_TIMEOUT_SEC: "260"
+  AGENT_DIAGNOSTICIAN_TIMEOUT_SEC: "100"
+  AGENT_SOLVER_TIMEOUT_SEC: "80"
   # ADR-105 P1: OpenClaw Agent Loop shadow canary.
   # 只給 read-only MCP tools，使用本地 Ollama，結果只附加 metadata 不改決策。
   # 回滾: kubectl set env deployment/awoooi-api -n awoooi-prod ENABLE_OPENCLAW_AGENT_LOOP_SHADOW=false

k8s/awoooi-prod/06-deployment-api.yaml

@@ -85,6 +85,10 @@ spec:
               value: "qwen2.5:7b-instruct"
             - name: OPENCLAW_TIMEOUT
               value: "120"
+            - name: INCIDENT_LLM_TIMEOUT_SECONDS
+              value: "240"
+            - name: AGENT_DEBATE_GLOBAL_TIMEOUT_SEC
+              value: "260"
             - name: AGENT_DIAGNOSTICIAN_TIMEOUT_SEC
               value: "100"
             - name: AGENT_SOLVER_TIMEOUT_SEC