fix(alerts): let GCP Ollama finish before cloud fallback

apps/api/src/services/agent_orchestrator.py

@@ -27,7 +27,7 @@ from __future__ import annotations
 
 import asyncio
 import dataclasses
-import json
+import os
 import time
 import uuid
 from datetime import UTC, datetime
@@ -63,11 +63,25 @@ if TYPE_CHECKING:
 
 logger = structlog.get_logger(__name__)
 
+def _agent_debate_global_timeout_seconds() -> float:
+    """Return the full Phase 2 debate timeout.
+
+    GCP Ollama incident analysis can legitimately take longer than the old
+    90s guard. Keep a hard ceiling, but make it an explicit deployment knob.
+    """
+
+    raw = os.environ.get("AGENT_DEBATE_GLOBAL_TIMEOUT_SEC", "260.0")
+    try:
+        timeout = float(raw)
+    except (TypeError, ValueError):
+        timeout = 260.0
+    return max(timeout, 90.0)
+
+
 # 全局超時（所有 Agent 加起來）
-# 2026-04-16 Claude Sonnet 4.6: deepseek-r1:14b 實測 2.2-27.3s avg 10.6s
-# 原 30s 對 3 個序列 Agent 每個只剩 10s → 頻繁 timeout → confidence=20%
-# 調整: 每 Agent 25s, 3個序列+1組並行 = 最差 75s + buffer = 90s
-GLOBAL_TIMEOUT_SEC = 90.0
+# 2026-05-06 Codex: configurable for GCP-A/GCP-B/111 Ollama-first incident
+# diagnosis. The old 90s guard was cutting off valid deep diagnosis runs.
+GLOBAL_TIMEOUT_SEC = _agent_debate_global_timeout_seconds()
 
 # 2026-04-16 ogt + Claude Sonnet 4.6: 移除 _PER_AGENT_TIMEOUT_SEC
 # LLM 必須等到完整回應，不得人工截斷。降級只在真正異常（連線失敗、模型崩潰）觸發。

apps/api/src/services/decision_manager.py

@@ -89,6 +89,22 @@ def _phase2_fallback_reason(package: Any) -> str | None:
     return None
 
 
+def _incident_llm_timeout_seconds() -> float:
+    """Return the outer timeout for incident LLM proposals.
+
+    The provider layer already has per-provider timeouts. This outer guard must
+    not be shorter than the GCP Ollama lane, or alert diagnosis will be cut off
+    before the free/local-first route can answer.
+    """
+
+    configured = getattr(settings, "INCIDENT_LLM_TIMEOUT_SECONDS", None)
+    try:
+        timeout = float(configured)
+    except (TypeError, ValueError):
+        timeout = 240.0
+    return max(timeout, float(getattr(settings, "OPENCLAW_TIMEOUT", 30)))
+
+
 def _should_escalate_auto_approve_rejection(reason: Any) -> bool:
     """Return True for manual gates that mean the automation path went blind."""
 
@@ -841,7 +857,6 @@ async def _resolve_target_from_k8s(incident: "Incident", namespace: str) -> str
                 reason="alertname 有對應但 keywords=[]，走 fallback 取第一個非 infra pod",
             )
 
-        import re as _re
         for line in pod_lines:
             pod = line.removeprefix("pod/").strip()
             if not pod:
@@ -2708,9 +2723,10 @@ class DecisionManager:
             if context_parts:
                 llm_expert_context["diagnosis_context"] = "\n\n".join(context_parts)
 
-            # GAP-B4 (2026-04-14 Claude Sonnet 4.6): LLM 25s hard timeout，
-            # 比外層 decide() 30s wait_for 更嚴格，留 5s 給 YAML risk override + NemoClaw second opinion
-            # Timeout → 明確 llm_timeout_fallback 日誌，返回 expert_result 而非等外層觸發
+            # 2026-05-06 Codex: The alert goal is resolution quality, not a
+            # fast-but-paid card. The outer guard is configurable and must allow
+            # the GCP-A → GCP-B → 111 Ollama lane to finish before cloud backup.
+            llm_timeout_seconds = _incident_llm_timeout_seconds()
             llm_result, provider, success = await asyncio.wait_for(
                 self._openclaw.generate_incident_proposal_with_tools(
                     incident_id=incident.incident_id,
@@ -2719,7 +2735,7 @@ class DecisionManager:
                     affected_services=incident.affected_services,
                     expert_context=llm_expert_context if llm_expert_context else None,
                 ),
-                timeout=25.0,
+                timeout=llm_timeout_seconds,
             )
 
             if success and llm_result:
@@ -2786,7 +2802,7 @@ class DecisionManager:
             logger.warning(
                 "llm_timeout_fallback",
                 incident_id=incident.incident_id,
-                timeout_sec=25.0,
+                timeout_sec=llm_timeout_seconds,
                 action="降級 Expert System",
             )
         except Exception as e:

apps/api/src/services/ollama_endpoint_resolver.py

@@ -35,9 +35,6 @@ _GCP_A_PREFERRED_WORKLOADS = {
     "interactive",
     "healthcheck",
     "alert_fast",
-}
-
-_LOCAL_PREFERRED_WORKLOADS = {
     "batch",
     "embedding",
     "rag",
@@ -47,6 +44,9 @@ _LOCAL_PREFERRED_WORKLOADS = {
     "deep_rca",
     "image_analysis",
     "hermes",
+}
+
+_LOCAL_PREFERRED_WORKLOADS = {
     "local_required",
     "privacy_sensitive",
     "dr",
@@ -102,11 +102,27 @@ def resolve_ollama_selection(
             reason="gcp_b_default_non_alert_lane",
         )
 
+    if primary:
+        return OllamaEndpointSelection(
+            url=primary,
+            provider_name="ollama_gcp_a",
+            workload_type=workload_type,
+            reason="primary_interactive_lane",
+        )
+
+    if secondary:
+        return OllamaEndpointSelection(
+            url=secondary,
+            provider_name="ollama_gcp_b",
+            workload_type=workload_type,
+            reason="primary_missing_gcp_b_fallback",
+        )
+
     return OllamaEndpointSelection(
-        url=primary,
-        provider_name="ollama_gcp_a",
+        url=fallback,
+        provider_name="ollama_local",
         workload_type=workload_type,
-        reason="primary_interactive_lane",
+        reason="gcp_missing_local_fallback",
     )
 
 

apps/api/src/services/playbook_rag.py

@@ -29,6 +29,7 @@ from typing import Any
 import httpx
 import structlog
 
+from src.core.config import settings
 from src.models.playbook import Playbook, SymptomPattern
 from src.repositories.interfaces import IEmbeddingCacheRepository
 from src.services.ollama_endpoint_resolver import resolve_ollama_endpoint
@@ -45,6 +46,20 @@ EMBEDDING_MODEL = "nomic-embed-text"
 EMBEDDING_DIM = 768  # nomic-embed-text 向量維度
 
 
+def _dedupe_urls(urls: list[str]) -> list[str]:
+    """Return configured Ollama URLs in order without blanks or duplicates."""
+
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = (url or "").rstrip("/")
+        if not normalized or normalized in seen:
+            continue
+        deduped.append(normalized)
+        seen.add(normalized)
+    return deduped
+
+
 # =============================================================================
 # Data Models
 # =============================================================================
@@ -147,6 +162,14 @@ class PlaybookRAGService:
         self._http_client = http_client
         self._embedding_cache = embedding_cache
         self.ollama_url = resolve_ollama_endpoint("embedding")
+        self.ollama_urls = _dedupe_urls(
+            [
+                self.ollama_url,
+                getattr(settings, "OLLAMA_URL", ""),
+                getattr(settings, "OLLAMA_SECONDARY_URL", ""),
+                getattr(settings, "OLLAMA_FALLBACK_URL", ""),
+            ]
+        )
         self.embedding_model = EMBEDDING_MODEL
 
     # =========================================================================
@@ -180,31 +203,57 @@ class PlaybookRAGService:
         """
         try:
             client = await self._get_http_client()
-            response = await client.post(
-                f"{self.ollama_url}/api/embeddings",
-                json={
-                    "model": self.embedding_model,
-                    "prompt": text,
-                },
-                timeout=30.0,  # 單次請求 timeout
+            last_error = ""
+            for endpoint_url in self.ollama_urls:
+                try:
+                    response = await client.post(
+                        f"{endpoint_url}/api/embeddings",
+                        json={
+                            "model": self.embedding_model,
+                            "prompt": text,
+                        },
+                        timeout=30.0,  # 單次請求 timeout
+                    )
+
+                    if response.status_code != 200:
+                        last_error = f"http_{response.status_code}"
+                        logger.warning(
+                            "ollama_embedding_failed",
+                            endpoint=endpoint_url,
+                            status_code=response.status_code,
+                            text_preview=text[:50],
+                        )
+                        continue
+
+                    result = response.json()
+                    embedding = result.get("embedding", [])
+
+                    if not embedding:
+                        last_error = "empty_embedding"
+                        logger.warning(
+                            "ollama_embedding_empty",
+                            endpoint=endpoint_url,
+                            text_preview=text[:50],
+                        )
+                        continue
+
+                    logger.info("ollama_embedding_success", endpoint=endpoint_url)
+                    return normalize_vector(embedding)
+                except Exception as endpoint_error:
+                    last_error = str(endpoint_error)
+                    logger.warning(
+                        "ollama_embedding_endpoint_error",
+                        endpoint=endpoint_url,
+                        error=last_error,
+                        text_preview=text[:50],
+                    )
+
+            logger.warning(
+                "ollama_embedding_error",
+                error=last_error or "all endpoints failed",
+                text_preview=text[:50],
             )
-
-            if response.status_code != 200:
-                logger.warning(
-                    "ollama_embedding_failed",
-                    status_code=response.status_code,
-                    text_preview=text[:50],
-                )
-                return None
-
-            result = response.json()
-            embedding = result.get("embedding", [])
-
-            if not embedding:
-                logger.warning("ollama_embedding_empty", text_preview=text[:50])
-                return None
-
-            return normalize_vector(embedding)
+            return None
 
         except Exception as e:
             logger.warning(