From c2c0b1ec823198b3c0ac99e3e5851bfb29957a7d Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 6 May 2026 05:27:36 +0800 Subject: [PATCH] fix(alerts): let GCP Ollama finish before cloud fallback --- apps/api/src/core/config.py | 8 ++ apps/api/src/services/agent_orchestrator.py | 24 ++++- apps/api/src/services/decision_manager.py | 28 ++++-- .../src/services/ollama_endpoint_resolver.py | 28 ++++-- apps/api/src/services/playbook_rag.py | 97 ++++++++++++++----- apps/api/tests/test_agent_step_timeouts.py | 23 ++++- .../tests/test_ollama_endpoint_resolver.py | 14 +-- apps/api/tests/test_phase2_fallback.py | 20 +++- docs/LOGBOOK.md | 15 +++ k8s/awoooi-prod/04-configmap.yaml | 10 ++ k8s/awoooi-prod/06-deployment-api.yaml | 4 + 11 files changed, 221 insertions(+), 50 deletions(-) diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py index 77f54aea7..351bddc63 100644 --- a/apps/api/src/core/config.py +++ b/apps/api/src/core/config.py @@ -529,6 +529,14 @@ class Settings(BaseSettings): "GCP-B, and 111 fail." ), ) + INCIDENT_LLM_TIMEOUT_SECONDS: int = Field( + default=240, + description=( + "Outer timeout for incident OpenClaw proposal generation. This must " + "be long enough for the GCP-A/GCP-B/111 Ollama lane to complete " + "before Gemini backup is considered useful." + ), + ) # 2026-03-29 ogt: ADR-036 Nemotron Tool Calling 整合 NVIDIA_API_KEY: str = Field( default="", diff --git a/apps/api/src/services/agent_orchestrator.py b/apps/api/src/services/agent_orchestrator.py index 35d3b5d82..a4605304d 100644 --- a/apps/api/src/services/agent_orchestrator.py +++ b/apps/api/src/services/agent_orchestrator.py @@ -27,7 +27,7 @@ from __future__ import annotations import asyncio import dataclasses -import json +import os import time import uuid from datetime import UTC, datetime @@ -63,11 +63,25 @@ if TYPE_CHECKING: logger = structlog.get_logger(__name__) +def _agent_debate_global_timeout_seconds() -> float: + """Return the full Phase 2 debate timeout. + + GCP Ollama incident analysis can legitimately take longer than the old + 90s guard. Keep a hard ceiling, but make it an explicit deployment knob. + """ + + raw = os.environ.get("AGENT_DEBATE_GLOBAL_TIMEOUT_SEC", "260.0") + try: + timeout = float(raw) + except (TypeError, ValueError): + timeout = 260.0 + return max(timeout, 90.0) + + # 全局超時（所有 Agent 加起來） -# 2026-04-16 Claude Sonnet 4.6: deepseek-r1:14b 實測 2.2-27.3s avg 10.6s -# 原 30s 對 3 個序列 Agent 每個只剩 10s → 頻繁 timeout → confidence=20% -# 調整: 每 Agent 25s, 3個序列+1組並行 = 最差 75s + buffer = 90s -GLOBAL_TIMEOUT_SEC = 90.0 +# 2026-05-06 Codex: configurable for GCP-A/GCP-B/111 Ollama-first incident +# diagnosis. The old 90s guard was cutting off valid deep diagnosis runs. +GLOBAL_TIMEOUT_SEC = _agent_debate_global_timeout_seconds() # 2026-04-16 ogt + Claude Sonnet 4.6: 移除 _PER_AGENT_TIMEOUT_SEC # LLM 必須等到完整回應，不得人工截斷。降級只在真正異常（連線失敗、模型崩潰）觸發。 diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index c3a835885..96d8ec268 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -89,6 +89,22 @@ def _phase2_fallback_reason(package: Any) -> str | None: return None +def _incident_llm_timeout_seconds() -> float: + """Return the outer timeout for incident LLM proposals. + + The provider layer already has per-provider timeouts. This outer guard must + not be shorter than the GCP Ollama lane, or alert diagnosis will be cut off + before the free/local-first route can answer. + """ + + configured = getattr(settings, "INCIDENT_LLM_TIMEOUT_SECONDS", None) + try: + timeout = float(configured) + except (TypeError, ValueError): + timeout = 240.0 + return max(timeout, float(getattr(settings, "OPENCLAW_TIMEOUT", 30))) + + def _should_escalate_auto_approve_rejection(reason: Any) -> bool: """Return True for manual gates that mean the automation path went blind.""" @@ -841,7 +857,6 @@ async def _resolve_target_from_k8s(incident: "Incident", namespace: str) -> str reason="alertname 有對應但 keywords=[]，走 fallback 取第一個非 infra pod", ) - import re as _re for line in pod_lines: pod = line.removeprefix("pod/").strip() if not pod: @@ -2708,9 +2723,10 @@ class DecisionManager: if context_parts: llm_expert_context["diagnosis_context"] = "\n\n".join(context_parts) - # GAP-B4 (2026-04-14 Claude Sonnet 4.6): LLM 25s hard timeout， - # 比外層 decide() 30s wait_for 更嚴格，留 5s 給 YAML risk override + NemoClaw second opinion - # Timeout → 明確 llm_timeout_fallback 日誌，返回 expert_result 而非等外層觸發 + # 2026-05-06 Codex: The alert goal is resolution quality, not a + # fast-but-paid card. The outer guard is configurable and must allow + # the GCP-A → GCP-B → 111 Ollama lane to finish before cloud backup. + llm_timeout_seconds = _incident_llm_timeout_seconds() llm_result, provider, success = await asyncio.wait_for( self._openclaw.generate_incident_proposal_with_tools( incident_id=incident.incident_id, @@ -2719,7 +2735,7 @@ class DecisionManager: affected_services=incident.affected_services, expert_context=llm_expert_context if llm_expert_context else None, ), - timeout=25.0, + timeout=llm_timeout_seconds, ) if success and llm_result: @@ -2786,7 +2802,7 @@ class DecisionManager: logger.warning( "llm_timeout_fallback", incident_id=incident.incident_id, - timeout_sec=25.0, + timeout_sec=llm_timeout_seconds, action="降級 Expert System", ) except Exception as e: diff --git a/apps/api/src/services/ollama_endpoint_resolver.py b/apps/api/src/services/ollama_endpoint_resolver.py index d51dd38b3..1ca0d7172 100644 --- a/apps/api/src/services/ollama_endpoint_resolver.py +++ b/apps/api/src/services/ollama_endpoint_resolver.py @@ -35,9 +35,6 @@ _GCP_A_PREFERRED_WORKLOADS = { "interactive", "healthcheck", "alert_fast", -} - -_LOCAL_PREFERRED_WORKLOADS = { "batch", "embedding", "rag", @@ -47,6 +44,9 @@ _LOCAL_PREFERRED_WORKLOADS = { "deep_rca", "image_analysis", "hermes", +} + +_LOCAL_PREFERRED_WORKLOADS = { "local_required", "privacy_sensitive", "dr", @@ -102,11 +102,27 @@ def resolve_ollama_selection( reason="gcp_b_default_non_alert_lane", ) + if primary: + return OllamaEndpointSelection( + url=primary, + provider_name="ollama_gcp_a", + workload_type=workload_type, + reason="primary_interactive_lane", + ) + + if secondary: + return OllamaEndpointSelection( + url=secondary, + provider_name="ollama_gcp_b", + workload_type=workload_type, + reason="primary_missing_gcp_b_fallback", + ) + return OllamaEndpointSelection( - url=primary, - provider_name="ollama_gcp_a", + url=fallback, + provider_name="ollama_local", workload_type=workload_type, - reason="primary_interactive_lane", + reason="gcp_missing_local_fallback", ) diff --git a/apps/api/src/services/playbook_rag.py b/apps/api/src/services/playbook_rag.py index 70694ffbf..62fe3b57a 100644 --- a/apps/api/src/services/playbook_rag.py +++ b/apps/api/src/services/playbook_rag.py @@ -29,6 +29,7 @@ from typing import Any import httpx import structlog +from src.core.config import settings from src.models.playbook import Playbook, SymptomPattern from src.repositories.interfaces import IEmbeddingCacheRepository from src.services.ollama_endpoint_resolver import resolve_ollama_endpoint @@ -45,6 +46,20 @@ EMBEDDING_MODEL = "nomic-embed-text" EMBEDDING_DIM = 768 # nomic-embed-text 向量維度 +def _dedupe_urls(urls: list[str]) -> list[str]: + """Return configured Ollama URLs in order without blanks or duplicates.""" + + deduped: list[str] = [] + seen: set[str] = set() + for url in urls: + normalized = (url or "").rstrip("/") + if not normalized or normalized in seen: + continue + deduped.append(normalized) + seen.add(normalized) + return deduped + + # ============================================================================= # Data Models # ============================================================================= @@ -147,6 +162,14 @@ class PlaybookRAGService: self._http_client = http_client self._embedding_cache = embedding_cache self.ollama_url = resolve_ollama_endpoint("embedding") + self.ollama_urls = _dedupe_urls( + [ + self.ollama_url, + getattr(settings, "OLLAMA_URL", ""), + getattr(settings, "OLLAMA_SECONDARY_URL", ""), + getattr(settings, "OLLAMA_FALLBACK_URL", ""), + ] + ) self.embedding_model = EMBEDDING_MODEL # ========================================================================= @@ -180,31 +203,57 @@ class PlaybookRAGService: """ try: client = await self._get_http_client() - response = await client.post( - f"{self.ollama_url}/api/embeddings", - json={ - "model": self.embedding_model, - "prompt": text, - }, - timeout=30.0, # 單次請求 timeout + last_error = "" + for endpoint_url in self.ollama_urls: + try: + response = await client.post( + f"{endpoint_url}/api/embeddings", + json={ + "model": self.embedding_model, + "prompt": text, + }, + timeout=30.0, # 單次請求 timeout + ) + + if response.status_code != 200: + last_error = f"http_{response.status_code}" + logger.warning( + "ollama_embedding_failed", + endpoint=endpoint_url, + status_code=response.status_code, + text_preview=text[:50], + ) + continue + + result = response.json() + embedding = result.get("embedding", []) + + if not embedding: + last_error = "empty_embedding" + logger.warning( + "ollama_embedding_empty", + endpoint=endpoint_url, + text_preview=text[:50], + ) + continue + + logger.info("ollama_embedding_success", endpoint=endpoint_url) + return normalize_vector(embedding) + except Exception as endpoint_error: + last_error = str(endpoint_error) + logger.warning( + "ollama_embedding_endpoint_error", + endpoint=endpoint_url, + error=last_error, + text_preview=text[:50], + ) + + logger.warning( + "ollama_embedding_error", + error=last_error or "all endpoints failed", + text_preview=text[:50], ) - - if response.status_code != 200: - logger.warning( - "ollama_embedding_failed", - status_code=response.status_code, - text_preview=text[:50], - ) - return None - - result = response.json() - embedding = result.get("embedding", []) - - if not embedding: - logger.warning("ollama_embedding_empty", text_preview=text[:50]) - return None - - return normalize_vector(embedding) + return None except Exception as e: logger.warning( diff --git a/apps/api/tests/test_agent_step_timeouts.py b/apps/api/tests/test_agent_step_timeouts.py index 84bddb79c..1ee1a49db 100644 --- a/apps/api/tests/test_agent_step_timeouts.py +++ b/apps/api/tests/test_agent_step_timeouts.py @@ -17,7 +17,6 @@ from __future__ import annotations import asyncio import importlib import sys -import time from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -72,6 +71,17 @@ class TestTimeoutDefaults: f"Critic default timeout 期望 15.0，實際 {mod.AGENT_CRITIC_TIMEOUT_SEC}" ) + def test_agent_debate_global_timeout_default_is_260(self, monkeypatch): + """Agent debate global timeout defaults to the GCP Ollama-first budget.""" + monkeypatch.delenv("AGENT_DEBATE_GLOBAL_TIMEOUT_SEC", raising=False) + + if "src.services.agent_orchestrator" in sys.modules: + del sys.modules["src.services.agent_orchestrator"] + import src.services.agent_orchestrator as mod + importlib.reload(mod) + + assert mod.GLOBAL_TIMEOUT_SEC == 260.0 + def test_deprecated_alias_matches_new_constant_diagnostician(self, monkeypatch): """PHASE2_STEP_TIMEOUT_SEC alias 應等於 AGENT_DIAGNOSTICIAN_TIMEOUT_SEC（相容性保證）""" monkeypatch.delenv("AGENT_DIAGNOSTICIAN_TIMEOUT_SEC", raising=False) @@ -172,6 +182,17 @@ class TestEnvOverride: assert mod.PHASE2_STEP_TIMEOUT_SEC == 8.0 assert mod.PHASE2_STEP_TIMEOUT_SEC == mod.AGENT_CRITIC_TIMEOUT_SEC + def test_agent_debate_global_timeout_env_override(self, monkeypatch): + """AGENT_DEBATE_GLOBAL_TIMEOUT_SEC=300 覆蓋 default 260.0""" + monkeypatch.setenv("AGENT_DEBATE_GLOBAL_TIMEOUT_SEC", "300") + + if "src.services.agent_orchestrator" in sys.modules: + del sys.modules["src.services.agent_orchestrator"] + import src.services.agent_orchestrator as mod + importlib.reload(mod) + + assert mod.GLOBAL_TIMEOUT_SEC == 300.0 + # ============================================================================= # Section 3: Metric Histogram observe 驗證 diff --git a/apps/api/tests/test_ollama_endpoint_resolver.py b/apps/api/tests/test_ollama_endpoint_resolver.py index 0f5d57d42..937c95a95 100644 --- a/apps/api/tests/test_ollama_endpoint_resolver.py +++ b/apps/api/tests/test_ollama_endpoint_resolver.py @@ -20,7 +20,7 @@ def _settings( ) -def test_heavy_workloads_prefer_local_lane() -> None: +def test_non_sensitive_workloads_prefer_gcp_a_lane() -> None: cfg = _settings() for workload in ( @@ -35,9 +35,9 @@ def test_heavy_workloads_prefer_local_lane() -> None: "hermes", ): selection = resolve_ollama_selection(workload, config=cfg) - assert selection.url == "http://192.168.0.110:11437" - assert selection.provider_name == "ollama_local" - assert selection.reason == "local_heavy_or_privacy_lane" + assert selection.url == "http://192.168.0.110:11435" + assert selection.provider_name == "ollama_gcp_a" + assert selection.reason == "primary_interactive_lane" def test_interactive_workloads_stay_on_gcp_a() -> None: @@ -58,10 +58,10 @@ def test_local_required_workloads_use_local_lane() -> None: assert selection.provider_name == "ollama_local" -def test_heavy_workloads_fall_back_to_gcp_b_when_local_missing() -> None: - cfg = _settings(fallback="") +def test_non_sensitive_workloads_fall_back_to_gcp_b_when_primary_missing() -> None: + cfg = _settings(primary="") selection = resolve_ollama_selection("embedding", config=cfg) assert selection.url == "http://192.168.0.110:11436" assert selection.provider_name == "ollama_gcp_b" - assert selection.reason == "local_missing_gcp_b_fallback" + assert selection.reason == "primary_missing_gcp_b_fallback" diff --git a/apps/api/tests/test_phase2_fallback.py b/apps/api/tests/test_phase2_fallback.py index ca4ad9b8e..f3f9775cb 100644 --- a/apps/api/tests/test_phase2_fallback.py +++ b/apps/api/tests/test_phase2_fallback.py @@ -1,7 +1,11 @@ from types import SimpleNamespace from src.agents.protocol import AgentSessionStatus -from src.services.decision_manager import _phase2_fallback_reason +from src.services import decision_manager as decision_manager_module +from src.services.decision_manager import ( + _incident_llm_timeout_seconds, + _phase2_fallback_reason, +) def _package(**kwargs): @@ -35,3 +39,17 @@ def test_phase2_actionable_package_stays_primary() -> None: pkg = _package() assert _phase2_fallback_reason(pkg) is None + + +def test_incident_llm_timeout_uses_configured_value(monkeypatch) -> None: + monkeypatch.setattr(decision_manager_module.settings, "INCIDENT_LLM_TIMEOUT_SECONDS", 240) + monkeypatch.setattr(decision_manager_module.settings, "OPENCLAW_TIMEOUT", 120) + + assert _incident_llm_timeout_seconds() == 240.0 + + +def test_incident_llm_timeout_never_below_openclaw_timeout(monkeypatch) -> None: + monkeypatch.setattr(decision_manager_module.settings, "INCIDENT_LLM_TIMEOUT_SECONDS", 60) + monkeypatch.setattr(decision_manager_module.settings, "OPENCLAW_TIMEOUT", 120) + + assert _incident_llm_timeout_seconds() == 120.0 diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index dc7d2d65f..78dc65e36 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -6,6 +6,21 @@ --- +## 2026-05-06 | Incident Ollama-first path stops timing out before GCP answers + +**背景**：production log 顯示告警 provider order 已是 `ollama_gcp_a -> ollama_gcp_b -> ollama_local -> gemini`，且 GCP-A 可用 `qwen3:14b` 成功回應（52s/75s），但 DecisionManager 仍用 25s 外層 timeout、Phase 2 debate 仍用 90s 全局 timeout，導致合法的 GCP Ollama 深度診斷被提前截斷；同時 RAG/embedding resolver 仍優先打目前不可達的 111，造成大量 `ollama_embedding_error`。 + +**本次修補**： +- 新增 `INCIDENT_LLM_TIMEOUT_SECONDS`，production 設為 240s；Incident LLM 外層 guard 不再硬編 25s，且不得低於 `OPENCLAW_TIMEOUT`。 +- 新增 `AGENT_DEBATE_GLOBAL_TIMEOUT_SEC`，production 設為 260s；Phase 2 debate 不再被 90s 固定值卡死。 +- `ollama_endpoint_resolver` 改為非敏感工作（embedding/RAG/deep_rca/Hermes/code_review 等）GCP-A 優先、GCP-B 備援、111 兜底；只有 `local_required` / `privacy_sensitive` / `dr` 維持 local-first。 +- `PlaybookRAGService.embed_text()` 改為依序嘗試配置的 Ollama endpoints，單一 endpoint 失敗不再直接放棄 RAG。 + +**驗證**： +- `py_compile` touched backend files OK；ruff `E9,F401,F821,F841` OK。 +- 相關測試：timeout/resolver 32 passed（1 個既有 coroutine warning）、OpenClaw Ollama route 13 passed、action/parser/learning guard 74 passed、Ollama failover/recovery 73 passed。 +- 現場確認 GCP-A/GCP-B 均可列出 `qwen3:14b`、`qwen2.5:7b-instruct`、`bge-m3`、`gemma3:4b`；111 `/api/tags` 目前 timeout，仍需後續修 111 連通性，但 Gemini 已回到 GCP-A/GCP-B/111 之後的最後備援角色。 + ## 2026-05-06 | Decision Telegram dedup no longer reads missing Incident.title **背景**：新 Ollama-first 部署後，production log 顯示 alert diagnosis 已走 `ollama_gcp_a -> ollama_gcp_b -> ollama_local -> gemini` 且 `phase24_ai_router_used` provider=`ollama`，但 DecisionManager 推送 Telegram decision card 時出現 `telegram_decision_push_failed: 'Incident' object has no attribute 'title'`。 diff --git a/k8s/awoooi-prod/04-configmap.yaml b/k8s/awoooi-prod/04-configmap.yaml index 1a90554ed..dd75197d5 100644 --- a/k8s/awoooi-prod/04-configmap.yaml +++ b/k8s/awoooi-prod/04-configmap.yaml @@ -60,6 +60,16 @@ data: # 回滾: kubectl set env deployment/awoooi-api -n awoooi-prod USE_AI_ROUTER=false # ============================================================================ USE_AI_ROUTER: "true" + ALERT_AI_ALLOW_CLOUD_FALLBACK: "true" + ALERT_AI_ENFORCE_OLLAMA_FIRST: "true" + ALERT_OLLAMA_MODEL: "qwen3:14b" + OLLAMA_HEALTH_CHECK_MODEL: "gemma3:4b" + OPENCLAW_DEFAULT_MODEL: "qwen2.5:7b-instruct" + OPENCLAW_TIMEOUT: "120" + INCIDENT_LLM_TIMEOUT_SECONDS: "240" + AGENT_DEBATE_GLOBAL_TIMEOUT_SEC: "260" + AGENT_DIAGNOSTICIAN_TIMEOUT_SEC: "100" + AGENT_SOLVER_TIMEOUT_SEC: "80" # ADR-105 P1: OpenClaw Agent Loop shadow canary. # 只給 read-only MCP tools，使用本地 Ollama，結果只附加 metadata 不改決策。 # 回滾: kubectl set env deployment/awoooi-api -n awoooi-prod ENABLE_OPENCLAW_AGENT_LOOP_SHADOW=false diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 378cf79d8..209f36954 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -85,6 +85,10 @@ spec: value: "qwen2.5:7b-instruct" - name: OPENCLAW_TIMEOUT value: "120" + - name: INCIDENT_LLM_TIMEOUT_SECONDS + value: "240" + - name: AGENT_DEBATE_GLOBAL_TIMEOUT_SEC + value: "260" - name: AGENT_DIAGNOSTICIAN_TIMEOUT_SEC value: "100" - name: AGENT_SOLVER_TIMEOUT_SEC