diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py index 26b97a564..77f54aea7 100644 --- a/apps/api/src/core/config.py +++ b/apps/api/src/core/config.py @@ -522,11 +522,11 @@ class Settings(BaseSettings): ), ) ALERT_OLLAMA_MODEL: str = Field( - default="gemma3:4b", + default="qwen3:14b", description=( - "Ollama model used for incident/alert diagnosis. Keep this separate " - "from the general RCA model so alert cards stay on the fast local " - "lane before Gemini backup is considered." + "Ollama model used for incident/alert deep diagnosis. Alert cards " + "may wait for this model; Gemini remains a backup after GCP-A, " + "GCP-B, and 111 fail." ), ) # 2026-03-29 ogt: ADR-036 Nemotron Tool Calling 整合 diff --git a/apps/api/src/services/ai_providers/ollama.py b/apps/api/src/services/ai_providers/ollama.py index 1bd45fc7d..19f8d8880 100644 --- a/apps/api/src/services/ai_providers/ollama.py +++ b/apps/api/src/services/ai_providers/ollama.py @@ -29,7 +29,7 @@ from src.services.model_registry import get_model_registry logger = structlog.get_logger(__name__) settings = get_settings() -_GCP_SAFE_MODELS = { +_GCP_LIGHTWEIGHT_MODELS = { "gemma3:4b", } @@ -54,26 +54,34 @@ def _resolve_model_for_endpoint( context: dict | None, ) -> str: """ - Keep GCP-A/B on the fast alert model unless explicitly allowed. + Keep non-diagnosis calls from polluting the GCP diagnosis lane. - The GCP hosts currently expose CPU-only Ollama. Loading 7B/14B/32B models on - that lane blocks synchronous alerts long enough to fall through to Gemini. - Heavy/deep workloads must use 111 or the future AwoooP Inference Gateway. + GCP-A/B are allowed to run the deep incident diagnosis model because the + alert goal is correctness and resolution, not the fastest Telegram card. + Accidental non-diagnosis workloads still fall back to the lightweight health + model so embedding/Hermes/background calls cannot occupy the same lane. """ model_name = requested_model.strip() context = context or {} allow_gcp_heavy = bool(context.get("allow_gcp_heavy_model")) + task_type = str(context.get("task_type") or context.get("intent_hint") or "").lower() + is_deep_diagnosis = task_type in {"diagnose", "alert_deep", "incident_diagnosis"} - if _is_gcp_alert_lane(endpoint_url) and not allow_gcp_heavy and model_name not in _GCP_SAFE_MODELS: - alert_model = str(getattr(settings, "ALERT_OLLAMA_MODEL", "gemma3:4b")).strip() or "gemma3:4b" + if ( + _is_gcp_alert_lane(endpoint_url) + and not allow_gcp_heavy + and not is_deep_diagnosis + and model_name not in _GCP_LIGHTWEIGHT_MODELS + ): + fallback_model = str(getattr(settings, "OLLAMA_HEALTH_CHECK_MODEL", "gemma3:4b")).strip() or "gemma3:4b" logger.warning( - "ollama_gcp_heavy_model_coerced", + "ollama_gcp_non_diagnosis_model_coerced", endpoint=endpoint_url, requested_model=model_name, - safe_model=alert_model, - task_type=context.get("task_type"), + safe_model=fallback_model, + task_type=task_type, ) - return alert_model + return fallback_model return model_name diff --git a/apps/api/src/services/ai_router.py b/apps/api/src/services/ai_router.py index 7344edf50..d8c4935d9 100644 --- a/apps/api/src/services/ai_router.py +++ b/apps/api/src/services/ai_router.py @@ -1142,6 +1142,10 @@ class AIRouterExecutor: _lf_trace_ctx = None errors: list[str] = [] + attempted_providers: set[str] = set() + alert_requires_ollama_before_cloud = bool( + (context or {}).get("alert_requires_ollama_before_cloud") + ) # 2026-04-27 Claude Sonnet 4.6: A2 INC-20260425 — DIAGNOSE fallback metric 追蹤 # 透過 context.get("intent_hint") 判斷是否為 DIAGNOSE，避免改動 execute() 簽名 @@ -1191,13 +1195,31 @@ class AIRouterExecutor: errors.append(f"{provider_name}: privacy_skip(non_local)") continue + if alert_requires_ollama_before_cloud and provider.privacy_level == "cloud": + if "ollama_local" not in attempted_providers: + errors.append(f"{provider_name}: blocked_until_ollama_local_attempted") + logger.warning( + "ai_router_cloud_blocked_until_ollama_local_attempted", + provider=provider_name, + provider_order=provider_order, + attempted_providers=sorted(attempted_providers), + ) + continue + # 閘門 1: Circuit Breaker (per-provider, C2 修復) cb = self._get_circuit_breaker(provider_name) if cb.is_open(): - errors.append(f"{provider_name}: circuit_open") - logger.warning("ai_router_circuit_open", provider=provider_name) - # 2026-04-27 Claude Sonnet 4.6: F6 — circuit_open 不設 _last_attempted_provider（未嘗試） - continue + if alert_requires_ollama_before_cloud and provider_name.startswith("ollama"): + logger.warning( + "ai_router_alert_ollama_circuit_bypassed", + provider=provider_name, + reason="alert_requires_ollama_before_cloud", + ) + else: + errors.append(f"{provider_name}: circuit_open") + logger.warning("ai_router_circuit_open", provider=provider_name) + # 2026-04-27 Claude Sonnet 4.6: F6 — circuit_open 不設 _last_attempted_provider（未嘗試） + continue # 閘門 2: Rate Limiter # 2026-04-02 Claude Code: Phase 24 B3 + C1 修復 — Rate Limiter (含 openclaw_nemo) @@ -1217,6 +1239,7 @@ class AIRouterExecutor: sem = self._get_semaphore(provider_name) async with sem: try: + attempted_providers.add(provider_name) result = await provider.analyze(prompt, context) if result.success: diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index 60292e381..e0d5b6963 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -535,7 +535,7 @@ class OpenClawService: registry = get_model_registry() model_name = registry.get_model("ollama", "rca") if ollama_only: - model_name = getattr(settings, "ALERT_OLLAMA_MODEL", "gemma3:4b") + model_name = getattr(settings, "ALERT_OLLAMA_MODEL", "qwen3:14b") options = registry.get_provider_options("ollama") timeout_seconds = max( float(settings.OPENCLAW_TIMEOUT), @@ -1145,7 +1145,9 @@ class OpenClawService: if decision.intent == IntentType.DIAGNOSE: exec_context["task_type"] = "diagnose" if self._is_incident_alert_context(alert_context): - exec_context["ollama_model"] = getattr(settings, "ALERT_OLLAMA_MODEL", "gemma3:4b") + exec_context["ollama_model"] = getattr(settings, "ALERT_OLLAMA_MODEL", "qwen3:14b") + exec_context["allow_gcp_heavy_model"] = True + exec_context["alert_requires_ollama_before_cloud"] = True result = await executor.execute( prompt=prompt, diff --git a/apps/api/tests/test_ai_router_cache_provider_policy.py b/apps/api/tests/test_ai_router_cache_provider_policy.py index 2d799bf75..5309850f1 100644 --- a/apps/api/tests/test_ai_router_cache_provider_policy.py +++ b/apps/api/tests/test_ai_router_cache_provider_policy.py @@ -43,6 +43,34 @@ class _FakeProvider: ) +class _FailingLocalProvider: + privacy_level = "local" + is_enabled = True + capabilities = {"rca", "chat"} + + def __init__(self, name: str) -> None: + self.name = name + self.calls = 0 + + async def analyze(self, prompt: str, context: dict[str, Any] | None = None) -> AIResult: + self.calls += 1 + return AIResult(raw_response="", success=False, provider=self.name, error="forced failure") + + +class _CloudProvider: + name = "gemini" + privacy_level = "cloud" + is_enabled = True + capabilities = {"rca", "chat"} + + def __init__(self) -> None: + self.calls = 0 + + async def analyze(self, prompt: str, context: dict[str, Any] | None = None) -> AIResult: + self.calls += 1 + return AIResult(raw_response='{"provider":"gemini"}', success=True, provider=self.name) + + @pytest.mark.asyncio async def test_executor_skips_cached_cloud_provider_when_ollama_lane_is_required( monkeypatch: pytest.MonkeyPatch, @@ -88,3 +116,65 @@ async def test_executor_allows_cached_ollama_provider_for_ollama_lane( assert result.provider == "ollama" assert result.from_cache is True assert fake_provider.calls == 0 + + +@pytest.mark.asyncio +async def test_alert_cloud_backup_waits_for_real_ollama_local_attempt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_redis = _FakeRedis(cached_provider="none") + gcp_a = _FailingLocalProvider("ollama_gcp_a") + gemini = _CloudProvider() + registry = AIProviderRegistry() + registry.register(gcp_a) + registry.register(gemini) + + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: fake_redis) + + result = await AIRouterExecutor(registry).execute( + prompt="diagnose alert", + provider_order=["ollama_gcp_a", "gemini"], + context={ + "intent_hint": "diagnose", + "alert_type": "HostHighCpuLoad", + "alert_requires_ollama_before_cloud": True, + }, + ) + + assert result.success is False + assert gcp_a.calls == 1 + assert gemini.calls == 0 + + +@pytest.mark.asyncio +async def test_alert_cloud_backup_allowed_after_ollama_local_attempt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake_redis = _FakeRedis(cached_provider="none") + gcp_a = _FailingLocalProvider("ollama_gcp_a") + local = _FailingLocalProvider("ollama_local") + gemini = _CloudProvider() + registry = AIProviderRegistry() + registry.register(gcp_a) + registry.register(local) + registry.register(gemini) + + monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False) + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: fake_redis) + + result = await AIRouterExecutor(registry).execute( + prompt="diagnose alert", + provider_order=["ollama_gcp_a", "ollama_local", "gemini"], + context={ + "intent_hint": "diagnose", + "alert_type": "HostHighCpuLoad", + "alert_requires_ollama_before_cloud": True, + }, + ) + + assert result.success is True + assert result.provider == "gemini" + assert gcp_a.calls == 1 + assert local.calls == 1 + assert gemini.calls == 1 diff --git a/apps/api/tests/test_ollama_provider_endpoints.py b/apps/api/tests/test_ollama_provider_endpoints.py index dbe4de829..8527500be 100644 --- a/apps/api/tests/test_ollama_provider_endpoints.py +++ b/apps/api/tests/test_ollama_provider_endpoints.py @@ -55,7 +55,7 @@ async def test_ollama_gcp_b_analyze_uses_secondary_url(monkeypatch: pytest.Monke "OLLAMA_SECONDARY_URL", "http://secondary:11436", ) - monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "gemma3:4b") + monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "qwen3:14b") client = _FakeClient() provider = OllamaGcpBProvider() @@ -70,17 +70,17 @@ async def test_ollama_gcp_b_analyze_uses_secondary_url(monkeypatch: pytest.Monke assert result.success is True assert result.provider == "ollama_gcp_b" assert client.posted_urls == ["http://secondary:11436/api/generate"] - assert client.posted_payloads[0]["model"] == "gemma3:4b" + assert client.posted_payloads[0]["model"] == "qwen3:14b" @pytest.mark.asyncio -async def test_ollama_gcp_a_coerces_heavy_diagnose_model_to_alert_model( +async def test_ollama_gcp_a_allows_heavy_diagnose_model( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(ollama_module, "get_model_registry", lambda: _FakeRegistry()) monkeypatch.setattr(ollama_module.settings, "OLLAMA_URL", "http://primary:11435") monkeypatch.setattr(ollama_module.settings, "OLLAMA_SECONDARY_URL", "http://secondary:11436") - monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "gemma3:4b") + monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "qwen3:14b") client = _FakeClient() provider = OllamaProvider() @@ -92,6 +92,30 @@ async def test_ollama_gcp_a_coerces_heavy_diagnose_model_to_alert_model( result = await provider.analyze("diagnose", context={"task_type": "diagnose"}) + assert result.success is True + assert client.posted_urls == ["http://primary:11435/api/generate"] + assert client.posted_payloads[0]["model"] == "qwen3:14b" + + +@pytest.mark.asyncio +async def test_ollama_gcp_a_coerces_non_diagnosis_heavy_model_to_health_model( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(ollama_module, "get_model_registry", lambda: _FakeRegistry()) + monkeypatch.setattr(ollama_module.settings, "OLLAMA_URL", "http://primary:11435") + monkeypatch.setattr(ollama_module.settings, "OLLAMA_SECONDARY_URL", "http://secondary:11436") + monkeypatch.setattr(ollama_module.settings, "OLLAMA_HEALTH_CHECK_MODEL", "gemma3:4b") + + client = _FakeClient() + provider = OllamaProvider() + + async def _get_client() -> _FakeClient: + return client + + monkeypatch.setattr(provider, "_get_client", _get_client) + + result = await provider.analyze("background summary", context={"task_type": "summary"}) + assert result.success is True assert client.posted_urls == ["http://primary:11435/api/generate"] assert client.posted_payloads[0]["model"] == "gemma3:4b" @@ -104,7 +128,7 @@ async def test_ollama_gcp_a_can_explicitly_allow_heavy_model( monkeypatch.setattr(ollama_module, "get_model_registry", lambda: _FakeRegistry()) monkeypatch.setattr(ollama_module.settings, "OLLAMA_URL", "http://primary:11435") monkeypatch.setattr(ollama_module.settings, "OLLAMA_SECONDARY_URL", "http://secondary:11436") - monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "gemma3:4b") + monkeypatch.setattr(ollama_module.settings, "ALERT_OLLAMA_MODEL", "qwen3:14b") client = _FakeClient() provider = OllamaProvider() diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index d85ecbef8..3f3424001 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -6,6 +6,16 @@ --- +## 2026-05-05 | Alert diagnosis prioritizes resolution over speed + +**背景**：統帥明確修正策略：告警不是為了快速發卡片，而是為了把問題想清楚並完成 AI 自動化解決；GCP-A/GCP-B 有 SSD，可承擔深度診斷等待時間，Gemini 只能作 GCP-A → GCP-B → 111 全失敗後的備援。 + +**本次修補**： +- `ALERT_OLLAMA_MODEL` 從 `gemma3:4b` 改回 `qwen3:14b`，告警診斷允許等待專業模型。 +- incident/alert context 會帶 `allow_gcp_heavy_model=true`，避免 GCP-A/B 的深度診斷被誤降級到健康檢查模型。 +- 新增 `alert_requires_ollama_before_cloud` 硬閘門：進 Gemini 前必須實際嘗試過 `ollama_local`（111）；告警 Ollama chain 不因 circuit breaker 直接跳過。 +- 非診斷背景任務仍會被攔到 `OLLAMA_HEALTH_CHECK_MODEL`，避免 Hermes/embedding/background 任務污染 GCP 診斷 lane。 + ## 2026-05-05 | GCP Ollama alert lane model isolation fix **背景**：Telegram 告警卡片仍看到 Gemini / GCP-A 逾時；live log 顯示 Phase24 AI Router 的 `diagnose` 路徑已選到 `ollama_gcp_a`，但模型仍使用 `qwen3:14b`，導致 CPU-only GCP-A 載入重模型後 `gemma3:4b` 健康檢查也 timeout。 diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index e5aa26e7f..378cf79d8 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -78,7 +78,7 @@ spec: - name: ALERT_AI_ENFORCE_OLLAMA_FIRST value: "true" # 告警診斷強制先走 GCP-A → GCP-B → 111 - name: ALERT_OLLAMA_MODEL - value: "gemma3:4b" # 2026-05-05 Codex: qwen3:14b 告警 JSON prompt 會拖到 504 + value: "qwen3:14b" # 2026-05-05 Codex: 告警以解決問題為目標，可等待深度診斷 - name: OLLAMA_HEALTH_CHECK_MODEL value: "gemma3:4b" # 2026-05-05 Codex: 避免 health probe 載入 qwen2.5 7B 污染 GCP alert lane - name: OPENCLAW_DEFAULT_MODEL