From 22344883780ae30667fcadce7ff87307d854fe87 Mon Sep 17 00:00:00 2001 From: OoO Date: Wed, 13 May 2026 21:41:43 +0800 Subject: [PATCH] fix: route special ollama fallbacks through cascade --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 3 +- .../claude_inventory_validation_20260513.md | 1 + routes/openclaw_bot_routes.py | 24 ++++---- services/mcp_collector_service.py | 24 ++++---- services/ollama_service.py | 18 ++++-- services/ppt_vision_service.py | 57 ++++--------------- tests/test_openclaw_bot_menu_keyboards.py | 1 + tests/test_ppt_vision_service.py | 17 +++--- 9 files changed, 63 insertions(+), 84 deletions(-) diff --git a/config.py b/config.py index 6935ecb..89470fd 100644 --- a/config.py +++ b/config.py @@ -320,7 +320,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.128" +SYSTEM_VERSION = "V10.129" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 4d34d0b..d728344 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-05-13 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯，Gemini 僅備援 / 鎖定場景 -> **適用版本**: V10.128 +> **適用版本**: V10.129 --- @@ -15,6 +15,7 @@ - 通用 AI 文案、關鍵字、商品洞察與 Telegram Q&A 第一響應不得 Gemini-first。 - Hermes intent / analyst 路徑不得手刻 `/api/generate` 或只 resolve 單次 host；必須走 `OllamaService`，讓同一請求可依序 retry GCP-A → GCP-B → 111。 - NemoTron qwen3 dispatch 的 `/api/chat` tool-calling 路徑也必須同一請求最多嘗試三台 Ollama，第一台失敗要 `mark_unhealthy()` 後再試下一台，最後才 fallback NIM。 +- PPT vision、PPT 文案 final fallback、MCP 離線 final fallback 等特殊 Ollama 路徑也不得只打單一 host；如需 `/api/generate`，一律透過 `OllamaService.generate()`。 - Code Review pipeline 也必須 Ollama-first：Hermes scan 與 OpenClaw assessment 都走 `OllamaService` 三主機 retry；Gemini telemetry 只能以 `code_review_openclaw_gemini` 出現，表示 Ollama/可選 Claude 備援都失敗後才啟用。 - OpenClaw Telegram Q&A 主路徑也不得綁單一 host：`_call_qwen3_qa()` 必須透過 `OllamaService` 跑 GCP-A → GCP-B → 111，並把實際落點寫入 `ai_calls.provider`。 diff --git a/docs/memory/claude_inventory_validation_20260513.md b/docs/memory/claude_inventory_validation_20260513.md index e15699c..3ccb267 100644 --- a/docs/memory/claude_inventory_validation_20260513.md +++ b/docs/memory/claude_inventory_validation_20260513.md @@ -53,6 +53,7 @@ - Ollama host env 已加白名單護欄：`OLLAMA_HOST*` / `EMBEDDING_HOST` 只接受 GCP-A、GCP-B、111 或 110 proxy，誤設 188/localhost 會回到核准主機。 - Hermes intent 與批量 analyst 已從單次 `resolve_ollama_host()` + raw `requests.post('/api/generate')` 改為 `OllamaService.generate()`，同一請求會依序 retry GCP-A → GCP-B → 111，並保留 `HERMES_KEEP_ALIVE` 與實際 provider 回寫測試。 - NemoTron qwen3 dispatch 的 `/api/chat` tool-calling 路徑已補同一請求三主機 retry：host 失敗會 `mark_unhealthy()` 後再 resolve 下一台，GCP-A 失敗可在同次 dispatch 接 GCP-B/111，三台都失敗才 fallback NIM。 +- PPT vision、PPT 文案 final fallback 與 MCP 離線 final fallback 已改走 `OllamaService.generate()`；`OllamaService.generate()` 支援 `options`、`keep_alive` 與 vision `images`，特殊 `/api/generate` 路徑同樣取得三主機 retry。 - OpenClaw QA / daily Hermes template / NemoTron qwen3 的 flag 文件與測試已對齊 Ollama-first 預設 ON；顯式 `false` 才是 Gemini/NIM legacy 緊急退路。OpenClaw QA 已移除單一 `OPENCLAW_QA_OLLAMA_HOST` 主機覆寫，`_call_qwen3_qa()` 改走 `OllamaService` 的 GCP-A → GCP-B → 111 retry 並回寫實際 provider。 - Code Review pipeline 已對齊 Ollama-first：`_hermes_scan()` 與 `_openclaw_assess()` 都先走 `OllamaService` 的 GCP-A → GCP-B → 111 retry；Gemini 僅在 Ollama（與可選 Claude）失敗後以 `code_review_openclaw_gemini` caller 記錄備援，不再以 `code_review_openclaw` 直接 Gemini-first。 - `.env.example` 已補齊 Python runtime 實際讀取的環境變數，`tests/test_phase3f_cleanup_contracts.py::test_env_example_documents_runtime_os_env_keys` 會掃 `app.py/config.py/scheduler.py/run_scheduler.py/routes/services/utils` 的 `os.getenv()` / `os.environ.get()`；只允許 `PYTEST_CURRENT_TEST` 與 `MOMO_ALLOW_INSECURE_CONFIG_FOR_TESTS` 兩個測試內部 key 不進範例。 diff --git a/routes/openclaw_bot_routes.py b/routes/openclaw_bot_routes.py index 6d5f900..8898eff 100644 --- a/routes/openclaw_bot_routes.py +++ b/routes/openclaw_bot_routes.py @@ -91,7 +91,7 @@ except ImportError: # V-New: 引入 Ollama 探測機制 try: - from services.ollama_service import resolve_ollama_host + from services.ollama_service import OllamaService _OLLAMA_AVAILABLE = True except ImportError: _OLLAMA_AVAILABLE = False @@ -2487,21 +2487,19 @@ def _ppt_ai_analysis(prompt_data: str, report_type: str = '') -> str: if not _OLLAMA_AVAILABLE: return "" try: - host = resolve_ollama_host() # 簡報分析使用 qwen2.5-coder:7b (已升級 GCP) 或 hermes3 model = os.getenv('OPENCLAW_OLLAMA_MODEL', 'qwen2.5-coder:7b') - r = requests.post( - f"{host}/api/generate", - json={ - 'model': model, - 'prompt': prompt, - 'stream': False, - 'options': {'num_predict': tokens, 'temperature': 0.3} - }, - timeout=90 + resp = OllamaService(model=model).generate( + prompt=prompt, + model=model, + temperature=0.3, + timeout=90, + options={'num_predict': tokens}, ) - r.raise_for_status() - return r.json().get('response', '').strip() + if not resp.success: + sys_log.warning(f"[PPT] Ollama cascade failed: {resp.error}") + return "" + return (resp.content or '').strip() except Exception as e: sys_log.warning(f"[PPT] Ollama error: {e}") return "" diff --git a/services/mcp_collector_service.py b/services/mcp_collector_service.py index 06183bb..066dba5 100644 --- a/services/mcp_collector_service.py +++ b/services/mcp_collector_service.py @@ -35,7 +35,7 @@ MCP_MODEL = os.getenv("MCP_GEMINI_MODEL", "gemini-2.0-flash") MCP_FALLBACK_MODEL = "gemini-1.5-flash" try: - from services.ollama_service import resolve_ollama_host + from services.ollama_service import OllamaService _OLLAMA_AVAILABLE = True except ImportError: _OLLAMA_AVAILABLE = False @@ -250,28 +250,26 @@ class MCPCollectorService: if _OLLAMA_AVAILABLE: try: logger.info("[MCP] Using GCP Ollama for market insight fallback") - host = resolve_ollama_host() + ollama_model = os.getenv('OPENCLAW_OLLAMA_MODEL', 'qwen2.5-coder:7b') ollama_prompt = ( f"你是一位精通台灣電商市場的分析師。目前無法取得即時搜尋結果，" f"請根據你的知識儲備，針對以下主題提供 2026 年可能的市場動態或洞察（繁體中文，300字以內）：\n" f"主題：{query}\n\n" "請註明：『(此為基於歷史趨勢的預測性洞察)』" ) - r = requests.post( - f"{host}/api/generate", - json={ - 'model': os.getenv('OPENCLAW_OLLAMA_MODEL', 'qwen2.5-coder:7b'), - 'prompt': ollama_prompt, - 'stream': False, - 'options': {'num_predict': 800, 'temperature': 0.4} - }, - timeout=45 + resp = OllamaService(model=ollama_model).generate( + prompt=ollama_prompt, + model=ollama_model, + temperature=0.4, + timeout=45, + options={'num_predict': 800}, ) - r.raise_for_status() - content = r.json().get('response', '').strip() + content = (resp.content or '').strip() if resp.success else '' if content: # 不進快取，因為這是預測性內容 return content + if not resp.success: + logger.warning("[MCP] Ollama cascade fallback failed: %s", resp.error) except Exception as e3: logger.warning("[MCP] Ollama fallback also failed: %s", e3) diff --git a/services/ollama_service.py b/services/ollama_service.py index 10ed9e2..ca2c28b 100644 --- a/services/ollama_service.py +++ b/services/ollama_service.py @@ -83,6 +83,13 @@ def mark_unhealthy(host: str) -> None: logger.warning(f"[OllamaHost] 主機標記為 unhealthy（30s 跳過）：{host}") +def _mark_unhealthy_best_effort(host: str) -> None: + try: + mark_unhealthy(host) + except Exception: + logger.debug("[OllamaHost] mark_unhealthy failed for host=%s", host, exc_info=True) + + def _is_unhealthy(host: str) -> bool: """檢查 host 是否在 unhealthy TTL 內""" import time @@ -287,7 +294,8 @@ class OllamaService: def generate(self, prompt: str, model: str = None, system_prompt: str = None, temperature: float = 0.7, timeout: int = None, keep_alive: str = None, - options: Optional[Dict[str, Any]] = None) -> OllamaResponse: + options: Optional[Dict[str, Any]] = None, + images: Optional[List[str]] = None) -> OllamaResponse: """ 生成文字 — 含三主機自動 retry（HOTFIX 2026-05-04） @@ -309,6 +317,8 @@ class OllamaService: payload["system"] = system_prompt if keep_alive: payload["keep_alive"] = keep_alive + if images: + payload["images"] = images # HOTFIX 三主機 retry 鏈 attempted_hosts: List[str] = [] @@ -344,15 +354,15 @@ class OllamaService: # HTTP 非 200：標 unhealthy + 嘗試下一台 last_error = f"HTTP {response.status_code}: {response.text[:200]}" logger.warning(f"[Ollama] {current_host} HTTP 失敗 → mark_unhealthy + retry: {last_error}") - mark_unhealthy(current_host) + _mark_unhealthy_best_effort(current_host) except requests.Timeout: last_error = f"timeout ({request_timeout}s)" logger.warning(f"[Ollama] {current_host} timeout → mark_unhealthy + retry") - mark_unhealthy(current_host) + _mark_unhealthy_best_effort(current_host) except Exception as e: last_error = f"{type(e).__name__}: {str(e)[:200]}" logger.error(f"[Ollama] {current_host} error → mark_unhealthy + retry: {last_error}") - mark_unhealthy(current_host) + _mark_unhealthy_best_effort(current_host) # 三台都失敗 return OllamaResponse( diff --git a/services/ppt_vision_service.py b/services/ppt_vision_service.py index a713fcc..e22f038 100644 --- a/services/ppt_vision_service.py +++ b/services/ppt_vision_service.py @@ -24,8 +24,6 @@ import logging from dataclasses import dataclass, field from typing import Optional, Dict, Any, List -import requests - logger = logging.getLogger(__name__) # ───────────────────────────────────────────────────────────────────────────── @@ -40,13 +38,6 @@ def is_ppt_vision_enabled() -> bool: return os.getenv('PPT_VISION_ENABLED', 'false').strip().lower() in ('true', '1', 'yes', 'on') -def _mark_unhealthy_best_effort(mark_unhealthy, host: str) -> None: - try: - mark_unhealthy(host) - except Exception: - logger.debug("[PPTVision] mark_unhealthy failed for host=%s", host, exc_info=True) - - # ───────────────────────────────────────────────────────────────────────────── # 結果容器 # ───────────────────────────────────────────────────────────────────────────── @@ -289,44 +280,28 @@ class PPTVisionService: error=f'read image failed: {type(e).__name__}: {str(e)[:200]}', ) - # 透過 resolve_ollama_host 取主機（享受三主機 retry 鏈） try: - from services.ollama_service import resolve_ollama_host, mark_unhealthy - host = resolve_ollama_host() - except Exception as e: - return VisionResult( - success=False, - error=f'resolve host failed: {e}', - ) + from services.ollama_service import OllamaService - # Ollama /api/generate 支援 images 欄位（base64 list） - payload = { - 'model': self.model, - 'system': PPT_VISION_SYSTEM_PROMPT, - 'prompt': '請檢查這張 momo 電商 PPT 截圖，找出視覺異常。', - 'images': [img_b64], - 'stream': False, - 'options': {'temperature': 0.2, 'num_predict': 512}, - } - - try: - resp = requests.post( - f"{host.rstrip('/')}/api/generate", - json=payload, + ollama = OllamaService(model=self.model) + resp = ollama.generate( + prompt='請檢查這張 momo 電商 PPT 截圖，找出視覺異常。', + model=self.model, + system_prompt=PPT_VISION_SYSTEM_PROMPT, + temperature=0.2, timeout=PPT_VISION_TIMEOUT, + options={'num_predict': 512}, + images=[img_b64], ) duration_ms = int((time.monotonic() - start) * 1000) - if resp.status_code != 200: - # mark_unhealthy 讓下次自動切其他主機 - _mark_unhealthy_best_effort(mark_unhealthy, host) + if not resp.success: return VisionResult( success=False, duration_ms=duration_ms, - error=f'HTTP {resp.status_code}: {resp.text[:200]}', + error=resp.error or 'ollama vision failed', ) - data = resp.json() - raw = (data.get('response') or '').strip() + raw = (resp.content or '').strip() # 解析輸出：每行一個 ⚠️ 開頭的視為 issue；✅ 無視覺異常則空 list issues = [] @@ -350,15 +325,7 @@ class PPTVisionService: duration_ms=duration_ms, ) - except requests.Timeout: - _mark_unhealthy_best_effort(mark_unhealthy, host) - duration_ms = int((time.monotonic() - start) * 1000) - return VisionResult( - success=False, duration_ms=duration_ms, - error=f'timeout ({PPT_VISION_TIMEOUT}s)', - ) except Exception as e: - _mark_unhealthy_best_effort(mark_unhealthy, host) duration_ms = int((time.monotonic() - start) * 1000) return VisionResult( success=False, duration_ms=duration_ms, diff --git a/tests/test_openclaw_bot_menu_keyboards.py b/tests/test_openclaw_bot_menu_keyboards.py index c180065..d118ccb 100644 --- a/tests/test_openclaw_bot_menu_keyboards.py +++ b/tests/test_openclaw_bot_menu_keyboards.py @@ -134,6 +134,7 @@ def test_mcp_collector_has_stable_fallbacks(): assert "def _fallback_topic_content" in source assert "def _looks_unreliable" in source assert '["google_search"]' in source + assert "OllamaService(model=ollama_model).generate" in source assert "return self._fallback_topic_content" in source diff --git a/tests/test_ppt_vision_service.py b/tests/test_ppt_vision_service.py index 92447ad..e755e5e 100644 --- a/tests/test_ppt_vision_service.py +++ b/tests/test_ppt_vision_service.py @@ -36,7 +36,7 @@ def test_flag_off_returns_disabled_error(fake_image): from services.ppt_vision_service import PPTVisionService svc = PPTVisionService() - with patch('services.ppt_vision_service.requests.post') as mock_post: + with patch('services.ollama_service.requests.post') as mock_post: result = svc.check_image(fake_image) assert result.success is False @@ -65,13 +65,16 @@ def test_no_issues_response(fake_image, monkeypatch): with patch('services.ollama_service.resolve_ollama_host', return_value='http://test:11434'), \ - patch('services.ppt_vision_service.requests.post', return_value=fake_resp): + patch('services.ollama_service.requests.post', return_value=fake_resp) as mock_post: svc = PPTVisionService() result = svc.check_image(fake_image) assert result.success is True assert result.issues_found == [] assert result.confidence == 1.0 + payload = mock_post.call_args.kwargs['json'] + assert payload['model'] == 'minicpm-v:latest' + assert payload['images'] and isinstance(payload['images'][0], str) def test_issues_detected(fake_image, monkeypatch): @@ -88,7 +91,7 @@ def test_issues_detected(fake_image, monkeypatch): with patch('services.ollama_service.resolve_ollama_host', return_value='http://test:11434'), \ - patch('services.ppt_vision_service.requests.post', return_value=fake_resp): + patch('services.ollama_service.requests.post', return_value=fake_resp): svc = PPTVisionService() result = svc.check_image(fake_image) @@ -110,7 +113,7 @@ def test_http_500_marks_unhealthy(fake_image, monkeypatch): with patch('services.ollama_service.resolve_ollama_host', return_value='http://test:11434'), \ patch('services.ollama_service.mark_unhealthy') as mock_mark, \ - patch('services.ppt_vision_service.requests.post', return_value=fake_resp): + patch('services.ollama_service.requests.post', return_value=fake_resp): svc = PPTVisionService() result = svc.check_image(fake_image) @@ -129,11 +132,11 @@ def test_http_500_preserves_error_when_mark_unhealthy_fails(fake_image, monkeypa def broken_mark(_host): raise RuntimeError('mark failed') - caplog.set_level(logging.DEBUG, logger="services.ppt_vision_service") + caplog.set_level(logging.DEBUG, logger="services.ollama_service") with patch('services.ollama_service.resolve_ollama_host', return_value='http://test:11434'), \ patch('services.ollama_service.mark_unhealthy', side_effect=broken_mark), \ - patch('services.ppt_vision_service.requests.post', return_value=fake_resp): + patch('services.ollama_service.requests.post', return_value=fake_resp): svc = PPTVisionService() result = svc.check_image(fake_image) @@ -150,7 +153,7 @@ def test_timeout_returns_failure(fake_image, monkeypatch): with patch('services.ollama_service.resolve_ollama_host', return_value='http://test:11434'), \ patch('services.ollama_service.mark_unhealthy'), \ - patch('services.ppt_vision_service.requests.post', + patch('services.ollama_service.requests.post', side_effect=requests.Timeout('60s')): svc = PPTVisionService() result = svc.check_image(fake_image)