From d6ae216c8c2dd33a1012072867574e6024555585 Mon Sep 17 00:00:00 2001 From: OoO Date: Thu, 21 May 2026 12:23:14 +0800 Subject: [PATCH] =?UTF-8?q?=E9=99=90=E5=88=B6=20111=20Ollama=20fallback=20?= =?UTF-8?q?=E8=B3=87=E6=BA=90=E5=8D=A0=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 5 ++ config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 11 +++-- docs/memory/history_logs.md | 1 + services/ollama_service.py | 77 ++++++++++++++++++++++++++---- tests/test_ollama_resolve.py | 43 +++++++++++++++++ 6 files changed, 123 insertions(+), 16 deletions(-) diff --git a/.env.example b/.env.example index 2f325fb..024d556 100644 --- a/.env.example +++ b/.env.example @@ -360,6 +360,11 @@ OLLAMA_MODEL=gemma3:4b OLLAMA_TIMEOUT=120 OLLAMA_COPY_TIMEOUT=180 OLLAMA_EMBED_TIMEOUT=45 +# 111 是 Mac final fallback，不承接 14B+ 重模型長駐；落到 111 時自動降級與縮短常駐。 +OLLAMA_111_MODEL_FALLBACK=qwen2.5:7b-instruct +OLLAMA_111_MODEL_DOWNGRADE_PATTERNS=qwen3:14b,deepseek-r1:14b,*:32b,*:70b +OLLAMA_111_KEEP_ALIVE=5m +OLLAMA_111_MAX_TIMEOUT=45 # [預設 true] OpenClaw Q&A 先走 Ollama，品質不足或失敗時才 fallback Gemini/NIM # 主機不提供單 caller override；一律走 OLLAMA_HOST_PRIMARY → OLLAMA_HOST_SECONDARY → OLLAMA_HOST_FALLBACK diff --git a/config.py b/config.py index 87482a4..0c47130 100644 --- a/config.py +++ b/config.py @@ -323,7 +323,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.360" +SYSTEM_VERSION = "V10.361" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 8924c61..3495190 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -1,8 +1,8 @@ # MOMO PRO — AI 競價情報模組 Single Source of Truth -> **最後更新**: 2026-05-20 (台北時間) +> **最後更新**: 2026-05-21 (台北時間) > **狀態**: 🟢 四 AI Agent 自動化閉環已落地；LLM 路由紅線升級為 Ollama-first 三主機級聯，Gemini 備援預設關閉 -> **適用版本**: V10.350 +> **適用版本**: V10.361 --- @@ -25,6 +25,7 @@ - OpenClaw 週報、月報、Meta analysis、日報洞察、Telegram PPT 分析與 MCP fallback 也必須 Ollama-first；Gemini caller 只能帶 `_gemini_fallback` 或明確 fallback caller 語意，且不得先於 Ollama/NIM 被呼叫。 - OpenClaw 週報、月報、Meta analysis、日報洞察與每日報告的 Gemini/NIM 備援 caller 必須登錄在 caller registry、AI 觀測台 agent group 與 Telegram 狀態統計，避免 fallback 用量被歸類為未知或漏算。 - Gemini API 出站有第二道 kill switch：`GEMINI_FALLBACK_ENABLED` 預設為 `false`。即使 `GEMINI_API_KEY` 存在，通用 AI fallback、OpenClaw 報告/QA/PPT/圖片、MCP Grounding 與 Code Review L3 都不得呼叫 Gemini；只有操作員明確設為 `true` 時，Gemini 才能作緊急備援。 +- 111 `192.168.0.111` 只是最後一道 Mac fallback，不承接 14B+ 重模型長駐；`OllamaService.generate()` 落到 111 時會將 `qwen3:14b` / `deepseek-r1:14b` / 32B+ / 70B+ 依 `OLLAMA_111_MODEL_DOWNGRADE_PATTERNS` 降級到 `OLLAMA_111_MODEL_FALLBACK`，並以 `OLLAMA_111_KEEP_ALIVE=5m`、`OLLAMA_111_MAX_TIMEOUT=45` 封頂，避免 16GB RAM 主機被 14B 模型與 24h keep-alive 壓到 swap。 ## 一、四 AI Agent 路由架構 @@ -36,7 +37,7 @@ SQL漏斗(~300筆) 任務: 競價威脅分類 → TOP 20 HIGH/MED/LOW ↓ [NemoTron / qwen3] — 派發器 - 主路徑: qwen3:14b @ Ollama 三主機級聯 + 主路徑: qwen3:14b @ GCP-A/GCP-B；落到 111 時自動降級 7B 備援: NVIDIA NIM meta/llama-3.1-8b-instruct 任務: Tool Calling → Telegram 告警 / DB 寫入 ↓ @@ -64,8 +65,8 @@ SQL漏斗(~300筆) | 角色 | 模型 | 主機 | 成本 | 每日限額 | |------|------|------|------|---------| | Hermes 分析師 | hermes3:latest / bge-m3 | GCP-A → GCP-B → 111 Ollama | 零 | 無限 | -| NemoTron 派發器 | qwen3:14b；NIM fallback | GCP-A → GCP-B → 111；NVIDIA NIM 備援 | Ollama 零；NIM 配額內免費 | NIM 80 | -| OpenClaw 策略師 | qwen3:14b / Gemini 鎖定場景 | Ollama-first；Gemini 備援 | Ollama 零；Gemini 需控管 | — | +| NemoTron 派發器 | qwen3:14b；111 fallback 降級 7B；NIM fallback | GCP-A → GCP-B → 111；NVIDIA NIM 備援 | Ollama 零；NIM 配額內免費 | NIM 80 | +| OpenClaw 策略師 | qwen3:14b；111 fallback 降級 7B；Gemini 鎖定場景 | Ollama-first；Gemini 備援 | Ollama 零；Gemini 需控管 | — | | ElephantAlpha 編排者 | ElephantAlpha | 依部署環境 | 受控 | HITL / 任務制 | --- diff --git a/docs/memory/history_logs.md b/docs/memory/history_logs.md index 0f2aa13..564b718 100644 --- a/docs/memory/history_logs.md +++ b/docs/memory/history_logs.md @@ -13,6 +13,7 @@ ## 📅 詳細更新日誌 (考古存檔) ### 2026-05-21：瀏覽器測試守門與 PChome 熱路徑優化 +- **V10.361 111 fallback resource guard**: 實測 111 Mac 高 load 主要來自 Codex app / WindowServer 前台負載，且 Ollama 曾因 fallback 載入 `qwen3:14b` 造成 16GB RAM / swap 壓力；已手動 unload 111 上的重模型，並讓 `OllamaService.generate()` 落到 111 時自動把 14B+ 模型降到 `OLLAMA_111_MODEL_FALLBACK`、`keep_alive` 縮至 `OLLAMA_111_KEEP_ALIVE=5m`、timeout 封頂 `OLLAMA_111_MAX_TIMEOUT=45`。GCP-A/GCP-B 仍可跑 `qwen3:14b`，111 只做短時最後備援。 - **V10.360 browser smoke guard**: `tests/test_image_fetch.py` 改為預設 skip，只有 `RUN_MOMO_BROWSER_TESTS=1` 才會打開外部 MOMO 網站；手動執行時預設 headless，並關閉 Chrome password manager/autofill，避免一般 pytest 觸發瀏覽器與密碼允許提示。 - **Scheduler Selenium 防彈窗**: `managed_scraper_resources()` 補 `credentials_enable_service=false`、`profile.password_manager_enabled=false` 與 Autofill/PasswordManager feature disable，降低背景 Selenium 觸發密碼管理提示的機率。 - **PChome dashboard hot path**: `competitor_intel_repository.py` 的 coverage / review queue 最新 MOMO 價格查詢改用 `JOIN LATERAL ... ORDER BY pr.timestamp DESC, pr.id DESC LIMIT 1`，避免 window function 掃描造成首頁與覆核隊列熱路徑變慢；Dashing Diva 召回搜尋補品線與 `magicpress` broad terms。 diff --git a/services/ollama_service.py b/services/ollama_service.py index 302e166..e1a6843 100644 --- a/services/ollama_service.py +++ b/services/ollama_service.py @@ -9,6 +9,7 @@ import os import requests import json import logging +import fnmatch from typing import Optional, Dict, Any, List, Tuple from dataclasses import dataclass @@ -55,6 +56,17 @@ DEFAULT_MODEL = os.getenv('OLLAMA_MODEL', 'llama3.1:8b') # 較快速的模型 TIMEOUT = int(os.getenv('OLLAMA_TIMEOUT', '120')) # 秒 - 2 分鐘 COPY_TIMEOUT = int(os.getenv('OLLAMA_COPY_TIMEOUT', '180')) # 文案生成專用超時 - 3 分鐘 EMBED_TIMEOUT = int(os.getenv('OLLAMA_EMBED_TIMEOUT', os.getenv('EMBEDDING_TIMEOUT', '45'))) +FALLBACK_111_KEEP_ALIVE = os.getenv('OLLAMA_111_KEEP_ALIVE', '5m') +FALLBACK_111_MAX_TIMEOUT = int(os.getenv('OLLAMA_111_MAX_TIMEOUT', '45')) +FALLBACK_111_MODEL = os.getenv('OLLAMA_111_MODEL_FALLBACK', 'qwen2.5:7b-instruct') +FALLBACK_111_MODEL_PATTERNS = tuple( + pattern.strip().lower() + for pattern in os.getenv( + 'OLLAMA_111_MODEL_DOWNGRADE_PATTERNS', + 'qwen3:14b,deepseek-r1:14b,*:32b,*:70b', + ).split(',') + if pattern.strip() +) # ── GCP 優先 / 111 備援：解析實際可用的 Ollama 主機 ────────────────────────── # ADR-027 Phase 2 強化： @@ -94,6 +106,36 @@ def _normalize_host(host: str) -> str: return (host or '').rstrip('/') +def _is_111_fallback_host(host: str) -> bool: + return '192.168.0.111:11434' in (host or '') + + +def _effective_model_for_host(model: str, host: str) -> str: + """ + 111 是 Mac/HDD final fallback，不承接 14B+ 等重模型。 + GCP-A/GCP-B 仍照 caller 指定模型；只有落到 111 才降級，避免 16GB RAM + 被 qwen3:14b / deepseek-r1:14b 長時間壓到 swap。 + """ + if not _is_111_fallback_host(host): + return model + model_lower = (model or '').lower() + if any(fnmatch.fnmatch(model_lower, pattern) for pattern in FALLBACK_111_MODEL_PATTERNS): + logger.warning( + "[Ollama] 111 fallback 不承接重模型 model=%s，改用 %s", + model, + FALLBACK_111_MODEL, + ) + return FALLBACK_111_MODEL + return model + + +def _effective_timeout_for_host(timeout_s: int, host: str) -> int: + """111 final fallback timeout 封頂，避免單次 fallback 拖住 Mac 數分鐘。""" + if _is_111_fallback_host(host): + return min(timeout_s, FALLBACK_111_MAX_TIMEOUT) + return timeout_s + + def _canonical_host_chain() -> List[str]: """Return the approved static fallback chain without duplicates.""" chain: List[str] = [] @@ -320,20 +362,18 @@ class OllamaService: model = model or self.model request_timeout = timeout or TIMEOUT - payload = { + base_payload = { "model": model, "prompt": prompt, "stream": False, "options": {"temperature": temperature}, } if options: - payload["options"].update(options) + base_payload["options"].update(options) if system_prompt: - payload["system"] = system_prompt - if keep_alive: - payload["keep_alive"] = keep_alive + base_payload["system"] = system_prompt if images: - payload["images"] = images + base_payload["images"] = images # HOTFIX 三主機 retry 鏈 attempted_hosts: List[str] = [] @@ -361,20 +401,37 @@ class OllamaService: current_host = next_host attempted_hosts.append(current_host) - logger.info(f"[Ollama] 嘗試 #{attempt+1}/3 host={current_host} model={model} timeout={request_timeout}s") + effective_model = _effective_model_for_host(model, current_host) + effective_timeout = _effective_timeout_for_host(request_timeout, current_host) + payload = dict(base_payload) + payload["options"] = dict(base_payload["options"]) + payload["model"] = effective_model + if _is_111_fallback_host(current_host): + payload["keep_alive"] = FALLBACK_111_KEEP_ALIVE + elif keep_alive: + payload["keep_alive"] = keep_alive + + logger.info( + "[Ollama] 嘗試 #%s/3 host=%s model=%s timeout=%ss keep_alive=%s", + attempt + 1, + current_host, + effective_model, + effective_timeout, + payload.get("keep_alive", ""), + ) try: response = requests.post( f"{current_host}/api/generate", json=payload, - timeout=request_timeout, + timeout=effective_timeout, ) if response.status_code == 200: data = response.json() return OllamaResponse( success=True, content=data.get('response', ''), - model=model, + model=effective_model, total_duration=data.get('total_duration', 0) / 1e9, host=current_host, # Phase 13 補強：解 token=0 假數據（A4 已知 limitation） @@ -386,7 +443,7 @@ class OllamaService: logger.warning(f"[Ollama] {current_host} HTTP 失敗 → mark_unhealthy + retry: {last_error}") _mark_unhealthy_best_effort(current_host) except requests.Timeout: - last_error = f"timeout ({request_timeout}s)" + last_error = f"timeout ({effective_timeout}s)" logger.warning(f"[Ollama] {current_host} timeout → mark_unhealthy + retry") _mark_unhealthy_best_effort(current_host) except Exception as e: diff --git a/tests/test_ollama_resolve.py b/tests/test_ollama_resolve.py index d4abfda..a2d626b 100644 --- a/tests/test_ollama_resolve.py +++ b/tests/test_ollama_resolve.py @@ -235,3 +235,46 @@ def test_resolve_uses_cache_within_ttl(): ) # cache 命中 → 第二次不打 HTTP assert mock_get.call_count == 1 + + +def test_111_fallback_downgrades_heavy_model_and_shortens_keep_alive(monkeypatch): + """111 是 final fallback；14B+ 模型不得長駐這台 16GB Mac。""" + from services import ollama_service as oss + + monkeypatch.setattr(oss, "FALLBACK_111_MODEL", "qwen2.5:7b-instruct") + monkeypatch.setattr(oss, "FALLBACK_111_KEEP_ALIVE", "5m") + monkeypatch.setattr(oss, "FALLBACK_111_MAX_TIMEOUT", 45) + monkeypatch.setattr(oss, "FALLBACK_111_MODEL_PATTERNS", ("qwen3:14b",)) + + fake_resp = MagicMock(status_code=200) + fake_resp.json.return_value = { + "response": "ok", + "prompt_eval_count": 3, + "eval_count": 2, + "total_duration": 1_000_000_000, + } + svc = oss.OllamaService(host="http://192.168.0.111:11434", model="qwen3:14b") + + with patch("services.ollama_service.requests.post", return_value=fake_resp) as mock_post: + resp = svc.generate("hi", timeout=120, keep_alive="24h") + + payload = mock_post.call_args.kwargs["json"] + assert payload["model"] == "qwen2.5:7b-instruct" + assert payload["keep_alive"] == "5m" + assert mock_post.call_args.kwargs["timeout"] == 45 + assert resp.model == "qwen2.5:7b-instruct" + + +def test_111_fallback_keeps_light_model_but_caps_timeout(monkeypatch): + from requests import Timeout + from services import ollama_service as oss + + monkeypatch.setattr(oss, "FALLBACK_111_KEEP_ALIVE", "5m") + monkeypatch.setattr(oss, "FALLBACK_111_MAX_TIMEOUT", 45) + svc = oss.OllamaService(host="http://192.168.0.111:11434", model="hermes3:latest") + + with patch("services.ollama_service.requests.post", side_effect=Timeout): + resp = svc.generate("hi", timeout=120, keep_alive="24h") + + assert resp.success is False + assert "timeout (45s)" in resp.error