feat(p1): Ollama 多層容災系統 — P1.1 健康檢測 + P1.2 ai_router 整合 + P1.5 容災告警
ADR-092 P1 飛輪閉環的 Ollama 失敗轉移子系統,全部 Engineer-A2/C/C2 補上。 新服務 (1581 行): - ollama_health_monitor.py (356):3 層健康檢測(TCP/HTTP/推理) - ollama_failover_manager.py (571):111→188 自動切換 + Redis 持久化 + recovery callback - ollama_auto_recovery.py (436):30s 背景監控 + 連續 3 次 HEALTHY → 切回 + clear_cache - failover_alerter.py (218):P1.5 Telegram 容災告警 服務整合: - ai_router.py: AIProviderEnum.OLLAMA_188 + 120s budget + failover fallback chain - main.py lifespan: 啟動時 wire callback + start recovery,關閉時優雅 stop - config.py: OLLAMA_FALLBACK_URL / OLLAMA_HEALTH_CHECK_MODEL / GEMINI_DAILY_QUOTA(帳單熔斷) K8s 配置: - 04-configmap.yaml.patch-188-fallback:注入 OLLAMA_FALLBACK_URL=http://192.168.0.188:11434 測試 (2082 行): - test_ollama_health_monitor.py (402) - test_ollama_failover_manager.py (707) - test_ollama_auto_recovery.py (580) - test_ai_router_failover_integration.py (257) - test_lifespan_failover_wiring.py (136) 依賴鏈:service 三件套 + ai_router + main.py 一起 commit,缺一就 ImportError。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -68,10 +68,14 @@ logger = structlog.get_logger(__name__)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
class AIProviderEnum(Enum):
|
||||
class AIProviderEnum(str, Enum):
|
||||
"""AI 提供者"""
|
||||
|
||||
OLLAMA = "ollama"
|
||||
# 2026-04-25 critic-fix Part2 B2 by Claude Engineer-C2
|
||||
# P1.1b OllamaFailoverManager 使用 provider_name="ollama_188",
|
||||
# 但 AIProviderEnum 沒有此值 → P1.2 整合時 lookup 失敗
|
||||
OLLAMA_188 = "ollama_188" # 188 CPU-only 備援節點(P1.1b)
|
||||
GEMINI = "gemini"
|
||||
CLAUDE = "claude"
|
||||
# 2026-04-02 ogt: C1 修復 — 對齊 Registry 實際名稱
|
||||
@@ -85,6 +89,8 @@ class AIProviderEnum(Enum):
|
||||
# Provider 對應延遲預算 (ms)
|
||||
PROVIDER_LATENCY_BUDGET: dict[AIProviderEnum, int] = {
|
||||
AIProviderEnum.OLLAMA: 60000, # 本地,允許較長處理時間
|
||||
# 2026-04-25 critic-fix Part2 B2 by Claude Engineer-C2 — 188 CPU-only 推理較慢
|
||||
AIProviderEnum.OLLAMA_188: 120000, # 120s budget for CPU inference
|
||||
AIProviderEnum.GEMINI: 30000, # 雲端,較低延遲
|
||||
AIProviderEnum.CLAUDE: 30000, # 雲端,較低延遲
|
||||
# 2026-04-02 ogt: C1 修復 — 對齊 Registry 名稱
|
||||
@@ -212,6 +218,10 @@ class AIRouter:
|
||||
self._intent_classifier = get_intent_classifier()
|
||||
self._complexity_scorer = get_complexity_scorer()
|
||||
self._model_registry = get_model_registry()
|
||||
# 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan
|
||||
# 延遲 import 避免循環依賴(ollama_failover_manager 不 import ai_router)
|
||||
from src.services.ollama_failover_manager import get_ollama_failover_manager
|
||||
self._failover_manager = get_ollama_failover_manager()
|
||||
|
||||
# 從 ModelRegistry 取得模型配置
|
||||
self._ollama_default = self._model_registry.get_model("ollama", "default")
|
||||
@@ -372,10 +382,51 @@ class AIRouter:
|
||||
intent, intent_result, complexity
|
||||
)
|
||||
|
||||
# Step 3b: 若 initial decision 選到 OLLAMA,交由 FailoverManager 重評
|
||||
# 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan
|
||||
# 只在 OLLAMA 時觸發,不干擾 NEMOTRON / OPENCLAW_NEMO / CLAUDE / GEMINI 路由
|
||||
failover_fallback: list[tuple[AIProviderEnum, str]] | None = None
|
||||
if provider == AIProviderEnum.OLLAMA:
|
||||
try:
|
||||
failover_result = await self._failover_manager.select_provider(
|
||||
task_type=intent.value if intent else "general"
|
||||
)
|
||||
primary_str = failover_result.primary.provider_name
|
||||
try:
|
||||
provider = AIProviderEnum(primary_str)
|
||||
model = failover_result.primary.model
|
||||
reason = f"{reason} [failover→{primary_str}]"
|
||||
except ValueError:
|
||||
# provider_name 無法對應已知 enum(理論上不應發生,OLLAMA_188 已加)
|
||||
logger.warning(
|
||||
"ai_router_unknown_failover_provider",
|
||||
provider=primary_str,
|
||||
)
|
||||
# 重建 fallback chain:從 failover_result 轉換
|
||||
fb_list: list[tuple[AIProviderEnum, str]] = []
|
||||
for ep in failover_result.fallback_chain:
|
||||
try:
|
||||
fb_provider = AIProviderEnum(ep.provider_name)
|
||||
fb_list.append((fb_provider, ep.model))
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"ai_router_unknown_failover_fallback_provider",
|
||||
provider=ep.provider_name,
|
||||
)
|
||||
failover_fallback = fb_list
|
||||
except Exception as e:
|
||||
# failover_manager 異常 → 保留原始 provider(fail-open)
|
||||
logger.warning("ai_router_failover_manager_error", error=str(e))
|
||||
|
||||
# Step 4: 建立 Fallback 鏈
|
||||
# 2026-04-05 Claude Code: v4.3 — DIAGNOSE 改回 _full_fallback_chain
|
||||
# NIM 從 Phase 22 起就是主力,無隱私問題;Ollama CPU-only 不可用(實測 238s)
|
||||
fallback_chain = self._build_fallback_chain(provider)
|
||||
# 2026-04-25 P1.2: 若 failover_manager 回傳了 fallback chain,優先使用
|
||||
fallback_chain = (
|
||||
failover_fallback
|
||||
if failover_fallback is not None
|
||||
else self._build_fallback_chain(provider)
|
||||
)
|
||||
|
||||
# Step 5: 計算延遲預算
|
||||
latency_budget = PROVIDER_LATENCY_BUDGET.get(provider, 30000)
|
||||
|
||||
Reference in New Issue
Block a user