fix(failover): 188 完全移出 routing chain，備援只用 Gemini
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / build-and-deploy (push) Has been cancelled
統帥鐵律 2026-04-26： - 唯一 Ollama = 111（M1 Pro Metal 加速） - 188 CPU-only (0.45 tok/s) 禁止即時回應，移出所有 fallback chain - 111 HEALTHY → fallback=[Gemini] - 111 非HEALTHY → primary=Gemini, fallback=[Nemotron, Claude] - Gemini quota exceeded → Nemotron → Claude（不落 188） - OllamaRoutingResult 移除 health_188 欄位 - select_provider 只 check 111（不再 asyncio.gather 兩節點） - 測試全部對齊新規則（1451 passed） Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Your Name
@@ -26,9 +26,8 @@ Ollama 自動容災管理 - P1.1b
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
|
||||
# 用標準庫 timezone(timedelta(hours=8)) 取代 zoneinfo，保證一定有 +8 時區
|
||||
# 原 zoneinfo.ZoneInfo("Asia/Taipei") 失敗時 = None → datetime.now(None) 為 UTC
|
||||
@@ -102,7 +101,6 @@ class OllamaRoutingResult:
|
||||
],
|
||||
"routing_reason": self.routing_reason,
|
||||
"health_111": self.health_111.to_dict(),
|
||||
"health_188": self.health_188.to_dict() if self.health_188 else None,
|
||||
}
|
||||
|
||||
|
||||
@@ -167,53 +165,22 @@ class OllamaFailoverManager:
|
||||
|
||||
async def select_provider(
|
||||
self,
|
||||
task_type: str = "",
|
||||
context: dict | None = None,
|
||||
task_type: str = "", # noqa: ARG002
|
||||
context: dict | None = None, # noqa: ARG002
|
||||
) -> OllamaRoutingResult:
|
||||
"""
|
||||
並行檢查 111 + 188，返回路由結果
|
||||
檢查 111 健康狀態，返回路由結果。
|
||||
|
||||
Args:
|
||||
task_type: 任務類型（預留，目前未影響路由邏輯）
|
||||
context: 額外上下文（預留）
|
||||
|
||||
Returns:
|
||||
OllamaRoutingResult
|
||||
2026-04-26 統帥鐵律：唯一 Ollama = 111，188 禁止用於即時回應。
|
||||
"""
|
||||
url_111 = self._settings.OLLAMA_URL
|
||||
url_188 = self._settings.OLLAMA_FALLBACK_URL or ""
|
||||
|
||||
# 並行檢查
|
||||
# 2026-04-25 critic-fix Part2 H4 by Claude Engineer-C2
|
||||
# return_exceptions=True 防止任一 check 例外導致整個 select_provider 炸
|
||||
if url_188:
|
||||
results = await asyncio.gather(
|
||||
self._monitor.check(url_111),
|
||||
self._monitor.check(url_188),
|
||||
return_exceptions=True,
|
||||
)
|
||||
# 處理 exception — 任一失敗視為 OFFLINE
|
||||
health_111_raw, health_188_raw = results
|
||||
health_111: HealthReport = (
|
||||
HealthReport(status=HealthStatus.OFFLINE, reason=f"check error: {health_111_raw}")
|
||||
if isinstance(health_111_raw, Exception)
|
||||
else health_111_raw
|
||||
)
|
||||
health_188: HealthReport | None = (
|
||||
HealthReport(status=HealthStatus.OFFLINE, reason=f"check error: {health_188_raw}")
|
||||
if isinstance(health_188_raw, Exception)
|
||||
else health_188_raw
|
||||
)
|
||||
else:
|
||||
try:
|
||||
health_111 = await self._monitor.check(url_111)
|
||||
health_188 = None
|
||||
except Exception as e:
|
||||
health_111 = HealthReport(status=HealthStatus.OFFLINE, reason=f"check error: {e}")
|
||||
|
||||
result = self._decide_route(
|
||||
health_111=health_111,
|
||||
health_188=health_188,
|
||||
url_111=url_111,
|
||||
url_188=url_188,
|
||||
)
|
||||
result = self._decide_route(health_111=health_111, url_111=url_111)
|
||||
|
||||
# Gemini 帳單熔斷（quota gate）
|
||||
# 2026-04-25 critic-fix Part2 H7 by Claude Engineer-C2
|
||||
@@ -226,12 +193,7 @@ class OllamaFailoverManager:
|
||||
quota=quota,
|
||||
health_111=health_111.status.value,
|
||||
)
|
||||
result = self._build_quota_exceeded_route(
|
||||
health_111=health_111,
|
||||
health_188=health_188,
|
||||
url_111=url_111,
|
||||
url_188=url_188,
|
||||
)
|
||||
result = self._build_quota_exceeded_route(health_111=health_111)
|
||||
# 2026-04-26 P1.5 整合點 3 by Claude Opus 4.7 — 配額耗盡 Telegram 告警
|
||||
# alerter 內部 24h dedup（QUOTA_DEDUP_TTL_SEC），即使每次 quota exceeded
|
||||
# 都呼叫，當日只會發送一次告警。失敗 fail-open（不阻擋 routing）。
|
||||
@@ -267,7 +229,6 @@ class OllamaFailoverManager:
|
||||
reason=result.routing_reason,
|
||||
fallback_count=len(result.fallback_chain),
|
||||
health_111=health_111.status.value,
|
||||
health_188=health_188.status.value if health_188 else "not_configured",
|
||||
)
|
||||
|
||||
# 通知 recovery service 當前 primary（跨重啟持久化）
|
||||
@@ -290,102 +251,44 @@ class OllamaFailoverManager:
|
||||
def _decide_route(
|
||||
self,
|
||||
health_111: HealthReport,
|
||||
health_188: HealthReport | None,
|
||||
url_111: str,
|
||||
url_188: str,
|
||||
) -> OllamaRoutingResult:
|
||||
"""
|
||||
決策矩陣（2026-04-25 統帥指令：Gemini 優先，188 最後備援）：
|
||||
決策矩陣（2026-04-26 統帥鐵律：唯一 Ollama=111，備援只用 Gemini）：
|
||||
|
||||
111 HEALTHY → primary=111, fallback=[Gemini, 188, Nemotron]
|
||||
111 SLOW → primary=Gemini, fallback=[111, 188]
|
||||
111 DEGRADED → primary=Gemini, fallback=[188, Nemotron, Claude]
|
||||
111 OFFLINE → primary=Gemini, fallback=[188, Nemotron, Claude]
|
||||
111 OFFLINE + 188 OFFLINE → primary=Gemini, fallback=[Nemotron, Claude]
|
||||
111 HEALTHY → primary=111, fallback=[Gemini]
|
||||
111 SLOW → primary=Gemini, fallback=[111, Nemotron, Claude]
|
||||
111 DEGRADED → primary=Gemini, fallback=[Nemotron, Claude]
|
||||
111 OFFLINE → primary=Gemini, fallback=[Nemotron, Claude]
|
||||
|
||||
關鍵原則：
|
||||
- 111 非 HEALTHY 時，primary 必為 Gemini（快速雲端，不等 188 慢推理）
|
||||
- 188 永遠在 fallback chain，作為 Gemini 額度耗盡的最後備援
|
||||
- degradation_reason 記錄切換原因 + 時間戳
|
||||
|
||||
2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
|
||||
188 完全移出（CPU-only 0.45 tok/s，禁止即時回應）。
|
||||
"""
|
||||
model_111 = self._settings.OLLAMA_HEALTH_CHECK_MODEL
|
||||
model_188 = "qwen2.5:7b-instruct" # 188 CPU-only 備援推薦模型（plan 方案 C）
|
||||
|
||||
ep_111 = OllamaEndpoint(url=url_111, provider_name="ollama", model=model_111)
|
||||
ep_188 = (
|
||||
OllamaEndpoint(url=url_188, provider_name="ollama_188", model=model_188)
|
||||
if url_188
|
||||
else None
|
||||
)
|
||||
|
||||
# 188 可用性判斷（僅供 fallback 使用）
|
||||
has_188 = ep_188 is not None and (
|
||||
health_188 is not None and health_188.status != HealthStatus.OFFLINE
|
||||
)
|
||||
|
||||
# 切換時間戳（台北時區 +8，標準庫保證）
|
||||
# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
|
||||
now_ts = datetime.datetime.now(TAIPEI_TZ).isoformat()
|
||||
|
||||
# ==========================================================
|
||||
# 111 HEALTHY → 主 111，Gemini 作為第一 fallback（快速雲端）
|
||||
# ==========================================================
|
||||
if health_111.status == HealthStatus.HEALTHY:
|
||||
fallback: list[OllamaEndpoint] = [_GEMINI_ENDPOINT]
|
||||
if has_188 and ep_188:
|
||||
fallback.append(ep_188)
|
||||
fallback.append(_NEMOTRON_ENDPOINT)
|
||||
return OllamaRoutingResult(
|
||||
primary=ep_111,
|
||||
fallback_chain=fallback,
|
||||
fallback_chain=[_GEMINI_ENDPOINT],
|
||||
routing_reason="111 HEALTHY → 主 111",
|
||||
health_111=health_111,
|
||||
health_188=health_188,
|
||||
)
|
||||
|
||||
# ==========================================================
|
||||
# 111 SLOW → primary=Gemini，fallback=[111, 188]
|
||||
# 111 實測 eval rate 0.09 token/s，~111s 推理，Gemini 更快
|
||||
# ==========================================================
|
||||
if health_111.status == HealthStatus.SLOW:
|
||||
fallback_slow: list[OllamaEndpoint] = [ep_111]
|
||||
if has_188 and ep_188:
|
||||
fallback_slow.append(ep_188)
|
||||
degradation_reason = (
|
||||
f"111 SLOW（eval ~0.09 token/s, ~111s）→ 切 Gemini at {now_ts}"
|
||||
)
|
||||
return OllamaRoutingResult(
|
||||
primary=_GEMINI_ENDPOINT,
|
||||
fallback_chain=fallback_slow,
|
||||
routing_reason=degradation_reason,
|
||||
fallback_chain=[ep_111, _NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
|
||||
routing_reason=f"111 SLOW → 切 Gemini at {now_ts}",
|
||||
health_111=health_111,
|
||||
health_188=health_188,
|
||||
)
|
||||
|
||||
# ==========================================================
|
||||
# 111 DEGRADED 或 OFFLINE → primary=Gemini，188 在 fallback
|
||||
# ==========================================================
|
||||
status_label = health_111.status.value # "degraded" / "offline"
|
||||
degradation_reason = f"111 {status_label} → 切 Gemini at {now_ts}"
|
||||
if has_188 and ep_188:
|
||||
return OllamaRoutingResult(
|
||||
primary=_GEMINI_ENDPOINT,
|
||||
fallback_chain=[ep_188, _NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
|
||||
routing_reason=degradation_reason,
|
||||
health_111=health_111,
|
||||
health_188=health_188,
|
||||
)
|
||||
|
||||
# 188 也不可用 → Gemini 主力，最後備援 Nemotron / Claude
|
||||
degradation_reason = f"111 {status_label} + 188 不可用 → 切 Gemini at {now_ts}"
|
||||
status_label = health_111.status.value
|
||||
return OllamaRoutingResult(
|
||||
primary=_GEMINI_ENDPOINT,
|
||||
fallback_chain=[_NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
|
||||
routing_reason=degradation_reason,
|
||||
routing_reason=f"111 {status_label} → 切 Gemini at {now_ts}",
|
||||
health_111=health_111,
|
||||
health_188=health_188,
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
@@ -464,40 +367,13 @@ class OllamaFailoverManager:
|
||||
def _build_quota_exceeded_route(
|
||||
self,
|
||||
health_111: HealthReport,
|
||||
health_188: HealthReport | None,
|
||||
url_111: str, # noqa: ARG002 — 保留供 OllamaRoutingResult 結構完整性（health_111 對應）
|
||||
url_188: str,
|
||||
) -> OllamaRoutingResult:
|
||||
"""
|
||||
Gemini 配額耗盡時的備援路由：primary=OLLAMA_188，fallback=[Nemotron, Claude]
|
||||
若 188 也不可用，則 primary=Nemotron。
|
||||
"""
|
||||
model_188 = "qwen2.5:7b-instruct"
|
||||
ep_188 = (
|
||||
OllamaEndpoint(url=url_188, provider_name="ollama_188", model=model_188)
|
||||
if url_188
|
||||
else None
|
||||
)
|
||||
has_188 = ep_188 is not None and (
|
||||
health_188 is not None and health_188.status != HealthStatus.OFFLINE
|
||||
)
|
||||
|
||||
if has_188 and ep_188:
|
||||
return OllamaRoutingResult(
|
||||
primary=ep_188,
|
||||
fallback_chain=[_NEMOTRON_ENDPOINT, _CLAUDE_ENDPOINT],
|
||||
routing_reason="Gemini quota exceeded → 188 CPU-only 備援",
|
||||
health_111=health_111,
|
||||
health_188=health_188,
|
||||
)
|
||||
|
||||
# 188 也不可用
|
||||
"""Gemini 配額耗盡 → Nemotron 備援。2026-04-26 統帥鐵律：188 移出。"""
|
||||
return OllamaRoutingResult(
|
||||
primary=_NEMOTRON_ENDPOINT,
|
||||
fallback_chain=[_CLAUDE_ENDPOINT],
|
||||
routing_reason="Gemini quota exceeded + 188 不可用 → Nemotron 備援",
|
||||
routing_reason="Gemini quota exceeded → Nemotron 備援",
|
||||
health_111=health_111,
|
||||
health_188=health_188,
|
||||
)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user