676 lines
28 KiB
Python
676 lines
28 KiB
Python
"""
|
||
Ollama 自動容災管理 - P1.1b
|
||
============================
|
||
依 OllamaHealthMonitor 健康狀態決定 Ollama 路由方案。
|
||
|
||
路由邏輯（global_product_governance_v2）：
|
||
selected primary 永遠是 GCP-A，executable chain 永遠是
|
||
GCP-A → GCP-B → Local 111 → Gemini。健康探測只會產生
|
||
observed_fallback，不得將 Gemini 呈現為 selected primary。
|
||
|
||
設計說明：
|
||
- GCP-A 主機：34.143.170.20（SSD，9x 載速 + 2x 推理）
|
||
- GCP-B 備援：34.21.145.224（SSD，9x 載速 + 2x 推理）
|
||
- Local 最後防線：192.168.0.111（M1 Pro, Metal 加速，HDD）
|
||
- 不直接依賴 AIProviderEnum（P1.2 Engineer-A 整合時再對齊）
|
||
- 返回輕量 OllamaRoutingResult，含主 endpoint + fallback 清單
|
||
- 並行檢查三台 Ollama 主機健康狀態
|
||
- 切換觸發時寫 audit_logs service="ollama_failover"
|
||
- clear_cache() 方法供 OllamaAutoRecoveryService 切回後清空路由快取
|
||
|
||
版本: v3.0
|
||
建立: 2026-04-25 (台北時區)
|
||
建立者: Claude Engineer-C (P1.1b)
|
||
更新: 2026-05-03 ogt — GCP 三層容災（ADR-110），GCP-A → GCP-B → Local → Gemini
|
||
# Created 2026-04-25 P1.1 by Claude Engineer-C
|
||
# 2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
|
||
# 2026-05-03 ogt: GCP 三層容災（ADR-110），GCP-A → GCP-B → Local → Gemini
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import datetime
|
||
from dataclasses import dataclass
|
||
from datetime import timedelta, timezone
|
||
|
||
import structlog
|
||
|
||
from src.core.config import get_settings
|
||
from src.services.ollama_endpoint_resolver import resolve_ollama_order
|
||
from src.services.ollama_health_monitor import (
|
||
HealthReport,
|
||
HealthStatus,
|
||
OllamaHealthMonitor,
|
||
get_ollama_health_monitor,
|
||
)
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
# 台北時區 +8（標準庫保險絲，100% 可用）
|
||
# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
|
||
# 用標準庫 timezone(timedelta(hours=8)) 取代 zoneinfo，保證一定有 +8 時區
|
||
# 原 zoneinfo.ZoneInfo("Asia/Taipei") 失敗時 = None → datetime.now(None) 為 UTC
|
||
TAIPEI_TZ = timezone(timedelta(hours=8))
|
||
|
||
|
||
# =============================================================================
|
||
# 路由結果模型（輕量，P1.2 整合時轉換為 RoutingDecision）
|
||
# =============================================================================
|
||
|
||
|
||
@dataclass
|
||
class OllamaEndpoint:
|
||
"""Ollama 端點描述"""
|
||
|
||
url: str
|
||
provider_name: str # 給 AIRouterExecutor 用的 provider 名稱
|
||
model: str
|
||
|
||
def to_dict(self) -> dict:
|
||
return {"url": self.url, "provider_name": self.provider_name, "model": self.model}
|
||
|
||
|
||
@dataclass
|
||
class OllamaRoutingResult:
|
||
"""
|
||
Ollama 容災路由結果（2026-05-03 ogt: 更新為三層 GCP 容災，ADR-110）
|
||
|
||
P1.2 Engineer-A 整合時，將此結果轉換為 ai_router.RoutingDecision：
|
||
- selected_provider = AIProviderEnum[result.primary.provider_name.upper()]
|
||
- selected_model = result.primary.model
|
||
- fallback_chain = [(AIProviderEnum[p.provider_name.upper()], p.model) for p in result.fallback_chain]
|
||
"""
|
||
|
||
primary: OllamaEndpoint
|
||
fallback_chain: list[OllamaEndpoint]
|
||
routing_reason: str
|
||
health_gcp_a: HealthReport # GCP-A 健康狀態（原 health_111）
|
||
health_gcp_b: HealthReport | None = None # GCP-B 健康狀態
|
||
health_local: HealthReport | None = None # Local(111) 健康狀態
|
||
observed_fallback: OllamaEndpoint | None = None
|
||
selection_mode: str = "fixed_production_order"
|
||
|
||
@property
|
||
def health_111(self) -> HealthReport:
|
||
"""向後相容屬性（舊測試 / log 使用）"""
|
||
return self.health_gcp_a
|
||
|
||
def all_endpoints_in_order(self) -> list[OllamaEndpoint]:
|
||
"""返回完整 executable chain，不受健康 observation 重排。"""
|
||
return [self.primary, *self.fallback_chain]
|
||
|
||
def to_dict(self) -> dict:
|
||
selected_primary = {
|
||
"url": self.primary.url,
|
||
"provider": self.primary.provider_name,
|
||
"model": self.primary.model,
|
||
}
|
||
observed_fallback = (
|
||
{
|
||
"url": self.observed_fallback.url,
|
||
"provider": self.observed_fallback.provider_name,
|
||
"model": self.observed_fallback.model,
|
||
"observation_only": True,
|
||
}
|
||
if self.observed_fallback
|
||
else None
|
||
)
|
||
return {
|
||
# ``primary`` remains as a compatibility alias, but it now always
|
||
# means selected execution primary rather than observed health lane.
|
||
"primary": selected_primary,
|
||
"selected_primary": selected_primary,
|
||
"observed_fallback": observed_fallback,
|
||
"selection_mode": self.selection_mode,
|
||
"provider_order": [
|
||
endpoint.provider_name for endpoint in self.all_endpoints_in_order()
|
||
],
|
||
"fallback_chain": [
|
||
{"url": e.url, "provider": e.provider_name, "model": e.model} # noqa: E501
|
||
for e in self.fallback_chain
|
||
],
|
||
"routing_reason": self.routing_reason,
|
||
"health_gcp_a": self.health_gcp_a.to_dict(),
|
||
"health_gcp_b": self.health_gcp_b.to_dict() if self.health_gcp_b else None,
|
||
"health_local": self.health_local.to_dict() if self.health_local else None,
|
||
}
|
||
|
||
|
||
# =============================================================================
|
||
# Final paid fallback endpoint
|
||
# =============================================================================
|
||
|
||
# 以 provider_name 對應 ai_router.AIProviderEnum 的 value
|
||
_GEMINI_ENDPOINT = OllamaEndpoint(
|
||
url="",
|
||
provider_name="gemini",
|
||
model="gemini-2.5-flash-lite",
|
||
)
|
||
_PAID_FALLBACK_BLOCKED_ENDPOINT = OllamaEndpoint(
|
||
url="",
|
||
provider_name="paid_fallback_blocked",
|
||
model="",
|
||
)
|
||
# =============================================================================
|
||
# OllamaFailoverManager
|
||
# =============================================================================
|
||
|
||
|
||
class OllamaFailoverManager:
|
||
"""
|
||
Ollama 自動容災管理器
|
||
|
||
並行檢查 111 + 188，依健康狀態選擇最佳路由。
|
||
|
||
使用方式：
|
||
manager = OllamaFailoverManager()
|
||
result = await manager.select_provider()
|
||
# result.primary.url → 使用的 Ollama URL
|
||
# result.fallback_chain → 依序 fallback
|
||
|
||
2026-04-25 Claude Engineer-C (P1.1b)
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
health_monitor: OllamaHealthMonitor | None = None,
|
||
recovery_callback=None,
|
||
) -> None:
|
||
self._monitor = health_monitor or get_ollama_health_monitor()
|
||
self._settings = get_settings()
|
||
# 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
|
||
# recovery_callback: async callable(provider_name: str) → None
|
||
# OllamaAutoRecoveryService.set_current_primary 在 failover 時被通知，
|
||
# 避免重啟後 _current_primary 停留在 "ollama" 而永不啟動恢復監控
|
||
self._recovery_callback = recovery_callback
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Public API
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def select_provider(
|
||
self,
|
||
task_type: str = "",
|
||
context: dict | None = None,
|
||
) -> OllamaRoutingResult:
|
||
"""
|
||
三層 Ollama 容災路由（ADR-110 修正版 2026-05-04）。所有工作固定
|
||
GCP-A → GCP-B → 111；Gemini 由實際 generation call 的原子費控
|
||
決定是否允許。
|
||
|
||
Production URL slots are GCP-A primary, GCP-B secondary, and 111 local
|
||
fallback; the ordered provider contract does not change by task type.
|
||
|
||
Args:
|
||
task_type: 任務類型（僅供 audit，不改變 provider 順序）
|
||
context: 額外上下文（預留）
|
||
|
||
Returns:
|
||
OllamaRoutingResult
|
||
"""
|
||
# 2026-05-04 ogt: 改用語意中性名稱 primary/secondary/tertiary，
|
||
# 避免 gcp_a/gcp_b/local 與實際 URL 脫鉤造成 log 誤導
|
||
url_primary = self._settings.OLLAMA_URL # 110:11435 → GCP-A (nginx proxy)
|
||
url_secondary = self._settings.OLLAMA_SECONDARY_URL # 110:11436 → GCP-B (nginx proxy)
|
||
url_tertiary = self._settings.OLLAMA_FALLBACK_URL # 110:11437 → Local 111 (nginx proxy)
|
||
|
||
def _to_health(r, label: str) -> HealthReport:
|
||
if isinstance(r, Exception):
|
||
return HealthReport(status=HealthStatus.OFFLINE, reason=f"{label} check error: {r}")
|
||
return r
|
||
|
||
def _short(url: str) -> str:
|
||
from urllib.parse import urlparse
|
||
return urlparse(url).hostname or url
|
||
|
||
# 2026-05-19 Codex: alert-fast path must not wait for the slow local lane
|
||
# when GCP-A is already healthy. The old gather(GCP-A/GCP-B/111) path made
|
||
# 111's 45s health timeout dominate every routing decision.
|
||
try:
|
||
primary_raw = await self._monitor.check(url_primary)
|
||
except Exception as exc:
|
||
primary_raw = exc
|
||
health_gcp_a = _to_health(primary_raw, f"primary({url_primary})")
|
||
health_gcp_b: HealthReport | None = None
|
||
health_local: HealthReport | None = None
|
||
|
||
if health_gcp_a.status == HealthStatus.HEALTHY:
|
||
model = self._settings.OLLAMA_HEALTH_CHECK_MODEL
|
||
fallback_chain = [
|
||
OllamaEndpoint(url=url_secondary, provider_name="ollama_gcp_b", model=model),
|
||
OllamaEndpoint(url=url_tertiary, provider_name="ollama_local", model=model),
|
||
_GEMINI_ENDPOINT,
|
||
]
|
||
result = OllamaRoutingResult(
|
||
primary=OllamaEndpoint(url=url_primary, provider_name="ollama_gcp_a", model=model),
|
||
fallback_chain=fallback_chain,
|
||
routing_reason=f"primary({_short(url_primary)}) HEALTHY",
|
||
health_gcp_a=health_gcp_a,
|
||
health_gcp_b=None,
|
||
health_local=None,
|
||
observed_fallback=None,
|
||
)
|
||
else:
|
||
# Primary 不健康時才並行檢查後兩層，保留 GCP-B/Local 容災。
|
||
results_raw = await asyncio.gather(
|
||
self._monitor.check(url_secondary),
|
||
self._monitor.check(url_tertiary),
|
||
return_exceptions=True,
|
||
)
|
||
health_gcp_b = _to_health(results_raw[0], f"secondary({url_secondary})")
|
||
health_local = _to_health(results_raw[1], f"tertiary({url_tertiary})")
|
||
|
||
result = self._decide_route(
|
||
health_gcp_a=health_gcp_a,
|
||
health_gcp_b=health_gcp_b,
|
||
health_local=health_local,
|
||
url_gcp_a=url_primary,
|
||
url_gcp_b=url_secondary,
|
||
url_local=url_tertiary,
|
||
task_type=task_type,
|
||
)
|
||
|
||
# Provider selection never consumes paid quota. GeminiProvider performs
|
||
# the only atomic requests/tokens/cost reservation immediately before
|
||
# generateContent and fails closed when the guard cannot write a receipt.
|
||
|
||
# 寫入 audit_log（best-effort）
|
||
await self._write_failover_audit(result)
|
||
|
||
def _status(report: HealthReport | None) -> str:
|
||
return report.status.value if report else "not_checked"
|
||
|
||
logger.info(
|
||
"ollama_failover_decision",
|
||
selected_primary=result.primary.provider_name,
|
||
selected_primary_url=result.primary.url,
|
||
observed_fallback=(
|
||
result.observed_fallback.provider_name
|
||
if result.observed_fallback
|
||
else None
|
||
),
|
||
reason=result.routing_reason,
|
||
fallback_count=len(result.fallback_chain),
|
||
health_gcp_a=health_gcp_a.status.value,
|
||
health_gcp_b=_status(health_gcp_b),
|
||
health_local=_status(health_local),
|
||
)
|
||
|
||
# Recovery tracks the first health-observed fallback candidate, while
|
||
# the selected execution primary remains fixed at GCP-A.
|
||
# 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
|
||
if self._recovery_callback is not None:
|
||
try:
|
||
recovery_observation = (
|
||
result.observed_fallback.provider_name
|
||
if result.observed_fallback
|
||
else result.primary.provider_name
|
||
)
|
||
await self._recovery_callback(recovery_observation)
|
||
except Exception as e:
|
||
logger.warning(
|
||
"ollama_failover_recovery_callback_failed",
|
||
error=str(e),
|
||
)
|
||
|
||
return result
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 路由決策邏輯
|
||
# -------------------------------------------------------------------------
|
||
|
||
def _decide_route(
|
||
self,
|
||
health_gcp_a: HealthReport,
|
||
health_gcp_b: HealthReport,
|
||
health_local: HealthReport,
|
||
url_gcp_a: str,
|
||
url_gcp_b: str,
|
||
url_local: str,
|
||
task_type: str = "",
|
||
) -> OllamaRoutingResult:
|
||
"""Build fixed execution order plus a separate health observation."""
|
||
model = self._settings.OLLAMA_HEALTH_CHECK_MODEL
|
||
ep_gcp_a = OllamaEndpoint(url=url_gcp_a, provider_name="ollama_gcp_a", model=model)
|
||
ep_gcp_b = OllamaEndpoint(url=url_gcp_b, provider_name="ollama_gcp_b", model=model)
|
||
ep_local = OllamaEndpoint(url=url_local, provider_name="ollama_local", model=model)
|
||
|
||
now_ts = datetime.datetime.now(TAIPEI_TZ).isoformat()
|
||
|
||
# 用實際 URL 取最後一段作為 log 標識（IP 或 hostname）
|
||
def _short(url: str) -> str:
|
||
from urllib.parse import urlparse
|
||
return urlparse(url).hostname or url
|
||
|
||
lbl_p = _short(url_gcp_a) # primary label
|
||
lbl_s = _short(url_gcp_b) # secondary label
|
||
lbl_t = _short(url_local) # tertiary label
|
||
_ = task_type
|
||
|
||
fixed_fallback_chain = [ep_gcp_b, ep_local, _GEMINI_ENDPOINT]
|
||
|
||
def _result(
|
||
*,
|
||
observed_fallback: OllamaEndpoint | None,
|
||
reason: str,
|
||
) -> OllamaRoutingResult:
|
||
return OllamaRoutingResult(
|
||
primary=ep_gcp_a,
|
||
fallback_chain=fixed_fallback_chain.copy(),
|
||
routing_reason=reason,
|
||
health_gcp_a=health_gcp_a,
|
||
health_gcp_b=health_gcp_b,
|
||
health_local=health_local,
|
||
observed_fallback=observed_fallback,
|
||
)
|
||
|
||
# Primary HEALTHY → no fallback observation.
|
||
if health_gcp_a.status == HealthStatus.HEALTHY:
|
||
return _result(
|
||
observed_fallback=None,
|
||
reason=f"selected_primary({lbl_p}) HEALTHY; fixed production order",
|
||
)
|
||
|
||
# Health probes identify the first likely fallback without changing
|
||
# the selected primary or truncating the executable chain.
|
||
if health_gcp_b.status == HealthStatus.HEALTHY:
|
||
return _result(
|
||
observed_fallback=ep_gcp_b,
|
||
reason=(
|
||
f"selected_primary({lbl_p}) remains fixed; "
|
||
f"observed_fallback=secondary({lbl_s}) HEALTHY after "
|
||
f"{health_gcp_a.status.value} at {now_ts}"
|
||
),
|
||
)
|
||
|
||
if health_local.status == HealthStatus.HEALTHY:
|
||
return _result(
|
||
observed_fallback=ep_local,
|
||
reason=(
|
||
f"selected_primary({lbl_p}) remains fixed; "
|
||
f"observed_fallback=tertiary({lbl_t}) HEALTHY after "
|
||
f"secondary({lbl_s}) {health_gcp_b.status.value} at {now_ts}"
|
||
),
|
||
)
|
||
|
||
# 2026-05-04 ogt: SLOW 容災備援（外網同時抖動時，SLOW Ollama 仍優於 Gemini quota 耗盡）
|
||
if health_gcp_a.status == HealthStatus.SLOW:
|
||
return _result(
|
||
observed_fallback=None,
|
||
reason=(
|
||
f"selected_primary({lbl_p}) SLOW but remains first "
|
||
f"execution hop at {now_ts}"
|
||
),
|
||
)
|
||
if health_gcp_b.status == HealthStatus.SLOW:
|
||
return _result(
|
||
observed_fallback=ep_gcp_b,
|
||
reason=(
|
||
f"selected_primary({lbl_p}) remains fixed; "
|
||
f"observed_fallback=secondary({lbl_s}) SLOW at {now_ts}"
|
||
),
|
||
)
|
||
if health_local.status == HealthStatus.SLOW:
|
||
return _result(
|
||
observed_fallback=ep_local,
|
||
reason=(
|
||
f"selected_primary({lbl_p}) remains fixed; "
|
||
f"observed_fallback=tertiary({lbl_t}) SLOW at {now_ts}"
|
||
),
|
||
)
|
||
|
||
# Gemini is only an observed final policy candidate here. Actual
|
||
# selection occurs after all Ollama attempts and its own durable cost
|
||
# reservation; this readback must never label Gemini as primary.
|
||
return _result(
|
||
observed_fallback=_GEMINI_ENDPOINT,
|
||
reason=(
|
||
f"selected_primary({lbl_p}) remains fixed; all Ollama health "
|
||
f"observations unavailable ({health_gcp_a.status.value}, "
|
||
f"{health_gcp_b.status.value}, {health_local.status.value}); "
|
||
f"observed_fallback=gemini policy candidate at {now_ts}"
|
||
),
|
||
)
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Gemini 帳單熔斷（quota gate）
|
||
# 2026-04-25 critic-fix Part2 H7 by Claude Engineer-C2
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def _check_gemini_quota(self) -> bool:
|
||
"""
|
||
Read-only compatibility check against the unified Gemini guard.
|
||
|
||
Provider selection must not consume quota. The atomic reservation is
|
||
owned by GeminiProvider immediately before generateContent.
|
||
"""
|
||
try:
|
||
from src.services.ai_rate_limiter import get_ai_rate_limiter
|
||
|
||
stats = await get_ai_rate_limiter().get_usage_stats("gemini")
|
||
request_gate = stats["daily_requests"]["current"] < stats["daily_requests"]["limit"]
|
||
token_gate = (
|
||
stats["daily_tokens"]["current"] + stats["daily_tokens"].get("reserved", 0)
|
||
< stats["daily_tokens"]["limit"]
|
||
)
|
||
daily_cost = stats.get("daily_cost_usd") or {}
|
||
daily_cost_gate = (
|
||
float(daily_cost.get("current", 0.0))
|
||
+ float(daily_cost.get("reserved", 0.0))
|
||
< float(daily_cost.get("limit", 0.0))
|
||
)
|
||
return bool(request_gate and token_gate and daily_cost_gate and not stats["cost_exceeded"])
|
||
except Exception as e:
|
||
logger.exception(
|
||
"gemini_quota_check_failed_failing_closed",
|
||
error=str(e),
|
||
security_note="Redis 異常時拒絕 Gemini；不插入其他付費 provider",
|
||
)
|
||
return False
|
||
|
||
def _build_quota_exceeded_route(
|
||
self,
|
||
health_gcp_a: HealthReport,
|
||
) -> OllamaRoutingResult:
|
||
"""
|
||
Legacy compatibility route for a blocked paid fallback.
|
||
|
||
It deliberately has no executable provider or fallback: cost exhaustion
|
||
must not insert Nemo/NVIDIA/Claude or make another paid API call.
|
||
"""
|
||
model = self._settings.OLLAMA_HEALTH_CHECK_MODEL
|
||
resolved = {
|
||
endpoint.provider_name: endpoint
|
||
for endpoint in resolve_ollama_order(
|
||
"interactive",
|
||
config=self._settings,
|
||
)
|
||
}
|
||
|
||
def _endpoint(provider_name: str) -> OllamaEndpoint:
|
||
selection = resolved.get(provider_name)
|
||
return OllamaEndpoint(
|
||
url=selection.url if selection else "",
|
||
provider_name=provider_name,
|
||
model=model,
|
||
)
|
||
|
||
return OllamaRoutingResult(
|
||
primary=_endpoint("ollama_gcp_a"),
|
||
fallback_chain=[
|
||
_endpoint("ollama_gcp_b"),
|
||
_endpoint("ollama_local"),
|
||
_GEMINI_ENDPOINT,
|
||
],
|
||
routing_reason=(
|
||
"selected primary remains GCP-A; Gemini cost guard observed blocked"
|
||
),
|
||
health_gcp_a=health_gcp_a,
|
||
observed_fallback=_PAID_FALLBACK_BLOCKED_ENDPOINT,
|
||
)
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Recovery API（供 OllamaAutoRecoveryService 呼叫）
|
||
# -------------------------------------------------------------------------
|
||
|
||
def set_recovery_callback(self, callback) -> None:
|
||
"""
|
||
設定 recovery callback（供 lifespan wiring 使用）。
|
||
callback signature: async (provider_name: str) -> None
|
||
|
||
# 2026-04-25 P1.2 by Claude Engineer-A2 — failover 整合到 ai_router + lifespan
|
||
"""
|
||
self._recovery_callback = callback
|
||
|
||
async def clear_cache(self) -> None:
|
||
"""
|
||
清空路由決策快取，讓下次 select_provider 重新評估健康狀態。
|
||
OllamaAutoRecoveryService 在偵測 111 恢復後呼叫此方法。
|
||
|
||
2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
|
||
# 2026-04-25 P1.2 by Claude Engineer-A2 — 改用 make_cache_key 動態組 key，消除硬編碼 IP
|
||
"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
from src.services.ollama_health_monitor import make_cache_key
|
||
redis = get_redis()
|
||
if redis is None:
|
||
return
|
||
# 動態由 settings URL 組 cache key，避免硬編碼 IP
|
||
# 2026-05-03 ogt: 新增 OLLAMA_SECONDARY_URL（ADR-110 GCP-B）
|
||
keys = [
|
||
make_cache_key(self._settings.OLLAMA_URL),
|
||
make_cache_key(self._settings.OLLAMA_SECONDARY_URL or ""),
|
||
make_cache_key(self._settings.OLLAMA_FALLBACK_URL or ""),
|
||
]
|
||
for k in keys:
|
||
if k and k != "ollama_health:": # 空 URL 會產生無意義的 key，跳過
|
||
await redis.delete(k)
|
||
logger.info(
|
||
"ollama_failover_cache_cleared",
|
||
service="ollama_failover",
|
||
reason="recovery_triggered",
|
||
)
|
||
except Exception as e:
|
||
logger.debug("ollama_failover_clear_cache_failed", error=str(e))
|
||
|
||
def notify_recovery(self, provider: str) -> None:
|
||
"""
|
||
預留：P1.5 Engineer 接入 Telegram alerter 時使用。
|
||
目前僅寫 structlog audit。
|
||
|
||
2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
|
||
"""
|
||
logger.info(
|
||
"ollama_recovery_notified",
|
||
service="ollama_failover",
|
||
provider=provider,
|
||
action="recovery_received",
|
||
)
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Audit Log
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def _write_failover_audit(self, result: OllamaRoutingResult) -> None:
|
||
"""
|
||
切換觸發時寫 structlog audit（best-effort）+ Telegram 告警
|
||
|
||
# 2026-04-25 critic-fix Part2 B1 by Claude Engineer-C2
|
||
# 原 AuditLog DB 寫入使用不存在的欄位（service/action/target/status/metadata）
|
||
# → SQLAlchemy crash → except 吃掉 → 零稽核
|
||
# 修法：刪除 DB 寫入路徑，改用 structlog only（audit 不依賴 DB schema）
|
||
|
||
# 2026-04-25 P1.5 by Claude Engineer-D — 新增 Telegram 告警（dedup 10min）
|
||
|
||
service="ollama_failover"（per 任務規格）
|
||
僅在 primary 非 111 時記錄（真正發生切換）
|
||
"""
|
||
observed = result.observed_fallback
|
||
if observed is None:
|
||
return
|
||
|
||
# 2026-04-26 P2.3 by Claude Sonnet 4.6 (tool-expert) — 記錄 failover Prometheus metric
|
||
try:
|
||
from src.core.metrics import (
|
||
OLLAMA_CURRENT_PRIMARY_IS_OLLAMA,
|
||
OLLAMA_FAILOVER_TRIGGERED_TOTAL,
|
||
)
|
||
OLLAMA_FAILOVER_TRIGGERED_TOTAL.labels(
|
||
from_provider=result.primary.provider_name,
|
||
to_provider=observed.provider_name,
|
||
).inc()
|
||
OLLAMA_CURRENT_PRIMARY_IS_OLLAMA.set(
|
||
1 if observed.provider_name.startswith("ollama") else 0
|
||
)
|
||
except Exception as _metric_err:
|
||
logger.debug("ollama_failover_metric_error", error=str(_metric_err))
|
||
|
||
logger.info(
|
||
"ollama_failover_triggered",
|
||
service="ollama_failover",
|
||
action="failover_triggered",
|
||
selected_primary=result.primary.provider_name,
|
||
observed_fallback=observed.provider_name,
|
||
reason=result.routing_reason,
|
||
selected_primary_url=result.primary.url or result.primary.provider_name,
|
||
observed_fallback_url=observed.url or observed.provider_name,
|
||
health_gcp_a=result.health_gcp_a.status.value,
|
||
health_gcp_b=result.health_gcp_b.status.value if result.health_gcp_b else "not_configured",
|
||
health_local=result.health_local.status.value if result.health_local else "not_configured",
|
||
)
|
||
|
||
# Telegram 告警（首次切換才通知，dedup 10min 內建）
|
||
# 2026-04-25 P1.5 by Claude Engineer-D — 告警失敗不阻斷主路由邏輯
|
||
# 2026-05-03 ogt: ADR-110 — failed_host 動態計算，顯示哪台 GCP/Local 故障
|
||
try:
|
||
from src.services.failover_alerter import get_failover_alerter
|
||
from src.services.ollama_health_monitor import HealthStatus
|
||
fallback_chain_str = " → ".join(
|
||
p.provider_name for p in result.fallback_chain
|
||
)
|
||
# 計算故障主機描述（哪層 Ollama 不健康，用實際 URL 不用硬編碼標籤）
|
||
_failed = []
|
||
if result.health_gcp_a.status != HealthStatus.HEALTHY:
|
||
_failed.append(self._settings.OLLAMA_URL)
|
||
if result.health_gcp_b and result.health_gcp_b.status != HealthStatus.HEALTHY:
|
||
_failed.append(self._settings.OLLAMA_SECONDARY_URL or "secondary")
|
||
if result.health_local and result.health_local.status != HealthStatus.HEALTHY:
|
||
_failed.append(self._settings.OLLAMA_FALLBACK_URL or "tertiary")
|
||
failed_host = " + ".join(_failed) if _failed else "Ollama"
|
||
alerter = get_failover_alerter()
|
||
await alerter.alert_failover({
|
||
"to_provider": observed.provider_name,
|
||
"selected_primary": result.primary.provider_name,
|
||
"observation_only": True,
|
||
"model": observed.model,
|
||
"reason": result.routing_reason,
|
||
"timestamp": datetime.datetime.now(TAIPEI_TZ).isoformat(),
|
||
"fallback_chain_str": fallback_chain_str,
|
||
"failed_host": failed_host,
|
||
})
|
||
except Exception as e:
|
||
logger.warning("failover_alert_failed", error=str(e))
|
||
|
||
|
||
# =============================================================================
|
||
# Singleton
|
||
# =============================================================================
|
||
|
||
_failover_manager: OllamaFailoverManager | None = None
|
||
|
||
|
||
def get_ollama_failover_manager() -> OllamaFailoverManager:
|
||
"""取得 OllamaFailoverManager singleton"""
|
||
global _failover_manager
|
||
if _failover_manager is None:
|
||
_failover_manager = OllamaFailoverManager()
|
||
return _failover_manager
|
||
|
||
|
||
def reset_ollama_failover_manager() -> None:
|
||
"""重置 singleton（測試用）"""
|
||
global _failover_manager
|
||
_failover_manager = None
|