497 lines
19 KiB
Python
497 lines
19 KiB
Python
"""
|
||
Ollama 自動恢復服務 - P1.1d
|
||
============================
|
||
背景監控：偵測 111 從 OFFLINE/SLOW/DEGRADED 恢復為 HEALTHY 後，立刻切回 Ollama。
|
||
|
||
核心設計：
|
||
- 30s 輪詢 111 健康狀態
|
||
- 防抖：連續 3 次 HEALTHY 才觸發切回（30s × 3 = 90s 穩定視窗）
|
||
- 中途若又 OFFLINE，counter 歸零，重新計數
|
||
- 切回後呼叫 failover_manager.clear_cache()，讓下次路由重新評估
|
||
- 預留 telegram_alerter 介面（P1.5 Engineer 接入）
|
||
- structlog audit service="ollama_auto_recovery"
|
||
|
||
整合方式（FastAPI lifespan）：
|
||
from src.services.ollama_auto_recovery import OllamaAutoRecoveryService
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(app: FastAPI):
|
||
recovery_svc = OllamaAutoRecoveryService()
|
||
await recovery_svc.start()
|
||
yield
|
||
await recovery_svc.stop()
|
||
|
||
版本: v1.0
|
||
建立: 2026-04-25 (台北時區)
|
||
建立者: Claude Engineer-C (P1.1d)
|
||
# 2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import datetime
|
||
from datetime import timezone, timedelta
|
||
from typing import Any, Protocol, runtime_checkable
|
||
|
||
import structlog
|
||
|
||
# 台北時區 +8（標準庫保險絲，100% 可用）
|
||
# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
|
||
# 原 zoneinfo.ZoneInfo("Asia/Taipei") 失敗時 = None → datetime.now(None) 為 UTC
|
||
TAIPEI_TZ = timezone(timedelta(hours=8))
|
||
|
||
from src.core.config import get_settings
|
||
from src.services.ollama_health_monitor import (
|
||
HealthStatus,
|
||
OllamaHealthMonitor,
|
||
get_ollama_health_monitor,
|
||
)
|
||
from src.services.ollama_failover_manager import (
|
||
OllamaFailoverManager,
|
||
get_ollama_failover_manager,
|
||
)
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
# =============================================================================
|
||
# 常數
|
||
# =============================================================================
|
||
|
||
RECOVERY_CHECK_INTERVAL_SEC = 30 # 每 30s 檢查一次
|
||
STABLE_COUNT_REQUIRED = 3 # 連續 3 次 HEALTHY 才切回（防抖）
|
||
|
||
# =============================================================================
|
||
# Telegram Alerter Protocol（預留介面，P1.5 Engineer 實作）
|
||
# =============================================================================
|
||
|
||
|
||
@runtime_checkable
|
||
class TelegramAlerter(Protocol):
|
||
"""Telegram 通知介面（預留，P1.5 Engineer 接入）"""
|
||
|
||
async def alert_recovery(self, payload: dict[str, Any]) -> None:
|
||
"""111 恢復通知"""
|
||
...
|
||
|
||
|
||
# =============================================================================
|
||
# OllamaAutoRecoveryService
|
||
# =============================================================================
|
||
|
||
|
||
class OllamaAutoRecoveryService:
|
||
"""
|
||
Ollama 111 自動恢復服務
|
||
|
||
偵測 111 從 OFFLINE/SLOW/DEGRADED 恢復為 HEALTHY 後，立刻切回 Ollama 路由。
|
||
|
||
防抖邏輯：
|
||
- 連續 3 次 HEALTHY（90s 穩定視窗）才觸發切回
|
||
- 中途任一次非 HEALTHY → counter 歸零
|
||
- 已是 ollama primary 時不重複觸發
|
||
|
||
2026-04-25 統帥指令 by Claude Engineer-C — 自動切 Gemini + 自動恢復
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
health_monitor: OllamaHealthMonitor | None = None,
|
||
failover_manager: OllamaFailoverManager | None = None,
|
||
telegram_alerter: TelegramAlerter | None = None,
|
||
recovery_check_interval_sec: int = RECOVERY_CHECK_INTERVAL_SEC,
|
||
stable_count_required: int = STABLE_COUNT_REQUIRED,
|
||
) -> None:
|
||
self._monitor = health_monitor or get_ollama_health_monitor()
|
||
self._failover_manager = failover_manager or get_ollama_failover_manager()
|
||
self._alerter = telegram_alerter
|
||
self._recovery_check_interval_sec = recovery_check_interval_sec
|
||
self._stable_count_required = stable_count_required
|
||
self._settings = get_settings()
|
||
|
||
# 狀態追蹤
|
||
# 2026-05-04 ogt: B5 修復 — 改用與 failover_manager callback 一致的命名
|
||
# failover_manager 傳入的是 "ollama_gcp_a"/"ollama_gcp_b"/"ollama_local"/"gemini"
|
||
# 原設計用 "ollama" 造成 != "ollama" 永遠成立 → recovery loop 永遠以為在 Gemini
|
||
# 修法：判斷改用 _is_ollama_primary()，比對 startswith("ollama_")
|
||
self._current_primary: str = "ollama_gcp_a" # fallback_manager 傳入的 provider_name
|
||
self._consecutive_healthy: int = 0
|
||
self._task: asyncio.Task | None = None
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Lifecycle
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def start(self) -> None:
|
||
"""
|
||
啟動背景監控任務。
|
||
|
||
在 FastAPI lifespan 的 startup 階段呼叫：
|
||
recovery_svc = OllamaAutoRecoveryService()
|
||
await recovery_svc.start()
|
||
|
||
# 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
|
||
# 啟動時從 Redis 載入狀態（跨重啟持久化），
|
||
# 若 primary != "ollama" 立刻執行一次 check（不等 30s）。
|
||
"""
|
||
if self._task is not None and not self._task.done():
|
||
logger.warning(
|
||
"ollama_auto_recovery_already_running",
|
||
service="ollama_auto_recovery",
|
||
)
|
||
return
|
||
|
||
# 從 Redis 載入持久化狀態（跨重啟恢復）
|
||
# 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
|
||
self._current_primary = await self._load_primary()
|
||
logger.info(
|
||
"ollama_auto_recovery_bootstrap",
|
||
service="ollama_auto_recovery",
|
||
current_primary=self._current_primary,
|
||
)
|
||
|
||
# 若 primary 不是 Ollama，立刻評估一次（不等 30s interval）
|
||
if not self._is_ollama_primary(self._current_primary):
|
||
try:
|
||
await self._check_and_recover()
|
||
except Exception:
|
||
logger.exception(
|
||
"ollama_auto_recovery_immediate_check_error",
|
||
service="ollama_auto_recovery",
|
||
)
|
||
|
||
self._task = asyncio.create_task(
|
||
self._monitor_loop(),
|
||
name="ollama_auto_recovery_loop",
|
||
)
|
||
logger.info(
|
||
"ollama_auto_recovery_started",
|
||
service="ollama_auto_recovery",
|
||
check_interval_sec=self._recovery_check_interval_sec,
|
||
stable_count_required=self._stable_count_required,
|
||
)
|
||
|
||
async def stop(self) -> None:
|
||
"""
|
||
優雅關閉背景任務。
|
||
|
||
在 FastAPI lifespan 的 shutdown 階段呼叫：
|
||
await recovery_svc.stop()
|
||
"""
|
||
if self._task is None or self._task.done():
|
||
return
|
||
|
||
self._task.cancel()
|
||
try:
|
||
await self._task
|
||
except asyncio.CancelledError:
|
||
pass
|
||
finally:
|
||
self._task = None
|
||
|
||
logger.info(
|
||
"ollama_auto_recovery_stopped",
|
||
service="ollama_auto_recovery",
|
||
)
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 狀態控制（供整合層設定當前 primary，例如 failover 觸發時更新）
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def set_current_primary(self, provider: str) -> None:
|
||
"""
|
||
通知 recovery service 當前路由 primary 是哪個 provider。
|
||
OllamaFailoverManager.select_provider() 觸發 failover 後呼叫此方法更新狀態。
|
||
|
||
# 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
|
||
# 改為 async，切換時同步寫 Redis 持久化（跨重啟恢復）
|
||
"""
|
||
if self._current_primary != provider:
|
||
logger.info(
|
||
"ollama_auto_recovery_primary_changed",
|
||
service="ollama_auto_recovery",
|
||
from_primary=self._current_primary,
|
||
to_primary=provider,
|
||
)
|
||
self._current_primary = provider
|
||
# Redis 持久化（跨重啟恢復）
|
||
await self._persist_primary(provider)
|
||
|
||
# 2026-05-04 ogt: B5/B6 修復 — 判斷改用 _is_ollama_primary()
|
||
# 原設計：provider != "ollama"，但 callback 傳 "ollama_gcp_a" → 永遠觸發 tracking
|
||
if not self._is_ollama_primary(provider):
|
||
# 切換到非 Ollama（gemini/nemotron/claude）→ 重置 counter，開始監控恢復
|
||
self._consecutive_healthy = 0
|
||
logger.info(
|
||
"ollama_auto_recovery_tracking_started",
|
||
service="ollama_auto_recovery",
|
||
current_primary=provider,
|
||
)
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Redis 持久化（跨重啟恢復）
|
||
# 2026-04-25 critic-fix Part2 H5+H6 by Claude Engineer-C2
|
||
# 2026-05-04 P0-11 Phase A 雙寫遷移（ADR-110/ADR-118）：
|
||
# 舊 key "ollama:current_primary" → 新 key "platform:ollama:current_primary"
|
||
# Ollama 是 platform_resource（非 tenant 資源），加 platform: 前綴明確分類
|
||
# Phase A（30 天）：同時寫入新舊 key，讀取以新 key 為主，舊 key 作 fallback
|
||
# Phase C（~2026-06-04）：停止寫入舊 key，刪除舊 key
|
||
# -------------------------------------------------------------------------
|
||
|
||
_REDIS_PRIMARY_KEY = "platform:ollama:current_primary" # 新 key（Phase A 起生效）
|
||
_REDIS_PRIMARY_KEY_LEGACY = "ollama:current_primary" # 舊 key（Phase C 前持續雙寫）
|
||
|
||
async def _persist_primary(self, primary: str) -> None:
|
||
"""持久化 current_primary 到 Redis（無 TTL，跨重啟恢復）
|
||
Phase A 雙寫：同時寫入新舊 key，確保舊版 Pod 滾動期間不失效。
|
||
"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
redis = get_redis()
|
||
await redis.set(self._REDIS_PRIMARY_KEY, primary)
|
||
await redis.set(self._REDIS_PRIMARY_KEY_LEGACY, primary) # Phase A 雙寫
|
||
except Exception as e:
|
||
logger.warning(
|
||
"ollama_auto_recovery_persist_failed",
|
||
service="ollama_auto_recovery",
|
||
error=str(e),
|
||
)
|
||
|
||
async def _load_primary(self) -> str:
|
||
"""從 Redis 載入 current_primary（找不到時預設 "ollama"）
|
||
Phase A 讀取：優先讀新 key，不存在時 fallback 舊 key。
|
||
"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
redis = get_redis()
|
||
val = await redis.get(self._REDIS_PRIMARY_KEY)
|
||
if not val:
|
||
# Phase A fallback：新 key 尚未有值（舊版 Pod 只寫了舊 key）
|
||
val = await redis.get(self._REDIS_PRIMARY_KEY_LEGACY)
|
||
if val:
|
||
logger.info(
|
||
"ollama_auto_recovery_loaded_from_legacy_key",
|
||
service="ollama_auto_recovery",
|
||
note="Phase A fallback，預計 2026-06-04 移除舊 key",
|
||
)
|
||
if val:
|
||
decoded = val.decode() if isinstance(val, bytes) else val
|
||
logger.info(
|
||
"ollama_auto_recovery_loaded_from_redis",
|
||
service="ollama_auto_recovery",
|
||
current_primary=decoded,
|
||
)
|
||
return decoded
|
||
except Exception as e:
|
||
logger.warning(
|
||
"ollama_auto_recovery_load_failed",
|
||
service="ollama_auto_recovery",
|
||
error=str(e),
|
||
)
|
||
return "ollama" # 預設：正常路由
|
||
|
||
@property
|
||
def current_primary(self) -> str:
|
||
return self._current_primary
|
||
|
||
@property
|
||
def consecutive_healthy(self) -> int:
|
||
return self._consecutive_healthy
|
||
|
||
@property
|
||
def is_running(self) -> bool:
|
||
return self._task is not None and not self._task.done()
|
||
|
||
@staticmethod
|
||
def _is_ollama_primary(provider: str) -> bool:
|
||
"""Return true for both legacy "ollama" and ADR-110 provider names."""
|
||
return provider == "ollama" or provider.startswith("ollama_")
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 背景監控迴圈
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def _monitor_loop(self) -> None:
|
||
"""
|
||
每 30s 檢查 111 健康狀態。
|
||
連續 3 次 HEALTHY 且 current_primary != "ollama" → 觸發切回。
|
||
"""
|
||
while True:
|
||
try:
|
||
await asyncio.sleep(self._recovery_check_interval_sec)
|
||
await self._check_and_recover()
|
||
except asyncio.CancelledError:
|
||
raise # 必須重新 raise，讓 stop() 的 await 正常結束
|
||
except Exception:
|
||
logger.exception(
|
||
"ollama_auto_recovery_loop_error",
|
||
service="ollama_auto_recovery",
|
||
)
|
||
# 不 break，繼續監控（異常不應停止監控迴圈）
|
||
|
||
async def _check_and_recover(self) -> None:
|
||
"""
|
||
單次健康狀態檢查 + 切回邏輯。
|
||
抽取為獨立方法，方便單元測試。
|
||
"""
|
||
host = self._settings.OLLAMA_URL
|
||
|
||
try:
|
||
health = await self._monitor.check(host)
|
||
except Exception:
|
||
logger.exception(
|
||
"ollama_auto_recovery_check_failed",
|
||
service="ollama_auto_recovery",
|
||
host=host,
|
||
)
|
||
self._consecutive_healthy = 0
|
||
return
|
||
|
||
# 2026-05-04 ogt: SLOW 視為可用（GCP 高負載 10-30s 仍優於 Gemini quota 限制）
|
||
# 原設計只接受 HEALTHY，GCP 若因負載偏高落入 SLOW 區會永久卡在 Gemini
|
||
is_usable = health.status in (HealthStatus.HEALTHY, HealthStatus.SLOW)
|
||
|
||
if is_usable:
|
||
self._consecutive_healthy += 1
|
||
logger.debug(
|
||
"ollama_auto_recovery_healthy_tick",
|
||
service="ollama_auto_recovery",
|
||
consecutive=self._consecutive_healthy,
|
||
required=self._stable_count_required,
|
||
current_primary=self._current_primary,
|
||
actual_status=health.status.value,
|
||
)
|
||
|
||
if (
|
||
self._consecutive_healthy >= self._stable_count_required
|
||
and not self._is_ollama_primary(self._current_primary)
|
||
):
|
||
await self._switch_back_to_ollama()
|
||
else:
|
||
# DEGRADED / OFFLINE → counter 歸零，繼續等
|
||
if self._consecutive_healthy > 0:
|
||
logger.debug(
|
||
"ollama_auto_recovery_counter_reset",
|
||
service="ollama_auto_recovery",
|
||
previous_count=self._consecutive_healthy,
|
||
status=health.status.value,
|
||
)
|
||
self._consecutive_healthy = 0
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 切回 Ollama 111
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def _switch_back_to_ollama(self) -> None:
|
||
"""
|
||
切回 Ollama 111：
|
||
1. 更新 _current_primary
|
||
2. 呼叫 failover_manager.clear_cache() 清空路由快取
|
||
3. 通知 failover_manager（notify_recovery）
|
||
4. Telegram 通知（若 alerter 已設定）
|
||
5. structlog audit
|
||
"""
|
||
from_provider = self._current_primary
|
||
self._current_primary = "ollama"
|
||
stable_count = self._consecutive_healthy
|
||
# counter 不立即清零：保留供 audit，下次 loop 開始後自然計數
|
||
self._consecutive_healthy = 0
|
||
|
||
# 取台北時間（標準庫保證 +8，禁 UTC）
|
||
# 2026-04-25 critic-fix Part2 B4 by Claude Engineer-C2
|
||
recovery_time = datetime.datetime.now(TAIPEI_TZ)
|
||
|
||
# 清空 failover manager 快取（讓下次 select_provider 重新評估）
|
||
try:
|
||
await self._failover_manager.clear_cache()
|
||
except Exception:
|
||
logger.exception(
|
||
"ollama_auto_recovery_clear_cache_error",
|
||
service="ollama_auto_recovery",
|
||
)
|
||
|
||
# 通知 failover_manager
|
||
try:
|
||
# 2026-05-03 ogt: ADR-110 — provider name 改為 "ollama_gcp_a"（GCP-A Primary）
|
||
self._failover_manager.notify_recovery("ollama_gcp_a")
|
||
except Exception:
|
||
logger.exception(
|
||
"ollama_auto_recovery_notify_error",
|
||
service="ollama_auto_recovery",
|
||
)
|
||
|
||
# Telegram 通知（P1.5 Engineer-D 接入）
|
||
# 2026-04-25 P1.5 by Claude Engineer-D — 接 FailoverAlerter，self._alerter 優先，
|
||
# 無則 fallback 到 get_failover_alerter() singleton
|
||
try:
|
||
alerter = self._alerter
|
||
if alerter is None:
|
||
from src.services.failover_alerter import get_failover_alerter
|
||
alerter = get_failover_alerter()
|
||
# 2026-05-05 ogt C1 修復：動態判斷 provider label，避免 hardcode "GCP-A"
|
||
# 依 OLLAMA_URL port 判斷：11435=GCP-A、11436=GCP-B、11437/111=Local
|
||
_recovered_url = getattr(self._settings, "OLLAMA_URL", "Ollama Primary")
|
||
if "11435" in _recovered_url:
|
||
_provider_label = "GCP-A"
|
||
elif "11436" in _recovered_url:
|
||
_provider_label = "GCP-B"
|
||
elif "11437" in _recovered_url or "192.168.0.111" in _recovered_url:
|
||
_provider_label = "Local"
|
||
else:
|
||
_provider_label = "Ollama"
|
||
await alerter.alert_recovery({
|
||
"from": from_provider,
|
||
"to": "ollama",
|
||
"recovered_host": f"{_provider_label} {_recovered_url}",
|
||
"stable_count": stable_count,
|
||
"recovery_time": recovery_time.isoformat(),
|
||
})
|
||
except Exception:
|
||
logger.exception(
|
||
"ollama_auto_recovery_telegram_alert_error",
|
||
service="ollama_auto_recovery",
|
||
)
|
||
|
||
# 2026-04-26 P2.3 by Claude Sonnet 4.6 (tool-expert) — 記錄 recovery Prometheus metric
|
||
try:
|
||
from src.core.metrics import (
|
||
OLLAMA_RECOVERY_TRIGGERED_TOTAL,
|
||
OLLAMA_CURRENT_PRIMARY_IS_OLLAMA,
|
||
)
|
||
OLLAMA_RECOVERY_TRIGGERED_TOTAL.labels(from_provider=from_provider).inc()
|
||
OLLAMA_CURRENT_PRIMARY_IS_OLLAMA.set(1)
|
||
except Exception as _metric_err:
|
||
logger.debug("ollama_recovery_metric_error", error=str(_metric_err))
|
||
|
||
# structlog audit（必須記錄）
|
||
logger.info(
|
||
"ollama_auto_recovery_switched_back",
|
||
service="ollama_auto_recovery",
|
||
action="switched_back",
|
||
from_provider=from_provider,
|
||
to_provider="ollama_gcp_a", # 2026-05-03 ogt: ADR-110
|
||
stable_count=stable_count,
|
||
recovery_time=recovery_time.isoformat(),
|
||
)
|
||
|
||
|
||
# =============================================================================
|
||
# Singleton
|
||
# =============================================================================
|
||
|
||
_recovery_service: OllamaAutoRecoveryService | None = None
|
||
|
||
|
||
def get_ollama_auto_recovery_service() -> OllamaAutoRecoveryService:
|
||
"""取得 OllamaAutoRecoveryService singleton"""
|
||
global _recovery_service
|
||
if _recovery_service is None:
|
||
_recovery_service = OllamaAutoRecoveryService()
|
||
return _recovery_service
|
||
|
||
|
||
def reset_ollama_auto_recovery_service() -> None:
|
||
"""重置 singleton（測試用）"""
|
||
global _recovery_service
|
||
_recovery_service = None
|