55c6b4e2d9
ADR-092 P1 飛輪閉環的 Ollama 失敗轉移子系統，全部 Engineer-A2/C/C2 補上。 新服務 (1581 行)： - ollama_health_monitor.py (356)：3 層健康檢測（TCP/HTTP/推理） - ollama_failover_manager.py (571)：111→188 自動切換 + Redis 持久化 + recovery callback - ollama_auto_recovery.py (436)：30s 背景監控 + 連續 3 次 HEALTHY → 切回 + clear_cache - failover_alerter.py (218)：P1.5 Telegram 容災告警 服務整合： - ai_router.py: AIProviderEnum.OLLAMA_188 + 120s budget + failover fallback chain - main.py lifespan: 啟動時 wire callback + start recovery，關閉時優雅 stop - config.py: OLLAMA_FALLBACK_URL / OLLAMA_HEALTH_CHECK_MODEL / GEMINI_DAILY_QUOTA（帳單熔斷） K8s 配置： - 04-configmap.yaml.patch-188-fallback：注入 OLLAMA_FALLBACK_URL=http://192.168.0.188:11434 測試 (2082 行)： - test_ollama_health_monitor.py (402) - test_ollama_failover_manager.py (707) - test_ollama_auto_recovery.py (580) - test_ai_router_failover_integration.py (257) - test_lifespan_failover_wiring.py (136) 依賴鏈：service 三件套 + ai_router + main.py 一起 commit，缺一就 ImportError。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
357 lines
12 KiB
Python
357 lines
12 KiB
Python
"""
|
||
Ollama 健康檢測模組 - P1.1a
|
||
============================
|
||
三層健康檢查：連通性 → 推理測試 → (optional) GPU 記憶體
|
||
|
||
設計原則：
|
||
- Redis 30s 快取防 health check storm
|
||
- 降級安全：快取失敗不影響功能，直接執行檢查
|
||
- 用 settings 取 host 配置（禁硬編碼 IP）
|
||
- 用 httpx.AsyncClient（非 requests）
|
||
|
||
版本: v1.1
|
||
建立: 2026-04-25 (台北時區)
|
||
建立者: Claude Engineer-C (P1.1a)
|
||
# Created 2026-04-25 P1.1 by Claude Engineer-C
|
||
# 2026-04-25 critic-fix by Claude Engineer-C — 修 BLOCKER B3(make_cache_key 公開) + H3(timeout 45s)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from enum import Enum
|
||
from urllib.parse import urlparse
|
||
|
||
import httpx
|
||
import structlog
|
||
|
||
from src.core.config import get_settings
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
# =============================================================================
|
||
# 常數
|
||
# =============================================================================
|
||
|
||
REDIS_CACHE_KEY_PREFIX = "ollama_health:"
|
||
REDIS_CACHE_TTL_SECONDS = 30 # 防 health check storm
|
||
|
||
CONNECTIVITY_TIMEOUT_SECONDS = 5.0
|
||
# 2026-04-25 critic-fix H3 by Claude Engineer-C — 45s 讓 SLOW 門檻(30s)真的能觀察到
|
||
INFERENCE_TIMEOUT_SECONDS = 45.0
|
||
|
||
# 推理延遲分級門檻（毫秒）
|
||
LATENCY_HEALTHY_THRESHOLD_MS = 10_000 # <10s → HEALTHY
|
||
LATENCY_SLOW_THRESHOLD_MS = 30_000 # 10-30s → SLOW
|
||
# >30s → DEGRADED
|
||
|
||
|
||
# =============================================================================
|
||
# 公開工具函數
|
||
# =============================================================================
|
||
|
||
def make_cache_key(url: str) -> str:
|
||
"""
|
||
由 URL 產生 Redis cache key。
|
||
|
||
取 host:port 部分（netloc）作為 key，避免 scheme 差異干擾。
|
||
例：http://192.168.0.111:11434 → "ollama_health:192.168.0.111:11434"
|
||
|
||
公開函數供 OllamaFailoverManager.clear_cache() 動態組 key，
|
||
確保與內部 _cache_key() 使用相同邏輯。
|
||
|
||
# 2026-04-25 critic-fix B3 by Claude Engineer-C — 避免 clear_cache 硬編碼 IP
|
||
"""
|
||
parsed = urlparse(url)
|
||
netloc = parsed.netloc or url
|
||
return f"{REDIS_CACHE_KEY_PREFIX}{netloc}"
|
||
|
||
|
||
# =============================================================================
|
||
# 資料模型
|
||
# =============================================================================
|
||
|
||
|
||
class HealthStatus(Enum):
|
||
"""Ollama 健康狀態分級"""
|
||
|
||
HEALTHY = "healthy" # <10s 推理，正常使用
|
||
SLOW = "slow" # 10-30s，降級使用
|
||
DEGRADED = "degraded" # >30s 或推理超時，優先切換
|
||
OFFLINE = "offline" # 連通性失敗，必須切換
|
||
|
||
|
||
@dataclass
|
||
class HealthReport:
|
||
"""Ollama 健康檢測報告"""
|
||
|
||
status: HealthStatus
|
||
host: str = ""
|
||
latency_ms: float = 0.0
|
||
reason: str = ""
|
||
checked_at: float = field(default_factory=time.time)
|
||
from_cache: bool = False
|
||
mcp_health: dict[str, bool] = field(default_factory=dict)
|
||
|
||
def is_usable(self) -> bool:
|
||
"""是否可用（不含 OFFLINE）"""
|
||
return self.status != HealthStatus.OFFLINE
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"status": self.status.value,
|
||
"host": self.host,
|
||
"latency_ms": round(self.latency_ms, 1),
|
||
"reason": self.reason,
|
||
"checked_at": self.checked_at,
|
||
"from_cache": self.from_cache,
|
||
}
|
||
|
||
|
||
# =============================================================================
|
||
# OllamaHealthMonitor
|
||
# =============================================================================
|
||
|
||
|
||
class OllamaHealthMonitor:
|
||
"""
|
||
Ollama 三層健康檢測
|
||
|
||
層 1：連通性（/api/tags，5s timeout）
|
||
層 2：推理測試（/api/generate，35s timeout）
|
||
層 3：GPU 記憶體（optional，via SSH MCP）
|
||
|
||
結果 Redis 快取 30s，防 health check storm。
|
||
|
||
2026-04-25 Claude Engineer-C (P1.1a)
|
||
"""
|
||
|
||
def __init__(self) -> None:
|
||
self._settings = get_settings()
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Public API
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def check(self, host: str) -> HealthReport:
|
||
"""
|
||
執行健康檢測（含 Redis 快取）
|
||
|
||
Args:
|
||
host: Ollama 主機 URL，e.g. "http://192.168.0.111:11434"
|
||
|
||
Returns:
|
||
HealthReport
|
||
"""
|
||
# 嘗試從快取取得
|
||
cached = await self._get_cached(host)
|
||
if cached is not None:
|
||
cached.from_cache = True
|
||
logger.debug("ollama_health_cache_hit", host=host, status=cached.status.value)
|
||
return cached
|
||
|
||
# 執行實際健康檢測
|
||
report = await self._run_checks(host)
|
||
report.host = host
|
||
|
||
# 寫入快取（失敗不影響功能，外部 exception 也靜默）
|
||
try:
|
||
await self._set_cached(host, report)
|
||
except Exception as e:
|
||
logger.debug("ollama_health_set_cached_outer_failed", host=host, error=str(e))
|
||
|
||
logger.info(
|
||
"ollama_health_checked",
|
||
host=host,
|
||
status=report.status.value,
|
||
latency_ms=round(report.latency_ms, 1),
|
||
reason=report.reason,
|
||
)
|
||
|
||
# 寫入 audit_log（best-effort）
|
||
await self._write_audit_log(host, report)
|
||
|
||
return report
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 三層檢查
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def _run_checks(self, host: str) -> HealthReport:
|
||
"""依序執行三層檢查"""
|
||
# 層 1：連通性
|
||
connectivity_ok = await self._check_connectivity(host)
|
||
if not connectivity_ok:
|
||
return HealthReport(
|
||
status=HealthStatus.OFFLINE,
|
||
host=host,
|
||
reason="連通性失敗：/api/tags 無回應",
|
||
)
|
||
|
||
# 層 2：推理測試
|
||
report = await self._check_inference(host)
|
||
return report
|
||
|
||
async def _check_connectivity(self, host: str) -> bool:
|
||
"""
|
||
層 1：連通性檢查
|
||
GET /api/tags，5s timeout，回傳 200 即通過
|
||
"""
|
||
try:
|
||
async with httpx.AsyncClient(
|
||
timeout=httpx.Timeout(CONNECTIVITY_TIMEOUT_SECONDS)
|
||
) as client:
|
||
resp = await client.get(f"{host}/api/tags")
|
||
return resp.status_code == 200
|
||
except (httpx.TimeoutException, httpx.ConnectError, httpx.NetworkError):
|
||
return False
|
||
except Exception as e:
|
||
logger.warning("ollama_connectivity_check_error", host=host, error=str(e))
|
||
return False
|
||
|
||
async def _check_inference(self, host: str) -> HealthReport:
|
||
"""
|
||
層 2：推理測試
|
||
POST /api/generate，35s timeout，依延遲分級
|
||
"""
|
||
model = self._settings.OLLAMA_HEALTH_CHECK_MODEL
|
||
start = time.perf_counter()
|
||
|
||
try:
|
||
async with httpx.AsyncClient(
|
||
timeout=httpx.Timeout(INFERENCE_TIMEOUT_SECONDS)
|
||
) as client:
|
||
resp = await client.post(
|
||
f"{host}/api/generate",
|
||
json={
|
||
"model": model,
|
||
"prompt": "hi",
|
||
"stream": False,
|
||
"options": {"num_predict": 1},
|
||
},
|
||
)
|
||
latency_ms = (time.perf_counter() - start) * 1000
|
||
|
||
if resp.status_code != 200:
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
latency_ms=latency_ms,
|
||
reason=f"推理回傳 HTTP {resp.status_code}",
|
||
)
|
||
|
||
# 分級判斷
|
||
if latency_ms < LATENCY_HEALTHY_THRESHOLD_MS:
|
||
return HealthReport(
|
||
status=HealthStatus.HEALTHY,
|
||
latency_ms=latency_ms,
|
||
)
|
||
elif latency_ms < LATENCY_SLOW_THRESHOLD_MS:
|
||
return HealthReport(
|
||
status=HealthStatus.SLOW,
|
||
latency_ms=latency_ms,
|
||
reason=f"推理延遲 {latency_ms:.0f}ms（slow zone）",
|
||
)
|
||
else:
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
latency_ms=latency_ms,
|
||
reason=f"推理延遲 {latency_ms:.0f}ms（>30s）",
|
||
)
|
||
|
||
except (httpx.TimeoutException, asyncio.TimeoutError):
|
||
latency_ms = (time.perf_counter() - start) * 1000
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
latency_ms=latency_ms,
|
||
reason="推理超時 >35s",
|
||
)
|
||
except (httpx.ConnectError, httpx.NetworkError) as e:
|
||
return HealthReport(
|
||
status=HealthStatus.OFFLINE,
|
||
reason=f"推理連接失敗：{e}",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("ollama_inference_check_error", host=host, error=str(e))
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
reason=f"推理測試例外：{e}",
|
||
)
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Redis 快取
|
||
# -------------------------------------------------------------------------
|
||
|
||
def _cache_key(self, host: str) -> str:
|
||
"""生成 Redis cache key（內部使用，委派至 make_cache_key）"""
|
||
return make_cache_key(host)
|
||
|
||
async def _get_cached(self, host: str) -> HealthReport | None:
|
||
"""從 Redis 取得快取的 HealthReport，失敗返回 None"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
redis = get_redis()
|
||
raw = await redis.get(self._cache_key(host))
|
||
if not raw:
|
||
return None
|
||
data = json.loads(raw)
|
||
return HealthReport(
|
||
status=HealthStatus(data["status"]),
|
||
host=data.get("host", host),
|
||
latency_ms=data.get("latency_ms", 0.0),
|
||
reason=data.get("reason", ""),
|
||
checked_at=data.get("checked_at", time.time()),
|
||
)
|
||
except Exception as e:
|
||
logger.debug("ollama_health_cache_get_failed", host=host, error=str(e))
|
||
return None
|
||
|
||
async def _set_cached(self, host: str, report: HealthReport) -> None:
|
||
"""寫入 Redis 快取，失敗靜默（不影響功能）"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
redis = get_redis()
|
||
data = json.dumps(report.to_dict())
|
||
await redis.set(self._cache_key(host), data, ex=REDIS_CACHE_TTL_SECONDS)
|
||
except Exception as e:
|
||
logger.debug("ollama_health_cache_set_failed", host=host, error=str(e))
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Audit Log (structured log，AuditLog 表設計用於 K8s 操作審計，不適合此場景)
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def _write_audit_log(self, host: str, report: HealthReport) -> None:
|
||
"""記錄健康檢測結果（structlog，service=ollama_health_monitor）"""
|
||
logger.info(
|
||
"ollama_health_monitor_audit",
|
||
service="ollama_health_monitor",
|
||
host=host,
|
||
status=report.status.value,
|
||
latency_ms=round(report.latency_ms, 1),
|
||
reason=report.reason,
|
||
is_usable=report.is_usable(),
|
||
)
|
||
|
||
|
||
# =============================================================================
|
||
# Singleton
|
||
# =============================================================================
|
||
|
||
_monitor: OllamaHealthMonitor | None = None
|
||
|
||
|
||
def get_ollama_health_monitor() -> OllamaHealthMonitor:
|
||
"""取得 OllamaHealthMonitor singleton"""
|
||
global _monitor
|
||
if _monitor is None:
|
||
_monitor = OllamaHealthMonitor()
|
||
return _monitor
|
||
|
||
|
||
def reset_ollama_health_monitor() -> None:
|
||
"""重置 singleton（測試用）"""
|
||
global _monitor
|
||
_monitor = None
|