410 lines
15 KiB
Python
410 lines
15 KiB
Python
"""
|
||
Ollama 健康檢測模組 - P1.1a
|
||
============================
|
||
三層健康檢查：連通性 → 推理測試 → (optional) GPU 記憶體
|
||
|
||
設計原則：
|
||
- Redis 30s 快取防 health check storm
|
||
- 降級安全：快取失敗不影響功能，直接執行檢查
|
||
- 用 settings 取 host 配置（禁硬編碼 IP）
|
||
- 用 httpx.AsyncClient（非 requests）
|
||
|
||
版本: v1.1
|
||
建立: 2026-04-25 (台北時區)
|
||
建立者: Claude Engineer-C (P1.1a)
|
||
# Created 2026-04-25 P1.1 by Claude Engineer-C
|
||
# 2026-04-25 critic-fix by Claude Engineer-C — 修 BLOCKER B3(make_cache_key 公開) + H3(timeout 45s)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from enum import Enum
|
||
from urllib.parse import urlparse
|
||
|
||
import httpx
|
||
import structlog
|
||
|
||
from src.core.config import get_settings
|
||
from src.services.ollama_endpoint_circuit_breaker import (
|
||
get_ollama_endpoint_cooldown_remaining_seconds,
|
||
record_ollama_endpoint_failure,
|
||
record_ollama_endpoint_success,
|
||
)
|
||
|
||
logger = structlog.get_logger(__name__)
|
||
|
||
# =============================================================================
|
||
# 常數
|
||
# =============================================================================
|
||
|
||
REDIS_CACHE_KEY_PREFIX = "ollama_health:"
|
||
REDIS_CACHE_TTL_SECONDS = 30 # HEALTHY/SLOW/DEGRADED 快取 30s 防 check storm
|
||
# 2026-05-04 ogt: B1 修復 — OFFLINE 快取 TTL 縮短至 5s
|
||
# 根因：NetworkPolicy reload/CNI 瞬態抖動導致三台同時 OFFLINE，被 30s cache 放大
|
||
# OFFLINE 狀態快取越短越好，讓系統儘快重新評估並切回
|
||
REDIS_CACHE_TTL_OFFLINE_SECONDS = 5 # OFFLINE 只快取 5s，儘速重試
|
||
|
||
CONNECTIVITY_TIMEOUT_SECONDS = 5.0
|
||
# 2026-04-25 critic-fix H3 by Claude Engineer-C — 45s 讓 SLOW 門檻(30s)真的能觀察到
|
||
INFERENCE_TIMEOUT_SECONDS = 45.0
|
||
|
||
# 推理延遲分級門檻（毫秒）
|
||
LATENCY_HEALTHY_THRESHOLD_MS = 10_000 # <10s → HEALTHY
|
||
LATENCY_SLOW_THRESHOLD_MS = 30_000 # 10-30s → SLOW
|
||
# >30s → DEGRADED
|
||
|
||
|
||
# =============================================================================
|
||
# 公開工具函數
|
||
# =============================================================================
|
||
|
||
def make_cache_key(url: str) -> str:
|
||
"""
|
||
由 URL 產生 Redis cache key。
|
||
|
||
取 host:port 部分（netloc）作為 key，避免 scheme 差異干擾。
|
||
例：http://192.168.0.111:11434 → "ollama_health:192.168.0.111:11434"
|
||
|
||
公開函數供 OllamaFailoverManager.clear_cache() 動態組 key，
|
||
確保與內部 _cache_key() 使用相同邏輯。
|
||
|
||
# 2026-04-25 critic-fix B3 by Claude Engineer-C — 避免 clear_cache 硬編碼 IP
|
||
"""
|
||
parsed = urlparse(url)
|
||
netloc = parsed.netloc or url
|
||
return f"{REDIS_CACHE_KEY_PREFIX}{netloc}"
|
||
|
||
|
||
# =============================================================================
|
||
# 資料模型
|
||
# =============================================================================
|
||
|
||
|
||
class HealthStatus(Enum):
|
||
"""Ollama 健康狀態分級"""
|
||
|
||
HEALTHY = "healthy" # <10s 推理，正常使用
|
||
SLOW = "slow" # 10-30s，降級使用
|
||
DEGRADED = "degraded" # >30s 或推理超時，優先切換
|
||
OFFLINE = "offline" # 連通性失敗，必須切換
|
||
|
||
|
||
@dataclass
|
||
class HealthReport:
|
||
"""Ollama 健康檢測報告"""
|
||
|
||
status: HealthStatus
|
||
host: str = ""
|
||
latency_ms: float = 0.0
|
||
reason: str = ""
|
||
checked_at: float = field(default_factory=time.time)
|
||
from_cache: bool = False
|
||
mcp_health: dict[str, bool] = field(default_factory=dict)
|
||
|
||
def is_usable(self) -> bool:
|
||
"""是否可用（不含 OFFLINE）"""
|
||
return self.status != HealthStatus.OFFLINE
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"status": self.status.value,
|
||
"host": self.host,
|
||
"latency_ms": round(self.latency_ms, 1),
|
||
"reason": self.reason,
|
||
"checked_at": self.checked_at,
|
||
"from_cache": self.from_cache,
|
||
}
|
||
|
||
|
||
# =============================================================================
|
||
# OllamaHealthMonitor
|
||
# =============================================================================
|
||
|
||
|
||
class OllamaHealthMonitor:
|
||
"""
|
||
Ollama 三層健康檢測
|
||
|
||
層 1：連通性（/api/tags，5s timeout）
|
||
層 2：推理測試（/api/generate，35s timeout）
|
||
層 3：GPU 記憶體（optional，via SSH MCP）
|
||
|
||
結果 Redis 快取 30s，防 health check storm。
|
||
|
||
2026-04-25 Claude Engineer-C (P1.1a)
|
||
"""
|
||
|
||
def __init__(self) -> None:
|
||
self._settings = get_settings()
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Public API
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def check(self, host: str) -> HealthReport:
|
||
"""
|
||
執行健康檢測（含 Redis 快取）
|
||
|
||
Args:
|
||
host: Ollama 主機 URL，e.g. "http://192.168.0.111:11434"
|
||
|
||
Returns:
|
||
HealthReport
|
||
"""
|
||
# 嘗試從快取取得
|
||
cached = await self._get_cached(host)
|
||
if cached is not None:
|
||
cached.from_cache = True
|
||
logger.debug("ollama_health_cache_hit", host=host, status=cached.status.value)
|
||
return cached
|
||
|
||
# 執行實際健康檢測
|
||
report = await self._run_checks(host)
|
||
report.host = host
|
||
|
||
# 寫入快取（失敗不影響功能，外部 exception 也靜默）
|
||
try:
|
||
await self._set_cached(host, report)
|
||
except Exception as e:
|
||
logger.debug("ollama_health_set_cached_outer_failed", host=host, error=str(e))
|
||
|
||
logger.info(
|
||
"ollama_health_checked",
|
||
host=host,
|
||
status=report.status.value,
|
||
latency_ms=round(report.latency_ms, 1),
|
||
reason=report.reason,
|
||
)
|
||
|
||
# 2026-04-26 P2.3 by Claude Sonnet 4.6 (tool-expert) — 更新 Prometheus health gauge
|
||
# host label 取 "111" / "188" 短標識（從 URL 解析）
|
||
try:
|
||
from src.core.metrics import OLLAMA_HEALTH_STATUS
|
||
from urllib.parse import urlparse as _urlparse
|
||
_netloc = _urlparse(host).hostname or host
|
||
# 2026-05-03 ogt: GCP 三層容災（ADR-110）— 通用 host label 萃取
|
||
# 私網 IP（192.168.x.y）→ 取最後一段（"111"/"188"）維持短標識
|
||
# GCP 公網 IP（34.x.y.z）→ 取前 8 字元避免 label 過長（"34.143.1"）
|
||
if _netloc.startswith("192.168."):
|
||
_host_label = _netloc.split(".")[-1]
|
||
elif "." in _netloc:
|
||
_host_label = _netloc.replace(".", "_")[:12] # e.g. "34_143_170_2"
|
||
else:
|
||
_host_label = _netloc
|
||
_is_healthy = 1 if report.status == HealthStatus.HEALTHY else 0
|
||
OLLAMA_HEALTH_STATUS.labels(host=_host_label).set(_is_healthy)
|
||
except Exception as _metric_err:
|
||
logger.debug("ollama_health_metric_error", host=host, error=str(_metric_err))
|
||
|
||
# 寫入 audit_log（best-effort）
|
||
await self._write_audit_log(host, report)
|
||
|
||
return report
|
||
|
||
# -------------------------------------------------------------------------
|
||
# 三層檢查
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def _run_checks(self, host: str) -> HealthReport:
|
||
"""依序執行三層檢查"""
|
||
cooldown_remaining = get_ollama_endpoint_cooldown_remaining_seconds(host)
|
||
if cooldown_remaining > 0:
|
||
return HealthReport(
|
||
status=HealthStatus.OFFLINE,
|
||
host=host,
|
||
reason=(
|
||
"recent_endpoint_failure_cooldown:"
|
||
f"{cooldown_remaining:.0f}s"
|
||
),
|
||
)
|
||
|
||
# 層 1：連通性
|
||
connectivity_ok = await self._check_connectivity(host)
|
||
if not connectivity_ok:
|
||
record_ollama_endpoint_failure(host)
|
||
return HealthReport(
|
||
status=HealthStatus.OFFLINE,
|
||
host=host,
|
||
reason="連通性失敗：/api/tags 無回應",
|
||
)
|
||
|
||
# 層 2：推理測試
|
||
report = await self._check_inference(host)
|
||
record_ollama_endpoint_success(host)
|
||
return report
|
||
|
||
async def _check_connectivity(self, host: str) -> bool:
|
||
"""
|
||
層 1：連通性檢查
|
||
GET /api/tags，5s timeout，回傳 200 即通過
|
||
"""
|
||
try:
|
||
async with httpx.AsyncClient(
|
||
timeout=httpx.Timeout(CONNECTIVITY_TIMEOUT_SECONDS)
|
||
) as client:
|
||
resp = await client.get(f"{host}/api/tags")
|
||
return resp.status_code == 200
|
||
except (httpx.TimeoutException, httpx.ConnectError, httpx.NetworkError):
|
||
return False
|
||
except Exception as e:
|
||
logger.warning("ollama_connectivity_check_error", host=host, error=str(e))
|
||
return False
|
||
|
||
async def _check_inference(self, host: str) -> HealthReport:
|
||
"""
|
||
層 2：推理測試
|
||
POST /api/generate，35s timeout，依延遲分級
|
||
"""
|
||
model = self._settings.OLLAMA_HEALTH_CHECK_MODEL
|
||
start = time.perf_counter()
|
||
|
||
try:
|
||
async with httpx.AsyncClient(
|
||
timeout=httpx.Timeout(INFERENCE_TIMEOUT_SECONDS)
|
||
) as client:
|
||
resp = await client.post(
|
||
f"{host}/api/generate",
|
||
json={
|
||
"model": model,
|
||
"prompt": "hi",
|
||
"stream": False,
|
||
"options": {"num_predict": 1},
|
||
},
|
||
)
|
||
latency_ms = (time.perf_counter() - start) * 1000
|
||
|
||
if resp.status_code != 200:
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
latency_ms=latency_ms,
|
||
reason=f"推理回傳 HTTP {resp.status_code}",
|
||
)
|
||
|
||
# 分級判斷
|
||
if latency_ms < LATENCY_HEALTHY_THRESHOLD_MS:
|
||
return HealthReport(
|
||
status=HealthStatus.HEALTHY,
|
||
latency_ms=latency_ms,
|
||
)
|
||
elif latency_ms < LATENCY_SLOW_THRESHOLD_MS:
|
||
return HealthReport(
|
||
status=HealthStatus.SLOW,
|
||
latency_ms=latency_ms,
|
||
reason=f"推理延遲 {latency_ms:.0f}ms（slow zone）",
|
||
)
|
||
else:
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
latency_ms=latency_ms,
|
||
reason=f"推理延遲 {latency_ms:.0f}ms（>30s）",
|
||
)
|
||
|
||
except (httpx.TimeoutException, asyncio.TimeoutError):
|
||
latency_ms = (time.perf_counter() - start) * 1000
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
latency_ms=latency_ms,
|
||
reason="推理超時 >35s",
|
||
)
|
||
except (httpx.ConnectError, httpx.NetworkError) as e:
|
||
# 2026-05-04 ogt: B3 修復 — connectivity 已通過，推理階段 ConnectError 改判 DEGRADED
|
||
# 原設計：ConnectError → OFFLINE，但 /api/tags 已成功，表示主機存活
|
||
# 根因：socket 半開（GCP LB 回收 idle conn）或 Ollama 進程重啟，屬瞬態
|
||
# DEGRADED 不觸發 30s OFFLINE cache，下次請求立刻重試
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
reason=f"推理連接失敗（主機可達，socket 瞬斷）：{e}",
|
||
)
|
||
except Exception as e:
|
||
logger.warning("ollama_inference_check_error", host=host, error=str(e))
|
||
return HealthReport(
|
||
status=HealthStatus.DEGRADED,
|
||
reason=f"推理測試例外：{e}",
|
||
)
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Redis 快取
|
||
# -------------------------------------------------------------------------
|
||
|
||
def _cache_key(self, host: str) -> str:
|
||
"""生成 Redis cache key（內部使用，委派至 make_cache_key）"""
|
||
return make_cache_key(host)
|
||
|
||
async def _get_cached(self, host: str) -> HealthReport | None:
|
||
"""從 Redis 取得快取的 HealthReport，失敗返回 None"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
redis = get_redis()
|
||
raw = await redis.get(self._cache_key(host))
|
||
if not raw:
|
||
return None
|
||
data = json.loads(raw)
|
||
return HealthReport(
|
||
status=HealthStatus(data["status"]),
|
||
host=data.get("host", host),
|
||
latency_ms=data.get("latency_ms", 0.0),
|
||
reason=data.get("reason", ""),
|
||
checked_at=data.get("checked_at", time.time()),
|
||
)
|
||
except Exception as e:
|
||
logger.debug("ollama_health_cache_get_failed", host=host, error=str(e))
|
||
return None
|
||
|
||
async def _set_cached(self, host: str, report: HealthReport) -> None:
|
||
"""寫入 Redis 快取，失敗靜默（不影響功能）
|
||
2026-05-04 ogt: OFFLINE 結果快取 5s（縮短），其他狀態快取 30s
|
||
"""
|
||
try:
|
||
from src.core.redis_client import get_redis
|
||
redis = get_redis()
|
||
data = json.dumps(report.to_dict())
|
||
ttl = (
|
||
REDIS_CACHE_TTL_OFFLINE_SECONDS
|
||
if report.status == HealthStatus.OFFLINE
|
||
else REDIS_CACHE_TTL_SECONDS
|
||
)
|
||
await redis.set(self._cache_key(host), data, ex=ttl)
|
||
except Exception as e:
|
||
logger.debug("ollama_health_cache_set_failed", host=host, error=str(e))
|
||
|
||
# -------------------------------------------------------------------------
|
||
# Audit Log (structured log，AuditLog 表設計用於 K8s 操作審計，不適合此場景)
|
||
# -------------------------------------------------------------------------
|
||
|
||
async def _write_audit_log(self, host: str, report: HealthReport) -> None:
|
||
"""記錄健康檢測結果（structlog，service=ollama_health_monitor）"""
|
||
logger.info(
|
||
"ollama_health_monitor_audit",
|
||
service="ollama_health_monitor",
|
||
host=host,
|
||
status=report.status.value,
|
||
latency_ms=round(report.latency_ms, 1),
|
||
reason=report.reason,
|
||
is_usable=report.is_usable(),
|
||
)
|
||
|
||
|
||
# =============================================================================
|
||
# Singleton
|
||
# =============================================================================
|
||
|
||
_monitor: OllamaHealthMonitor | None = None
|
||
|
||
|
||
def get_ollama_health_monitor() -> OllamaHealthMonitor:
|
||
"""取得 OllamaHealthMonitor singleton"""
|
||
global _monitor
|
||
if _monitor is None:
|
||
_monitor = OllamaHealthMonitor()
|
||
return _monitor
|
||
|
||
|
||
def reset_ollama_health_monitor() -> None:
|
||
"""重置 singleton（測試用）"""
|
||
global _monitor
|
||
_monitor = None
|