feat(p1): Ollama 多層容災系統 — P1.1 健康檢測 + P1.2 ai_router 整合 + P1.5 容災告警
ADR-092 P1 飛輪閉環的 Ollama 失敗轉移子系統，全部 Engineer-A2/C/C2 補上。 新服務 (1581 行)： - ollama_health_monitor.py (356)：3 層健康檢測（TCP/HTTP/推理） - ollama_failover_manager.py (571)：111→188 自動切換 + Redis 持久化 + recovery callback - ollama_auto_recovery.py (436)：30s 背景監控 + 連續 3 次 HEALTHY → 切回 + clear_cache - failover_alerter.py (218)：P1.5 Telegram 容災告警 服務整合： - ai_router.py: AIProviderEnum.OLLAMA_188 + 120s budget + failover fallback chain - main.py lifespan: 啟動時 wire callback + start recovery，關閉時優雅 stop - config.py: OLLAMA_FALLBACK_URL / OLLAMA_HEALTH_CHECK_MODEL / GEMINI_DAILY_QUOTA（帳單熔斷） K8s 配置： - 04-configmap.yaml.patch-188-fallback：注入 OLLAMA_FALLBACK_URL=http://192.168.0.188:11434 測試 (2082 行)： - test_ollama_health_monitor.py (402) - test_ollama_failover_manager.py (707) - test_ollama_auto_recovery.py (580) - test_ai_router_failover_integration.py (257) - test_lifespan_failover_wiring.py (136) 依賴鏈：service 三件套 + ai_router + main.py 一起 commit，缺一就 ImportError。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Your Name
218
apps/api/src/services/failover_alerter.py
Normal file
218
apps/api/src/services/failover_alerter.py
Normal file
@@ -0,0 +1,218 @@
|
||||
"""Ollama 容災 / 自動恢復 / Gemini 帳單 Telegram 告警
|
||||
|
||||
設計原則：
|
||||
- 每次 failover 觸發都通知（用戶明確指示）
|
||||
- 但用 10min Redis dedup TTL 防同樣狀態重複告警
|
||||
- 三種告警類型：failover_triggered / recovery_succeeded / gemini_quota_exceeded
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D — Telegram Alerter for Ollama Failover
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
TAIPEI_TZ = timezone(timedelta(hours=8))
|
||||
DEDUP_TTL_SEC = 600 # 10 min
|
||||
QUOTA_DEDUP_TTL_SEC = 86400 # 24h（每日 quota 告警只發一次）
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
class FailoverAlerter:
|
||||
"""Ollama 容災 Telegram 告警
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D — Telegram Alerter for Ollama Failover
|
||||
"""
|
||||
|
||||
def __init__(self, redis_client=None) -> None:
|
||||
# telegram_gateway 從 singleton 取，不注入（lifespan 已確保初始化）
|
||||
self._redis = redis_client
|
||||
|
||||
async def alert_failover(self, event: dict[str, Any]) -> None:
|
||||
"""111 故障切換到 Gemini/188 — 10min dedup"""
|
||||
to_provider = event.get("to_provider", "unknown")
|
||||
dedup_key = f"alert:failover:{to_provider}"
|
||||
if not await self._check_dedup(dedup_key, ttl=DEDUP_TTL_SEC):
|
||||
logger.debug("failover_alert_dedup_skipped", to_provider=to_provider)
|
||||
return
|
||||
|
||||
reason = event.get("reason", "unknown")
|
||||
model = event.get("model", "?")
|
||||
timestamp = event.get("timestamp", datetime.now(TAIPEI_TZ).isoformat())
|
||||
fallback_chain_str = event.get("fallback_chain_str", "?")
|
||||
|
||||
msg = (
|
||||
f"*Ollama 容災激活*\n\n"
|
||||
f"故障主機：Ollama 111 \\(GPU\\)\n"
|
||||
f"故障狀態：{_escape_md(reason)}\n"
|
||||
f"切換目標：{_escape_md(to_provider)} \\(model: {_escape_md(model)}\\)\n"
|
||||
f"切換時間：{_escape_md(timestamp)}\n\n"
|
||||
f"Fallback 鏈：{_escape_md(fallback_chain_str)}\n\n"
|
||||
f"自動恢復服務持續監控，3 次 HEALTHY 後自動切回"
|
||||
)
|
||||
await self._send(msg)
|
||||
logger.info("failover_alert_sent", to_provider=to_provider)
|
||||
|
||||
async def alert_recovery(self, event: dict[str, Any]) -> None:
|
||||
"""111 恢復後自動切回 — 10min dedup
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D — 用戶明確要求要通知
|
||||
"""
|
||||
dedup_key = "alert:recovery"
|
||||
if not await self._check_dedup(dedup_key, ttl=DEDUP_TTL_SEC):
|
||||
logger.debug("recovery_alert_dedup_skipped")
|
||||
return
|
||||
|
||||
stable_count = event.get("stable_count", event.get("to", "?"))
|
||||
# 相容 auto_recovery 傳入的 {"from": ..., "to": ...} 格式
|
||||
from_provider = event.get("from_provider", event.get("from", "?"))
|
||||
to_provider = event.get("to_provider", event.get("to", "ollama_111"))
|
||||
recovery_time = event.get("recovery_time", datetime.now(TAIPEI_TZ).isoformat())
|
||||
|
||||
msg = (
|
||||
f"*Ollama 自動恢復*\n\n"
|
||||
f"恢復主機：Ollama 111 \\(GPU\\)\n"
|
||||
f"穩定計數：連續 {stable_count} 次 HEALTHY\n"
|
||||
f"切回時間：{_escape_md(str(recovery_time))}\n"
|
||||
f"切換路徑：{_escape_md(str(from_provider))} → {_escape_md(str(to_provider))}\n\n"
|
||||
f"自動化飛輪已恢復至 GPU 推理模式"
|
||||
)
|
||||
await self._send(msg)
|
||||
logger.info("recovery_alert_sent", from_provider=from_provider)
|
||||
|
||||
async def alert_gemini_quota_exceeded(self, event: dict[str, Any]) -> None:
|
||||
"""Gemini 每日上限觸發，降級到 188 CPU 備援 — 24h dedup"""
|
||||
dedup_key = "alert:gemini_quota_exceeded"
|
||||
if not await self._check_dedup(dedup_key, ttl=QUOTA_DEDUP_TTL_SEC):
|
||||
logger.debug("quota_alert_dedup_skipped")
|
||||
return
|
||||
|
||||
quota = event.get("quota", "?")
|
||||
current_count = event.get("current_count", "?")
|
||||
date_str = datetime.now(TAIPEI_TZ).date().isoformat()
|
||||
|
||||
msg = (
|
||||
f"*Gemini 每日配額耗盡*\n\n"
|
||||
f"日期：{date_str}\n"
|
||||
f"上限：{quota} calls/day\n"
|
||||
f"當前用量：{current_count}\n"
|
||||
f"降級目標：OLLAMA\\_188 \\(CPU，推理較慢\\)\n\n"
|
||||
f"進入慢速模式至明日 0:00\n"
|
||||
f"建議檢查是否有異常流量，評估是否升級 Gemini 配額"
|
||||
)
|
||||
await self._send(msg)
|
||||
logger.info("quota_alert_sent", quota=quota, current_count=current_count)
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Dedup（Redis SET NX EX）
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
async def _check_dedup(self, key: str, ttl: int) -> bool:
|
||||
"""
|
||||
Redis SET NX EX 防止重複告警。
|
||||
True = 第一次（應送出），False = 已送過（跳過）。
|
||||
|
||||
fail-open：Redis 不可用時允許送出（不阻擋通知）
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D — Telegram dedup 鐵律 10min/24h TTL
|
||||
"""
|
||||
if self._redis is None:
|
||||
return True # fail-open
|
||||
try:
|
||||
ok = await self._redis.set(f"{key}:dedup", "1", ex=ttl, nx=True)
|
||||
return bool(ok)
|
||||
except Exception as e:
|
||||
logger.warning("dedup_check_failed", error=str(e))
|
||||
return True # fail-open
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# 發送（透過 TelegramGateway singleton）
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
async def _send(self, message: str) -> None:
|
||||
"""發送至 Telegram OPENCLAW_TG_CHAT_ID
|
||||
|
||||
使用現有 TelegramGateway singleton（不另建 HTTP client），
|
||||
parse_mode=MarkdownV2 對應 MarkdownV2 escape 規則。
|
||||
|
||||
alerter 失敗不阻斷主路由邏輯（exception 吞掉只 log）
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D — 告警失敗不能阻斷主流程
|
||||
"""
|
||||
try:
|
||||
from src.services.telegram_gateway import get_telegram_gateway
|
||||
from src.core.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
chat_id = getattr(settings, "OPENCLAW_TG_CHAT_ID", None)
|
||||
if not chat_id:
|
||||
logger.warning("telegram_chat_id_missing_failover_alert")
|
||||
return
|
||||
|
||||
gateway = get_telegram_gateway()
|
||||
await gateway.send_notification(text=message, parse_mode="MarkdownV2")
|
||||
logger.info("telegram_failover_alert_sent", message_len=len(message))
|
||||
except Exception as e:
|
||||
# 不 raise — 告警失敗不該阻斷主流程（鐵律）
|
||||
logger.exception("telegram_failover_send_failed", error=str(e))
|
||||
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# MarkdownV2 escape 工具
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
_MD2_SPECIAL = r"\_*[]()~`>#+-=|{}.!"
|
||||
|
||||
|
||||
def _escape_md(text: str) -> str:
|
||||
"""Escape MarkdownV2 特殊字元，防止 Telegram parse error。
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D — MarkdownV2 安全逸出
|
||||
"""
|
||||
for ch in _MD2_SPECIAL:
|
||||
text = text.replace(ch, f"\\{ch}")
|
||||
return text
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Singleton
|
||||
# =============================================================================
|
||||
|
||||
_alerter_instance: FailoverAlerter | None = None
|
||||
|
||||
|
||||
def get_failover_alerter() -> FailoverAlerter:
|
||||
"""取得 FailoverAlerter singleton（lifespan 中注入依賴前也可安全呼叫，僅 fail-open）
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D — Singleton 取得 alerter
|
||||
"""
|
||||
global _alerter_instance
|
||||
if _alerter_instance is None:
|
||||
_alerter_instance = FailoverAlerter()
|
||||
return _alerter_instance
|
||||
|
||||
|
||||
def configure_alerter(redis_client) -> None:
|
||||
"""Lifespan 注入：redis_client 就緒後呼叫，讓 dedup 功能生效。
|
||||
|
||||
telegram 不另外注入，直接從 get_telegram_gateway() singleton 取得
|
||||
（lifespan startup 已保證 TelegramGateway 初始化完成）。
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D — Lifespan 注入
|
||||
"""
|
||||
global _alerter_instance
|
||||
_alerter_instance = FailoverAlerter(redis_client=redis_client)
|
||||
logger.info("failover_alerter_configured")
|
||||
|
||||
|
||||
def reset_failover_alerter() -> None:
|
||||
"""重置 singleton（測試用）
|
||||
|
||||
2026-04-25 P1.5 by Claude Engineer-D
|
||||
"""
|
||||
global _alerter_instance
|
||||
_alerter_instance = None
|
||||
Reference in New Issue
Block a user