diff --git a/apps/api/src/services/failover_alerter.py b/apps/api/src/services/failover_alerter.py index e758df18b..7e184fd7e 100644 --- a/apps/api/src/services/failover_alerter.py +++ b/apps/api/src/services/failover_alerter.py @@ -18,7 +18,8 @@ from typing import Any import structlog TAIPEI_TZ = timezone(timedelta(hours=8)) -DEDUP_TTL_SEC = 600 # 10 min +DEDUP_TTL_SEC = 600 # 10 min(故障切換用) +RECOVERY_DEDUP_TTL_SEC = 3600 # 1h — GCP 健康閃爍時 1 小時內不重複告警 QUOTA_DEDUP_TTL_SEC = 86400 # 24h(每日 quota 告警只發一次) logger = structlog.get_logger(__name__) @@ -69,11 +70,16 @@ class FailoverAlerter: logger.info("failover_alert_sent", to_provider=to_provider) async def alert_recovery(self, event: dict[str, Any]) -> None: - """Ollama 自動恢復告警 — 10min dedup + """Ollama 自動恢復告警 — 1h dedup per host # 2026-05-03 ogt: ADR-110 三層容災,恢復主機從 event["recovered_host"] 動態讀取 + # 2026-05-04 ogt: dedup key 加 recovered_host + 改 1h TTL + # 原 key 固定 "alert:recovery" → GCP-A 每 10min 閃爍就重發 """ - dedup_key = "alert:recovery" - if not await self._check_dedup(dedup_key, ttl=DEDUP_TTL_SEC): + recovered_host = event.get("recovered_host", event.get("to_provider", "ollama")) + # sanitize host → 只留 IP/hostname 部分,拿掉 http:// 前綴與 port + safe_host = str(recovered_host).replace("http://", "").replace("/", "_").replace(":", "_") + dedup_key = f"alert:recovery:{safe_host}" + if not await self._check_dedup(dedup_key, ttl=RECOVERY_DEDUP_TTL_SEC): logger.debug("recovery_alert_dedup_skipped") return @@ -82,8 +88,6 @@ class FailoverAlerter: from_provider = event.get("from_provider", event.get("from", "?")) to_provider = event.get("to_provider", event.get("to", "ollama")) recovery_time = event.get("recovery_time", datetime.now(TAIPEI_TZ).isoformat()) - # 2026-05-03 ogt: ADR-110 — 恢復主機動態,不再硬編碼 111 - recovered_host = event.get("recovered_host", to_provider) msg = ( f"*Ollama 自動恢復*\n\n"