From 3f853accf2591b1e6ba5834c4cf8edb2c6ef2237 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 5 May 2026 01:22:01 +0800 Subject: [PATCH] =?UTF-8?q?fix(alerter):=20Ollama=20=E6=81=A2=E5=BE=A9?= =?UTF-8?q?=E5=91=8A=E8=AD=A6=E5=8E=BB=E9=87=8D=E4=BF=AE=E5=BE=A9=20?= =?UTF-8?q?=E2=80=94=20per-host=20key=20+=201h=20TTL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根因： 1. dedup_key 固定為 "alert:recovery"，GCP-A 每 10min 健康閃爍就觸發重發 2. 三層容災下不同主機恢復共用同一個 key，互相污染 修法： - dedup key 改為 "alert:recovery:{safe_host}"，各主機獨立 dedup - RECOVERY_DEDUP_TTL_SEC = 3600（1h），GCP 持續閃爍只報一次 Co-Authored-By: Claude Sonnet 4.6 --- apps/api/src/services/failover_alerter.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/apps/api/src/services/failover_alerter.py b/apps/api/src/services/failover_alerter.py index e758df18b..7e184fd7e 100644 --- a/apps/api/src/services/failover_alerter.py +++ b/apps/api/src/services/failover_alerter.py @@ -18,7 +18,8 @@ from typing import Any import structlog TAIPEI_TZ = timezone(timedelta(hours=8)) -DEDUP_TTL_SEC = 600 # 10 min +DEDUP_TTL_SEC = 600 # 10 min（故障切換用） +RECOVERY_DEDUP_TTL_SEC = 3600 # 1h — GCP 健康閃爍時 1 小時內不重複告警 QUOTA_DEDUP_TTL_SEC = 86400 # 24h（每日 quota 告警只發一次） logger = structlog.get_logger(__name__) @@ -69,11 +70,16 @@ class FailoverAlerter: logger.info("failover_alert_sent", to_provider=to_provider) async def alert_recovery(self, event: dict[str, Any]) -> None: - """Ollama 自動恢復告警 — 10min dedup + """Ollama 自動恢復告警 — 1h dedup per host # 2026-05-03 ogt: ADR-110 三層容災，恢復主機從 event["recovered_host"] 動態讀取 + # 2026-05-04 ogt: dedup key 加 recovered_host + 改 1h TTL + # 原 key 固定 "alert:recovery" → GCP-A 每 10min 閃爍就重發 """ - dedup_key = "alert:recovery" - if not await self._check_dedup(dedup_key, ttl=DEDUP_TTL_SEC): + recovered_host = event.get("recovered_host", event.get("to_provider", "ollama")) + # sanitize host → 只留 IP/hostname 部分，拿掉 http:// 前綴與 port + safe_host = str(recovered_host).replace("http://", "").replace("/", "_").replace(":", "_") + dedup_key = f"alert:recovery:{safe_host}" + if not await self._check_dedup(dedup_key, ttl=RECOVERY_DEDUP_TTL_SEC): logger.debug("recovery_alert_dedup_skipped") return @@ -82,8 +88,6 @@ class FailoverAlerter: from_provider = event.get("from_provider", event.get("from", "?")) to_provider = event.get("to_provider", event.get("to", "ollama")) recovery_time = event.get("recovery_time", datetime.now(TAIPEI_TZ).isoformat()) - # 2026-05-03 ogt: ADR-110 — 恢復主機動態，不再硬編碼 111 - recovered_host = event.get("recovered_host", to_provider) msg = ( f"*Ollama 自動恢復*\n\n"