fix(alerter): Ollama 恢復告警去重修復 — per-host key + 1h TTL
根因:
1. dedup_key 固定為 "alert:recovery",GCP-A 每 10min 健康閃爍就觸發重發
2. 三層容災下不同主機恢復共用同一個 key,互相污染
修法:
- dedup key 改為 "alert:recovery:{safe_host}",各主機獨立 dedup
- RECOVERY_DEDUP_TTL_SEC = 3600(1h),GCP 持續閃爍只報一次
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -18,7 +18,8 @@ from typing import Any
|
||||
import structlog
|
||||
|
||||
TAIPEI_TZ = timezone(timedelta(hours=8))
|
||||
DEDUP_TTL_SEC = 600 # 10 min
|
||||
DEDUP_TTL_SEC = 600 # 10 min(故障切換用)
|
||||
RECOVERY_DEDUP_TTL_SEC = 3600 # 1h — GCP 健康閃爍時 1 小時內不重複告警
|
||||
QUOTA_DEDUP_TTL_SEC = 86400 # 24h(每日 quota 告警只發一次)
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
@@ -69,11 +70,16 @@ class FailoverAlerter:
|
||||
logger.info("failover_alert_sent", to_provider=to_provider)
|
||||
|
||||
async def alert_recovery(self, event: dict[str, Any]) -> None:
|
||||
"""Ollama 自動恢復告警 — 10min dedup
|
||||
"""Ollama 自動恢復告警 — 1h dedup per host
|
||||
# 2026-05-03 ogt: ADR-110 三層容災,恢復主機從 event["recovered_host"] 動態讀取
|
||||
# 2026-05-04 ogt: dedup key 加 recovered_host + 改 1h TTL
|
||||
# 原 key 固定 "alert:recovery" → GCP-A 每 10min 閃爍就重發
|
||||
"""
|
||||
dedup_key = "alert:recovery"
|
||||
if not await self._check_dedup(dedup_key, ttl=DEDUP_TTL_SEC):
|
||||
recovered_host = event.get("recovered_host", event.get("to_provider", "ollama"))
|
||||
# sanitize host → 只留 IP/hostname 部分,拿掉 http:// 前綴與 port
|
||||
safe_host = str(recovered_host).replace("http://", "").replace("/", "_").replace(":", "_")
|
||||
dedup_key = f"alert:recovery:{safe_host}"
|
||||
if not await self._check_dedup(dedup_key, ttl=RECOVERY_DEDUP_TTL_SEC):
|
||||
logger.debug("recovery_alert_dedup_skipped")
|
||||
return
|
||||
|
||||
@@ -82,8 +88,6 @@ class FailoverAlerter:
|
||||
from_provider = event.get("from_provider", event.get("from", "?"))
|
||||
to_provider = event.get("to_provider", event.get("to", "ollama"))
|
||||
recovery_time = event.get("recovery_time", datetime.now(TAIPEI_TZ).isoformat())
|
||||
# 2026-05-03 ogt: ADR-110 — 恢復主機動態,不再硬編碼 111
|
||||
recovered_host = event.get("recovered_host", to_provider)
|
||||
|
||||
msg = (
|
||||
f"*Ollama 自動恢復*\n\n"
|
||||
|
||||
Reference in New Issue
Block a user