""" 告警聚合引擎 (Alert Grouping Engine) ===================================== ADR-076: 告警風暴防禦 — 滑動視窗聚合 建立: 2026-04-14 (台北時區) Claude Haiku 4.5 目標: - 防止告警風暴:同一 namespace/alertname 在 5 分鐘內爆出多個告警 → 聚合為 Parent Alert - 節省 LLM token 費用 - 避免 Telegram 被洗版 設計原則: - Redis Sorted Set 滑動視窗(同 anomaly_counter.py ADR-037 模式) - 遵循 leWOOOgo 積木化鐵律 - 只用 Redis,不直接存取 DB - Graceful Degradation:Redis 失敗不阻斷主流程 - 統帥設定 THRESHOLD=3(5 分鐘內 3 個以上才聚合) Redis Key 設計: - alert_group:{group_key}:count — Sorted Set (timestamp → timestamp) - alert_group:{group_key}:meta — Hash (parent_fingerprint, first_seen, count) TTL: 10 分鐘(略長於 5 分鐘視窗) """ from __future__ import annotations import time from dataclasses import dataclass from typing import TYPE_CHECKING import structlog if TYPE_CHECKING: import redis.asyncio as redis logger = structlog.get_logger(__name__) def _decode_redis_member(value: object, fallback: str) -> str: """Redis client 可能回 bytes 或 str;統一成 str 供 DB / log 使用。""" if isinstance(value, bytes): return value.decode("utf-8", errors="replace") if isinstance(value, str): return value if value is None: return fallback return str(value) # ============================================================================= # Data Types # ============================================================================= @dataclass class GroupingResult: """聚合評估結果""" is_grouped: bool """是否已被聚合(True = 此告警是子告警,應跳過 LLM)""" group_key: str """聚合分組 key""" count: int """目前視窗內的告警數量""" parent_fingerprint: str | None """父告警的指紋(第一個進來的告警)""" is_parent: bool """是否為父告警(第一個進來觸發聚合的那個)""" # ============================================================================= # AlertGroupingService # ============================================================================= class AlertGroupingService: """ 告警聚合引擎 統帥指令 (2026-04-14): - "防禦告警風暴:同一 namespace/deployment 在 5 分鐘內炸出 10 個相同告警 → 搓合成 1 個 Parent Alert" - "大幅節省 LLM Token 費用,避免 Telegram 被洗版" 滑動視窗設計(同 anomaly_counter.py ADR-037): - ZADD alert_group:{key}:window {ts} {ts} - ZCOUNT alert_group:{key}:window {cutoff} +inf - ZREMRANGEBYSCORE alert_group:{key}:window -inf {cutoff} """ # 5 分鐘滑動視窗 WINDOW_SECONDS: int = 300 # 觸發聚合的閾值:保留第一張主卡,第二個同組告警開始收斂。 # 2026-05-07 Codex — Telegram 群組噪音治理:舊值 3 會讓前兩張同類告警仍進 AI/Telegram。 GROUP_THRESHOLD: int = 2 # Redis Key 前綴 PREFIX_WINDOW = "alert_group:window:" PREFIX_META = "alert_group:meta:" # TTL(視窗 + 5 分鐘緩衝) TTL_SECONDS: int = 600 def __init__(self, redis_client: redis.Redis) -> None: self.redis = redis_client @staticmethod def build_group_key(alertname: str, namespace: str) -> str: """ 從 alertname + namespace 建構聚合分組 key 分組邏輯:取 alertname 的前綴(去掉數字後綴)+ namespace 例:PodCrashLoopBackOff-pod-1 + awoooi-prod → PodCrashLoopBackOff:awoooi-prod Args: alertname: 告警名稱 namespace: K8s namespace Returns: 分組 key 字串 """ import re # 取 alertname 前綴(去掉尾端的數字或 UUID 後綴) prefix = re.split(r"[-_]\d+$|[-_][0-9a-f]{8,}$", alertname, maxsplit=1)[0] return f"{prefix}:{namespace}" async def evaluate( self, alertname: str, namespace: str, fingerprint: str, ) -> GroupingResult: """ 評估告警是否應被聚合 流程: 1. 計算 group_key 2. 將此告警加入滑動視窗 3. 計算視窗內告警數量 4. 若數量 >= THRESHOLD,標記為子告警(is_grouped=True) 5. 第一個告警(count==1)為父告警 Graceful Degradation: Redis 失敗 → 返回 is_grouped=False,不阻斷主流程 Args: alertname: 告警名稱 namespace: K8s namespace fingerprint: 此告警的指紋 Returns: GroupingResult """ group_key = self.build_group_key(alertname, namespace) try: return await self._do_evaluate(group_key, fingerprint) except Exception: logger.warning( "alert_grouping_redis_error", group_key=group_key, alertname=alertname, namespace=namespace, ) # Graceful Degradation:Redis 失敗不阻斷主流程 return GroupingResult( is_grouped=False, group_key=group_key, count=0, parent_fingerprint=None, is_parent=True, ) async def _do_evaluate(self, group_key: str, fingerprint: str) -> GroupingResult: """ 核心聚合邏輯(內部方法) 使用 Redis Pipeline 保證原子性 """ now_ts = time.time() cutoff_ts = now_ts - self.WINDOW_SECONDS window_key = f"{self.PREFIX_WINDOW}{group_key}" async with self.redis.pipeline(transaction=True) as pipe: # 1. 清理過期記錄 pipe.zremrangebyscore(window_key, "-inf", cutoff_ts) # 2. 加入當前告警(score=timestamp, member=fingerprint) pipe.zadd(window_key, {fingerprint: now_ts}) # 3. 計算視窗內告警數量 pipe.zcount(window_key, cutoff_ts, "+inf") # 4. 取第一個告警(父告警) pipe.zrange(window_key, 0, 0) # 5. 設定 TTL pipe.expire(window_key, self.TTL_SECONDS) results = await pipe.execute() count = results[2] first_members = results[3] parent_fingerprint = _decode_redis_member( first_members[0] if first_members else None, fallback=fingerprint, ) # 是否為父告警(第一個) is_parent = parent_fingerprint == fingerprint or count == 1 # 是否觸發聚合(count >= THRESHOLD 且非父告警) is_grouped = count >= self.GROUP_THRESHOLD and not is_parent if is_grouped: logger.info( "alert_grouped_as_child", group_key=group_key, fingerprint=fingerprint, parent_fingerprint=parent_fingerprint, count=count, threshold=self.GROUP_THRESHOLD, ) elif count >= self.GROUP_THRESHOLD and is_parent: # 父告警 + 超過閾值:表示新的父告警開始聚合 logger.info( "alert_grouping_parent_promoted", group_key=group_key, fingerprint=fingerprint, count=count, ) return GroupingResult( is_grouped=is_grouped, group_key=group_key, count=count, parent_fingerprint=parent_fingerprint, is_parent=is_parent, ) async def get_group_count(self, alertname: str, namespace: str) -> int: """ 查詢分組當前視窗內的告警數量 Args: alertname: 告警名稱 namespace: K8s namespace Returns: 視窗內告警數量(Redis 失敗返回 0) """ group_key = self.build_group_key(alertname, namespace) window_key = f"{self.PREFIX_WINDOW}{group_key}" try: now_ts = time.time() cutoff_ts = now_ts - self.WINDOW_SECONDS count = await self.redis.zcount(window_key, cutoff_ts, "+inf") return int(count) except Exception: logger.warning("alert_grouping_count_error", group_key=group_key) return 0 # ============================================================================= # Factory Function # ============================================================================= _instance: AlertGroupingService | None = None def get_alert_grouping_service() -> AlertGroupingService: """ 取得 AlertGroupingService 單例 依賴注入:需要在 Redis 初始化後呼叫 Returns: AlertGroupingService 實例 """ global _instance if _instance is None: from src.core.redis_client import get_redis redis_client = get_redis() _instance = AlertGroupingService(redis_client) return _instance