"""Throttled Telegram notices for converged alert fingerprints.""" import hashlib import html from src.core.logging import get_logger from src.core.redis_client import get_redis from src.services.telegram_gateway import ( _telegram_send_delivery_succeeded, get_telegram_gateway, ) logger = get_logger("awoooi.converged_alert_recurrence") CONVERGED_ALERT_RECURRENCE_REDIS_PREFIX = "awoooi:tg:converged_alert_recurrence:" CONVERGED_ALERT_RECURRENCE_TTL_SECONDS = 30 * 60 CONVERGED_ALERT_RECURRENCE_FALLBACK_MILESTONES = frozenset({2, 3, 5, 10, 20, 50, 100}) def shorten_alert_text(value: str | None, *, limit: int = 80) -> str: """Keep Telegram recurrence notices readable and HTML-safe.""" text = " ".join(str(value or "-").split()) if len(text) <= limit: return html.escape(text) return html.escape(f"{text[: limit - 3]}...") def is_converged_alert_recurrence_milestone(hit_count: int) -> bool: """Fallback dedupe when Redis is unavailable.""" if hit_count <= 1: return False return hit_count in CONVERGED_ALERT_RECURRENCE_FALLBACK_MILESTONES or hit_count % 100 == 0 def converged_alert_recurrence_key(fingerprint: str) -> str: digest = hashlib.sha256(str(fingerprint or "missing").encode()).hexdigest()[:32] return f"{CONVERGED_ALERT_RECURRENCE_REDIS_PREFIX}{digest}" async def should_notify_converged_alert_recurrence( *, fingerprint: str, hit_count: int, ) -> bool: """Throttle converged alert notices without making active incidents disappear.""" if hit_count <= 1: return False key = converged_alert_recurrence_key(fingerprint) try: redis = get_redis() acquired = await redis.set( key, str(hit_count), ex=CONVERGED_ALERT_RECURRENCE_TTL_SECONDS, nx=True, ) return bool(acquired) except Exception as exc: logger.warning( "converged_alert_recurrence_dedup_unavailable", fingerprint_hash=key.rsplit(":", 1)[-1], hit_count=hit_count, fallback="milestone", error=str(exc), ) return is_converged_alert_recurrence_milestone(hit_count) async def _release_converged_alert_recurrence_reservation( fingerprint: str, ) -> None: """Release a pre-send reservation when provider delivery is not proven.""" key = converged_alert_recurrence_key(fingerprint) try: await get_redis().delete(key) except Exception as exc: logger.warning( "converged_alert_recurrence_reservation_release_failed", fingerprint_hash=key.rsplit(":", 1)[-1], error=str(exc), ) def format_converged_alert_recurrence_message( *, source: str, alertname: str, severity: str, namespace: str, target_resource: str, hit_count: int, incident_id: str | None, approval_id: str | None, alert_category: str = "", notification_type: str = "", recurrence_stage: str = "converged", ) -> str: """Build a concise recurrence notice for an already-open alert fingerprint.""" is_llm_inflight = recurrence_stage == "llm_inflight" title = "告警仍在發生:AI 分析中" if is_llm_inflight else "告警仍在發生" summary = ( "同一指紋已有背景 AI 分析鎖,重複告警已節流提醒,不再完全靜音。" if is_llm_inflight else "同一指紋已收斂,系統保留去重,但不再完全靜音。" ) next_step = ( "下一步:等待背景分析完成;若持續超過分析鎖 TTL,請查看 AwoooP 事件時間線與 API logs。" if is_llm_inflight else "下一步:請查看 AwoooP 事件時間線。" ) return "\n".join( [ f"{title}", summary, "", f"來源:{shorten_alert_text(source, limit=40)}", f"告警:{shorten_alert_text(alertname, limit=80)}", f"嚴重度:{shorten_alert_text(severity, limit=24)}", f"目標:{shorten_alert_text(target_resource, limit=80)}", f"命名空間:{shorten_alert_text(namespace, limit=48)}", f"累計次數:{hit_count}", f"事件:{shorten_alert_text(incident_id, limit=48)}", f"簽核:{shorten_alert_text(approval_id, limit=48)}", f"分類:{shorten_alert_text(alert_category or '-', limit=48)}", f"通知型別:{shorten_alert_text(notification_type or '-', limit=48)}", "", f"{next_step} 這不是新的自動修復授權。", ] ) async def notify_converged_alert_recurrence( *, source: str, fingerprint: str, alertname: str, severity: str, namespace: str, target_resource: str, hit_count: int, incident_id: str | None, approval_id: str | None, alert_category: str = "", notification_type: str = "", recurrence_stage: str = "converged", ) -> None: """Send a throttled recurrence notice for an already-open alert fingerprint.""" if not await should_notify_converged_alert_recurrence( fingerprint=fingerprint, hit_count=hit_count, ): logger.info( "converged_alert_recurrence_throttled", source=source, hit_count=hit_count, approval_id=approval_id, recurrence_stage=recurrence_stage, ) return text = format_converged_alert_recurrence_message( source=source, alertname=alertname, severity=severity, namespace=namespace, target_resource=target_resource, hit_count=hit_count, incident_id=incident_id, approval_id=approval_id, alert_category=alert_category, notification_type=notification_type, recurrence_stage=recurrence_stage, ) gateway = get_telegram_gateway() sent_count = 0 failures: list[str] = [] try: result = await gateway.send_alert_notification( text, product_id="awoooi", signal_family="incident_lifecycle", severity="P1", ) if _telegram_send_delivery_succeeded(result): sent_count += 1 else: await _release_converged_alert_recurrence_reservation(fingerprint) result_payload = result if isinstance(result, dict) else {} receipt = result_payload.get("_awoooi_canonical_route_receipt") or {} failures.append( f"route:{receipt.get('classifier') or 'blocked_no_egress'}" ) except Exception as exc: await _release_converged_alert_recurrence_reservation(fingerprint) failures.append(f"primary:{type(exc).__name__}") logger.warning( "converged_alert_recurrence_primary_failed", source=source, approval_id=approval_id, recurrence_stage=recurrence_stage, error=str(exc), ) if sent_count: logger.info( "converged_alert_recurrence_sent", source=source, hit_count=hit_count, approval_id=approval_id, recurrence_stage=recurrence_stage, mirrored_to_private=False, sent_count=sent_count, ) else: logger.error( "converged_alert_recurrence_failed", source=source, hit_count=hit_count, approval_id=approval_id, recurrence_stage=recurrence_stage, failures=failures, )