"""Throttled Telegram notices for converged alert fingerprints."""
import hashlib
import html
from src.core.logging import get_logger
from src.core.redis_client import get_redis
from src.services.telegram_gateway import (
_telegram_send_delivery_succeeded,
get_telegram_gateway,
)
logger = get_logger("awoooi.converged_alert_recurrence")
CONVERGED_ALERT_RECURRENCE_REDIS_PREFIX = "awoooi:tg:converged_alert_recurrence:"
CONVERGED_ALERT_RECURRENCE_TTL_SECONDS = 30 * 60
CONVERGED_ALERT_RECURRENCE_FALLBACK_MILESTONES = frozenset({2, 3, 5, 10, 20, 50, 100})
def shorten_alert_text(value: str | None, *, limit: int = 80) -> str:
"""Keep Telegram recurrence notices readable and HTML-safe."""
text = " ".join(str(value or "-").split())
if len(text) <= limit:
return html.escape(text)
return html.escape(f"{text[: limit - 3]}...")
def is_converged_alert_recurrence_milestone(hit_count: int) -> bool:
"""Fallback dedupe when Redis is unavailable."""
if hit_count <= 1:
return False
return hit_count in CONVERGED_ALERT_RECURRENCE_FALLBACK_MILESTONES or hit_count % 100 == 0
def converged_alert_recurrence_key(fingerprint: str) -> str:
digest = hashlib.sha256(str(fingerprint or "missing").encode()).hexdigest()[:32]
return f"{CONVERGED_ALERT_RECURRENCE_REDIS_PREFIX}{digest}"
async def should_notify_converged_alert_recurrence(
*,
fingerprint: str,
hit_count: int,
) -> bool:
"""Throttle converged alert notices without making active incidents disappear."""
if hit_count <= 1:
return False
key = converged_alert_recurrence_key(fingerprint)
try:
redis = get_redis()
acquired = await redis.set(
key,
str(hit_count),
ex=CONVERGED_ALERT_RECURRENCE_TTL_SECONDS,
nx=True,
)
return bool(acquired)
except Exception as exc:
logger.warning(
"converged_alert_recurrence_dedup_unavailable",
fingerprint_hash=key.rsplit(":", 1)[-1],
hit_count=hit_count,
fallback="milestone",
error=str(exc),
)
return is_converged_alert_recurrence_milestone(hit_count)
async def _release_converged_alert_recurrence_reservation(
fingerprint: str,
) -> None:
"""Release a pre-send reservation when provider delivery is not proven."""
key = converged_alert_recurrence_key(fingerprint)
try:
await get_redis().delete(key)
except Exception as exc:
logger.warning(
"converged_alert_recurrence_reservation_release_failed",
fingerprint_hash=key.rsplit(":", 1)[-1],
error=str(exc),
)
def format_converged_alert_recurrence_message(
*,
source: str,
alertname: str,
severity: str,
namespace: str,
target_resource: str,
hit_count: int,
incident_id: str | None,
approval_id: str | None,
alert_category: str = "",
notification_type: str = "",
recurrence_stage: str = "converged",
) -> str:
"""Build a concise recurrence notice for an already-open alert fingerprint."""
is_llm_inflight = recurrence_stage == "llm_inflight"
title = "告警仍在發生:AI 分析中" if is_llm_inflight else "告警仍在發生"
summary = (
"同一指紋已有背景 AI 分析鎖,重複告警已節流提醒,不再完全靜音。"
if is_llm_inflight
else "同一指紋已收斂,系統保留去重,但不再完全靜音。"
)
next_step = (
"下一步:等待背景分析完成;若持續超過分析鎖 TTL,請查看 AwoooP 事件時間線與 API logs。"
if is_llm_inflight
else "下一步:請查看 AwoooP 事件時間線。"
)
return "\n".join(
[
f"{title}",
summary,
"",
f"來源:{shorten_alert_text(source, limit=40)}",
f"告警:{shorten_alert_text(alertname, limit=80)}",
f"嚴重度:{shorten_alert_text(severity, limit=24)}",
f"目標:{shorten_alert_text(target_resource, limit=80)}",
f"命名空間:{shorten_alert_text(namespace, limit=48)}",
f"累計次數:{hit_count}",
f"事件:{shorten_alert_text(incident_id, limit=48)}",
f"簽核:{shorten_alert_text(approval_id, limit=48)}",
f"分類:{shorten_alert_text(alert_category or '-', limit=48)}",
f"通知型別:{shorten_alert_text(notification_type or '-', limit=48)}",
"",
f"{next_step} 這不是新的自動修復授權。",
]
)
async def notify_converged_alert_recurrence(
*,
source: str,
fingerprint: str,
alertname: str,
severity: str,
namespace: str,
target_resource: str,
hit_count: int,
incident_id: str | None,
approval_id: str | None,
alert_category: str = "",
notification_type: str = "",
recurrence_stage: str = "converged",
) -> None:
"""Send a throttled recurrence notice for an already-open alert fingerprint."""
if not await should_notify_converged_alert_recurrence(
fingerprint=fingerprint,
hit_count=hit_count,
):
logger.info(
"converged_alert_recurrence_throttled",
source=source,
hit_count=hit_count,
approval_id=approval_id,
recurrence_stage=recurrence_stage,
)
return
text = format_converged_alert_recurrence_message(
source=source,
alertname=alertname,
severity=severity,
namespace=namespace,
target_resource=target_resource,
hit_count=hit_count,
incident_id=incident_id,
approval_id=approval_id,
alert_category=alert_category,
notification_type=notification_type,
recurrence_stage=recurrence_stage,
)
gateway = get_telegram_gateway()
sent_count = 0
failures: list[str] = []
try:
result = await gateway.send_alert_notification(
text,
product_id="awoooi",
signal_family="incident_lifecycle",
severity="P1",
)
if _telegram_send_delivery_succeeded(result):
sent_count += 1
else:
await _release_converged_alert_recurrence_reservation(fingerprint)
result_payload = result if isinstance(result, dict) else {}
receipt = result_payload.get("_awoooi_canonical_route_receipt") or {}
failures.append(
f"route:{receipt.get('classifier') or 'blocked_no_egress'}"
)
except Exception as exc:
await _release_converged_alert_recurrence_reservation(fingerprint)
failures.append(f"primary:{type(exc).__name__}")
logger.warning(
"converged_alert_recurrence_primary_failed",
source=source,
approval_id=approval_id,
recurrence_stage=recurrence_stage,
error=str(exc),
)
if sent_count:
logger.info(
"converged_alert_recurrence_sent",
source=source,
hit_count=hit_count,
approval_id=approval_id,
recurrence_stage=recurrence_stage,
mirrored_to_private=False,
sent_count=sent_count,
)
else:
logger.error(
"converged_alert_recurrence_failed",
source=source,
hit_count=hit_count,
approval_id=approval_id,
recurrence_stage=recurrence_stage,
failures=failures,
)