"""Emergency escalation service for automation blockers. Keeps Redis dedup, Telegram fanout, and operation-log writes out of API routers while giving auto-repair / drift paths a single emergency channel. """ from __future__ import annotations from typing import Any import structlog from src.core.config import settings from src.core.redis_client import get_redis logger = structlog.get_logger(__name__) async def escalate_auto_repair_unavailable( *, incident_id: str, approval_id: str | None, alert_type: str, target_resource: str, namespace: str, failure_reason: str, attempted_actions: str, ) -> None: """Open an emergency channel when auto repair cannot safely continue.""" # 2026-05-02 Claude Opus 4.7 + 統帥 ogt:dedup key 從 incident_id → fingerprint(alertname+target) # 鐵證:4 條 ESCALATION 卡 17:35-17:36 連發(HostOutOfDiskSpace + 3×HostDiskUsageHigh,全 target=node-exporter-110) # 原本 incident_id 是 uuid4 隨機,TTL 900s 太短 → 同症狀換 INC ID 完全不去重 # 改成 alertname+target fingerprint + TTL 86400s,與 decision_manager.py:218 對齊。 _alertname_fp = (alert_type or "AutoRepairBlocked").strip().lower().replace(" ", "_")[:60] _target_fp = (target_resource or "unknown").lower()[:40] dedup_key = f"auto_repair:emergency_escalated:fp:{_alertname_fp}:{_target_fp}" if not await _dedup_first_send(dedup_key, ttl=86400, event="auto_repair"): logger.info( "auto_repair_escalation_dedup_skipped", incident_id=incident_id, approval_id=approval_id, fingerprint=f"{_alertname_fp}:{_target_fp}", ) return try: from src.repositories.alert_operation_log_repository import ( get_alert_operation_log_repository, ) from src.services.approval_db import get_timeline_service from src.services.telegram_gateway import get_telegram_gateway await get_telegram_gateway().send_escalation_card( incident_id=incident_id, original_alertname=alert_type or "AutoRepairBlocked", duration_min=0, priority=0, attempted_actions=attempted_actions, failure_reason=failure_reason, current_impact=f"target={target_resource or 'unknown'} namespace={namespace or 'unknown'}", group_chat_id=settings.SRE_GROUP_CHAT_ID or None, ) await get_alert_operation_log_repository().append( "APPROVAL_ESCALATED", incident_id=incident_id, approval_id=approval_id, actor="auto_repair", action_detail="auto_repair_unavailable_emergency_channel", success=True, context={ "alert_type": alert_type, "target_resource": target_resource, "namespace": namespace, "failure_reason": failure_reason, "attempted_actions": attempted_actions, }, ) try: await get_timeline_service().add_event( event_type="agent", status="warning", title="AI emergency intervention requested", description=( f"{failure_reason} | attempted={attempted_actions}" )[:500], actor="auto_repair", actor_role="emergency_intervention", approval_id=approval_id, incident_id=incident_id, ) except Exception as timeline_exc: logger.warning( "auto_repair_emergency_timeline_failed", incident_id=incident_id, approval_id=approval_id, error=str(timeline_exc), ) logger.warning( "auto_repair_emergency_escalated", incident_id=incident_id, approval_id=approval_id, reason=failure_reason, ) except Exception as exc: logger.warning( "auto_repair_emergency_escalation_failed", incident_id=incident_id, approval_id=approval_id, error=str(exc), ) async def escalate_drift_auto_adopt_blocked( *, report: Any, reason: str, interpretation: Any, ) -> None: """Notify the emergency channel when drift cannot be auto-adopted safely.""" dedup_key = f"drift:auto_adopt_emergency:{report.report_id}" if not await _dedup_first_send(dedup_key, ttl=3600, event="drift"): logger.info("drift_emergency_escalation_dedup_skipped", report_id=report.report_id) return try: from src.repositories.alert_operation_log_repository import ( get_alert_operation_log_repository, ) from src.services.approval_db import get_timeline_service from src.services.telegram_gateway import get_telegram_gateway actionable_count = sum( 1 for item in report.items if not getattr(item, "is_allowlisted", False) ) intent = getattr(getattr(interpretation, "intent", None), "value", "unknown") confidence = getattr(interpretation, "confidence", 0.0) if interpretation else 0.0 risk = getattr(interpretation, "risk", "unknown") if interpretation else "unknown" await get_telegram_gateway().send_escalation_card( incident_id=report.report_id, original_alertname="ConfigDriftAutoAdoptBlocked", duration_min=0, priority=0 if report.high_count > 0 else 1, attempted_actions="drift_interpreter -> auto_adopt_if_safe -> emergency_escalation", failure_reason=reason, current_impact=( f"namespace={report.namespace} high={report.high_count} " f"medium={report.medium_count} actionable={actionable_count} " f"intent={intent} confidence={confidence:.0%} risk={risk}" ), group_chat_id=settings.SRE_GROUP_CHAT_ID or None, ) await get_alert_operation_log_repository().append( "APPROVAL_ESCALATED", incident_id=report.report_id, actor="drift_auto_adopt", action_detail="drift_auto_adopt_blocked_emergency_channel", success=True, context={ "namespace": report.namespace, "reason": reason, "high_count": report.high_count, "medium_count": report.medium_count, "actionable_count": actionable_count, "intent": intent, "confidence": confidence, "risk": risk, }, ) try: await get_timeline_service().add_event( event_type="agent", status="warning", title="Drift emergency intervention requested", description=( f"{reason} | namespace={report.namespace} " f"high={report.high_count} medium={report.medium_count} " f"intent={intent} confidence={confidence:.0%}" )[:500], actor="drift_auto_adopt", actor_role="emergency_intervention", incident_id=report.report_id, ) except Exception as timeline_exc: logger.warning( "drift_emergency_timeline_failed", report_id=report.report_id, error=str(timeline_exc), ) logger.warning( "drift_auto_adopt_emergency_escalated", report_id=report.report_id, reason=reason, high=report.high_count, medium=report.medium_count, actionable=actionable_count, ) except Exception as exc: logger.warning( "drift_emergency_escalation_failed", report_id=report.report_id, error=str(exc), ) async def _dedup_first_send(key: str, *, ttl: int, event: str) -> bool: """Return True when this is the first escalation in the dedup window.""" try: redis = get_redis() return bool(await redis.set(key, "1", ex=ttl, nx=True)) except Exception as exc: logger.warning( "emergency_escalation_dedup_failed_open", key=key, event=event, error=str(exc), ) return True