fix(aiops): persist emergency intervention traces
This commit is contained in:
@@ -23,6 +23,7 @@ Phase 5 Sprint 5.0-5.1 — 2026-04-14 Claude Sonnet 4.6
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
@@ -264,7 +265,12 @@ async def dispatch_action(
|
||||
try:
|
||||
# internal provider: 特殊 URL builder(無 MCP call)
|
||||
if spec.mcp_provider == "internal":
|
||||
result_text = _handle_internal_action(spec, resolved_params)
|
||||
result_text = await _handle_internal_action(
|
||||
spec,
|
||||
resolved_params,
|
||||
incident_id=incident_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
logger.info("dispatch_action_internal", action=action_name, duration_ms=round(duration, 1))
|
||||
return DispatchResult(
|
||||
@@ -361,7 +367,13 @@ async def dispatch_action(
|
||||
)
|
||||
|
||||
|
||||
def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
|
||||
async def _handle_internal_action(
|
||||
spec: ActionSpec,
|
||||
params: dict,
|
||||
*,
|
||||
incident_id: str,
|
||||
user_id: int | None,
|
||||
) -> str:
|
||||
"""
|
||||
Internal actions — 不走 MCP,直接產生 URL/文字回覆
|
||||
|
||||
@@ -379,12 +391,20 @@ def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
|
||||
return f"{spec.emoji} <b>{spec.label}</b>\nhttps://awoooi.wooo.work/flywheel"
|
||||
|
||||
if tool == "record_authorization":
|
||||
# Sprint 5.4 會實作真實授權記錄,這裡先返回確認
|
||||
user_id = params.get("user_id", 0)
|
||||
recorded = await _record_authorization_audit(
|
||||
spec=spec,
|
||||
params=params,
|
||||
incident_id=incident_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
_user_id = params.get("user_id", user_id or 0)
|
||||
source = params.get("source", "unknown")
|
||||
action = params.get("action", "authorize")
|
||||
suffix = "已寫入審計與時間線" if recorded else "已受理;審計寫入將由後續補償"
|
||||
return (
|
||||
f"{spec.emoji} <b>{spec.label}</b>\n"
|
||||
f"已記錄 user={user_id} 授權 source={source}(24h 內同源告警將靜默)"
|
||||
f"已記錄 user={_user_id} 授權 source={source} action={action}(24h 內同源告警將靜默)\n"
|
||||
f"{suffix}"
|
||||
)
|
||||
|
||||
# 未知的 internal tool
|
||||
@@ -394,6 +414,104 @@ def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
|
||||
)
|
||||
|
||||
|
||||
async def _record_authorization_audit(
|
||||
*,
|
||||
spec: ActionSpec,
|
||||
params: dict,
|
||||
incident_id: str,
|
||||
user_id: int | None,
|
||||
) -> bool:
|
||||
"""Best-effort persistence for internal authorization actions."""
|
||||
|
||||
source = str(params.get("source") or "unknown")
|
||||
requested_action = str(params.get("action") or spec.name)
|
||||
source_ip = str(params.get("source_ip") or "")
|
||||
actor = f"telegram:{user_id or params.get('user_id') or 0}"
|
||||
context = {
|
||||
"action": spec.name,
|
||||
"label": spec.label,
|
||||
"risk": spec.risk,
|
||||
"category": spec.category,
|
||||
"requested_action": requested_action,
|
||||
"source": source,
|
||||
"source_ip": source_ip,
|
||||
"user_id": user_id or params.get("user_id") or 0,
|
||||
"requires_multi_sig": spec.requires_multi_sig,
|
||||
}
|
||||
wrote_any = False
|
||||
|
||||
try:
|
||||
from src.core.redis_client import get_redis
|
||||
|
||||
redis = get_redis()
|
||||
redis_key = f"secops:authorization:{source}"
|
||||
await redis.set(redis_key, json.dumps(context, ensure_ascii=False), ex=86400)
|
||||
wrote_any = True
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"record_authorization_redis_failed",
|
||||
incident_id=incident_id,
|
||||
source=source,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
try:
|
||||
from src.repositories.alert_operation_log_repository import (
|
||||
get_alert_operation_log_repository,
|
||||
)
|
||||
|
||||
event_type = "APPROVAL_ESCALATED" if spec.requires_multi_sig or spec.risk == "critical" else "USER_ACTION"
|
||||
record = await get_alert_operation_log_repository().append(
|
||||
event_type,
|
||||
incident_id=incident_id,
|
||||
actor=actor,
|
||||
action_detail=f"telegram_authorization:{requested_action}"[:200],
|
||||
success=True,
|
||||
context=context,
|
||||
)
|
||||
wrote_any = wrote_any or bool(record)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"record_authorization_aol_failed",
|
||||
incident_id=incident_id,
|
||||
source=source,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
try:
|
||||
from src.services.approval_db import get_timeline_service
|
||||
|
||||
await get_timeline_service().add_event(
|
||||
event_type="security",
|
||||
status="warning" if spec.requires_multi_sig or spec.risk == "critical" else "info",
|
||||
title="Telegram authorization recorded",
|
||||
description=(
|
||||
f"action={requested_action} source={source} source_ip={source_ip or 'unknown'}"
|
||||
)[:500],
|
||||
actor=actor,
|
||||
actor_role="secops_authorization",
|
||||
risk_level=spec.risk,
|
||||
incident_id=incident_id,
|
||||
)
|
||||
wrote_any = True
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"record_authorization_timeline_failed",
|
||||
incident_id=incident_id,
|
||||
source=source,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"record_authorization_audit_complete",
|
||||
incident_id=incident_id,
|
||||
source=source,
|
||||
action=requested_action,
|
||||
wrote_any=wrote_any,
|
||||
)
|
||||
return wrote_any
|
||||
|
||||
|
||||
def _format_reply(
|
||||
mcp_result: Any, reply_format: str, label: str, emoji: str
|
||||
) -> str:
|
||||
|
||||
@@ -119,6 +119,10 @@ async def escalate_drift_auto_adopt_blocked(
|
||||
return
|
||||
|
||||
try:
|
||||
from src.repositories.alert_operation_log_repository import (
|
||||
get_alert_operation_log_repository,
|
||||
)
|
||||
from src.services.approval_db import get_timeline_service
|
||||
from src.services.telegram_gateway import get_telegram_gateway
|
||||
|
||||
actionable_count = sum(
|
||||
@@ -143,6 +147,43 @@ async def escalate_drift_auto_adopt_blocked(
|
||||
),
|
||||
group_chat_id=settings.SRE_GROUP_CHAT_ID or None,
|
||||
)
|
||||
await get_alert_operation_log_repository().append(
|
||||
"APPROVAL_ESCALATED",
|
||||
incident_id=report.report_id,
|
||||
actor="drift_auto_adopt",
|
||||
action_detail="drift_auto_adopt_blocked_emergency_channel",
|
||||
success=True,
|
||||
context={
|
||||
"namespace": report.namespace,
|
||||
"reason": reason,
|
||||
"high_count": report.high_count,
|
||||
"medium_count": report.medium_count,
|
||||
"actionable_count": actionable_count,
|
||||
"intent": intent,
|
||||
"confidence": confidence,
|
||||
"risk": risk,
|
||||
},
|
||||
)
|
||||
try:
|
||||
await get_timeline_service().add_event(
|
||||
event_type="agent",
|
||||
status="warning",
|
||||
title="Drift emergency intervention requested",
|
||||
description=(
|
||||
f"{reason} | namespace={report.namespace} "
|
||||
f"high={report.high_count} medium={report.medium_count} "
|
||||
f"intent={intent} confidence={confidence:.0%}"
|
||||
)[:500],
|
||||
actor="drift_auto_adopt",
|
||||
actor_role="emergency_intervention",
|
||||
incident_id=report.report_id,
|
||||
)
|
||||
except Exception as timeline_exc:
|
||||
logger.warning(
|
||||
"drift_emergency_timeline_failed",
|
||||
report_id=report.report_id,
|
||||
error=str(timeline_exc),
|
||||
)
|
||||
logger.warning(
|
||||
"drift_auto_adopt_emergency_escalated",
|
||||
report_id=report.report_id,
|
||||
|
||||
Reference in New Issue
Block a user