fix(aiops): persist emergency intervention traces
Some checks failed
CD Pipeline / tests (push) Successful in 2m56s
Code Review / ai-code-review (push) Failing after 39s
CD Pipeline / build-and-deploy (push) Successful in 12m54s
CD Pipeline / post-deploy-checks (push) Successful in 4m40s

This commit is contained in:
Your Name
2026-05-01 20:34:33 +08:00
parent 8e49f2ea88
commit 7795f027d2
7 changed files with 297 additions and 9 deletions

View File

@@ -23,6 +23,7 @@ Phase 5 Sprint 5.0-5.1 — 2026-04-14 Claude Sonnet 4.6
from __future__ import annotations
import json
import time
from dataclasses import dataclass
from functools import lru_cache
@@ -264,7 +265,12 @@ async def dispatch_action(
try:
# internal provider: 特殊 URL builder無 MCP call
if spec.mcp_provider == "internal":
result_text = _handle_internal_action(spec, resolved_params)
result_text = await _handle_internal_action(
spec,
resolved_params,
incident_id=incident_id,
user_id=user_id,
)
duration = (time.perf_counter() - start) * 1000
logger.info("dispatch_action_internal", action=action_name, duration_ms=round(duration, 1))
return DispatchResult(
@@ -361,7 +367,13 @@ async def dispatch_action(
)
def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
async def _handle_internal_action(
spec: ActionSpec,
params: dict,
*,
incident_id: str,
user_id: int | None,
) -> str:
"""
Internal actions — 不走 MCP直接產生 URL/文字回覆
@@ -379,12 +391,20 @@ def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
return f"{spec.emoji} <b>{spec.label}</b>\nhttps://awoooi.wooo.work/flywheel"
if tool == "record_authorization":
# Sprint 5.4 會實作真實授權記錄,這裡先返回確認
user_id = params.get("user_id", 0)
recorded = await _record_authorization_audit(
spec=spec,
params=params,
incident_id=incident_id,
user_id=user_id,
)
_user_id = params.get("user_id", user_id or 0)
source = params.get("source", "unknown")
action = params.get("action", "authorize")
suffix = "已寫入審計與時間線" if recorded else "已受理;審計寫入將由後續補償"
return (
f"{spec.emoji} <b>{spec.label}</b>\n"
f"已記錄 user={user_id} 授權 source={source}24h 內同源告警將靜默)"
f"已記錄 user={_user_id} 授權 source={source} action={action}24h 內同源告警將靜默)\n"
f"{suffix}"
)
# 未知的 internal tool
@@ -394,6 +414,104 @@ def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
)
async def _record_authorization_audit(
*,
spec: ActionSpec,
params: dict,
incident_id: str,
user_id: int | None,
) -> bool:
"""Best-effort persistence for internal authorization actions."""
source = str(params.get("source") or "unknown")
requested_action = str(params.get("action") or spec.name)
source_ip = str(params.get("source_ip") or "")
actor = f"telegram:{user_id or params.get('user_id') or 0}"
context = {
"action": spec.name,
"label": spec.label,
"risk": spec.risk,
"category": spec.category,
"requested_action": requested_action,
"source": source,
"source_ip": source_ip,
"user_id": user_id or params.get("user_id") or 0,
"requires_multi_sig": spec.requires_multi_sig,
}
wrote_any = False
try:
from src.core.redis_client import get_redis
redis = get_redis()
redis_key = f"secops:authorization:{source}"
await redis.set(redis_key, json.dumps(context, ensure_ascii=False), ex=86400)
wrote_any = True
except Exception as exc:
logger.warning(
"record_authorization_redis_failed",
incident_id=incident_id,
source=source,
error=str(exc),
)
try:
from src.repositories.alert_operation_log_repository import (
get_alert_operation_log_repository,
)
event_type = "APPROVAL_ESCALATED" if spec.requires_multi_sig or spec.risk == "critical" else "USER_ACTION"
record = await get_alert_operation_log_repository().append(
event_type,
incident_id=incident_id,
actor=actor,
action_detail=f"telegram_authorization:{requested_action}"[:200],
success=True,
context=context,
)
wrote_any = wrote_any or bool(record)
except Exception as exc:
logger.warning(
"record_authorization_aol_failed",
incident_id=incident_id,
source=source,
error=str(exc),
)
try:
from src.services.approval_db import get_timeline_service
await get_timeline_service().add_event(
event_type="security",
status="warning" if spec.requires_multi_sig or spec.risk == "critical" else "info",
title="Telegram authorization recorded",
description=(
f"action={requested_action} source={source} source_ip={source_ip or 'unknown'}"
)[:500],
actor=actor,
actor_role="secops_authorization",
risk_level=spec.risk,
incident_id=incident_id,
)
wrote_any = True
except Exception as exc:
logger.warning(
"record_authorization_timeline_failed",
incident_id=incident_id,
source=source,
error=str(exc),
)
logger.info(
"record_authorization_audit_complete",
incident_id=incident_id,
source=source,
action=requested_action,
wrote_any=wrote_any,
)
return wrote_any
def _format_reply(
mcp_result: Any, reply_format: str, label: str, emoji: str
) -> str:

View File

@@ -119,6 +119,10 @@ async def escalate_drift_auto_adopt_blocked(
return
try:
from src.repositories.alert_operation_log_repository import (
get_alert_operation_log_repository,
)
from src.services.approval_db import get_timeline_service
from src.services.telegram_gateway import get_telegram_gateway
actionable_count = sum(
@@ -143,6 +147,43 @@ async def escalate_drift_auto_adopt_blocked(
),
group_chat_id=settings.SRE_GROUP_CHAT_ID or None,
)
await get_alert_operation_log_repository().append(
"APPROVAL_ESCALATED",
incident_id=report.report_id,
actor="drift_auto_adopt",
action_detail="drift_auto_adopt_blocked_emergency_channel",
success=True,
context={
"namespace": report.namespace,
"reason": reason,
"high_count": report.high_count,
"medium_count": report.medium_count,
"actionable_count": actionable_count,
"intent": intent,
"confidence": confidence,
"risk": risk,
},
)
try:
await get_timeline_service().add_event(
event_type="agent",
status="warning",
title="Drift emergency intervention requested",
description=(
f"{reason} | namespace={report.namespace} "
f"high={report.high_count} medium={report.medium_count} "
f"intent={intent} confidence={confidence:.0%}"
)[:500],
actor="drift_auto_adopt",
actor_role="emergency_intervention",
incident_id=report.report_id,
)
except Exception as timeline_exc:
logger.warning(
"drift_emergency_timeline_failed",
report_id=report.report_id,
error=str(timeline_exc),
)
logger.warning(
"drift_auto_adopt_emergency_escalated",
report_id=report.report_id,