fix(aiops): persist emergency intervention traces
This commit is contained in:
@@ -23,6 +23,7 @@ Phase 5 Sprint 5.0-5.1 — 2026-04-14 Claude Sonnet 4.6
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
@@ -264,7 +265,12 @@ async def dispatch_action(
|
||||
try:
|
||||
# internal provider: 特殊 URL builder(無 MCP call)
|
||||
if spec.mcp_provider == "internal":
|
||||
result_text = _handle_internal_action(spec, resolved_params)
|
||||
result_text = await _handle_internal_action(
|
||||
spec,
|
||||
resolved_params,
|
||||
incident_id=incident_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
duration = (time.perf_counter() - start) * 1000
|
||||
logger.info("dispatch_action_internal", action=action_name, duration_ms=round(duration, 1))
|
||||
return DispatchResult(
|
||||
@@ -361,7 +367,13 @@ async def dispatch_action(
|
||||
)
|
||||
|
||||
|
||||
def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
|
||||
async def _handle_internal_action(
|
||||
spec: ActionSpec,
|
||||
params: dict,
|
||||
*,
|
||||
incident_id: str,
|
||||
user_id: int | None,
|
||||
) -> str:
|
||||
"""
|
||||
Internal actions — 不走 MCP,直接產生 URL/文字回覆
|
||||
|
||||
@@ -379,12 +391,20 @@ def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
|
||||
return f"{spec.emoji} <b>{spec.label}</b>\nhttps://awoooi.wooo.work/flywheel"
|
||||
|
||||
if tool == "record_authorization":
|
||||
# Sprint 5.4 會實作真實授權記錄,這裡先返回確認
|
||||
user_id = params.get("user_id", 0)
|
||||
recorded = await _record_authorization_audit(
|
||||
spec=spec,
|
||||
params=params,
|
||||
incident_id=incident_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
_user_id = params.get("user_id", user_id or 0)
|
||||
source = params.get("source", "unknown")
|
||||
action = params.get("action", "authorize")
|
||||
suffix = "已寫入審計與時間線" if recorded else "已受理;審計寫入將由後續補償"
|
||||
return (
|
||||
f"{spec.emoji} <b>{spec.label}</b>\n"
|
||||
f"已記錄 user={user_id} 授權 source={source}(24h 內同源告警將靜默)"
|
||||
f"已記錄 user={_user_id} 授權 source={source} action={action}(24h 內同源告警將靜默)\n"
|
||||
f"{suffix}"
|
||||
)
|
||||
|
||||
# 未知的 internal tool
|
||||
@@ -394,6 +414,104 @@ def _handle_internal_action(spec: ActionSpec, params: dict) -> str:
|
||||
)
|
||||
|
||||
|
||||
async def _record_authorization_audit(
|
||||
*,
|
||||
spec: ActionSpec,
|
||||
params: dict,
|
||||
incident_id: str,
|
||||
user_id: int | None,
|
||||
) -> bool:
|
||||
"""Best-effort persistence for internal authorization actions."""
|
||||
|
||||
source = str(params.get("source") or "unknown")
|
||||
requested_action = str(params.get("action") or spec.name)
|
||||
source_ip = str(params.get("source_ip") or "")
|
||||
actor = f"telegram:{user_id or params.get('user_id') or 0}"
|
||||
context = {
|
||||
"action": spec.name,
|
||||
"label": spec.label,
|
||||
"risk": spec.risk,
|
||||
"category": spec.category,
|
||||
"requested_action": requested_action,
|
||||
"source": source,
|
||||
"source_ip": source_ip,
|
||||
"user_id": user_id or params.get("user_id") or 0,
|
||||
"requires_multi_sig": spec.requires_multi_sig,
|
||||
}
|
||||
wrote_any = False
|
||||
|
||||
try:
|
||||
from src.core.redis_client import get_redis
|
||||
|
||||
redis = get_redis()
|
||||
redis_key = f"secops:authorization:{source}"
|
||||
await redis.set(redis_key, json.dumps(context, ensure_ascii=False), ex=86400)
|
||||
wrote_any = True
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"record_authorization_redis_failed",
|
||||
incident_id=incident_id,
|
||||
source=source,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
try:
|
||||
from src.repositories.alert_operation_log_repository import (
|
||||
get_alert_operation_log_repository,
|
||||
)
|
||||
|
||||
event_type = "APPROVAL_ESCALATED" if spec.requires_multi_sig or spec.risk == "critical" else "USER_ACTION"
|
||||
record = await get_alert_operation_log_repository().append(
|
||||
event_type,
|
||||
incident_id=incident_id,
|
||||
actor=actor,
|
||||
action_detail=f"telegram_authorization:{requested_action}"[:200],
|
||||
success=True,
|
||||
context=context,
|
||||
)
|
||||
wrote_any = wrote_any or bool(record)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"record_authorization_aol_failed",
|
||||
incident_id=incident_id,
|
||||
source=source,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
try:
|
||||
from src.services.approval_db import get_timeline_service
|
||||
|
||||
await get_timeline_service().add_event(
|
||||
event_type="security",
|
||||
status="warning" if spec.requires_multi_sig or spec.risk == "critical" else "info",
|
||||
title="Telegram authorization recorded",
|
||||
description=(
|
||||
f"action={requested_action} source={source} source_ip={source_ip or 'unknown'}"
|
||||
)[:500],
|
||||
actor=actor,
|
||||
actor_role="secops_authorization",
|
||||
risk_level=spec.risk,
|
||||
incident_id=incident_id,
|
||||
)
|
||||
wrote_any = True
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"record_authorization_timeline_failed",
|
||||
incident_id=incident_id,
|
||||
source=source,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"record_authorization_audit_complete",
|
||||
incident_id=incident_id,
|
||||
source=source,
|
||||
action=requested_action,
|
||||
wrote_any=wrote_any,
|
||||
)
|
||||
return wrote_any
|
||||
|
||||
|
||||
def _format_reply(
|
||||
mcp_result: Any, reply_format: str, label: str, emoji: str
|
||||
) -> str:
|
||||
|
||||
@@ -119,6 +119,10 @@ async def escalate_drift_auto_adopt_blocked(
|
||||
return
|
||||
|
||||
try:
|
||||
from src.repositories.alert_operation_log_repository import (
|
||||
get_alert_operation_log_repository,
|
||||
)
|
||||
from src.services.approval_db import get_timeline_service
|
||||
from src.services.telegram_gateway import get_telegram_gateway
|
||||
|
||||
actionable_count = sum(
|
||||
@@ -143,6 +147,43 @@ async def escalate_drift_auto_adopt_blocked(
|
||||
),
|
||||
group_chat_id=settings.SRE_GROUP_CHAT_ID or None,
|
||||
)
|
||||
await get_alert_operation_log_repository().append(
|
||||
"APPROVAL_ESCALATED",
|
||||
incident_id=report.report_id,
|
||||
actor="drift_auto_adopt",
|
||||
action_detail="drift_auto_adopt_blocked_emergency_channel",
|
||||
success=True,
|
||||
context={
|
||||
"namespace": report.namespace,
|
||||
"reason": reason,
|
||||
"high_count": report.high_count,
|
||||
"medium_count": report.medium_count,
|
||||
"actionable_count": actionable_count,
|
||||
"intent": intent,
|
||||
"confidence": confidence,
|
||||
"risk": risk,
|
||||
},
|
||||
)
|
||||
try:
|
||||
await get_timeline_service().add_event(
|
||||
event_type="agent",
|
||||
status="warning",
|
||||
title="Drift emergency intervention requested",
|
||||
description=(
|
||||
f"{reason} | namespace={report.namespace} "
|
||||
f"high={report.high_count} medium={report.medium_count} "
|
||||
f"intent={intent} confidence={confidence:.0%}"
|
||||
)[:500],
|
||||
actor="drift_auto_adopt",
|
||||
actor_role="emergency_intervention",
|
||||
incident_id=report.report_id,
|
||||
)
|
||||
except Exception as timeline_exc:
|
||||
logger.warning(
|
||||
"drift_emergency_timeline_failed",
|
||||
report_id=report.report_id,
|
||||
error=str(timeline_exc),
|
||||
)
|
||||
logger.warning(
|
||||
"drift_auto_adopt_emergency_escalated",
|
||||
report_id=report.report_id,
|
||||
|
||||
@@ -15,6 +15,7 @@ Phase 5 Sprint 5.0-5.1 Callback Dispatcher 單元測試
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services import callback_dispatcher as callback_dispatcher_module
|
||||
from src.services.callback_dispatcher import (
|
||||
dispatch_action,
|
||||
get_action_spec,
|
||||
@@ -269,6 +270,36 @@ class TestInternalActions:
|
||||
assert result.success is True
|
||||
assert "12345" in result.result_text
|
||||
|
||||
async def test_record_authorization_persists_audit_intent(self, monkeypatch):
|
||||
captured = {}
|
||||
|
||||
async def fake_record_authorization_audit(*, spec, params, incident_id, user_id):
|
||||
captured["spec"] = spec
|
||||
captured["params"] = params
|
||||
captured["incident_id"] = incident_id
|
||||
captured["user_id"] = user_id
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
callback_dispatcher_module,
|
||||
"_record_authorization_audit",
|
||||
fake_record_authorization_audit,
|
||||
)
|
||||
|
||||
result = await dispatch_action(
|
||||
action_name="secops_isolate",
|
||||
incident_id="INC-SEC-AUTH",
|
||||
user_id=67890,
|
||||
labels={"instance": "192.168.0.110"},
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert "已寫入審計與時間線" in result.result_text
|
||||
assert captured["spec"].name == "secops_isolate"
|
||||
assert captured["params"]["action"] == "request_network_isolation"
|
||||
assert captured["incident_id"] == "INC-SEC-AUTH"
|
||||
assert captured["user_id"] == 67890
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Sprint 5.2 — MCP 呼叫失敗路徑(Provider 未註冊)
|
||||
|
||||
71
apps/api/tests/test_emergency_escalation_service.py
Normal file
71
apps/api/tests/test_emergency_escalation_service.py
Normal file
@@ -0,0 +1,71 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services import emergency_escalation_service as service
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drift_emergency_escalation_writes_aol_and_timeline(monkeypatch):
|
||||
sent_cards = []
|
||||
aol_calls = []
|
||||
timeline_calls = []
|
||||
|
||||
async def fake_dedup(*args, **kwargs):
|
||||
return True
|
||||
|
||||
class FakeGateway:
|
||||
async def send_escalation_card(self, **kwargs):
|
||||
sent_cards.append(kwargs)
|
||||
|
||||
class FakeRepo:
|
||||
async def append(self, *args, **kwargs):
|
||||
aol_calls.append((args, kwargs))
|
||||
return object()
|
||||
|
||||
class FakeTimeline:
|
||||
async def add_event(self, **kwargs):
|
||||
timeline_calls.append(kwargs)
|
||||
return {"id": "timeline-1"}
|
||||
|
||||
monkeypatch.setattr(service, "_dedup_first_send", fake_dedup)
|
||||
monkeypatch.setattr(
|
||||
"src.services.telegram_gateway.get_telegram_gateway",
|
||||
lambda: FakeGateway(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
|
||||
lambda: FakeRepo(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.services.approval_db.get_timeline_service",
|
||||
lambda: FakeTimeline(),
|
||||
)
|
||||
|
||||
report = SimpleNamespace(
|
||||
report_id="drift-123",
|
||||
namespace="awoooi-prod",
|
||||
high_count=1,
|
||||
medium_count=2,
|
||||
items=[
|
||||
SimpleNamespace(is_allowlisted=False),
|
||||
SimpleNamespace(is_allowlisted=True),
|
||||
],
|
||||
)
|
||||
interpretation = SimpleNamespace(
|
||||
intent=SimpleNamespace(value="emergency_hotfix"),
|
||||
confidence=0.72,
|
||||
risk="high",
|
||||
)
|
||||
|
||||
await service.escalate_drift_auto_adopt_blocked(
|
||||
report=report,
|
||||
reason="unsafe drift",
|
||||
interpretation=interpretation,
|
||||
)
|
||||
|
||||
assert sent_cards and sent_cards[0]["incident_id"] == "drift-123"
|
||||
assert aol_calls and aol_calls[0][0][0] == "APPROVAL_ESCALATED"
|
||||
assert aol_calls[0][1]["actor"] == "drift_auto_adopt"
|
||||
assert aol_calls[0][1]["context"]["intent"] == "emergency_hotfix"
|
||||
assert timeline_calls and timeline_calls[0]["actor_role"] == "emergency_intervention"
|
||||
Reference in New Issue
Block a user