feat(flywheel): expose incident processing timeline
All checks were successful
CD Pipeline / build-and-deploy (push) Successful in 10m56s
All checks were successful
CD Pipeline / build-and-deploy (push) Successful in 10m56s
This commit is contained in:
@@ -921,6 +921,7 @@ class TimelineDBService:
|
||||
actor_role: str | None = None,
|
||||
risk_level: str | None = None,
|
||||
approval_id: str | None = None,
|
||||
incident_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
新增時間軸事件
|
||||
@@ -935,6 +936,7 @@ class TimelineDBService:
|
||||
actor_role=actor_role,
|
||||
risk_level=risk_level,
|
||||
approval_id=approval_id,
|
||||
incident_id=incident_id,
|
||||
)
|
||||
db.add(event)
|
||||
await db.flush()
|
||||
@@ -945,6 +947,7 @@ class TimelineDBService:
|
||||
id=event.id,
|
||||
type=event_type,
|
||||
title=title,
|
||||
incident_id=incident_id,
|
||||
)
|
||||
|
||||
return {
|
||||
@@ -952,19 +955,28 @@ class TimelineDBService:
|
||||
"type": event.event_type,
|
||||
"status": event.status,
|
||||
"title": event.title,
|
||||
"incident_id": event.incident_id,
|
||||
"created_at": event.created_at.isoformat(),
|
||||
}
|
||||
|
||||
async def get_events(self, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async def get_events(
|
||||
self,
|
||||
limit: int = 50,
|
||||
incident_id: str | None = None,
|
||||
approval_ids: list[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
取得最近的時間軸事件
|
||||
"""
|
||||
async with get_db_context() as db:
|
||||
result = await db.execute(
|
||||
select(TimelineEvent)
|
||||
.order_by(TimelineEvent.created_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
query = select(TimelineEvent)
|
||||
if incident_id:
|
||||
from sqlalchemy import or_
|
||||
filters = [TimelineEvent.incident_id == incident_id]
|
||||
if approval_ids:
|
||||
filters.append(TimelineEvent.approval_id.in_(approval_ids))
|
||||
query = query.where(or_(*filters))
|
||||
result = await db.execute(query.order_by(TimelineEvent.created_at.desc()).limit(limit))
|
||||
events = result.scalars().all()
|
||||
|
||||
return [
|
||||
@@ -978,6 +990,7 @@ class TimelineDBService:
|
||||
"actor_role": e.actor_role,
|
||||
"risk_level": e.risk_level,
|
||||
"approval_id": e.approval_id,
|
||||
"incident_id": e.incident_id,
|
||||
"created_at": e.created_at.isoformat(),
|
||||
}
|
||||
for e in events
|
||||
|
||||
@@ -233,6 +233,7 @@ class ApprovalExecutionService:
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=approval.incident_id,
|
||||
)
|
||||
# 執行結果 reply 原告警卡片
|
||||
asyncio.create_task(
|
||||
@@ -269,6 +270,7 @@ class ApprovalExecutionService:
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=approval.incident_id,
|
||||
)
|
||||
|
||||
# Phase 6: 發送失敗通知 (fire-and-forget)
|
||||
@@ -346,6 +348,7 @@ class ApprovalExecutionService:
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=approval.incident_id,
|
||||
)
|
||||
await asyncio.sleep(self.RETRY_DELAY_SECONDS)
|
||||
result = await executor.execute_with_audit(
|
||||
@@ -385,6 +388,7 @@ class ApprovalExecutionService:
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=approval.incident_id,
|
||||
)
|
||||
|
||||
# Phase 6: 發送成功通知 (fire-and-forget)
|
||||
@@ -506,6 +510,7 @@ class ApprovalExecutionService:
|
||||
actor="leWOOOgo",
|
||||
actor_role="executor",
|
||||
approval_id=str(approval.id),
|
||||
incident_id=approval.incident_id,
|
||||
)
|
||||
|
||||
# Phase 6: 發送失敗通知 (fire-and-forget, 包含 Dry-Run 攔截)
|
||||
|
||||
@@ -398,6 +398,28 @@ async def create_incident_for_approval(
|
||||
error=str(_pg_err),
|
||||
)
|
||||
|
||||
try:
|
||||
from src.services.approval_db import get_timeline_service
|
||||
|
||||
await get_timeline_service().add_event(
|
||||
event_type="webhook",
|
||||
status="success",
|
||||
title=f"Webhook alert received: {alertname or alert_type}",
|
||||
description=message,
|
||||
actor=source,
|
||||
actor_role="webhook",
|
||||
risk_level=risk_level,
|
||||
approval_id=approval_id,
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
except Exception as _timeline_err:
|
||||
logger.warning(
|
||||
"incident_timeline_webhook_event_failed",
|
||||
incident_id=incident.incident_id,
|
||||
approval_id=approval_id,
|
||||
error=str(_timeline_err),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"incident_created_for_approval",
|
||||
incident_id=incident.incident_id,
|
||||
@@ -897,6 +919,30 @@ class IncidentService:
|
||||
persisted_to_pg=incident.persisted_to_pg,
|
||||
)
|
||||
|
||||
try:
|
||||
from src.services.approval_db import get_timeline_service
|
||||
|
||||
await get_timeline_service().add_event(
|
||||
event_type="webhook",
|
||||
status="success" if redis_success or pg_success else "warning",
|
||||
title=f"Signal received: {signal.alert_name}",
|
||||
description=(
|
||||
signal.annotations.get("message")
|
||||
or signal.annotations.get("description")
|
||||
or signal.annotations.get("summary")
|
||||
),
|
||||
actor=signal.source,
|
||||
actor_role="webhook",
|
||||
risk_level=incident.severity.value,
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
except Exception as timeline_error:
|
||||
logger.warning(
|
||||
"incident_timeline_signal_event_failed",
|
||||
incident_id=incident.incident_id,
|
||||
error=str(timeline_error),
|
||||
)
|
||||
|
||||
return incident
|
||||
|
||||
except Exception as e:
|
||||
|
||||
570
apps/api/src/services/incident_timeline_service.py
Normal file
570
apps/api/src/services/incident_timeline_service.py
Normal file
@@ -0,0 +1,570 @@
|
||||
"""Incident processing timeline aggregation.
|
||||
|
||||
Builds the operator-facing "what happened" timeline from the existing event
|
||||
tables without adding another schema hop. The raw `timeline_events` table is
|
||||
still the append-only audit rail; this service composes it with Incident,
|
||||
Approval, Evidence, Executor, and KM records so a single Incident detail view can
|
||||
show the full path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
from src.db.base import get_db_context
|
||||
from src.db.models import (
|
||||
AlertOperationLog,
|
||||
ApprovalRecord,
|
||||
AutoRepairExecution,
|
||||
IncidentEvidence,
|
||||
IncidentRecord,
|
||||
KnowledgeEntryRecord,
|
||||
TimelineEvent,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
STAGE_DEFS: tuple[tuple[str, str], ...] = (
|
||||
("webhook", "Webhook"),
|
||||
("investigator", "Investigator"),
|
||||
("ai_router", "AI Router"),
|
||||
("llm", "LLM"),
|
||||
("target", "Target"),
|
||||
("blast", "Blast Radius"),
|
||||
("safe", "Safety Gate"),
|
||||
("executor", "Executor"),
|
||||
("verifier", "Verifier"),
|
||||
("km", "KM"),
|
||||
("close", "Closure"),
|
||||
)
|
||||
|
||||
_STAGE_LABEL = dict(STAGE_DEFS)
|
||||
_STATUS_RANK = {
|
||||
"skipped": 0,
|
||||
"pending": 1,
|
||||
"info": 2,
|
||||
"completed": 3,
|
||||
"success": 4,
|
||||
"warning": 5,
|
||||
"error": 6,
|
||||
}
|
||||
_EVENT_STAGE_MAP = {
|
||||
"webhook": "webhook",
|
||||
"alert": "webhook",
|
||||
"system": "safe",
|
||||
"agent": "llm",
|
||||
"ai_router": "ai_router",
|
||||
"llm": "llm",
|
||||
"mcp_call": "investigator",
|
||||
"investigator": "investigator",
|
||||
"target": "target",
|
||||
"blast": "blast",
|
||||
"security": "safe",
|
||||
"safe": "safe",
|
||||
"human": "safe",
|
||||
"exec": "executor",
|
||||
"executor": "executor",
|
||||
"verify": "verifier",
|
||||
"verifier": "verifier",
|
||||
"km": "km",
|
||||
"learn": "km",
|
||||
"close": "close",
|
||||
"resolved": "close",
|
||||
}
|
||||
|
||||
|
||||
def _value(value: Any) -> Any:
|
||||
return value.value if hasattr(value, "value") else value
|
||||
|
||||
|
||||
def _iso(value: Any) -> str | None:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
return None
|
||||
|
||||
|
||||
def _compact(value: str | None, max_len: int = 500) -> str | None:
|
||||
if not value:
|
||||
return value
|
||||
return value if len(value) <= max_len else f"{value[:max_len - 3]}..."
|
||||
|
||||
|
||||
def _event(
|
||||
*,
|
||||
stage: str,
|
||||
status: str,
|
||||
title: str,
|
||||
timestamp: Any = None,
|
||||
description: str | None = None,
|
||||
actor: str | None = None,
|
||||
source_table: str,
|
||||
data: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"stage": stage,
|
||||
"status": status,
|
||||
"title": title,
|
||||
"description": _compact(description),
|
||||
"actor": actor,
|
||||
"timestamp": _iso(timestamp),
|
||||
"source_table": source_table,
|
||||
"data": data or {},
|
||||
}
|
||||
|
||||
|
||||
def _empty_stage(stage: str, label: str) -> dict[str, Any]:
|
||||
return {
|
||||
"stage": stage,
|
||||
"label": label,
|
||||
"status": "skipped",
|
||||
"timestamp": None,
|
||||
"title": f"{label} not recorded",
|
||||
"description": None,
|
||||
"actor": None,
|
||||
"source_table": None,
|
||||
"data": {},
|
||||
"events": [],
|
||||
}
|
||||
|
||||
|
||||
def _apply_event(stages: dict[str, dict[str, Any]], event: dict[str, Any]) -> None:
|
||||
stage_name = event["stage"]
|
||||
stage = stages.get(stage_name)
|
||||
if stage is None:
|
||||
return
|
||||
|
||||
stage["events"].append(event)
|
||||
current_rank = _STATUS_RANK.get(stage["status"], 0)
|
||||
incoming_rank = _STATUS_RANK.get(event["status"], 0)
|
||||
if incoming_rank >= current_rank:
|
||||
stage.update({
|
||||
"status": event["status"],
|
||||
"timestamp": event["timestamp"] or stage["timestamp"],
|
||||
"title": event["title"],
|
||||
"description": event["description"],
|
||||
"actor": event["actor"],
|
||||
"source_table": event["source_table"],
|
||||
"data": event["data"],
|
||||
})
|
||||
elif stage["timestamp"] is None and event["timestamp"]:
|
||||
stage["timestamp"] = event["timestamp"]
|
||||
|
||||
|
||||
def _stage_from_event_type(event_type: str | None) -> str:
|
||||
return _EVENT_STAGE_MAP.get((event_type or "").lower(), "safe")
|
||||
|
||||
|
||||
def format_ascii_timeline(stages: list[dict[str, Any]]) -> str:
|
||||
"""Compact ASCII line for Telegram and logs."""
|
||||
marks = {
|
||||
"success": "ok",
|
||||
"completed": "ok",
|
||||
"info": "ok",
|
||||
"warning": "warn",
|
||||
"error": "fail",
|
||||
"pending": "wait",
|
||||
"skipped": "skip",
|
||||
}
|
||||
parts = [
|
||||
f"{stage['stage']}:{marks.get(stage['status'], stage['status'])}"
|
||||
for stage in stages
|
||||
if stage["status"] != "skipped"
|
||||
]
|
||||
return " > ".join(parts) if parts else "webhook:skip > ai:skip > executor:skip"
|
||||
|
||||
|
||||
async def fetch_incident_timeline(incident_id: str) -> dict[str, Any] | None:
|
||||
"""Return a complete detail timeline for one incident."""
|
||||
stages = {stage: _empty_stage(stage, label) for stage, label in STAGE_DEFS}
|
||||
|
||||
async with get_db_context() as db:
|
||||
incident = (
|
||||
await db.execute(
|
||||
select(IncidentRecord).where(IncidentRecord.incident_id == incident_id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if incident is None:
|
||||
return None
|
||||
|
||||
approvals = (
|
||||
await db.execute(
|
||||
select(ApprovalRecord)
|
||||
.where(ApprovalRecord.incident_id == incident_id)
|
||||
.order_by(ApprovalRecord.created_at.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
approval_ids = [str(a.id) for a in approvals]
|
||||
|
||||
evidence_records = (
|
||||
await db.execute(
|
||||
select(IncidentEvidence)
|
||||
.where(IncidentEvidence.incident_id == incident_id)
|
||||
.order_by(IncidentEvidence.collected_at.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
executions = (
|
||||
await db.execute(
|
||||
select(AutoRepairExecution)
|
||||
.where(AutoRepairExecution.incident_id == incident_id)
|
||||
.order_by(AutoRepairExecution.created_at.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
km_entries = (
|
||||
await db.execute(
|
||||
select(KnowledgeEntryRecord)
|
||||
.where(KnowledgeEntryRecord.related_incident_id == incident_id)
|
||||
.order_by(KnowledgeEntryRecord.created_at.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
timeline_filter = TimelineEvent.incident_id == incident_id
|
||||
if approval_ids:
|
||||
timeline_filter = or_(timeline_filter, TimelineEvent.approval_id.in_(approval_ids))
|
||||
raw_timeline = (
|
||||
await db.execute(
|
||||
select(TimelineEvent)
|
||||
.where(timeline_filter)
|
||||
.order_by(TimelineEvent.created_at.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
aol_filter = AlertOperationLog.incident_id == incident_id
|
||||
if approval_ids:
|
||||
aol_filter = or_(aol_filter, AlertOperationLog.approval_id.in_(approval_ids))
|
||||
alert_ops = (
|
||||
await db.execute(
|
||||
select(AlertOperationLog)
|
||||
.where(aol_filter)
|
||||
.order_by(AlertOperationLog.created_at.asc())
|
||||
.limit(100)
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
events: list[dict[str, Any]] = []
|
||||
|
||||
alert_name = incident.alertname
|
||||
if not alert_name and incident.signals:
|
||||
first_signal = incident.signals[0] if isinstance(incident.signals, list) else {}
|
||||
alert_name = first_signal.get("alert_name") or first_signal.get("labels", {}).get("alertname")
|
||||
|
||||
events.append(_event(
|
||||
stage="webhook",
|
||||
status="completed",
|
||||
title=f"Alert received: {alert_name or 'unknown'}",
|
||||
timestamp=incident.created_at,
|
||||
description=f"source={_signal_source(incident.signals)} severity={_value(incident.severity)}",
|
||||
actor=_signal_source(incident.signals) or "alertmanager",
|
||||
source_table="incidents",
|
||||
data={
|
||||
"alertname": alert_name,
|
||||
"severity": _value(incident.severity),
|
||||
"signals": incident.signals or [],
|
||||
"affected_services": incident.affected_services or [],
|
||||
},
|
||||
))
|
||||
|
||||
for evidence in evidence_records:
|
||||
status = "completed" if (evidence.sensors_succeeded or 0) > 0 else "warning"
|
||||
events.append(_event(
|
||||
stage="investigator",
|
||||
status=status,
|
||||
title="Evidence snapshot collected",
|
||||
timestamp=evidence.collected_at,
|
||||
description=evidence.evidence_summary,
|
||||
actor="pre_decision_investigator",
|
||||
source_table="incident_evidence",
|
||||
data={
|
||||
"sensors_attempted": evidence.sensors_attempted,
|
||||
"sensors_succeeded": evidence.sensors_succeeded,
|
||||
"duration_ms": evidence.collection_duration_ms,
|
||||
"mcp_health": evidence.mcp_health,
|
||||
},
|
||||
))
|
||||
if evidence.verification_result:
|
||||
verification_status = (
|
||||
"success" if evidence.verification_result == "success"
|
||||
else "warning" if evidence.verification_result == "degraded"
|
||||
else "error"
|
||||
)
|
||||
events.append(_event(
|
||||
stage="verifier",
|
||||
status=verification_status,
|
||||
title=f"Post-execution verification: {evidence.verification_result}",
|
||||
timestamp=evidence.collected_at,
|
||||
description=evidence.self_healing_detail and str(evidence.self_healing_detail),
|
||||
actor="post_execution_verifier",
|
||||
source_table="incident_evidence",
|
||||
data={
|
||||
"verification_result": evidence.verification_result,
|
||||
"self_healing_score": evidence.self_healing_score,
|
||||
"self_healing_detail": evidence.self_healing_detail,
|
||||
},
|
||||
))
|
||||
|
||||
for approval in approvals:
|
||||
metadata = approval.extra_metadata or {}
|
||||
provider = metadata.get("source") or _provider_from_description(approval.description)
|
||||
if provider:
|
||||
events.append(_event(
|
||||
stage="ai_router",
|
||||
status="completed",
|
||||
title=f"AI route selected: {provider}",
|
||||
timestamp=approval.created_at,
|
||||
description=approval.description,
|
||||
actor="ai_router",
|
||||
source_table="approval_records",
|
||||
data={
|
||||
"provider": provider,
|
||||
"confidence_score": metadata.get("confidence_score"),
|
||||
"is_rule_based": metadata.get("is_rule_based"),
|
||||
},
|
||||
))
|
||||
events.append(_event(
|
||||
stage="llm",
|
||||
status="completed",
|
||||
title=f"LLM proposal generated: {provider}",
|
||||
timestamp=approval.created_at,
|
||||
description=approval.description,
|
||||
actor=provider,
|
||||
source_table="approval_records",
|
||||
data={
|
||||
"approval_id": approval.id,
|
||||
"matched_playbook_id": approval.matched_playbook_id,
|
||||
"playbook_id": metadata.get("playbook_id"),
|
||||
},
|
||||
))
|
||||
|
||||
events.append(_event(
|
||||
stage="target",
|
||||
status="completed",
|
||||
title="Target resource selected",
|
||||
timestamp=approval.created_at,
|
||||
description=approval.action,
|
||||
actor=approval.requested_by,
|
||||
source_table="approval_records",
|
||||
data={"action": approval.action},
|
||||
))
|
||||
|
||||
events.append(_event(
|
||||
stage="blast",
|
||||
status="completed" if approval.blast_radius else "warning",
|
||||
title="Blast radius evaluated",
|
||||
timestamp=approval.created_at,
|
||||
description=None,
|
||||
actor=approval.requested_by,
|
||||
source_table="approval_records",
|
||||
data=approval.blast_radius or {},
|
||||
))
|
||||
|
||||
events.append(_event(
|
||||
stage="safe",
|
||||
status=_approval_status_to_timeline_status(approval.status),
|
||||
title=f"Safety gate: {_value(approval.risk_level)} / {_value(approval.status)}",
|
||||
timestamp=approval.created_at,
|
||||
description=_dry_run_summary(approval.dry_run_checks),
|
||||
actor=approval.requested_by,
|
||||
source_table="approval_records",
|
||||
data={
|
||||
"approval_id": approval.id,
|
||||
"risk_level": _value(approval.risk_level),
|
||||
"status": _value(approval.status),
|
||||
"required_signatures": approval.required_signatures,
|
||||
"current_signatures": approval.current_signatures,
|
||||
"dry_run_checks": approval.dry_run_checks or [],
|
||||
},
|
||||
))
|
||||
|
||||
if str(_value(approval.status)).startswith("execution_"):
|
||||
success = _value(approval.status) == "execution_success"
|
||||
events.append(_event(
|
||||
stage="executor",
|
||||
status="success" if success else "error",
|
||||
title="Approval execution completed",
|
||||
timestamp=approval.resolved_at or approval.updated_at,
|
||||
description=approval.rejection_reason,
|
||||
actor="approval_execution",
|
||||
source_table="approval_records",
|
||||
data={
|
||||
"approval_id": approval.id,
|
||||
"status": _value(approval.status),
|
||||
},
|
||||
))
|
||||
|
||||
for execution in executions:
|
||||
events.append(_event(
|
||||
stage="executor",
|
||||
status="success" if execution.success else "error",
|
||||
title=f"Auto repair execution: {execution.playbook_name}",
|
||||
timestamp=execution.created_at,
|
||||
description=execution.error_message,
|
||||
actor=execution.triggered_by,
|
||||
source_table="auto_repair_executions",
|
||||
data={
|
||||
"playbook_id": execution.playbook_id,
|
||||
"success": execution.success,
|
||||
"execution_time_ms": execution.execution_time_ms,
|
||||
"similarity_score": execution.similarity_score,
|
||||
"risk_level": execution.risk_level,
|
||||
"executed_steps": execution.executed_steps,
|
||||
},
|
||||
))
|
||||
|
||||
for entry in km_entries:
|
||||
events.append(_event(
|
||||
stage="km",
|
||||
status="completed",
|
||||
title=f"KM entry written: {entry.title}",
|
||||
timestamp=entry.created_at,
|
||||
description=entry.content,
|
||||
actor=entry.created_by or _value(entry.source),
|
||||
source_table="knowledge_entries",
|
||||
data={
|
||||
"knowledge_id": entry.id,
|
||||
"entry_type": _value(entry.entry_type),
|
||||
"status": _value(entry.status),
|
||||
"path_type": entry.path_type,
|
||||
"related_approval_id": entry.related_approval_id,
|
||||
},
|
||||
))
|
||||
|
||||
if incident.resolved_at or incident.closed_at:
|
||||
events.append(_event(
|
||||
stage="close",
|
||||
status="success",
|
||||
title=f"Incident {_value(incident.status)}",
|
||||
timestamp=incident.closed_at or incident.resolved_at,
|
||||
description=None,
|
||||
actor="incident_service",
|
||||
source_table="incidents",
|
||||
data={
|
||||
"status": _value(incident.status),
|
||||
"outcome": incident.outcome,
|
||||
"resolved_at": _iso(incident.resolved_at),
|
||||
"closed_at": _iso(incident.closed_at),
|
||||
},
|
||||
))
|
||||
|
||||
for raw in raw_timeline:
|
||||
events.append(_event(
|
||||
stage=_stage_from_event_type(raw.event_type),
|
||||
status=raw.status,
|
||||
title=raw.title,
|
||||
timestamp=raw.created_at,
|
||||
description=raw.description,
|
||||
actor=raw.actor,
|
||||
source_table="timeline_events",
|
||||
data={
|
||||
"timeline_event_id": raw.id,
|
||||
"event_type": raw.event_type,
|
||||
"approval_id": raw.approval_id,
|
||||
"actor_role": raw.actor_role,
|
||||
"risk_level": raw.risk_level,
|
||||
},
|
||||
))
|
||||
|
||||
for op in alert_ops:
|
||||
events.append(_event(
|
||||
stage=_stage_from_aol(op.event_type),
|
||||
status="error" if op.success is False else "success" if op.success is True else "info",
|
||||
title=f"AOL: {_value(op.event_type)}",
|
||||
timestamp=op.created_at,
|
||||
description=op.action_detail or op.error_message,
|
||||
actor=op.actor,
|
||||
source_table="alert_operation_log",
|
||||
data={
|
||||
"operation_id": op.id,
|
||||
"event_type": _value(op.event_type),
|
||||
"approval_id": op.approval_id,
|
||||
"context": op.context,
|
||||
},
|
||||
))
|
||||
|
||||
events.sort(key=lambda e: e["timestamp"] or "")
|
||||
for event in events:
|
||||
_apply_event(stages, event)
|
||||
|
||||
stage_list = [stages[stage] for stage, _ in STAGE_DEFS]
|
||||
result = {
|
||||
"incident_id": incident.incident_id,
|
||||
"title": alert_name or incident.incident_id,
|
||||
"status": _value(incident.status),
|
||||
"severity": _value(incident.severity),
|
||||
"started_at": _iso(incident.created_at),
|
||||
"updated_at": _iso(incident.updated_at),
|
||||
"resolved_at": _iso(incident.resolved_at),
|
||||
"affected_services": incident.affected_services or [],
|
||||
"approval_ids": approval_ids,
|
||||
"timeline": stage_list,
|
||||
"events": events,
|
||||
"ascii_timeline": format_ascii_timeline(stage_list),
|
||||
}
|
||||
logger.info(
|
||||
"incident_timeline_fetched",
|
||||
incident_id=incident_id,
|
||||
stages_recorded=sum(1 for stage in stage_list if stage["status"] != "skipped"),
|
||||
event_count=len(events),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _signal_source(signals: Any) -> str | None:
|
||||
if not signals or not isinstance(signals, list):
|
||||
return None
|
||||
first_signal = signals[0] if signals else {}
|
||||
if not isinstance(first_signal, dict):
|
||||
return None
|
||||
return first_signal.get("source")
|
||||
|
||||
|
||||
def _provider_from_description(description: str | None) -> str | None:
|
||||
if not description:
|
||||
return None
|
||||
if description.startswith("[AI:"):
|
||||
return description.split("]", 1)[0].replace("[AI:", "").strip()
|
||||
return None
|
||||
|
||||
|
||||
def _approval_status_to_timeline_status(status: Any) -> str:
|
||||
value = str(_value(status))
|
||||
if value in {"rejected", "expired"}:
|
||||
return "error"
|
||||
if value in {"approved", "execution_success"}:
|
||||
return "success"
|
||||
if value == "execution_failed":
|
||||
return "warning"
|
||||
return "info"
|
||||
|
||||
|
||||
def _dry_run_summary(checks: Any) -> str | None:
|
||||
if not checks:
|
||||
return None
|
||||
passed = 0
|
||||
total = 0
|
||||
for check in checks:
|
||||
if isinstance(check, dict):
|
||||
total += 1
|
||||
if check.get("passed"):
|
||||
passed += 1
|
||||
return f"Dry-run checks: {passed}/{total} passed" if total else None
|
||||
|
||||
|
||||
def _stage_from_aol(event_type: Any) -> str:
|
||||
value = str(_value(event_type)).upper()
|
||||
if value == "ALERT_RECEIVED":
|
||||
return "webhook"
|
||||
if value in {"PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED", "GUARDRAIL_BLOCKED"}:
|
||||
return "safe"
|
||||
if value in {"EXECUTION_STARTED", "EXECUTION_COMPLETED", "AUTO_REPAIR_TRIGGERED", "CHANGE_APPLIED"}:
|
||||
return "executor"
|
||||
if value in {"TELEGRAM_SENT", "TELEGRAM_RESULT_SENT", "USER_ACTION", "APPROVAL_ESCALATED"}:
|
||||
return "safe"
|
||||
if value == "RESOLVED":
|
||||
return "close"
|
||||
return "safe"
|
||||
@@ -374,6 +374,7 @@ class TelegramMessage:
|
||||
f"{self.status_emoji} ACTION REQUIRED | <b>{html.escape(risk_zh)}</b>\n"
|
||||
f"──────────────────────\n"
|
||||
f"📋 <code>{html.escape(incident_id)}</code>\n"
|
||||
f"流程:<code>webhook>investigator>ai>safe>executor>verifier>km</code>\n"
|
||||
f"🎯 資源:<code>{safe_resource}</code>\n"
|
||||
f"{category_line}"
|
||||
f"\n"
|
||||
@@ -4593,6 +4594,7 @@ class TelegramGateway:
|
||||
"""
|
||||
# 延遲 import 避免循環依賴 (與 approval_service 同一模式)
|
||||
from src.repositories.incident_repository import get_incident_repository
|
||||
from src.services.incident_timeline_service import fetch_incident_timeline
|
||||
|
||||
try:
|
||||
repo = get_incident_repository()
|
||||
@@ -4606,8 +4608,8 @@ class TelegramGateway:
|
||||
confidence_bar = "█" * int((dc.confidence if dc else 0) * 10) + "░" * (10 - int((dc.confidence if dc else 0) * 10))
|
||||
|
||||
lines = [
|
||||
f"📋 <b>事件詳情</b>",
|
||||
f"",
|
||||
"📋 <b>事件詳情</b>",
|
||||
"",
|
||||
f"🔖 <b>ID:</b> <code>{html.escape(incident.incident_id)}</code>",
|
||||
f"📊 <b>狀態:</b> {incident.status.value}",
|
||||
f"⚡ <b>嚴重度:</b> {incident.severity.value}",
|
||||
@@ -4618,7 +4620,7 @@ class TelegramGateway:
|
||||
|
||||
if dc:
|
||||
lines += [
|
||||
f"",
|
||||
"",
|
||||
f"🤖 <b>AI 分析</b> ({html.escape(dc.model_used)})",
|
||||
f"💡 {html.escape(dc.hypothesis)}",
|
||||
f"📈 信心: [{confidence_bar}] {dc.confidence:.0%}",
|
||||
@@ -4630,7 +4632,7 @@ class TelegramGateway:
|
||||
from zoneinfo import ZoneInfo
|
||||
created_taipei = incident.created_at.astimezone(ZoneInfo("Asia/Taipei")) if incident.created_at else incident.created_at
|
||||
lines += [
|
||||
f"",
|
||||
"",
|
||||
f"🕐 <b>建立:</b> {created_taipei.strftime('%m/%d %H:%M') if created_taipei else 'N/A'}",
|
||||
]
|
||||
|
||||
@@ -4638,6 +4640,14 @@ class TelegramGateway:
|
||||
fs = incident.frequency_stats
|
||||
lines.append(f"📉 <b>頻率:</b> 1h={fs.count_1h} 24h={fs.count_24h} 7d={fs.count_7d}")
|
||||
|
||||
timeline = await fetch_incident_timeline(incident_id)
|
||||
if timeline and timeline.get("ascii_timeline"):
|
||||
lines += [
|
||||
"",
|
||||
"🧭 <b>處理歷程</b>",
|
||||
f"<code>{html.escape(timeline['ascii_timeline'])}</code>",
|
||||
]
|
||||
|
||||
await self.send_notification("\n".join(lines))
|
||||
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user