Files
awoooi/apps/api/src/services/incident_timeline_service.py
Your Name 8c40621d42
Some checks failed
CD Pipeline / tests (push) Successful in 1m22s
Code Review / ai-code-review (push) Successful in 12s
CD Pipeline / build-and-deploy (push) Successful in 4m14s
CD Pipeline / post-deploy-checks (push) Has been cancelled
fix(alerts): distinguish diagnostic ops from repair
2026-05-31 14:31:07 +08:00

832 lines
28 KiB
Python

"""Incident processing timeline aggregation.
Builds the operator-facing "what happened" timeline from the existing event
tables without adding another schema hop. The raw `timeline_events` table is
still the append-only audit rail; this service composes it with Incident,
Approval, Evidence, Executor, and KM records so a single Incident detail view can
show the full path.
"""
from __future__ import annotations
from datetime import datetime
from types import SimpleNamespace
from typing import Any
import structlog
from sqlalchemy import or_, select, text
from sqlalchemy.exc import SQLAlchemyError
from src.db.base import get_db_context
from src.db.models import (
AlertOperationLog,
ApprovalRecord,
AutoRepairExecution,
IncidentEvidence,
IncidentRecord,
KnowledgeEntryRecord,
TimelineEvent,
)
from src.services.approval_action_classifier import is_no_action_approval_action
from src.services.awooop_truth_chain_service import build_incident_reconciliation
logger = structlog.get_logger(__name__)
STAGE_DEFS: tuple[tuple[str, str], ...] = (
("webhook", "Webhook"),
("investigator", "Investigator"),
("ai_router", "AI Router"),
("llm", "LLM"),
("target", "Target"),
("blast", "Blast Radius"),
("safe", "Safety Gate"),
("executor", "Executor"),
("verifier", "Verifier"),
("km", "KM"),
("close", "Closure"),
)
_STAGE_LABEL = dict(STAGE_DEFS)
_STATUS_RANK = {
"skipped": 0,
"pending": 1,
"info": 2,
"completed": 3,
"success": 4,
"warning": 5,
"error": 6,
}
_EVENT_STAGE_MAP = {
"webhook": "webhook",
"alert": "webhook",
"system": "safe",
"agent": "llm",
"ai_router": "ai_router",
"llm": "llm",
"mcp_call": "investigator",
"investigator": "investigator",
"target": "target",
"blast": "blast",
"security": "safe",
"safe": "safe",
"human": "safe",
"exec": "executor",
"executor": "executor",
"verify": "verifier",
"verifier": "verifier",
"km": "km",
"learn": "km",
"close": "close",
"resolved": "close",
}
_AUTOMATION_STAGE_MAP = {
"monitor_configured": "investigator",
"monitor_removed": "safe",
"alert_fired": "webhook",
"alert_suppressed": "safe",
"alert_routed": "safe",
"rule_created": "km",
"rule_updated": "km",
"rule_matched": "ai_router",
"rule_rejected": "safe",
"rule_deprecated": "km",
"playbook_generated": "km",
"playbook_updated": "km",
"playbook_executed": "executor",
"remediation_executed": "executor",
"remediation_verified": "verifier",
"remediation_rolled_back": "executor",
"self_correction_attempted": "verifier",
"km_created": "km",
"km_updated": "km",
"km_linked": "km",
"asset_discovered": "investigator",
"coverage_recalculated": "verifier",
"capacity_recommendation": "investigator",
"quota_enforced": "safe",
"notification_formatted": "safe",
"ansible_candidate_matched": "ai_router",
"ansible_check_mode_executed": "executor",
"ansible_apply_executed": "executor",
"ansible_rollback_executed": "executor",
"ansible_execution_skipped": "safe",
}
_AUTOMATION_STATUS_MAP = {
"pending": "pending",
"success": "success",
"failed": "error",
"dry_run": "info",
"rolled_back": "warning",
}
def _value(value: Any) -> Any:
return value.value if hasattr(value, "value") else value
def _iso(value: Any) -> str | None:
if isinstance(value, datetime):
return value.isoformat()
return None
def _compact(value: str | None, max_len: int = 500) -> str | None:
if not value:
return value
return value if len(value) <= max_len else f"{value[:max_len - 3]}..."
def _event(
*,
stage: str,
status: str,
title: str,
timestamp: Any = None,
description: str | None = None,
actor: str | None = None,
source_table: str,
data: dict[str, Any] | None = None,
) -> dict[str, Any]:
return {
"stage": stage,
"status": status,
"title": title,
"description": _compact(description),
"actor": actor,
"timestamp": _iso(timestamp),
"source_table": source_table,
"data": data or {},
}
def _empty_stage(stage: str, label: str) -> dict[str, Any]:
return {
"stage": stage,
"label": label,
"status": "skipped",
"timestamp": None,
"title": f"{label} not recorded",
"description": None,
"actor": None,
"source_table": None,
"data": {},
"events": [],
}
def _apply_event(stages: dict[str, dict[str, Any]], event: dict[str, Any]) -> None:
stage_name = event["stage"]
stage = stages.get(stage_name)
if stage is None:
return
stage["events"].append(event)
current_rank = _STATUS_RANK.get(stage["status"], 0)
incoming_rank = _STATUS_RANK.get(event["status"], 0)
if incoming_rank >= current_rank:
stage.update({
"status": event["status"],
"timestamp": event["timestamp"] or stage["timestamp"],
"title": event["title"],
"description": event["description"],
"actor": event["actor"],
"source_table": event["source_table"],
"data": event["data"],
})
elif stage["timestamp"] is None and event["timestamp"]:
stage["timestamp"] = event["timestamp"]
def _stage_from_event_type(event_type: str | None) -> str:
return _EVENT_STAGE_MAP.get((event_type or "").lower(), "safe")
def _stage_from_automation_op(operation_type: Any) -> str:
return _AUTOMATION_STAGE_MAP.get(str(operation_type or "").lower(), "safe")
def _automation_status(status: Any) -> str:
return _AUTOMATION_STATUS_MAP.get(str(status or "").lower(), "info")
def _as_dict(value: Any) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
def _automation_summary(row: Any) -> str | None:
output = _as_dict(row.output)
input_data = _as_dict(row.input)
for key in ("summary", "message", "action", "rule_id", "playbook_id"):
value = output.get(key) or input_data.get(key)
if value:
return str(value)
return row.error
def _reconciliation_event(reconciliation: dict[str, Any]) -> dict[str, Any] | None:
"""Render truth-chain reconciliation into the operator timeline."""
if not reconciliation.get("applicable"):
return None
status = str(reconciliation.get("consistency_status") or "unknown")
mismatches = reconciliation.get("mismatches") or []
if status == "consistent" and not mismatches:
return None
stage_status = "error" if status == "blocked" else "warning"
codes = [str(row.get("code")) for row in mismatches if row.get("code")]
description = "; ".join(codes) if codes else None
return _event(
stage="safe",
status=stage_status,
title=f"Lifecycle reconciliation: {status}",
description=description,
actor="truth_chain_reconciliation",
source_table="truth_chain",
data=reconciliation,
)
async def _fetch_automation_ops(
db: Any,
incident_id: str,
approval_ids: list[str],
) -> list[Any]:
"""Best-effort ADR-090 automation_operation_log lookup for one incident."""
params: dict[str, Any] = {"incident_id": incident_id}
approval_clause = ""
if approval_ids:
placeholders = []
for idx, approval_id in enumerate(approval_ids):
key = f"approval_id_{idx}"
params[key] = approval_id
placeholders.append(f":{key}")
in_list = ", ".join(placeholders)
approval_clause = (
f" OR input ->> 'approval_id' IN ({in_list})"
f" OR output ->> 'approval_id' IN ({in_list})"
)
try:
rows = await db.execute(
text(f"""
SELECT
op_id::text AS op_id,
operation_type,
actor,
status,
input,
output,
error,
duration_ms,
tags,
created_at
FROM automation_operation_log
WHERE input ->> 'incident_id' = :incident_id
OR output ->> 'incident_id' = :incident_id
{approval_clause}
ORDER BY created_at ASC
LIMIT 100
"""),
params,
)
return [SimpleNamespace(**dict(row)) for row in rows.mappings().all()]
except SQLAlchemyError as exc:
logger.debug(
"incident_timeline_automation_log_skipped",
incident_id=incident_id,
error=str(exc),
)
return []
def format_ascii_timeline(stages: list[dict[str, Any]]) -> str:
"""Compact ASCII line for Telegram and logs."""
marks = {
"success": "ok",
"completed": "ok",
"info": "ok",
"warning": "warn",
"error": "fail",
"pending": "wait",
"skipped": "skip",
}
parts = [
f"{stage['stage']}:{marks.get(stage['status'], stage['status'])}"
for stage in stages
if stage["status"] != "skipped"
]
return " > ".join(parts) if parts else "webhook:skip > ai:skip > executor:skip"
async def fetch_incident_timeline(incident_id: str) -> dict[str, Any] | None:
"""Return a complete detail timeline for one incident."""
stages = {stage: _empty_stage(stage, label) for stage, label in STAGE_DEFS}
async with get_db_context() as db:
incident = (
await db.execute(
select(IncidentRecord).where(IncidentRecord.incident_id == incident_id)
)
).scalar_one_or_none()
if incident is None:
return None
approvals = (
await db.execute(
select(ApprovalRecord)
.where(ApprovalRecord.incident_id == incident_id)
.order_by(ApprovalRecord.created_at.asc())
)
).scalars().all()
approval_ids = [str(a.id) for a in approvals]
evidence_records = (
await db.execute(
select(IncidentEvidence)
.where(IncidentEvidence.incident_id == incident_id)
.order_by(IncidentEvidence.collected_at.asc())
)
).scalars().all()
executions = (
await db.execute(
select(AutoRepairExecution)
.where(AutoRepairExecution.incident_id == incident_id)
.order_by(AutoRepairExecution.created_at.asc())
)
).scalars().all()
km_entries = (
await db.execute(
select(KnowledgeEntryRecord)
.where(KnowledgeEntryRecord.related_incident_id == incident_id)
.order_by(KnowledgeEntryRecord.created_at.asc())
)
).scalars().all()
timeline_filter = TimelineEvent.incident_id == incident_id
if approval_ids:
timeline_filter = or_(timeline_filter, TimelineEvent.approval_id.in_(approval_ids))
raw_timeline = (
await db.execute(
select(TimelineEvent)
.where(timeline_filter)
.order_by(TimelineEvent.created_at.asc())
)
).scalars().all()
aol_filter = AlertOperationLog.incident_id == incident_id
if approval_ids:
aol_filter = or_(aol_filter, AlertOperationLog.approval_id.in_(approval_ids))
alert_ops = (
await db.execute(
select(AlertOperationLog)
.where(aol_filter)
.order_by(AlertOperationLog.created_at.asc())
.limit(100)
)
).scalars().all()
automation_ops = await _fetch_automation_ops(db, incident_id, approval_ids)
events: list[dict[str, Any]] = []
reconciliation = build_incident_reconciliation(
incident={
"incident_id": incident.incident_id,
"status": _value(incident.status),
},
approvals=[
{
"id": str(approval.id),
"status": _value(approval.status),
"action": approval.action,
"resolved_at": _iso(approval.resolved_at),
}
for approval in sorted(
approvals,
key=lambda row: row.created_at or datetime.min,
reverse=True,
)
],
evidence_rows=[
{
"sensors_attempted": evidence.sensors_attempted,
"sensors_succeeded": evidence.sensors_succeeded,
}
for evidence in evidence_records
],
automation_ops=[
{
"status": op.status,
"operation_type": op.operation_type,
"op_id": op.op_id,
}
for op in automation_ops
],
auto_repair_executions=[
{
"id": execution.id,
"success": execution.success,
"playbook_id": execution.playbook_id,
}
for execution in executions
],
timeline_events=[
{
"event_type": event.event_type,
"status": event.status,
}
for event in raw_timeline
],
)
if reconciliation_event := _reconciliation_event(reconciliation):
events.append(reconciliation_event)
alert_name = incident.alertname
if not alert_name and incident.signals:
first_signal = incident.signals[0] if isinstance(incident.signals, list) else {}
alert_name = first_signal.get("alert_name") or first_signal.get("labels", {}).get("alertname")
events.append(_event(
stage="webhook",
status="completed",
title=f"Alert received: {alert_name or 'unknown'}",
timestamp=incident.created_at,
description=f"source={_signal_source(incident.signals)} severity={_value(incident.severity)}",
actor=_signal_source(incident.signals) or "alertmanager",
source_table="incidents",
data={
"alertname": alert_name,
"severity": _value(incident.severity),
"signals": incident.signals or [],
"affected_services": incident.affected_services or [],
},
))
for evidence in evidence_records:
status = "completed" if (evidence.sensors_succeeded or 0) > 0 else "warning"
events.append(_event(
stage="investigator",
status=status,
title="Evidence snapshot collected",
timestamp=evidence.collected_at,
description=evidence.evidence_summary,
actor="pre_decision_investigator",
source_table="incident_evidence",
data={
"sensors_attempted": evidence.sensors_attempted,
"sensors_succeeded": evidence.sensors_succeeded,
"duration_ms": evidence.collection_duration_ms,
"mcp_health": evidence.mcp_health,
},
))
if evidence.verification_result:
verification_status = (
"success" if evidence.verification_result == "success"
else "warning" if evidence.verification_result == "degraded"
else "error"
)
events.append(_event(
stage="verifier",
status=verification_status,
title=f"Post-execution verification: {evidence.verification_result}",
timestamp=evidence.collected_at,
description=evidence.self_healing_detail and str(evidence.self_healing_detail),
actor="post_execution_verifier",
source_table="incident_evidence",
data={
"verification_result": evidence.verification_result,
"self_healing_score": evidence.self_healing_score,
"self_healing_detail": evidence.self_healing_detail,
},
))
for approval in approvals:
metadata = approval.extra_metadata or {}
provider = metadata.get("source") or _provider_from_description(approval.description)
if provider:
events.append(_event(
stage="ai_router",
status="completed",
title=f"AI route selected: {provider}",
timestamp=approval.created_at,
description=approval.description,
actor="ai_router",
source_table="approval_records",
data={
"provider": provider,
"confidence_score": metadata.get("confidence_score"),
"is_rule_based": metadata.get("is_rule_based"),
},
))
events.append(_event(
stage="llm",
status="completed",
title=f"LLM proposal generated: {provider}",
timestamp=approval.created_at,
description=approval.description,
actor=provider,
source_table="approval_records",
data={
"approval_id": approval.id,
"matched_playbook_id": approval.matched_playbook_id,
"playbook_id": metadata.get("playbook_id"),
},
))
events.append(_event(
stage="target",
status="completed",
title="Target resource selected",
timestamp=approval.created_at,
description=approval.action,
actor=approval.requested_by,
source_table="approval_records",
data={"action": approval.action},
))
events.append(_event(
stage="blast",
status="completed" if approval.blast_radius else "warning",
title="Blast radius evaluated",
timestamp=approval.created_at,
description=None,
actor=approval.requested_by,
source_table="approval_records",
data=approval.blast_radius or {},
))
execution_truth = _approval_execution_truth(approval)
events.append(_event(
stage="safe",
status=_approval_status_to_timeline_status(
approval.status,
repair_executed=execution_truth["repair_executed"],
),
title=f"Safety gate: {_value(approval.risk_level)} / {_value(approval.status)}",
timestamp=approval.created_at,
description=_dry_run_summary(approval.dry_run_checks),
actor=approval.requested_by,
source_table="approval_records",
data={
"approval_id": approval.id,
"risk_level": _value(approval.risk_level),
"status": _value(approval.status),
"required_signatures": approval.required_signatures,
"current_signatures": approval.current_signatures,
"dry_run_checks": approval.dry_run_checks or [],
"execution_kind": execution_truth["execution_kind"],
"repair_executed": execution_truth["repair_executed"],
},
))
if str(_value(approval.status)).startswith("execution_"):
success = (
_value(approval.status) == "execution_success"
and execution_truth["repair_executed"] is not False
)
no_repair_success = (
_value(approval.status) == "execution_success"
and execution_truth["repair_executed"] is False
)
events.append(_event(
stage="executor",
status="success" if success else ("info" if no_repair_success else "error"),
title=(
"Approval observation recorded"
if no_repair_success
else "Approval execution completed"
),
timestamp=approval.resolved_at or approval.updated_at,
description=(
approval.rejection_reason
or (
"Diagnostic or observation was recorded; no repair was executed."
if no_repair_success
else None
)
),
actor="approval_execution",
source_table="approval_records",
data={
"approval_id": approval.id,
"status": _value(approval.status),
"execution_kind": execution_truth["execution_kind"],
"repair_executed": execution_truth["repair_executed"],
},
))
for execution in executions:
events.append(_event(
stage="executor",
status="success" if execution.success else "error",
title=f"Auto repair execution: {execution.playbook_name}",
timestamp=execution.created_at,
description=execution.error_message,
actor=execution.triggered_by,
source_table="auto_repair_executions",
data={
"playbook_id": execution.playbook_id,
"success": execution.success,
"execution_time_ms": execution.execution_time_ms,
"similarity_score": execution.similarity_score,
"risk_level": execution.risk_level,
"executed_steps": execution.executed_steps,
},
))
for entry in km_entries:
events.append(_event(
stage="km",
status="completed",
title=f"KM entry written: {entry.title}",
timestamp=entry.created_at,
description=entry.content,
actor=entry.created_by or _value(entry.source),
source_table="knowledge_entries",
data={
"knowledge_id": entry.id,
"entry_type": _value(entry.entry_type),
"status": _value(entry.status),
"path_type": entry.path_type,
"related_approval_id": entry.related_approval_id,
},
))
if incident.resolved_at or incident.closed_at:
events.append(_event(
stage="close",
status="success",
title=f"Incident {_value(incident.status)}",
timestamp=incident.closed_at or incident.resolved_at,
description=None,
actor="incident_service",
source_table="incidents",
data={
"status": _value(incident.status),
"outcome": incident.outcome,
"resolved_at": _iso(incident.resolved_at),
"closed_at": _iso(incident.closed_at),
},
))
for raw in raw_timeline:
events.append(_event(
stage=_stage_from_event_type(raw.event_type),
status=raw.status,
title=raw.title,
timestamp=raw.created_at,
description=raw.description,
actor=raw.actor,
source_table="timeline_events",
data={
"timeline_event_id": raw.id,
"event_type": raw.event_type,
"approval_id": raw.approval_id,
"actor_role": raw.actor_role,
"risk_level": raw.risk_level,
},
))
for op in alert_ops:
events.append(_event(
stage=_stage_from_aol(op.event_type),
status="error" if op.success is False else "success" if op.success is True else "info",
title=f"AOL: {_value(op.event_type)}",
timestamp=op.created_at,
description=op.action_detail or op.error_message,
actor=op.actor,
source_table="alert_operation_log",
data={
"operation_id": op.id,
"event_type": _value(op.event_type),
"approval_id": op.approval_id,
"context": op.context,
},
))
for op in automation_ops:
events.append(_event(
stage=_stage_from_automation_op(op.operation_type),
status=_automation_status(op.status),
title=f"Automation: {op.operation_type}",
timestamp=op.created_at,
description=_automation_summary(op),
actor=op.actor,
source_table="automation_operation_log",
data={
"op_id": op.op_id,
"operation_type": op.operation_type,
"status": op.status,
"duration_ms": op.duration_ms,
"tags": op.tags or [],
},
))
events.sort(key=lambda e: e["timestamp"] or "")
for event in events:
_apply_event(stages, event)
stage_list = [stages[stage] for stage, _ in STAGE_DEFS]
result = {
"incident_id": incident.incident_id,
"title": alert_name or incident.incident_id,
"status": _value(incident.status),
"severity": _value(incident.severity),
"started_at": _iso(incident.created_at),
"updated_at": _iso(incident.updated_at),
"resolved_at": _iso(incident.resolved_at),
"affected_services": incident.affected_services or [],
"approval_ids": approval_ids,
"timeline": stage_list,
"events": events,
"ascii_timeline": format_ascii_timeline(stage_list),
"reconciliation": reconciliation,
}
logger.info(
"incident_timeline_fetched",
incident_id=incident_id,
stages_recorded=sum(1 for stage in stage_list if stage["status"] != "skipped"),
event_count=len(events),
)
return result
def _signal_source(signals: Any) -> str | None:
if not signals or not isinstance(signals, list):
return None
first_signal = signals[0] if signals else {}
if not isinstance(first_signal, dict):
return None
return first_signal.get("source")
def _provider_from_description(description: str | None) -> str | None:
if not description:
return None
if description.startswith("[AI:"):
return description.split("]", 1)[0].replace("[AI:", "").strip()
return None
def _approval_execution_truth(approval: Any) -> dict[str, Any]:
raw_metadata = getattr(approval, "extra_metadata", None)
metadata = raw_metadata if isinstance(raw_metadata, dict) else {}
execution_kind = str(metadata.get("execution_kind") or "").strip().lower()
repair_executed = metadata.get("repair_executed")
if repair_executed is None:
if execution_kind in {"no_action", "diagnostic", "parse_failed", "unsupported_action"}:
repair_executed = False
elif is_no_action_approval_action(getattr(approval, "action", None)):
repair_executed = False
return {
"execution_kind": execution_kind or None,
"repair_executed": repair_executed,
}
def _approval_status_to_timeline_status(
status: Any,
*,
repair_executed: bool | None = None,
) -> str:
value = str(_value(status))
if value in {"rejected", "expired"}:
return "error"
if value in {"approved", "execution_success"} and repair_executed is False:
return "info"
if value in {"approved", "execution_success"}:
return "success"
if value == "execution_failed":
return "warning"
return "info"
def _dry_run_summary(checks: Any) -> str | None:
if not checks:
return None
passed = 0
total = 0
for check in checks:
if isinstance(check, dict):
total += 1
if check.get("passed"):
passed += 1
return f"Dry-run checks: {passed}/{total} passed" if total else None
def _stage_from_aol(event_type: Any) -> str:
value = str(_value(event_type)).upper()
if value == "ALERT_RECEIVED":
return "webhook"
if value in {"PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED", "GUARDRAIL_BLOCKED"}:
return "safe"
if value in {"EXECUTION_STARTED", "EXECUTION_COMPLETED", "AUTO_REPAIR_TRIGGERED", "CHANGE_APPLIED"}:
return "executor"
if value in {"TELEGRAM_SENT", "TELEGRAM_RESULT_SENT", "USER_ACTION", "APPROVAL_ESCALATED"}:
return "safe"
if value == "RESOLVED":
return "close"
return "safe"