832 lines
28 KiB
Python
832 lines
28 KiB
Python
"""Incident processing timeline aggregation.
|
|
|
|
Builds the operator-facing "what happened" timeline from the existing event
|
|
tables without adding another schema hop. The raw `timeline_events` table is
|
|
still the append-only audit rail; this service composes it with Incident,
|
|
Approval, Evidence, Executor, and KM records so a single Incident detail view can
|
|
show the full path.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import datetime
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
import structlog
|
|
from sqlalchemy import or_, select, text
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|
|
|
from src.db.base import get_db_context
|
|
from src.db.models import (
|
|
AlertOperationLog,
|
|
ApprovalRecord,
|
|
AutoRepairExecution,
|
|
IncidentEvidence,
|
|
IncidentRecord,
|
|
KnowledgeEntryRecord,
|
|
TimelineEvent,
|
|
)
|
|
from src.services.approval_action_classifier import is_no_action_approval_action
|
|
from src.services.awooop_truth_chain_service import build_incident_reconciliation
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
|
|
STAGE_DEFS: tuple[tuple[str, str], ...] = (
|
|
("webhook", "Webhook"),
|
|
("investigator", "Investigator"),
|
|
("ai_router", "AI Router"),
|
|
("llm", "LLM"),
|
|
("target", "Target"),
|
|
("blast", "Blast Radius"),
|
|
("safe", "Safety Gate"),
|
|
("executor", "Executor"),
|
|
("verifier", "Verifier"),
|
|
("km", "KM"),
|
|
("close", "Closure"),
|
|
)
|
|
|
|
_STAGE_LABEL = dict(STAGE_DEFS)
|
|
_STATUS_RANK = {
|
|
"skipped": 0,
|
|
"pending": 1,
|
|
"info": 2,
|
|
"completed": 3,
|
|
"success": 4,
|
|
"warning": 5,
|
|
"error": 6,
|
|
}
|
|
_EVENT_STAGE_MAP = {
|
|
"webhook": "webhook",
|
|
"alert": "webhook",
|
|
"system": "safe",
|
|
"agent": "llm",
|
|
"ai_router": "ai_router",
|
|
"llm": "llm",
|
|
"mcp_call": "investigator",
|
|
"investigator": "investigator",
|
|
"target": "target",
|
|
"blast": "blast",
|
|
"security": "safe",
|
|
"safe": "safe",
|
|
"human": "safe",
|
|
"exec": "executor",
|
|
"executor": "executor",
|
|
"verify": "verifier",
|
|
"verifier": "verifier",
|
|
"km": "km",
|
|
"learn": "km",
|
|
"close": "close",
|
|
"resolved": "close",
|
|
}
|
|
_AUTOMATION_STAGE_MAP = {
|
|
"monitor_configured": "investigator",
|
|
"monitor_removed": "safe",
|
|
"alert_fired": "webhook",
|
|
"alert_suppressed": "safe",
|
|
"alert_routed": "safe",
|
|
"rule_created": "km",
|
|
"rule_updated": "km",
|
|
"rule_matched": "ai_router",
|
|
"rule_rejected": "safe",
|
|
"rule_deprecated": "km",
|
|
"playbook_generated": "km",
|
|
"playbook_updated": "km",
|
|
"playbook_executed": "executor",
|
|
"remediation_executed": "executor",
|
|
"remediation_verified": "verifier",
|
|
"remediation_rolled_back": "executor",
|
|
"self_correction_attempted": "verifier",
|
|
"km_created": "km",
|
|
"km_updated": "km",
|
|
"km_linked": "km",
|
|
"asset_discovered": "investigator",
|
|
"coverage_recalculated": "verifier",
|
|
"capacity_recommendation": "investigator",
|
|
"quota_enforced": "safe",
|
|
"notification_formatted": "safe",
|
|
"ansible_candidate_matched": "ai_router",
|
|
"ansible_check_mode_executed": "executor",
|
|
"ansible_apply_executed": "executor",
|
|
"ansible_rollback_executed": "executor",
|
|
"ansible_execution_skipped": "safe",
|
|
}
|
|
_AUTOMATION_STATUS_MAP = {
|
|
"pending": "pending",
|
|
"success": "success",
|
|
"failed": "error",
|
|
"dry_run": "info",
|
|
"rolled_back": "warning",
|
|
}
|
|
|
|
|
|
def _value(value: Any) -> Any:
|
|
return value.value if hasattr(value, "value") else value
|
|
|
|
|
|
def _iso(value: Any) -> str | None:
|
|
if isinstance(value, datetime):
|
|
return value.isoformat()
|
|
return None
|
|
|
|
|
|
def _compact(value: str | None, max_len: int = 500) -> str | None:
|
|
if not value:
|
|
return value
|
|
return value if len(value) <= max_len else f"{value[:max_len - 3]}..."
|
|
|
|
|
|
def _event(
|
|
*,
|
|
stage: str,
|
|
status: str,
|
|
title: str,
|
|
timestamp: Any = None,
|
|
description: str | None = None,
|
|
actor: str | None = None,
|
|
source_table: str,
|
|
data: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"stage": stage,
|
|
"status": status,
|
|
"title": title,
|
|
"description": _compact(description),
|
|
"actor": actor,
|
|
"timestamp": _iso(timestamp),
|
|
"source_table": source_table,
|
|
"data": data or {},
|
|
}
|
|
|
|
|
|
def _empty_stage(stage: str, label: str) -> dict[str, Any]:
|
|
return {
|
|
"stage": stage,
|
|
"label": label,
|
|
"status": "skipped",
|
|
"timestamp": None,
|
|
"title": f"{label} not recorded",
|
|
"description": None,
|
|
"actor": None,
|
|
"source_table": None,
|
|
"data": {},
|
|
"events": [],
|
|
}
|
|
|
|
|
|
def _apply_event(stages: dict[str, dict[str, Any]], event: dict[str, Any]) -> None:
|
|
stage_name = event["stage"]
|
|
stage = stages.get(stage_name)
|
|
if stage is None:
|
|
return
|
|
|
|
stage["events"].append(event)
|
|
current_rank = _STATUS_RANK.get(stage["status"], 0)
|
|
incoming_rank = _STATUS_RANK.get(event["status"], 0)
|
|
if incoming_rank >= current_rank:
|
|
stage.update({
|
|
"status": event["status"],
|
|
"timestamp": event["timestamp"] or stage["timestamp"],
|
|
"title": event["title"],
|
|
"description": event["description"],
|
|
"actor": event["actor"],
|
|
"source_table": event["source_table"],
|
|
"data": event["data"],
|
|
})
|
|
elif stage["timestamp"] is None and event["timestamp"]:
|
|
stage["timestamp"] = event["timestamp"]
|
|
|
|
|
|
def _stage_from_event_type(event_type: str | None) -> str:
|
|
return _EVENT_STAGE_MAP.get((event_type or "").lower(), "safe")
|
|
|
|
|
|
def _stage_from_automation_op(operation_type: Any) -> str:
|
|
return _AUTOMATION_STAGE_MAP.get(str(operation_type or "").lower(), "safe")
|
|
|
|
|
|
def _automation_status(status: Any) -> str:
|
|
return _AUTOMATION_STATUS_MAP.get(str(status or "").lower(), "info")
|
|
|
|
|
|
def _as_dict(value: Any) -> dict[str, Any]:
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def _automation_summary(row: Any) -> str | None:
|
|
output = _as_dict(row.output)
|
|
input_data = _as_dict(row.input)
|
|
for key in ("summary", "message", "action", "rule_id", "playbook_id"):
|
|
value = output.get(key) or input_data.get(key)
|
|
if value:
|
|
return str(value)
|
|
return row.error
|
|
|
|
|
|
def _reconciliation_event(reconciliation: dict[str, Any]) -> dict[str, Any] | None:
|
|
"""Render truth-chain reconciliation into the operator timeline."""
|
|
if not reconciliation.get("applicable"):
|
|
return None
|
|
status = str(reconciliation.get("consistency_status") or "unknown")
|
|
mismatches = reconciliation.get("mismatches") or []
|
|
if status == "consistent" and not mismatches:
|
|
return None
|
|
|
|
stage_status = "error" if status == "blocked" else "warning"
|
|
codes = [str(row.get("code")) for row in mismatches if row.get("code")]
|
|
description = "; ".join(codes) if codes else None
|
|
return _event(
|
|
stage="safe",
|
|
status=stage_status,
|
|
title=f"Lifecycle reconciliation: {status}",
|
|
description=description,
|
|
actor="truth_chain_reconciliation",
|
|
source_table="truth_chain",
|
|
data=reconciliation,
|
|
)
|
|
|
|
|
|
async def _fetch_automation_ops(
|
|
db: Any,
|
|
incident_id: str,
|
|
approval_ids: list[str],
|
|
) -> list[Any]:
|
|
"""Best-effort ADR-090 automation_operation_log lookup for one incident."""
|
|
params: dict[str, Any] = {"incident_id": incident_id}
|
|
approval_clause = ""
|
|
if approval_ids:
|
|
placeholders = []
|
|
for idx, approval_id in enumerate(approval_ids):
|
|
key = f"approval_id_{idx}"
|
|
params[key] = approval_id
|
|
placeholders.append(f":{key}")
|
|
in_list = ", ".join(placeholders)
|
|
approval_clause = (
|
|
f" OR input ->> 'approval_id' IN ({in_list})"
|
|
f" OR output ->> 'approval_id' IN ({in_list})"
|
|
)
|
|
|
|
try:
|
|
rows = await db.execute(
|
|
text(f"""
|
|
SELECT
|
|
op_id::text AS op_id,
|
|
operation_type,
|
|
actor,
|
|
status,
|
|
input,
|
|
output,
|
|
error,
|
|
duration_ms,
|
|
tags,
|
|
created_at
|
|
FROM automation_operation_log
|
|
WHERE input ->> 'incident_id' = :incident_id
|
|
OR output ->> 'incident_id' = :incident_id
|
|
{approval_clause}
|
|
ORDER BY created_at ASC
|
|
LIMIT 100
|
|
"""),
|
|
params,
|
|
)
|
|
return [SimpleNamespace(**dict(row)) for row in rows.mappings().all()]
|
|
except SQLAlchemyError as exc:
|
|
logger.debug(
|
|
"incident_timeline_automation_log_skipped",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
return []
|
|
|
|
|
|
def format_ascii_timeline(stages: list[dict[str, Any]]) -> str:
|
|
"""Compact ASCII line for Telegram and logs."""
|
|
marks = {
|
|
"success": "ok",
|
|
"completed": "ok",
|
|
"info": "ok",
|
|
"warning": "warn",
|
|
"error": "fail",
|
|
"pending": "wait",
|
|
"skipped": "skip",
|
|
}
|
|
parts = [
|
|
f"{stage['stage']}:{marks.get(stage['status'], stage['status'])}"
|
|
for stage in stages
|
|
if stage["status"] != "skipped"
|
|
]
|
|
return " > ".join(parts) if parts else "webhook:skip > ai:skip > executor:skip"
|
|
|
|
|
|
async def fetch_incident_timeline(incident_id: str) -> dict[str, Any] | None:
|
|
"""Return a complete detail timeline for one incident."""
|
|
stages = {stage: _empty_stage(stage, label) for stage, label in STAGE_DEFS}
|
|
|
|
async with get_db_context() as db:
|
|
incident = (
|
|
await db.execute(
|
|
select(IncidentRecord).where(IncidentRecord.incident_id == incident_id)
|
|
)
|
|
).scalar_one_or_none()
|
|
if incident is None:
|
|
return None
|
|
|
|
approvals = (
|
|
await db.execute(
|
|
select(ApprovalRecord)
|
|
.where(ApprovalRecord.incident_id == incident_id)
|
|
.order_by(ApprovalRecord.created_at.asc())
|
|
)
|
|
).scalars().all()
|
|
approval_ids = [str(a.id) for a in approvals]
|
|
|
|
evidence_records = (
|
|
await db.execute(
|
|
select(IncidentEvidence)
|
|
.where(IncidentEvidence.incident_id == incident_id)
|
|
.order_by(IncidentEvidence.collected_at.asc())
|
|
)
|
|
).scalars().all()
|
|
|
|
executions = (
|
|
await db.execute(
|
|
select(AutoRepairExecution)
|
|
.where(AutoRepairExecution.incident_id == incident_id)
|
|
.order_by(AutoRepairExecution.created_at.asc())
|
|
)
|
|
).scalars().all()
|
|
|
|
km_entries = (
|
|
await db.execute(
|
|
select(KnowledgeEntryRecord)
|
|
.where(KnowledgeEntryRecord.related_incident_id == incident_id)
|
|
.order_by(KnowledgeEntryRecord.created_at.asc())
|
|
)
|
|
).scalars().all()
|
|
|
|
timeline_filter = TimelineEvent.incident_id == incident_id
|
|
if approval_ids:
|
|
timeline_filter = or_(timeline_filter, TimelineEvent.approval_id.in_(approval_ids))
|
|
raw_timeline = (
|
|
await db.execute(
|
|
select(TimelineEvent)
|
|
.where(timeline_filter)
|
|
.order_by(TimelineEvent.created_at.asc())
|
|
)
|
|
).scalars().all()
|
|
|
|
aol_filter = AlertOperationLog.incident_id == incident_id
|
|
if approval_ids:
|
|
aol_filter = or_(aol_filter, AlertOperationLog.approval_id.in_(approval_ids))
|
|
alert_ops = (
|
|
await db.execute(
|
|
select(AlertOperationLog)
|
|
.where(aol_filter)
|
|
.order_by(AlertOperationLog.created_at.asc())
|
|
.limit(100)
|
|
)
|
|
).scalars().all()
|
|
automation_ops = await _fetch_automation_ops(db, incident_id, approval_ids)
|
|
|
|
events: list[dict[str, Any]] = []
|
|
reconciliation = build_incident_reconciliation(
|
|
incident={
|
|
"incident_id": incident.incident_id,
|
|
"status": _value(incident.status),
|
|
},
|
|
approvals=[
|
|
{
|
|
"id": str(approval.id),
|
|
"status": _value(approval.status),
|
|
"action": approval.action,
|
|
"resolved_at": _iso(approval.resolved_at),
|
|
}
|
|
for approval in sorted(
|
|
approvals,
|
|
key=lambda row: row.created_at or datetime.min,
|
|
reverse=True,
|
|
)
|
|
],
|
|
evidence_rows=[
|
|
{
|
|
"sensors_attempted": evidence.sensors_attempted,
|
|
"sensors_succeeded": evidence.sensors_succeeded,
|
|
}
|
|
for evidence in evidence_records
|
|
],
|
|
automation_ops=[
|
|
{
|
|
"status": op.status,
|
|
"operation_type": op.operation_type,
|
|
"op_id": op.op_id,
|
|
}
|
|
for op in automation_ops
|
|
],
|
|
auto_repair_executions=[
|
|
{
|
|
"id": execution.id,
|
|
"success": execution.success,
|
|
"playbook_id": execution.playbook_id,
|
|
}
|
|
for execution in executions
|
|
],
|
|
timeline_events=[
|
|
{
|
|
"event_type": event.event_type,
|
|
"status": event.status,
|
|
}
|
|
for event in raw_timeline
|
|
],
|
|
)
|
|
if reconciliation_event := _reconciliation_event(reconciliation):
|
|
events.append(reconciliation_event)
|
|
|
|
alert_name = incident.alertname
|
|
if not alert_name and incident.signals:
|
|
first_signal = incident.signals[0] if isinstance(incident.signals, list) else {}
|
|
alert_name = first_signal.get("alert_name") or first_signal.get("labels", {}).get("alertname")
|
|
|
|
events.append(_event(
|
|
stage="webhook",
|
|
status="completed",
|
|
title=f"Alert received: {alert_name or 'unknown'}",
|
|
timestamp=incident.created_at,
|
|
description=f"source={_signal_source(incident.signals)} severity={_value(incident.severity)}",
|
|
actor=_signal_source(incident.signals) or "alertmanager",
|
|
source_table="incidents",
|
|
data={
|
|
"alertname": alert_name,
|
|
"severity": _value(incident.severity),
|
|
"signals": incident.signals or [],
|
|
"affected_services": incident.affected_services or [],
|
|
},
|
|
))
|
|
|
|
for evidence in evidence_records:
|
|
status = "completed" if (evidence.sensors_succeeded or 0) > 0 else "warning"
|
|
events.append(_event(
|
|
stage="investigator",
|
|
status=status,
|
|
title="Evidence snapshot collected",
|
|
timestamp=evidence.collected_at,
|
|
description=evidence.evidence_summary,
|
|
actor="pre_decision_investigator",
|
|
source_table="incident_evidence",
|
|
data={
|
|
"sensors_attempted": evidence.sensors_attempted,
|
|
"sensors_succeeded": evidence.sensors_succeeded,
|
|
"duration_ms": evidence.collection_duration_ms,
|
|
"mcp_health": evidence.mcp_health,
|
|
},
|
|
))
|
|
if evidence.verification_result:
|
|
verification_status = (
|
|
"success" if evidence.verification_result == "success"
|
|
else "warning" if evidence.verification_result == "degraded"
|
|
else "error"
|
|
)
|
|
events.append(_event(
|
|
stage="verifier",
|
|
status=verification_status,
|
|
title=f"Post-execution verification: {evidence.verification_result}",
|
|
timestamp=evidence.collected_at,
|
|
description=evidence.self_healing_detail and str(evidence.self_healing_detail),
|
|
actor="post_execution_verifier",
|
|
source_table="incident_evidence",
|
|
data={
|
|
"verification_result": evidence.verification_result,
|
|
"self_healing_score": evidence.self_healing_score,
|
|
"self_healing_detail": evidence.self_healing_detail,
|
|
},
|
|
))
|
|
|
|
for approval in approvals:
|
|
metadata = approval.extra_metadata or {}
|
|
provider = metadata.get("source") or _provider_from_description(approval.description)
|
|
if provider:
|
|
events.append(_event(
|
|
stage="ai_router",
|
|
status="completed",
|
|
title=f"AI route selected: {provider}",
|
|
timestamp=approval.created_at,
|
|
description=approval.description,
|
|
actor="ai_router",
|
|
source_table="approval_records",
|
|
data={
|
|
"provider": provider,
|
|
"confidence_score": metadata.get("confidence_score"),
|
|
"is_rule_based": metadata.get("is_rule_based"),
|
|
},
|
|
))
|
|
events.append(_event(
|
|
stage="llm",
|
|
status="completed",
|
|
title=f"LLM proposal generated: {provider}",
|
|
timestamp=approval.created_at,
|
|
description=approval.description,
|
|
actor=provider,
|
|
source_table="approval_records",
|
|
data={
|
|
"approval_id": approval.id,
|
|
"matched_playbook_id": approval.matched_playbook_id,
|
|
"playbook_id": metadata.get("playbook_id"),
|
|
},
|
|
))
|
|
|
|
events.append(_event(
|
|
stage="target",
|
|
status="completed",
|
|
title="Target resource selected",
|
|
timestamp=approval.created_at,
|
|
description=approval.action,
|
|
actor=approval.requested_by,
|
|
source_table="approval_records",
|
|
data={"action": approval.action},
|
|
))
|
|
|
|
events.append(_event(
|
|
stage="blast",
|
|
status="completed" if approval.blast_radius else "warning",
|
|
title="Blast radius evaluated",
|
|
timestamp=approval.created_at,
|
|
description=None,
|
|
actor=approval.requested_by,
|
|
source_table="approval_records",
|
|
data=approval.blast_radius or {},
|
|
))
|
|
|
|
execution_truth = _approval_execution_truth(approval)
|
|
|
|
events.append(_event(
|
|
stage="safe",
|
|
status=_approval_status_to_timeline_status(
|
|
approval.status,
|
|
repair_executed=execution_truth["repair_executed"],
|
|
),
|
|
title=f"Safety gate: {_value(approval.risk_level)} / {_value(approval.status)}",
|
|
timestamp=approval.created_at,
|
|
description=_dry_run_summary(approval.dry_run_checks),
|
|
actor=approval.requested_by,
|
|
source_table="approval_records",
|
|
data={
|
|
"approval_id": approval.id,
|
|
"risk_level": _value(approval.risk_level),
|
|
"status": _value(approval.status),
|
|
"required_signatures": approval.required_signatures,
|
|
"current_signatures": approval.current_signatures,
|
|
"dry_run_checks": approval.dry_run_checks or [],
|
|
"execution_kind": execution_truth["execution_kind"],
|
|
"repair_executed": execution_truth["repair_executed"],
|
|
},
|
|
))
|
|
|
|
if str(_value(approval.status)).startswith("execution_"):
|
|
success = (
|
|
_value(approval.status) == "execution_success"
|
|
and execution_truth["repair_executed"] is not False
|
|
)
|
|
no_repair_success = (
|
|
_value(approval.status) == "execution_success"
|
|
and execution_truth["repair_executed"] is False
|
|
)
|
|
events.append(_event(
|
|
stage="executor",
|
|
status="success" if success else ("info" if no_repair_success else "error"),
|
|
title=(
|
|
"Approval observation recorded"
|
|
if no_repair_success
|
|
else "Approval execution completed"
|
|
),
|
|
timestamp=approval.resolved_at or approval.updated_at,
|
|
description=(
|
|
approval.rejection_reason
|
|
or (
|
|
"Diagnostic or observation was recorded; no repair was executed."
|
|
if no_repair_success
|
|
else None
|
|
)
|
|
),
|
|
actor="approval_execution",
|
|
source_table="approval_records",
|
|
data={
|
|
"approval_id": approval.id,
|
|
"status": _value(approval.status),
|
|
"execution_kind": execution_truth["execution_kind"],
|
|
"repair_executed": execution_truth["repair_executed"],
|
|
},
|
|
))
|
|
|
|
for execution in executions:
|
|
events.append(_event(
|
|
stage="executor",
|
|
status="success" if execution.success else "error",
|
|
title=f"Auto repair execution: {execution.playbook_name}",
|
|
timestamp=execution.created_at,
|
|
description=execution.error_message,
|
|
actor=execution.triggered_by,
|
|
source_table="auto_repair_executions",
|
|
data={
|
|
"playbook_id": execution.playbook_id,
|
|
"success": execution.success,
|
|
"execution_time_ms": execution.execution_time_ms,
|
|
"similarity_score": execution.similarity_score,
|
|
"risk_level": execution.risk_level,
|
|
"executed_steps": execution.executed_steps,
|
|
},
|
|
))
|
|
|
|
for entry in km_entries:
|
|
events.append(_event(
|
|
stage="km",
|
|
status="completed",
|
|
title=f"KM entry written: {entry.title}",
|
|
timestamp=entry.created_at,
|
|
description=entry.content,
|
|
actor=entry.created_by or _value(entry.source),
|
|
source_table="knowledge_entries",
|
|
data={
|
|
"knowledge_id": entry.id,
|
|
"entry_type": _value(entry.entry_type),
|
|
"status": _value(entry.status),
|
|
"path_type": entry.path_type,
|
|
"related_approval_id": entry.related_approval_id,
|
|
},
|
|
))
|
|
|
|
if incident.resolved_at or incident.closed_at:
|
|
events.append(_event(
|
|
stage="close",
|
|
status="success",
|
|
title=f"Incident {_value(incident.status)}",
|
|
timestamp=incident.closed_at or incident.resolved_at,
|
|
description=None,
|
|
actor="incident_service",
|
|
source_table="incidents",
|
|
data={
|
|
"status": _value(incident.status),
|
|
"outcome": incident.outcome,
|
|
"resolved_at": _iso(incident.resolved_at),
|
|
"closed_at": _iso(incident.closed_at),
|
|
},
|
|
))
|
|
|
|
for raw in raw_timeline:
|
|
events.append(_event(
|
|
stage=_stage_from_event_type(raw.event_type),
|
|
status=raw.status,
|
|
title=raw.title,
|
|
timestamp=raw.created_at,
|
|
description=raw.description,
|
|
actor=raw.actor,
|
|
source_table="timeline_events",
|
|
data={
|
|
"timeline_event_id": raw.id,
|
|
"event_type": raw.event_type,
|
|
"approval_id": raw.approval_id,
|
|
"actor_role": raw.actor_role,
|
|
"risk_level": raw.risk_level,
|
|
},
|
|
))
|
|
|
|
for op in alert_ops:
|
|
events.append(_event(
|
|
stage=_stage_from_aol(op.event_type),
|
|
status="error" if op.success is False else "success" if op.success is True else "info",
|
|
title=f"AOL: {_value(op.event_type)}",
|
|
timestamp=op.created_at,
|
|
description=op.action_detail or op.error_message,
|
|
actor=op.actor,
|
|
source_table="alert_operation_log",
|
|
data={
|
|
"operation_id": op.id,
|
|
"event_type": _value(op.event_type),
|
|
"approval_id": op.approval_id,
|
|
"context": op.context,
|
|
},
|
|
))
|
|
|
|
for op in automation_ops:
|
|
events.append(_event(
|
|
stage=_stage_from_automation_op(op.operation_type),
|
|
status=_automation_status(op.status),
|
|
title=f"Automation: {op.operation_type}",
|
|
timestamp=op.created_at,
|
|
description=_automation_summary(op),
|
|
actor=op.actor,
|
|
source_table="automation_operation_log",
|
|
data={
|
|
"op_id": op.op_id,
|
|
"operation_type": op.operation_type,
|
|
"status": op.status,
|
|
"duration_ms": op.duration_ms,
|
|
"tags": op.tags or [],
|
|
},
|
|
))
|
|
|
|
events.sort(key=lambda e: e["timestamp"] or "")
|
|
for event in events:
|
|
_apply_event(stages, event)
|
|
|
|
stage_list = [stages[stage] for stage, _ in STAGE_DEFS]
|
|
result = {
|
|
"incident_id": incident.incident_id,
|
|
"title": alert_name or incident.incident_id,
|
|
"status": _value(incident.status),
|
|
"severity": _value(incident.severity),
|
|
"started_at": _iso(incident.created_at),
|
|
"updated_at": _iso(incident.updated_at),
|
|
"resolved_at": _iso(incident.resolved_at),
|
|
"affected_services": incident.affected_services or [],
|
|
"approval_ids": approval_ids,
|
|
"timeline": stage_list,
|
|
"events": events,
|
|
"ascii_timeline": format_ascii_timeline(stage_list),
|
|
"reconciliation": reconciliation,
|
|
}
|
|
logger.info(
|
|
"incident_timeline_fetched",
|
|
incident_id=incident_id,
|
|
stages_recorded=sum(1 for stage in stage_list if stage["status"] != "skipped"),
|
|
event_count=len(events),
|
|
)
|
|
return result
|
|
|
|
|
|
def _signal_source(signals: Any) -> str | None:
|
|
if not signals or not isinstance(signals, list):
|
|
return None
|
|
first_signal = signals[0] if signals else {}
|
|
if not isinstance(first_signal, dict):
|
|
return None
|
|
return first_signal.get("source")
|
|
|
|
|
|
def _provider_from_description(description: str | None) -> str | None:
|
|
if not description:
|
|
return None
|
|
if description.startswith("[AI:"):
|
|
return description.split("]", 1)[0].replace("[AI:", "").strip()
|
|
return None
|
|
|
|
|
|
def _approval_execution_truth(approval: Any) -> dict[str, Any]:
|
|
raw_metadata = getattr(approval, "extra_metadata", None)
|
|
metadata = raw_metadata if isinstance(raw_metadata, dict) else {}
|
|
execution_kind = str(metadata.get("execution_kind") or "").strip().lower()
|
|
repair_executed = metadata.get("repair_executed")
|
|
if repair_executed is None:
|
|
if execution_kind in {"no_action", "diagnostic", "parse_failed", "unsupported_action"}:
|
|
repair_executed = False
|
|
elif is_no_action_approval_action(getattr(approval, "action", None)):
|
|
repair_executed = False
|
|
return {
|
|
"execution_kind": execution_kind or None,
|
|
"repair_executed": repair_executed,
|
|
}
|
|
|
|
|
|
def _approval_status_to_timeline_status(
|
|
status: Any,
|
|
*,
|
|
repair_executed: bool | None = None,
|
|
) -> str:
|
|
value = str(_value(status))
|
|
if value in {"rejected", "expired"}:
|
|
return "error"
|
|
if value in {"approved", "execution_success"} and repair_executed is False:
|
|
return "info"
|
|
if value in {"approved", "execution_success"}:
|
|
return "success"
|
|
if value == "execution_failed":
|
|
return "warning"
|
|
return "info"
|
|
|
|
|
|
def _dry_run_summary(checks: Any) -> str | None:
|
|
if not checks:
|
|
return None
|
|
passed = 0
|
|
total = 0
|
|
for check in checks:
|
|
if isinstance(check, dict):
|
|
total += 1
|
|
if check.get("passed"):
|
|
passed += 1
|
|
return f"Dry-run checks: {passed}/{total} passed" if total else None
|
|
|
|
|
|
def _stage_from_aol(event_type: Any) -> str:
|
|
value = str(_value(event_type)).upper()
|
|
if value == "ALERT_RECEIVED":
|
|
return "webhook"
|
|
if value in {"PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED", "GUARDRAIL_BLOCKED"}:
|
|
return "safe"
|
|
if value in {"EXECUTION_STARTED", "EXECUTION_COMPLETED", "AUTO_REPAIR_TRIGGERED", "CHANGE_APPLIED"}:
|
|
return "executor"
|
|
if value in {"TELEGRAM_SENT", "TELEGRAM_RESULT_SENT", "USER_ACTION", "APPROVAL_ESCALATED"}:
|
|
return "safe"
|
|
if value == "RESOLVED":
|
|
return "close"
|
|
return "safe"
|