"""Incident processing timeline aggregation. Builds the operator-facing "what happened" timeline from the existing event tables without adding another schema hop. The raw `timeline_events` table is still the append-only audit rail; this service composes it with Incident, Approval, Evidence, Executor, and KM records so a single Incident detail view can show the full path. """ from __future__ import annotations from datetime import datetime from types import SimpleNamespace from typing import Any import structlog from sqlalchemy import or_, select, text from sqlalchemy.exc import SQLAlchemyError from src.db.base import get_db_context from src.db.models import ( AlertOperationLog, ApprovalRecord, AutoRepairExecution, IncidentEvidence, IncidentRecord, KnowledgeEntryRecord, TimelineEvent, ) from src.services.approval_action_classifier import is_no_action_approval_action from src.services.awooop_truth_chain_service import build_incident_reconciliation logger = structlog.get_logger(__name__) STAGE_DEFS: tuple[tuple[str, str], ...] = ( ("webhook", "Webhook"), ("investigator", "Investigator"), ("ai_router", "AI Router"), ("llm", "LLM"), ("target", "Target"), ("blast", "Blast Radius"), ("safe", "Safety Gate"), ("executor", "Executor"), ("verifier", "Verifier"), ("km", "KM"), ("close", "Closure"), ) _STAGE_LABEL = dict(STAGE_DEFS) _STATUS_RANK = { "skipped": 0, "pending": 1, "info": 2, "completed": 3, "success": 4, "warning": 5, "error": 6, } _EVENT_STAGE_MAP = { "webhook": "webhook", "alert": "webhook", "system": "safe", "agent": "llm", "ai_router": "ai_router", "llm": "llm", "mcp_call": "investigator", "investigator": "investigator", "target": "target", "blast": "blast", "security": "safe", "safe": "safe", "human": "safe", "exec": "executor", "executor": "executor", "verify": "verifier", "verifier": "verifier", "km": "km", "learn": "km", "close": "close", "resolved": "close", } _AUTOMATION_STAGE_MAP = { "monitor_configured": "investigator", "monitor_removed": "safe", "alert_fired": "webhook", "alert_suppressed": "safe", "alert_routed": "safe", "rule_created": "km", "rule_updated": "km", "rule_matched": "ai_router", "rule_rejected": "safe", "rule_deprecated": "km", "playbook_generated": "km", "playbook_updated": "km", "playbook_executed": "executor", "remediation_executed": "executor", "remediation_verified": "verifier", "remediation_rolled_back": "executor", "self_correction_attempted": "verifier", "km_created": "km", "km_updated": "km", "km_linked": "km", "asset_discovered": "investigator", "coverage_recalculated": "verifier", "capacity_recommendation": "investigator", "quota_enforced": "safe", "notification_formatted": "safe", "ansible_candidate_matched": "ai_router", "ansible_check_mode_executed": "executor", "ansible_apply_executed": "executor", "ansible_rollback_executed": "executor", "ansible_execution_skipped": "safe", } _AUTOMATION_STATUS_MAP = { "pending": "pending", "success": "success", "failed": "error", "dry_run": "info", "rolled_back": "warning", } def _value(value: Any) -> Any: return value.value if hasattr(value, "value") else value def _iso(value: Any) -> str | None: if isinstance(value, datetime): return value.isoformat() return None def _compact(value: str | None, max_len: int = 500) -> str | None: if not value: return value return value if len(value) <= max_len else f"{value[:max_len - 3]}..." def _event( *, stage: str, status: str, title: str, timestamp: Any = None, description: str | None = None, actor: str | None = None, source_table: str, data: dict[str, Any] | None = None, ) -> dict[str, Any]: return { "stage": stage, "status": status, "title": title, "description": _compact(description), "actor": actor, "timestamp": _iso(timestamp), "source_table": source_table, "data": data or {}, } def _empty_stage(stage: str, label: str) -> dict[str, Any]: return { "stage": stage, "label": label, "status": "skipped", "timestamp": None, "title": f"{label} not recorded", "description": None, "actor": None, "source_table": None, "data": {}, "events": [], } def _apply_event(stages: dict[str, dict[str, Any]], event: dict[str, Any]) -> None: stage_name = event["stage"] stage = stages.get(stage_name) if stage is None: return stage["events"].append(event) current_rank = _STATUS_RANK.get(stage["status"], 0) incoming_rank = _STATUS_RANK.get(event["status"], 0) if incoming_rank >= current_rank: stage.update({ "status": event["status"], "timestamp": event["timestamp"] or stage["timestamp"], "title": event["title"], "description": event["description"], "actor": event["actor"], "source_table": event["source_table"], "data": event["data"], }) elif stage["timestamp"] is None and event["timestamp"]: stage["timestamp"] = event["timestamp"] def _stage_from_event_type(event_type: str | None) -> str: return _EVENT_STAGE_MAP.get((event_type or "").lower(), "safe") def _stage_from_automation_op(operation_type: Any) -> str: return _AUTOMATION_STAGE_MAP.get(str(operation_type or "").lower(), "safe") def _automation_status(status: Any) -> str: return _AUTOMATION_STATUS_MAP.get(str(status or "").lower(), "info") def _as_dict(value: Any) -> dict[str, Any]: return value if isinstance(value, dict) else {} def _automation_summary(row: Any) -> str | None: output = _as_dict(row.output) input_data = _as_dict(row.input) for key in ("summary", "message", "action", "rule_id", "playbook_id"): value = output.get(key) or input_data.get(key) if value: return str(value) return row.error def _reconciliation_event(reconciliation: dict[str, Any]) -> dict[str, Any] | None: """Render truth-chain reconciliation into the operator timeline.""" if not reconciliation.get("applicable"): return None status = str(reconciliation.get("consistency_status") or "unknown") mismatches = reconciliation.get("mismatches") or [] if status == "consistent" and not mismatches: return None stage_status = "error" if status == "blocked" else "warning" codes = [str(row.get("code")) for row in mismatches if row.get("code")] description = "; ".join(codes) if codes else None return _event( stage="safe", status=stage_status, title=f"Lifecycle reconciliation: {status}", description=description, actor="truth_chain_reconciliation", source_table="truth_chain", data=reconciliation, ) async def _fetch_automation_ops( db: Any, incident_id: str, approval_ids: list[str], ) -> list[Any]: """Best-effort ADR-090 automation_operation_log lookup for one incident.""" params: dict[str, Any] = {"incident_id": incident_id} approval_clause = "" if approval_ids: placeholders = [] for idx, approval_id in enumerate(approval_ids): key = f"approval_id_{idx}" params[key] = approval_id placeholders.append(f":{key}") in_list = ", ".join(placeholders) approval_clause = ( f" OR input ->> 'approval_id' IN ({in_list})" f" OR output ->> 'approval_id' IN ({in_list})" ) try: rows = await db.execute( text(f""" SELECT op_id::text AS op_id, operation_type, actor, status, input, output, error, duration_ms, tags, created_at FROM automation_operation_log WHERE input ->> 'incident_id' = :incident_id OR output ->> 'incident_id' = :incident_id {approval_clause} ORDER BY created_at ASC LIMIT 100 """), params, ) return [SimpleNamespace(**dict(row)) for row in rows.mappings().all()] except SQLAlchemyError as exc: logger.debug( "incident_timeline_automation_log_skipped", incident_id=incident_id, error=str(exc), ) return [] def format_ascii_timeline(stages: list[dict[str, Any]]) -> str: """Compact ASCII line for Telegram and logs.""" marks = { "success": "ok", "completed": "ok", "info": "ok", "warning": "warn", "error": "fail", "pending": "wait", "skipped": "skip", } parts = [ f"{stage['stage']}:{marks.get(stage['status'], stage['status'])}" for stage in stages if stage["status"] != "skipped" ] return " > ".join(parts) if parts else "webhook:skip > ai:skip > executor:skip" async def fetch_incident_timeline(incident_id: str) -> dict[str, Any] | None: """Return a complete detail timeline for one incident.""" stages = {stage: _empty_stage(stage, label) for stage, label in STAGE_DEFS} async with get_db_context() as db: incident = ( await db.execute( select(IncidentRecord).where(IncidentRecord.incident_id == incident_id) ) ).scalar_one_or_none() if incident is None: return None approvals = ( await db.execute( select(ApprovalRecord) .where(ApprovalRecord.incident_id == incident_id) .order_by(ApprovalRecord.created_at.asc()) ) ).scalars().all() approval_ids = [str(a.id) for a in approvals] evidence_records = ( await db.execute( select(IncidentEvidence) .where(IncidentEvidence.incident_id == incident_id) .order_by(IncidentEvidence.collected_at.asc()) ) ).scalars().all() executions = ( await db.execute( select(AutoRepairExecution) .where(AutoRepairExecution.incident_id == incident_id) .order_by(AutoRepairExecution.created_at.asc()) ) ).scalars().all() km_entries = ( await db.execute( select(KnowledgeEntryRecord) .where(KnowledgeEntryRecord.related_incident_id == incident_id) .order_by(KnowledgeEntryRecord.created_at.asc()) ) ).scalars().all() timeline_filter = TimelineEvent.incident_id == incident_id if approval_ids: timeline_filter = or_(timeline_filter, TimelineEvent.approval_id.in_(approval_ids)) raw_timeline = ( await db.execute( select(TimelineEvent) .where(timeline_filter) .order_by(TimelineEvent.created_at.asc()) ) ).scalars().all() aol_filter = AlertOperationLog.incident_id == incident_id if approval_ids: aol_filter = or_(aol_filter, AlertOperationLog.approval_id.in_(approval_ids)) alert_ops = ( await db.execute( select(AlertOperationLog) .where(aol_filter) .order_by(AlertOperationLog.created_at.asc()) .limit(100) ) ).scalars().all() automation_ops = await _fetch_automation_ops(db, incident_id, approval_ids) events: list[dict[str, Any]] = [] reconciliation = build_incident_reconciliation( incident={ "incident_id": incident.incident_id, "status": _value(incident.status), }, approvals=[ { "id": str(approval.id), "status": _value(approval.status), "action": approval.action, "resolved_at": _iso(approval.resolved_at), } for approval in sorted( approvals, key=lambda row: row.created_at or datetime.min, reverse=True, ) ], evidence_rows=[ { "sensors_attempted": evidence.sensors_attempted, "sensors_succeeded": evidence.sensors_succeeded, } for evidence in evidence_records ], automation_ops=[ { "status": op.status, "operation_type": op.operation_type, "op_id": op.op_id, } for op in automation_ops ], auto_repair_executions=[ { "id": execution.id, "success": execution.success, "playbook_id": execution.playbook_id, } for execution in executions ], timeline_events=[ { "event_type": event.event_type, "status": event.status, } for event in raw_timeline ], ) if reconciliation_event := _reconciliation_event(reconciliation): events.append(reconciliation_event) alert_name = incident.alertname if not alert_name and incident.signals: first_signal = incident.signals[0] if isinstance(incident.signals, list) else {} alert_name = first_signal.get("alert_name") or first_signal.get("labels", {}).get("alertname") events.append(_event( stage="webhook", status="completed", title=f"Alert received: {alert_name or 'unknown'}", timestamp=incident.created_at, description=f"source={_signal_source(incident.signals)} severity={_value(incident.severity)}", actor=_signal_source(incident.signals) or "alertmanager", source_table="incidents", data={ "alertname": alert_name, "severity": _value(incident.severity), "signals": incident.signals or [], "affected_services": incident.affected_services or [], }, )) for evidence in evidence_records: status = "completed" if (evidence.sensors_succeeded or 0) > 0 else "warning" events.append(_event( stage="investigator", status=status, title="Evidence snapshot collected", timestamp=evidence.collected_at, description=evidence.evidence_summary, actor="pre_decision_investigator", source_table="incident_evidence", data={ "sensors_attempted": evidence.sensors_attempted, "sensors_succeeded": evidence.sensors_succeeded, "duration_ms": evidence.collection_duration_ms, "mcp_health": evidence.mcp_health, }, )) if evidence.verification_result: verification_status = ( "success" if evidence.verification_result == "success" else "warning" if evidence.verification_result == "degraded" else "error" ) events.append(_event( stage="verifier", status=verification_status, title=f"Post-execution verification: {evidence.verification_result}", timestamp=evidence.collected_at, description=evidence.self_healing_detail and str(evidence.self_healing_detail), actor="post_execution_verifier", source_table="incident_evidence", data={ "verification_result": evidence.verification_result, "self_healing_score": evidence.self_healing_score, "self_healing_detail": evidence.self_healing_detail, }, )) for approval in approvals: metadata = approval.extra_metadata or {} provider = metadata.get("source") or _provider_from_description(approval.description) if provider: events.append(_event( stage="ai_router", status="completed", title=f"AI route selected: {provider}", timestamp=approval.created_at, description=approval.description, actor="ai_router", source_table="approval_records", data={ "provider": provider, "confidence_score": metadata.get("confidence_score"), "is_rule_based": metadata.get("is_rule_based"), }, )) events.append(_event( stage="llm", status="completed", title=f"LLM proposal generated: {provider}", timestamp=approval.created_at, description=approval.description, actor=provider, source_table="approval_records", data={ "approval_id": approval.id, "matched_playbook_id": approval.matched_playbook_id, "playbook_id": metadata.get("playbook_id"), }, )) events.append(_event( stage="target", status="completed", title="Target resource selected", timestamp=approval.created_at, description=approval.action, actor=approval.requested_by, source_table="approval_records", data={"action": approval.action}, )) events.append(_event( stage="blast", status="completed" if approval.blast_radius else "warning", title="Blast radius evaluated", timestamp=approval.created_at, description=None, actor=approval.requested_by, source_table="approval_records", data=approval.blast_radius or {}, )) execution_truth = _approval_execution_truth(approval) events.append(_event( stage="safe", status=_approval_status_to_timeline_status( approval.status, repair_executed=execution_truth["repair_executed"], ), title=f"Safety gate: {_value(approval.risk_level)} / {_value(approval.status)}", timestamp=approval.created_at, description=_dry_run_summary(approval.dry_run_checks), actor=approval.requested_by, source_table="approval_records", data={ "approval_id": approval.id, "risk_level": _value(approval.risk_level), "status": _value(approval.status), "required_signatures": approval.required_signatures, "current_signatures": approval.current_signatures, "dry_run_checks": approval.dry_run_checks or [], "execution_kind": execution_truth["execution_kind"], "repair_executed": execution_truth["repair_executed"], }, )) if str(_value(approval.status)).startswith("execution_"): success = ( _value(approval.status) == "execution_success" and execution_truth["repair_executed"] is not False ) no_repair_success = ( _value(approval.status) == "execution_success" and execution_truth["repair_executed"] is False ) events.append(_event( stage="executor", status="success" if success else ("info" if no_repair_success else "error"), title=( "Approval observation recorded" if no_repair_success else "Approval execution completed" ), timestamp=approval.resolved_at or approval.updated_at, description=( approval.rejection_reason or ( "Diagnostic or observation was recorded; no repair was executed." if no_repair_success else None ) ), actor="approval_execution", source_table="approval_records", data={ "approval_id": approval.id, "status": _value(approval.status), "execution_kind": execution_truth["execution_kind"], "repair_executed": execution_truth["repair_executed"], }, )) for execution in executions: events.append(_event( stage="executor", status="success" if execution.success else "error", title=f"Auto repair execution: {execution.playbook_name}", timestamp=execution.created_at, description=execution.error_message, actor=execution.triggered_by, source_table="auto_repair_executions", data={ "playbook_id": execution.playbook_id, "success": execution.success, "execution_time_ms": execution.execution_time_ms, "similarity_score": execution.similarity_score, "risk_level": execution.risk_level, "executed_steps": execution.executed_steps, }, )) for entry in km_entries: events.append(_event( stage="km", status="completed", title=f"KM entry written: {entry.title}", timestamp=entry.created_at, description=entry.content, actor=entry.created_by or _value(entry.source), source_table="knowledge_entries", data={ "knowledge_id": entry.id, "entry_type": _value(entry.entry_type), "status": _value(entry.status), "path_type": entry.path_type, "related_approval_id": entry.related_approval_id, }, )) if incident.resolved_at or incident.closed_at: events.append(_event( stage="close", status="success", title=f"Incident {_value(incident.status)}", timestamp=incident.closed_at or incident.resolved_at, description=None, actor="incident_service", source_table="incidents", data={ "status": _value(incident.status), "outcome": incident.outcome, "resolved_at": _iso(incident.resolved_at), "closed_at": _iso(incident.closed_at), }, )) for raw in raw_timeline: events.append(_event( stage=_stage_from_event_type(raw.event_type), status=raw.status, title=raw.title, timestamp=raw.created_at, description=raw.description, actor=raw.actor, source_table="timeline_events", data={ "timeline_event_id": raw.id, "event_type": raw.event_type, "approval_id": raw.approval_id, "actor_role": raw.actor_role, "risk_level": raw.risk_level, }, )) for op in alert_ops: events.append(_event( stage=_stage_from_aol(op.event_type), status="error" if op.success is False else "success" if op.success is True else "info", title=f"AOL: {_value(op.event_type)}", timestamp=op.created_at, description=op.action_detail or op.error_message, actor=op.actor, source_table="alert_operation_log", data={ "operation_id": op.id, "event_type": _value(op.event_type), "approval_id": op.approval_id, "context": op.context, }, )) for op in automation_ops: events.append(_event( stage=_stage_from_automation_op(op.operation_type), status=_automation_status(op.status), title=f"Automation: {op.operation_type}", timestamp=op.created_at, description=_automation_summary(op), actor=op.actor, source_table="automation_operation_log", data={ "op_id": op.op_id, "operation_type": op.operation_type, "status": op.status, "duration_ms": op.duration_ms, "tags": op.tags or [], }, )) events.sort(key=lambda e: e["timestamp"] or "") for event in events: _apply_event(stages, event) stage_list = [stages[stage] for stage, _ in STAGE_DEFS] result = { "incident_id": incident.incident_id, "title": alert_name or incident.incident_id, "status": _value(incident.status), "severity": _value(incident.severity), "started_at": _iso(incident.created_at), "updated_at": _iso(incident.updated_at), "resolved_at": _iso(incident.resolved_at), "affected_services": incident.affected_services or [], "approval_ids": approval_ids, "timeline": stage_list, "events": events, "ascii_timeline": format_ascii_timeline(stage_list), "reconciliation": reconciliation, } logger.info( "incident_timeline_fetched", incident_id=incident_id, stages_recorded=sum(1 for stage in stage_list if stage["status"] != "skipped"), event_count=len(events), ) return result def _signal_source(signals: Any) -> str | None: if not signals or not isinstance(signals, list): return None first_signal = signals[0] if signals else {} if not isinstance(first_signal, dict): return None return first_signal.get("source") def _provider_from_description(description: str | None) -> str | None: if not description: return None if description.startswith("[AI:"): return description.split("]", 1)[0].replace("[AI:", "").strip() return None def _approval_execution_truth(approval: Any) -> dict[str, Any]: raw_metadata = getattr(approval, "extra_metadata", None) metadata = raw_metadata if isinstance(raw_metadata, dict) else {} execution_kind = str(metadata.get("execution_kind") or "").strip().lower() repair_executed = metadata.get("repair_executed") if repair_executed is None: if execution_kind in {"no_action", "diagnostic", "parse_failed", "unsupported_action"}: repair_executed = False elif is_no_action_approval_action(getattr(approval, "action", None)): repair_executed = False return { "execution_kind": execution_kind or None, "repair_executed": repair_executed, } def _approval_status_to_timeline_status( status: Any, *, repair_executed: bool | None = None, ) -> str: value = str(_value(status)) if value in {"rejected", "expired"}: return "error" if value in {"approved", "execution_success"} and repair_executed is False: return "info" if value in {"approved", "execution_success"}: return "success" if value == "execution_failed": return "warning" return "info" def _dry_run_summary(checks: Any) -> str | None: if not checks: return None passed = 0 total = 0 for check in checks: if isinstance(check, dict): total += 1 if check.get("passed"): passed += 1 return f"Dry-run checks: {passed}/{total} passed" if total else None def _stage_from_aol(event_type: Any) -> str: value = str(_value(event_type)).upper() if value == "ALERT_RECEIVED": return "webhook" if value in {"PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED", "GUARDRAIL_BLOCKED"}: return "safe" if value in {"EXECUTION_STARTED", "EXECUTION_COMPLETED", "AUTO_REPAIR_TRIGGERED", "CHANGE_APPLIED"}: return "executor" if value in {"TELEGRAM_SENT", "TELEGRAM_RESULT_SENT", "USER_ACTION", "APPROVAL_ESCALATED"}: return "safe" if value == "RESOLVED": return "close" return "safe"