""" ADR-100 Remediation Service =========================== Safe operator entrypoints for verification remediation work items. T25: remediation queue items are now actionable without mutating incident state: - preview: show the selected guardrail path - dry-run: collect read-only current state and validate supported executor routing """ from __future__ import annotations import asyncio import hashlib import json from datetime import datetime, timedelta, timezone from typing import Any, Literal, Protocol from uuid import NAMESPACE_URL, uuid5 import structlog from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert as pg_insert from src.db.awooop_models import ( AwoooPRunIdempotency, AwoooPRunState, AwoooPRunStepJournal, ) from src.db.base import get_db_context from src.models.approval import ( ApprovalRequestCreate, BlastRadius, DataImpact, DryRunCheck, RiskLevel, ) from src.models.incident import Incident from src.repositories.incident_repository import IncidentDBRepository from src.services.adr100_slo_status_service import ( Adr100SloStatusService, get_adr100_slo_status_service, ) from src.services.auto_repair_service import AutoRepairService from src.services.post_execution_verifier import ( PostExecutionVerifier, _assess_recovery, _build_prometheus_query, get_post_execution_verifier, ) logger = structlog.get_logger(__name__) RemediationMode = Literal["auto", "reverify", "replay", "ticket", "approval"] _READY_STATUSES = {"ready_for_replay", "ready_for_reverify"} _TICKET_STATUSES = {"needs_playbook_ticket"} _TICKET_ACTIONS = {"create_playbook_ticket", "promote_diagnostic_to_repair_playbook"} _RUNTIME_REPLAY_STATUSES = {"ready_for_replay"} _RUNTIME_REPLAY_ACTIONS = {"replay_with_supported_executor"} _AWOOOP_GATE5_TRIGGER_TYPE = "adr100_runtime_replay_gate5" _AWOOOP_GATE5_CHANNEL_TYPE = "adr100_gate5_approval" _AWOOOP_GATE5_PROJECT_ID = "awoooi" class RemediationNotFoundError(LookupError): """Requested ADR-100 remediation work item is not in the current read model.""" class _IncidentRepository(Protocol): async def get_by_id(self, incident_id: str) -> Incident | None: ... class Adr100RemediationService: """Read-only remediation preview and dry-run service.""" def __init__( self, *, slo_service: Adr100SloStatusService | None = None, incident_repository: _IncidentRepository | None = None, auto_repair_service: AutoRepairService | None = None, playbook_service: Any | None = None, verifier: PostExecutionVerifier | None = None, approval_service: Any | None = None, awooop_approval_projector: Any | None = None, timeline_service: Any | None = None, alert_operation_log_repository: Any | None = None, record_history: bool = True, ) -> None: self._slo_service = slo_service or get_adr100_slo_status_service() self._incident_repository = incident_repository or IncidentDBRepository() self._auto_repair_service = auto_repair_service or AutoRepairService() self._playbook_service = playbook_service self._verifier = verifier or get_post_execution_verifier() self._approval_service = approval_service self._awooop_approval_projector = awooop_approval_projector self._timeline_service = timeline_service self._alert_operation_log_repository = alert_operation_log_repository self._record_history_enabled = record_history async def preview(self, work_item_id: str, mode: RemediationMode = "auto") -> dict[str, Any]: """Return the safe execution plan for a remediation queue item.""" item = await self._find_work_item(work_item_id) selected_mode = _select_mode(item, mode) checks = _base_checks(item) allowed = all(check["passed"] for check in checks) return { "schema_version": "adr100_remediation_preview_v1", "work_item_id": item.get("work_item_id"), "incident_id": item.get("incident_id"), "auto_repair_id": item.get("auto_repair_id"), "mode": selected_mode, "allowed": allowed, "safety_level": "read_only", "writes_incident_state": False, "writes_auto_repair_result": False, "checks": checks, "plan": _plan_for_item(item, selected_mode), "source": "adr100.verification_coverage.remediation_queue", } async def dry_run(self, work_item_id: str, mode: RemediationMode = "auto") -> dict[str, Any]: """Run a safe, read-only remediation dry-run for one queue item.""" item = await self._find_work_item(work_item_id) selected_mode = _select_mode(item, mode) checks = _base_checks(item) incident = await self._load_incident(item) checks.append({ "name": "incident_loaded", "passed": incident is not None, "detail": item.get("incident_id") or "missing incident_id", }) if incident is None or not all(check["passed"] for check in checks): payload = _dry_run_blocked_payload(item, selected_mode, checks) payload["history"] = await self._record_dry_run_history(item, payload) return payload if selected_mode == "ticket": return await self._dry_run_ticket_proposal(item, incident, checks) if selected_mode == "replay": return await self._dry_run_replay(item, incident, checks) return await self._dry_run_reverify(item, incident, checks) async def create_approval_request( self, work_item_id: str, mode: RemediationMode = "approval", ) -> dict[str, Any]: """Create a record-only approval for PlayBook authoring or runtime replay.""" item = await self._find_work_item(work_item_id) selected_mode = _select_mode(item, mode) checks = _base_checks(item) checks.append({ "name": "approval_request_supported", "passed": selected_mode in {"ticket", "approval"}, "detail": str(item.get("remediation_status") or "unknown"), }) incident = await self._load_incident(item) checks.append({ "name": "incident_loaded", "passed": incident is not None, "detail": item.get("incident_id") or "missing incident_id", }) if incident is None or not all(check["passed"] for check in checks): payload = _approval_blocked_payload(item, selected_mode, checks) payload["history"] = await self._record_dry_run_history(item, payload) return payload replay_gate: dict[str, Any] | None = None if selected_mode == "approval": replay_gate = await self._build_replay_gate(item, incident) checks.append({ "name": "runtime_replay_gate_ready", "passed": replay_gate.get("status") == "runtime_replay_ready", "detail": str(replay_gate.get("status") or "unknown"), }) if replay_gate.get("status") != "runtime_replay_ready": payload = _approval_blocked_payload( item, selected_mode, checks, extra={ "replay_gate": replay_gate, "verification_result_preview": "runtime_replay_gate_blocked", }, ) payload["history"] = await self._record_dry_run_history(item, payload) return payload approval_request = _runtime_replay_approval_request_for_item( item, incident, checks, replay_gate, ) else: approval_request = _approval_request_for_item(item, incident, checks) approval_svc = self._approval_service if approval_svc is None: from src.services.approval_db import get_approval_service approval_svc = get_approval_service() approval_kind = str((approval_request.metadata or {}).get("approval_kind") or "") fingerprint = _approval_fingerprint(item, approval_kind=approval_kind) approval = None if hasattr(approval_svc, "find_by_fingerprint"): try: approval = await approval_svc.find_by_fingerprint(fingerprint) except Exception as exc: logger.warning( "adr100_remediation_approval_dedupe_lookup_failed", fingerprint=fingerprint, error=str(exc), ) approval_created = approval is None if approval is None and hasattr(approval_svc, "create_approval_with_fingerprint"): approval = await approval_svc.create_approval_with_fingerprint( approval_request, fingerprint=fingerprint, ) elif approval is None: approval = await approval_svc.create_approval(approval_request) payload = _approval_result_payload( item=item, incident=incident, checks=checks, approval=approval, request=approval_request, approval_created=approval_created, fingerprint=fingerprint, ) if payload.get("approval_kind") == _AWOOOP_GATE5_TRIGGER_TYPE: payload["awooop_projection"] = await self._project_awooop_gate5_approval( item=item, incident=incident, request=approval_request, payload=payload, ) payload["history"] = await self._record_approval_history(item, payload) return payload async def history( self, *, limit: int = 50, incident_id: str | None = None, work_item_id: str | None = None, ) -> dict[str, Any]: """Return durable dry-run history written by this remediation service.""" safe_limit = max(1, min(limit, 200)) fetch_limit = min(max(safe_limit * 4, 50), 200) rows: list[Any] = [] repo = self._alert_operation_log_repository if repo is None: from src.repositories.alert_operation_log_repository import ( get_alert_operation_log_repository, ) repo = get_alert_operation_log_repository() for event_type in ("PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED", "APPROVAL_ESCALATED"): try: batch, _total = await repo.list_recent( limit=fetch_limit, event_type=event_type, incident_id=incident_id, ) rows.extend(batch) except Exception as exc: logger.warning( "adr100_remediation_history_fetch_failed", event_type=event_type, incident_id=incident_id, error=str(exc), ) rows.sort(key=_record_created_at, reverse=True) items: list[dict[str, Any]] = [] for row in rows: context = getattr(row, "context", None) or {} if context.get("schema_version") not in { "adr100_remediation_dry_run_history_v1", "adr100_remediation_approval_history_v1", }: continue if work_item_id and context.get("work_item_id") != work_item_id: continue items.append(_history_item(row, context)) if len(items) >= safe_limit: break return { "schema_version": "adr100_remediation_history_v1", "total": len(items), "limit": safe_limit, "filters": { "incident_id": incident_id, "work_item_id": work_item_id, }, "items": items, "by_work_item": _summarize_history_by_work_item(items), } async def _find_work_item(self, work_item_id: str) -> dict[str, Any]: report = await self._slo_service.fetch_report() coverage = report.get("verification_coverage") or {} queue = coverage.get("remediation_queue") or {} for item in queue.get("items") or []: if item.get("work_item_id") == work_item_id: return dict(item) raise RemediationNotFoundError(work_item_id) async def _load_incident(self, item: dict[str, Any]) -> Incident | None: incident_id = str(item.get("incident_id") or "") if not incident_id: return None return await self._incident_repository.get_by_id(incident_id) async def _dry_run_reverify( self, item: dict[str, Any], incident: Incident, checks: list[dict[str, Any]], ) -> dict[str, Any]: post_state = await self._collect_current_state(incident) action_taken = f"dry_run_reverify:{item.get('playbook_id') or 'unknown'}" result = _assess_recovery(None, post_state, action_taken) payload = _dry_run_result_payload( item=item, mode="reverify", checks=checks, post_state=post_state, verification_result_preview=result, extra={ "promql": _promql_for_incident(incident), "mcp_route": { "agent_id": "post_execution_verifier", "required_scope": "read", "is_shadow": True, "flywheel_node": "verify", }, }, ) payload["history"] = await self._record_dry_run_history(item, payload) return payload async def _dry_run_replay( self, item: dict[str, Any], incident: Incident, checks: list[dict[str, Any]], ) -> dict[str, Any]: diagnostic_command = _diagnostic_command_for_incident(incident) route = self._auto_repair_service.preview_read_only_ssh_mcp_route( incident, diagnostic_command, ) checks.append({ "name": "supported_executor_route", "passed": route is not None, "detail": "mcp:ssh_diagnose" if route else "missing host/container route", }) post_state = await self._collect_current_state(incident) replay_gate = await self._build_replay_gate(item, incident) action_taken = f"dry_run_replay:{item.get('playbook_id') or 'unknown'}" result = _assess_recovery(None, post_state, action_taken) payload = _dry_run_result_payload( item=item, mode="replay", checks=checks, post_state=post_state, verification_result_preview=result, extra={ "diagnostic_command_preview": diagnostic_command, "mcp_route": route, "replay_gate": replay_gate, "promql": _promql_for_incident(incident), }, ) payload["history"] = await self._record_dry_run_history(item, payload) return payload async def _build_replay_gate( self, item: dict[str, Any], incident: Incident, ) -> dict[str, Any]: playbook_id = str(item.get("playbook_id") or "") base: dict[str, Any] = { "schema_version": "adr100_replay_gate_v1", "playbook_id": playbook_id or None, "playbook_loaded": False, "status": "blocked_missing_playbook_id", "execution_authorized": False, "repair_executed": False, "writes_incident_state": False, "writes_auto_repair_result": False, "can_runtime_replay": False, "mutating_step_count": 0, "supported_write_route_count": 0, "unsupported_step_count": 0, "approval_required_count": 0, "next_step": "attach_playbook_to_work_item", "steps": [], } if not playbook_id: return base playbook_svc = self._playbook_service if playbook_svc is None: from src.services.playbook_service import get_playbook_service playbook_svc = get_playbook_service() playbook = await playbook_svc.get_by_id(playbook_id) if playbook is None: return { **base, "status": "blocked_playbook_not_found", "next_step": "reload_playbook_catalog_or_author_supported_executor", } steps: list[dict[str, Any]] = [] mutating_count = 0 supported_write_count = 0 unsupported_count = 0 approval_required_count = 0 for step in playbook.repair_steps: command = str(getattr(step, "command", "") or "") action_type = _step_action_type(step) write_route = None if action_type == "ssh_command": write_route = self._auto_repair_service.preview_write_ssh_mcp_route( incident, command, ) read_route = self._auto_repair_service.preview_read_only_ssh_mcp_route( incident, command, ) if action_type == "ssh_command" else None is_mutating = bool(write_route) or _step_looks_mutating(step) requires_approval = bool(getattr(step, "requires_approval", False)) if is_mutating: mutating_count += 1 if write_route: supported_write_count += 1 else: unsupported_count += 1 if requires_approval: approval_required_count += 1 steps.append({ "step_number": getattr(step, "step_number", None), "action_type": action_type, "risk_level": _step_risk_level(step), "requires_approval": requires_approval, "is_mutating": is_mutating, "write_route": write_route, "read_route": read_route, "supported": bool(write_route) or not is_mutating, "command_preview": _compact_command(command), }) if mutating_count == 0: status = "blocked_observe_only_playbook" next_step = "author_mutating_repair_step" can_runtime_replay = False elif unsupported_count > 0: status = "blocked_unsupported_write_route" next_step = "author_supported_executor_step" can_runtime_replay = False elif approval_required_count > 0: status = "approval_required" next_step = "request_runtime_replay_approval" can_runtime_replay = False else: status = "runtime_replay_ready" next_step = "queue_runtime_replay_with_gate5_projection" can_runtime_replay = True return { **base, "playbook_loaded": True, "status": status, "can_runtime_replay": can_runtime_replay, "mutating_step_count": mutating_count, "supported_write_route_count": supported_write_count, "unsupported_step_count": unsupported_count, "approval_required_count": approval_required_count, "next_step": next_step, "steps": steps, } async def _dry_run_ticket_proposal( self, item: dict[str, Any], incident: Incident, checks: list[dict[str, Any]], ) -> dict[str, Any]: ticket_preview = _ticket_preview_for_item(item, incident) checks.append({ "name": "external_ticket_not_created", "passed": True, "detail": "dry_run_records_internal_history_only", }) payload = _dry_run_result_payload( item=item, mode="ticket", checks=checks, post_state={}, verification_result_preview="ticket_proposal", extra={ "ticket_preview": ticket_preview, "writes_ticket": False, "creates_external_ticket": False, "plan": _plan_for_item(item, "ticket"), }, ) payload["history"] = await self._record_dry_run_history(item, payload) return payload async def _collect_current_state(self, incident: Incident) -> dict[str, Any]: try: return await asyncio.wait_for( self._verifier._collect_post_state(incident), timeout=12.0, ) except asyncio.TimeoutError: logger.warning( "adr100_remediation_dry_run_timeout", incident_id=incident.incident_id, ) return {} except Exception as exc: logger.warning( "adr100_remediation_dry_run_collect_failed", incident_id=incident.incident_id, error=str(exc), ) return {} async def _record_dry_run_history( self, item: dict[str, Any], payload: dict[str, Any], ) -> dict[str, Any]: if not self._record_history_enabled: return {"recorded": False, "reason": "disabled"} incident_id = str(item.get("incident_id") or "") if not incident_id: return {"recorded": False, "reason": "missing_incident_id"} history: dict[str, Any] = { "recorded": False, "alert_operation_id": None, "timeline_event_id": None, } context = _history_context(item, payload) allowed = bool(payload.get("allowed")) try: repo = self._alert_operation_log_repository if repo is None: from src.repositories.alert_operation_log_repository import ( get_alert_operation_log_repository, ) repo = get_alert_operation_log_repository() record = await repo.append( "PRE_FLIGHT_PASSED" if allowed else "PRE_FLIGHT_FAILED", incident_id=incident_id, auto_repair_id=str(item.get("auto_repair_id") or "") or None, actor="adr100_remediation_service", action_detail=f"adr100_remediation_dry_run:{payload.get('mode')}"[:200], success=allowed, context=context, ) if record is not None: history["alert_operation_id"] = getattr(record, "id", None) except Exception as exc: logger.warning( "adr100_remediation_alert_operation_history_failed", incident_id=incident_id, error=str(exc), ) try: timeline = self._timeline_service if timeline is None: from src.services.approval_db import get_timeline_service timeline = get_timeline_service() event = await timeline.add_event( event_type="verifier", status=_timeline_status(payload), title="ADR-100 remediation dry-run", description=_history_description(context), actor="adr100_remediation_service", actor_role=str(payload.get("mode") or "dry_run"), incident_id=incident_id, ) if event: history["timeline_event_id"] = event.get("id") except Exception as exc: logger.warning( "adr100_remediation_timeline_history_failed", incident_id=incident_id, error=str(exc), ) history["recorded"] = bool( history.get("alert_operation_id") or history.get("timeline_event_id") ) return history async def _project_awooop_gate5_approval( self, *, item: dict[str, Any], incident: Incident, request: ApprovalRequestCreate, payload: dict[str, Any], ) -> dict[str, Any]: projector = self._awooop_approval_projector if projector is not None: return await projector.project_runtime_replay_approval( item=item, incident=incident, request=request, payload=payload, ) try: return await _project_runtime_replay_approval_to_awooop( item=item, incident=incident, request=request, payload=payload, ) except Exception as exc: logger.warning( "adr100_gate5_awooop_projection_failed", incident_id=item.get("incident_id"), approval_id=payload.get("approval_id"), error=str(exc), ) return { "schema_version": "adr100_runtime_replay_awooop_projection_v1", "projected": False, "projection_mode": "approval_projection_only", "error": str(exc), "execution_authorized": False, "repair_executed": False, } async def _record_approval_history( self, item: dict[str, Any], payload: dict[str, Any], ) -> dict[str, Any]: if not self._record_history_enabled: return {"recorded": False, "reason": "disabled"} incident_id = str(item.get("incident_id") or "") approval_id = str(payload.get("approval_id") or "") history: dict[str, Any] = { "recorded": False, "alert_operation_id": None, "timeline_event_id": None, } context = _approval_history_context(item, payload) approval_kind = str(payload.get("approval_kind") or "") is_runtime_replay = approval_kind == "adr100_runtime_replay_gate5" try: repo = self._alert_operation_log_repository if repo is None: from src.repositories.alert_operation_log_repository import ( get_alert_operation_log_repository, ) repo = get_alert_operation_log_repository() record = await repo.append( "APPROVAL_ESCALATED", incident_id=incident_id or None, approval_id=approval_id or None, auto_repair_id=str(item.get("auto_repair_id") or "") or None, actor="adr100_remediation_service", action_detail=( "adr100_runtime_replay_gate5_approval_requested" if is_runtime_replay else "adr100_playbook_authoring_approval_requested" ), success=True, context=context, ) if record is not None: history["alert_operation_id"] = getattr(record, "id", None) except Exception as exc: logger.warning( "adr100_remediation_approval_history_failed", incident_id=incident_id, approval_id=approval_id, error=str(exc), ) try: timeline = self._timeline_service if timeline is None: from src.services.approval_db import get_timeline_service timeline = get_timeline_service() event = await timeline.add_event( event_type="human", status="warning", title=( "ADR-100 runtime replay Gate 5 approval requested" if is_runtime_replay else "ADR-100 PlayBook authoring approval requested" ), description=_approval_history_description(context), actor="adr100_remediation_service", actor_role="approval", approval_id=approval_id or None, incident_id=incident_id or None, ) if event: history["timeline_event_id"] = event.get("id") except Exception as exc: logger.warning( "adr100_remediation_approval_timeline_failed", incident_id=incident_id, approval_id=approval_id, error=str(exc), ) history["recorded"] = bool( history.get("alert_operation_id") or history.get("timeline_event_id") ) return history def _select_mode( item: dict[str, Any], requested: RemediationMode, ) -> Literal["reverify", "replay", "ticket", "approval"]: if requested == "approval": if item.get("remediation_status") in _TICKET_STATUSES: return "ticket" if item.get("remediation_action") in _TICKET_ACTIONS: return "ticket" if ( item.get("remediation_status") in _RUNTIME_REPLAY_STATUSES or item.get("remediation_action") in _RUNTIME_REPLAY_ACTIONS ): return "approval" return "replay" if requested in ("reverify", "replay"): return requested if requested == "ticket": return "ticket" if item.get("remediation_status") in _TICKET_STATUSES: return "ticket" if item.get("remediation_action") in _TICKET_ACTIONS: return "ticket" if item.get("remediation_status") == "ready_for_reverify": return "reverify" if item.get("remediation_action") == "reverify_with_promql_template": return "reverify" return "replay" def _base_checks(item: dict[str, Any]) -> list[dict[str, Any]]: status = str(item.get("remediation_status") or "unknown") action = str(item.get("remediation_action") or "unknown") return [ { "name": "queue_item_ready", "passed": status in _READY_STATUSES or status in _TICKET_STATUSES, "detail": status, }, { "name": "read_or_record_only_guardrail", "passed": action in { "replay_with_supported_executor", "reverify_with_promql_template", *_TICKET_ACTIONS, }, "detail": action, }, { "name": "no_state_mutation", "passed": True, "detail": "dry_run_does_not_update_incident_or_auto_repair_rows", }, ] def _plan_for_item(item: dict[str, Any], mode: str) -> dict[str, Any]: if mode == "reverify": return { "step": "collect_current_state_and_assess", "agent_id": "post_execution_verifier", "required_scope": "read", "writes": [], } if mode == "ticket": return { "step": "create_playbook_authoring_ticket_proposal", "agent_id": "openclaw_playbook_planner", "required_scope": "record_only", "writes": ["alert_operation_log", "timeline"], "target_action": item.get("remediation_action"), } if mode == "approval": if ( item.get("remediation_status") in _RUNTIME_REPLAY_STATUSES or item.get("remediation_action") in _RUNTIME_REPLAY_ACTIONS ): return { "step": "request_runtime_replay_gate5_approval", "agent_id": "auto_repair_executor", "required_scope": "record_only_until_approved", "writes": ["approval_records", "alert_operation_log", "timeline"], "target_action": item.get("remediation_action"), } return { "step": "request_playbook_authoring_approval", "agent_id": "openclaw_playbook_planner", "required_scope": "record_only", "writes": ["approval_records", "alert_operation_log", "timeline"], "target_action": item.get("remediation_action"), } return { "step": "validate_supported_executor_route_then_collect_current_state", "agent_id": "auto_repair_executor", "required_scope": "read", "writes": [], "target_action": item.get("remediation_action"), } def _dry_run_blocked_payload( item: dict[str, Any], mode: str, checks: list[dict[str, Any]], ) -> dict[str, Any]: return { "schema_version": "adr100_remediation_dry_run_v1", "work_item_id": item.get("work_item_id"), "incident_id": item.get("incident_id"), "auto_repair_id": item.get("auto_repair_id"), "mode": mode, "allowed": False, "executed": False, "safety_level": "read_only", "writes_incident_state": False, "writes_auto_repair_result": False, "writes_ticket": False, "creates_external_ticket": False, "checks": checks, "verification_result_preview": "blocked", "post_state_summary": {}, } def _approval_blocked_payload( item: dict[str, Any], mode: str, checks: list[dict[str, Any]], extra: dict[str, Any] | None = None, ) -> dict[str, Any]: return { "schema_version": "adr100_remediation_approval_v1", "work_item_id": item.get("work_item_id"), "incident_id": item.get("incident_id"), "auto_repair_id": item.get("auto_repair_id"), "mode": "approval", "requested_mode": mode, "allowed": False, "executed": False, "safety_level": "approval_record_only", "writes_incident_state": False, "writes_auto_repair_result": False, "writes_ticket": False, "writes_approval_record": False, "creates_external_ticket": False, "checks": checks, "verification_result_preview": "blocked", "approval": None, "approval_id": None, "plan": _plan_for_item(item, "approval"), **(extra or {}), } def _dry_run_result_payload( *, item: dict[str, Any], mode: str, checks: list[dict[str, Any]], post_state: dict[str, Any], verification_result_preview: str, extra: dict[str, Any], ) -> dict[str, Any]: return { "schema_version": "adr100_remediation_dry_run_v1", "work_item_id": item.get("work_item_id"), "incident_id": item.get("incident_id"), "auto_repair_id": item.get("auto_repair_id"), "mode": mode, "allowed": all(check["passed"] for check in checks), "executed": True, "safety_level": "read_only", "writes_incident_state": False, "writes_auto_repair_result": False, "writes_ticket": extra.get("writes_ticket", False), "creates_external_ticket": extra.get("creates_external_ticket", False), "checks": checks, "verification_result_preview": verification_result_preview, "post_state_summary": _summarize_post_state(post_state), **extra, } def _approval_request_for_item( item: dict[str, Any], incident: Incident, checks: list[dict[str, Any]], ) -> ApprovalRequestCreate: ticket_preview = _ticket_preview_for_item(item, incident) services = [svc for svc in (incident.affected_services or []) if svc] if not services: services = [str(item.get("alertname") or "unknown_alert")] playbook_id = str(item.get("playbook_id") or "unknown_playbook") work_item_id = str(item.get("work_item_id") or "") action = ( "PLAYBOOK_AUTHORING_RECORD_ONLY: " f"ADR-100 promote diagnostic PlayBook {playbook_id}" ) description = ( f"{ticket_preview.get('title')}\n\n" f"{ticket_preview.get('body_preview')}\n\n" "Approval scope: record-only PlayBook authoring. Signing this request " "does not execute a runtime repair, does not resolve the incident, and " "does not mark the old diagnostic run as verified_success." ) return ApprovalRequestCreate( action=action, description=description[:4000], risk_level=RiskLevel.MEDIUM, blast_radius=BlastRadius( affected_pods=0, estimated_downtime="0", related_services=services[:6], data_impact=DataImpact.READ_ONLY, ), dry_run_checks=[ DryRunCheck( name=str(check.get("name") or "check"), passed=bool(check.get("passed")), message=str(check.get("detail") or ""), ) for check in checks ], requested_by="adr100_remediation_service", expires_at=datetime.now(timezone.utc) + timedelta(hours=48), metadata={ "schema_version": "adr100_playbook_authoring_approval_v1", "approval_kind": "adr100_playbook_authoring", "execution_kind": "playbook_authoring_record_only", "execution_authorized": False, "repair_attempted": False, "repair_executed": False, "work_item_id": work_item_id, "auto_repair_id": item.get("auto_repair_id"), "source": "adr100.verification_coverage.remediation_queue", "ticket_preview": ticket_preview, "target_action": item.get("remediation_action"), "required_scope": "record_only", "next_step": "author_mutating_repair_step", "playbook_id": playbook_id, "flywheel_node": "approval", "agent_id": "openclaw_playbook_planner", "mcp_gate": "not_required_record_only", }, incident_id=str(item.get("incident_id") or incident.incident_id), matched_playbook_id=playbook_id if playbook_id != "unknown_playbook" else None, ) def _runtime_replay_approval_request_for_item( item: dict[str, Any], incident: Incident, checks: list[dict[str, Any]], replay_gate: dict[str, Any], ) -> ApprovalRequestCreate: services = [svc for svc in (incident.affected_services or []) if svc] if not services: services = [str(item.get("alertname") or "unknown_alert")] playbook_id = str(item.get("playbook_id") or "unknown_playbook") work_item_id = str(item.get("work_item_id") or "") write_routes = [ step.get("write_route") for step in replay_gate.get("steps") or [] if isinstance(step, dict) and step.get("write_route") ] route_names = [ str(route.get("tool_name") or "unknown_write_route") for route in write_routes if isinstance(route, dict) ] action = ( "RUNTIME_REPLAY_GATE5: " f"ADR-100 replay {playbook_id} via {', '.join(route_names) or 'mcp_write'}" ) description = ( f"Incident: {item.get('incident_id') or incident.incident_id}\n" f"Work item: {work_item_id or '-'}\n" f"PlayBook: {playbook_id}\n" f"Replay gate: {replay_gate.get('status')}\n" f"Write routes: {', '.join(route_names) or '-'}\n\n" "Approval scope: Gate 5 authorization for a controlled runtime replay. " "Creating this approval does not execute repair, does not restart a " "container, does not update incident state, and does not write an " "auto_repair_executions result. Execution must happen only after the " "approval status reaches approved and the executor re-validates the gate." ) return ApprovalRequestCreate( action=action, description=description[:4000], risk_level=RiskLevel.MEDIUM, blast_radius=BlastRadius( affected_pods=max(1, int(replay_gate.get("supported_write_route_count") or 1)), estimated_downtime="<1m", related_services=services[:6], data_impact=DataImpact.WRITE, ), dry_run_checks=[ DryRunCheck( name=str(check.get("name") or "check"), passed=bool(check.get("passed")), message=str(check.get("detail") or ""), ) for check in checks ], requested_by="adr100_remediation_service", expires_at=datetime.now(timezone.utc) + timedelta(hours=2), metadata={ "schema_version": "adr100_runtime_replay_gate5_approval_v1", "approval_kind": "adr100_runtime_replay_gate5", "execution_kind": "runtime_replay_gate5_pending", "execution_authorized": False, "repair_attempted": False, "repair_executed": False, "work_item_id": work_item_id, "auto_repair_id": item.get("auto_repair_id"), "source": "adr100.verification_coverage.remediation_queue", "target_action": item.get("remediation_action"), "required_scope": "write_after_approval", "next_step": "approve_then_dispatch_auto_repair_executor", "playbook_id": playbook_id, "flywheel_node": "approval", "agent_id": "auto_repair_executor", "mcp_gate": "gate5_required", "replay_gate": replay_gate, "write_routes": write_routes, }, incident_id=str(item.get("incident_id") or incident.incident_id), matched_playbook_id=playbook_id if playbook_id != "unknown_playbook" else None, ) def _approval_fingerprint( item: dict[str, Any], *, approval_kind: str = "adr100_playbook_authoring", ) -> str: work_item_id = str(item.get("work_item_id") or "") playbook_id = str(item.get("playbook_id") or "") incident_id = str(item.get("incident_id") or "") basis = work_item_id or f"{incident_id}:{playbook_id}:{item.get('remediation_action') or ''}" kind = approval_kind or "adr100_playbook_authoring" return hashlib.sha256(f"{kind}:{basis}".encode("utf-8")).hexdigest() def _approval_result_payload( *, item: dict[str, Any], incident: Incident, checks: list[dict[str, Any]], approval: Any, request: ApprovalRequestCreate, approval_created: bool, fingerprint: str, ) -> dict[str, Any]: metadata = request.metadata or {} approval_kind = str(metadata.get("approval_kind") or "adr100_playbook_authoring") ticket_preview = metadata.get("ticket_preview") if ticket_preview is None and approval_kind == "adr100_playbook_authoring": ticket_preview = _ticket_preview_for_item(item, incident) replay_gate = metadata.get("replay_gate") approval_id = str(getattr(approval, "id", "") or "") approval_status = getattr(getattr(approval, "status", None), "value", None) or getattr( approval, "status", None, ) risk_level = getattr(getattr(approval, "risk_level", None), "value", None) or getattr( approval, "risk_level", None, ) return { "schema_version": "adr100_remediation_approval_v1", "work_item_id": item.get("work_item_id"), "incident_id": item.get("incident_id") or incident.incident_id, "auto_repair_id": item.get("auto_repair_id"), "mode": "approval", "allowed": True, "executed": False, "safety_level": "approval_record_only", "writes_incident_state": False, "writes_auto_repair_result": False, "writes_ticket": False, "writes_approval_record": approval_created, "creates_external_ticket": False, "deduplicated": not approval_created, "fingerprint": fingerprint, "approval_kind": approval_kind, "checks": checks, "verification_result_preview": ( "runtime_replay_approval_requested" if approval_kind == "adr100_runtime_replay_gate5" else "approval_requested" ), "approval_id": approval_id or None, "approval": { "id": approval_id or None, "status": str(approval_status or ""), "risk_level": str(risk_level or ""), "required_signatures": getattr(approval, "required_signatures", None), "current_signatures": getattr(approval, "current_signatures", None), "requested_by": getattr(approval, "requested_by", None), "incident_id": getattr(approval, "incident_id", None), "matched_playbook_id": getattr(approval, "matched_playbook_id", None), }, "ticket_preview": ticket_preview, "replay_gate": replay_gate, "plan": _plan_for_item(item, "approval"), } async def _project_runtime_replay_approval_to_awooop( *, item: dict[str, Any], incident: Incident, request: ApprovalRequestCreate, payload: dict[str, Any], ) -> dict[str, Any]: approval_id = str(payload.get("approval_id") or "") if not approval_id: return { "schema_version": "adr100_runtime_replay_awooop_projection_v1", "projected": False, "projection_mode": "approval_projection_only", "reason": "missing_approval_id", "execution_authorized": False, "repair_executed": False, } project_id = _AWOOOP_GATE5_PROJECT_ID incident_id = str(item.get("incident_id") or incident.incident_id or "") work_item_id = str(item.get("work_item_id") or "") auto_repair_id = str(item.get("auto_repair_id") or "") playbook_id = str(item.get("playbook_id") or "unknown_playbook") run_id = uuid5( NAMESPACE_URL, f"awooop:{_AWOOOP_GATE5_TRIGGER_TYPE}:{project_id}:{approval_id}", ) trigger_ref = f"adr100_gate5:{incident_id}:{approval_id}"[:256] provider_event_id = f"adr100_gate5:{approval_id}" now = datetime.now(timezone.utc).replace(tzinfo=None) timeout_at = now + timedelta(hours=6) projection_input = { "schema_version": "adr100_runtime_replay_awooop_projection_input_v1", "approval_id": approval_id, "approval_kind": payload.get("approval_kind"), "incident_id": incident_id, "work_item_id": work_item_id, "auto_repair_id": auto_repair_id, "playbook_id": playbook_id, "replay_gate_status": (payload.get("replay_gate") or {}).get("status"), "write_route_tools": [ str(route.get("tool_name") or "") for route in (request.metadata or {}).get("write_routes") or [] if isinstance(route, dict) ], "execution_authorized": False, "repair_executed": False, "projection_mode": "approval_projection_only", } input_json = _stable_json(projection_input) input_hash = hashlib.sha256(input_json.encode("utf-8")).hexdigest() projection_metadata = { "schema_version": "adr100_runtime_replay_awooop_projection_v1", "approval_id": approval_id, "legacy_approval_status": (payload.get("approval") or {}).get("status"), "incident_id": incident_id, "work_item_id": work_item_id, "auto_repair_id": auto_repair_id, "playbook_id": playbook_id, "projection_mode": "approval_projection_only", "execution_authorized": False, "repair_attempted": False, "repair_executed": False, "required_handoff": "legacy_gate5_approval_to_auto_repair_executor", } async with get_db_context(project_id) as db: run_insert = ( pg_insert(AwoooPRunState) .values( run_id=run_id, project_id=project_id, agent_id="auto_repair_executor", state="waiting_approval", attempt_count=0, max_attempts=1, trace_id=f"gate5:{approval_id}", trigger_type=_AWOOOP_GATE5_TRIGGER_TYPE, trigger_ref=trigger_ref, is_shadow=True, input_sha256=input_hash, step_count=1, error_code="E-ADR100-GATE5-PROJECTION", error_detail=_stable_json(projection_metadata), timeout_at=timeout_at, ) .on_conflict_do_nothing(index_elements=[AwoooPRunState.run_id]) .returning(AwoooPRunState.run_id) ) run_result = await db.execute(run_insert) inserted = run_result.scalar_one_or_none() is not None idempotency_insert = ( pg_insert(AwoooPRunIdempotency) .values( project_id=project_id, channel_type=_AWOOOP_GATE5_CHANNEL_TYPE, provider_event_id=provider_event_id, run_id=run_id, ) .on_conflict_do_nothing(constraint="uix_run_idempotency_key") ) await db.execute(idempotency_insert) step_insert = ( pg_insert(AwoooPRunStepJournal) .values( run_id=run_id, project_id=project_id, step_seq=1, tool_name="adr100.runtime_replay_gate5.waiting_approval", input_hash=input_hash, compensation_json=projection_metadata, result_status="pending", error_code="E-ADR100-GATE5-PROJECTION", was_blocked=True, block_reason="approval_projection_only", ) .on_conflict_do_nothing(constraint="uix_run_step_seq") ) await db.execute(step_insert) state_result = await db.execute( select(AwoooPRunState.state, AwoooPRunState.timeout_at).where( AwoooPRunState.run_id == run_id, AwoooPRunState.project_id == project_id, ) ) state_row = state_result.one_or_none() state = str(state_row.state) if state_row else "unknown" projected = state == "waiting_approval" return { "schema_version": "adr100_runtime_replay_awooop_projection_v1", "projected": projected, "inserted": inserted, "deduplicated": not inserted, "projection_mode": "approval_projection_only", "run_id": str(run_id), "project_id": project_id, "state": state, "timeout_at": state_row.timeout_at.isoformat() if state_row and state_row.timeout_at else None, "trigger_type": _AWOOOP_GATE5_TRIGGER_TYPE, "trigger_ref": trigger_ref, "channel_type": _AWOOOP_GATE5_CHANNEL_TYPE, "provider_event_id": provider_event_id, "decision_endpoint_enabled": False, "execution_authorized": False, "repair_attempted": False, "repair_executed": False, "required_handoff": "legacy_gate5_approval_to_auto_repair_executor", "step_journal": { "step_seq": 1, "tool_name": "adr100.runtime_replay_gate5.waiting_approval", "result_status": "pending", "was_blocked": True, "block_reason": "approval_projection_only", }, } def _stable_json(value: dict[str, Any]) -> str: return json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str, ) def _summarize_post_state(post_state: dict[str, Any]) -> dict[str, Any]: keys = sorted(post_state.keys()) return { "tool_count": len(keys), "tools": keys[:8], "has_state": bool(post_state), } def _step_action_type(step: Any) -> str: action_type = getattr(step, "action_type", None) return str(getattr(action_type, "value", action_type) or "unknown") def _step_risk_level(step: Any) -> str: risk_level = getattr(step, "risk_level", None) return str(getattr(risk_level, "value", risk_level) or "unknown").lower() def _step_looks_mutating(step: Any) -> bool: command = str(getattr(step, "command", "") or "").strip().lower() action_type = _step_action_type(step) if not command or action_type == "manual": return False if action_type == "script": return True if action_type == "kubectl": return not command.startswith(( "kubectl get ", "kubectl describe ", "kubectl logs ", "kubectl top ", "kubectl explain ", )) if action_type == "ssh_command": return any(token in command for token in ( "docker restart", "docker start", "docker stop", "systemctl restart", "systemctl start", "certbot renew", )) return False def _compact_command(command: str) -> str: compact = " ".join(str(command or "").split()) return compact[:240] def _history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]: return { "schema_version": "adr100_remediation_dry_run_history_v1", "work_item_id": item.get("work_item_id"), "auto_repair_id": item.get("auto_repair_id"), "playbook_id": item.get("playbook_id"), "alertname": item.get("alertname"), "mode": payload.get("mode"), "allowed": payload.get("allowed"), "executed": payload.get("executed"), "safety_level": payload.get("safety_level"), "writes_incident_state": payload.get("writes_incident_state"), "writes_auto_repair_result": payload.get("writes_auto_repair_result"), "writes_ticket": payload.get("writes_ticket"), "creates_external_ticket": payload.get("creates_external_ticket"), "ticket_preview": payload.get("ticket_preview"), "plan": payload.get("plan"), "verification_result_preview": payload.get("verification_result_preview"), "post_state_summary": payload.get("post_state_summary"), "mcp_route": payload.get("mcp_route"), "replay_gate": payload.get("replay_gate"), "checks": payload.get("checks"), } def _approval_history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]: return { "schema_version": "adr100_remediation_approval_history_v1", "work_item_id": item.get("work_item_id"), "auto_repair_id": item.get("auto_repair_id"), "playbook_id": item.get("playbook_id"), "alertname": item.get("alertname"), "mode": payload.get("mode"), "allowed": payload.get("allowed"), "executed": payload.get("executed"), "safety_level": payload.get("safety_level"), "writes_incident_state": payload.get("writes_incident_state"), "writes_auto_repair_result": payload.get("writes_auto_repair_result"), "writes_ticket": payload.get("writes_ticket"), "writes_approval_record": payload.get("writes_approval_record"), "creates_external_ticket": payload.get("creates_external_ticket"), "deduplicated": payload.get("deduplicated"), "fingerprint": payload.get("fingerprint"), "approval_kind": payload.get("approval_kind"), "ticket_preview": payload.get("ticket_preview"), "replay_gate": payload.get("replay_gate"), "awooop_projection": payload.get("awooop_projection"), "approval": payload.get("approval"), "approval_id": payload.get("approval_id"), "plan": payload.get("plan"), "verification_result_preview": payload.get("verification_result_preview"), "checks": payload.get("checks"), } def _timeline_status(payload: dict[str, Any]) -> str: if not payload.get("allowed"): return "warning" if payload.get("verification_result_preview") == "success": return "success" return "warning" def _history_description(context: dict[str, Any]) -> str: tool_count = (context.get("post_state_summary") or {}).get("tool_count", 0) route = context.get("mcp_route") or {} agent = route.get("agent_id") or "unknown_agent" tool = route.get("tool_name") or "current_state" return ( f"mode={context.get('mode')} " f"preview={context.get('verification_result_preview')} " f"tools={tool_count} route={agent}/{tool} " f"writes_incident={context.get('writes_incident_state')} " f"writes_auto_repair={context.get('writes_auto_repair_result')}" )[:500] def _approval_history_description(context: dict[str, Any]) -> str: approval = context.get("approval") or {} return ( f"kind={context.get('approval_kind') or 'unknown'} " f"approval={approval.get('id') or context.get('approval_id') or 'unknown'} " f"status={approval.get('status') or 'unknown'} " f"preview={context.get('verification_result_preview')} " f"writes_approval={context.get('writes_approval_record')} " f"writes_incident={context.get('writes_incident_state')} " f"writes_auto_repair={context.get('writes_auto_repair_result')}" )[:500] def _record_created_at(record: Any) -> str: value = getattr(record, "created_at", None) if hasattr(value, "isoformat"): return value.isoformat() return str(value or "") def _history_item(record: Any, context: dict[str, Any]) -> dict[str, Any]: route = context.get("mcp_route") or {} post_state = context.get("post_state_summary") or {} approval = context.get("approval") or {} replay_gate = context.get("replay_gate") or {} awooop_projection = context.get("awooop_projection") or {} return { "id": str(getattr(record, "id", "")), "incident_id": getattr(record, "incident_id", None), "auto_repair_id": getattr(record, "auto_repair_id", None) or context.get("auto_repair_id"), "event_type": str(getattr(record, "event_type", "")), "actor": getattr(record, "actor", None), "success": getattr(record, "success", None), "created_at": _record_created_at(record), "work_item_id": context.get("work_item_id"), "playbook_id": context.get("playbook_id"), "alertname": context.get("alertname"), "mode": context.get("mode"), "allowed": context.get("allowed"), "executed": context.get("executed"), "safety_level": context.get("safety_level"), "verification_result_preview": context.get("verification_result_preview"), "tool_count": post_state.get("tool_count", 0), "tools": post_state.get("tools") or [], "agent_id": route.get("agent_id"), "tool_name": route.get("tool_name") or "current_state", "required_scope": route.get("required_scope"), "writes_incident_state": context.get("writes_incident_state"), "writes_auto_repair_result": context.get("writes_auto_repair_result"), "writes_ticket": context.get("writes_ticket"), "writes_approval_record": context.get("writes_approval_record"), "creates_external_ticket": context.get("creates_external_ticket"), "approval_kind": context.get("approval_kind"), "approval_id": context.get("approval_id") or approval.get("id"), "approval_status": approval.get("status"), "approval_risk_level": approval.get("risk_level"), "deduplicated": context.get("deduplicated"), "fingerprint": context.get("fingerprint"), "ticket_preview": context.get("ticket_preview"), "replay_gate": replay_gate or None, "replay_gate_status": replay_gate.get("status"), "replay_gate_next_step": replay_gate.get("next_step"), "awooop_projection": awooop_projection or None, "awooop_projection_run_id": awooop_projection.get("run_id"), "awooop_projection_state": awooop_projection.get("state"), "plan": context.get("plan"), "checks": context.get("checks") or [], } def _summarize_history_by_work_item(items: list[dict[str, Any]]) -> list[dict[str, Any]]: summary: dict[str, dict[str, Any]] = {} for item in items: key = str(item.get("work_item_id") or item.get("incident_id") or item.get("id")) if key not in summary: summary[key] = { "work_item_id": item.get("work_item_id"), "incident_id": item.get("incident_id"), "count": 0, "latest_at": item.get("created_at"), "latest_event_type": item.get("event_type"), "latest_success": item.get("success"), "latest_preview": item.get("verification_result_preview"), "latest_mode": item.get("mode"), "latest_agent_id": item.get("agent_id"), "latest_tool_name": item.get("tool_name"), "required_scope": item.get("required_scope"), "latest_replay_gate_status": item.get("replay_gate_status"), "latest_replay_gate_next_step": item.get("replay_gate_next_step"), } summary[key]["count"] += 1 return list(summary.values()) def _diagnostic_command_for_incident(incident: Incident) -> str: labels = _labels_for_incident(incident) host = str(labels.get("host") or labels.get("instance") or "{host}") container = str(labels.get("container_name") or labels.get("container") or "") if container: return f"ssh {host} 'uptime; docker stats --no-stream {container}'" return f"ssh {host} 'uptime; docker stats --no-stream'" def _ticket_preview_for_item(item: dict[str, Any], incident: Incident) -> dict[str, Any]: labels = _labels_for_incident(incident) alertname = str(item.get("alertname") or labels.get("alertname") or "unknown_alert") incident_id = str(item.get("incident_id") or incident.incident_id) playbook_id = str(item.get("playbook_id") or "unknown_playbook") host = str(labels.get("host") or labels.get("instance") or "unknown_host") container = str(labels.get("container_name") or labels.get("container") or "") target = f"host={host}" + (f" container={container}" if container else "") title = f"[ADR-100] Promote diagnostic PlayBook to repair: {alertname}" body = ( f"Incident: {incident_id}\n" f"Auto repair: {item.get('auto_repair_id') or 'unknown'}\n" f"PlayBook: {playbook_id}\n" f"Target: {target}\n" f"Failure class: {item.get('failure_class') or 'observe_only_playbook'}\n" "Required change: add a gated mutating repair step such as docker restart, " "Ansible check-mode/apply, or another approved executor action, then keep " "post-execution verification tied to the same target.\n" "Guardrail: do not mark the old diagnostic-only run as verified_success." ) return { "would_create": True, "external_ticket_created": False, "title": title, "labels": [ "adr100", "playbook-authoring", "observe-only-playbook", "needs-owner-review", ], "body_preview": body[:1000], "owner": item.get("remediation_owner") or "solver_or_operator", "next_step": "author_mutating_repair_step", "playbook_id": playbook_id, "target": target, } def _promql_for_incident(incident: Incident) -> str: labels = _labels_for_incident(incident) alertname = "" if incident.signals: signal = incident.signals[0] alertname = labels.get("alertname") or getattr(signal, "alert_name", "") return _build_prometheus_query(alertname, labels) def _labels_for_incident(incident: Incident) -> dict[str, Any]: if incident.signals: return incident.signals[0].labels or {} return {} _service: Adr100RemediationService | None = None def get_adr100_remediation_service() -> Adr100RemediationService: """Return singleton ADR-100 remediation service.""" global _service if _service is None: _service = Adr100RemediationService() return _service def set_adr100_remediation_service(service: Adr100RemediationService | None) -> None: """Inject ADR-100 remediation service for tests.""" global _service _service = service