1080 lines
41 KiB
Python
1080 lines
41 KiB
Python
"""
|
|
ADR-100 Remediation Service
|
|
===========================
|
|
Safe operator entrypoints for verification remediation work items.
|
|
|
|
T25: remediation queue items are now actionable without mutating incident state:
|
|
- preview: show the selected guardrail path
|
|
- dry-run: collect read-only current state and validate supported executor routing
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any, Literal, Protocol
|
|
|
|
import structlog
|
|
|
|
from src.models.approval import (
|
|
ApprovalRequestCreate,
|
|
BlastRadius,
|
|
DataImpact,
|
|
DryRunCheck,
|
|
RiskLevel,
|
|
)
|
|
from src.models.incident import Incident
|
|
from src.repositories.incident_repository import IncidentDBRepository
|
|
from src.services.adr100_slo_status_service import (
|
|
Adr100SloStatusService,
|
|
get_adr100_slo_status_service,
|
|
)
|
|
from src.services.auto_repair_service import AutoRepairService
|
|
from src.services.post_execution_verifier import (
|
|
PostExecutionVerifier,
|
|
_assess_recovery,
|
|
_build_prometheus_query,
|
|
get_post_execution_verifier,
|
|
)
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
RemediationMode = Literal["auto", "reverify", "replay", "ticket", "approval"]
|
|
|
|
_READY_STATUSES = {"ready_for_replay", "ready_for_reverify"}
|
|
_TICKET_STATUSES = {"needs_playbook_ticket"}
|
|
_TICKET_ACTIONS = {"create_playbook_ticket", "promote_diagnostic_to_repair_playbook"}
|
|
|
|
|
|
class RemediationNotFoundError(LookupError):
|
|
"""Requested ADR-100 remediation work item is not in the current read model."""
|
|
|
|
|
|
class _IncidentRepository(Protocol):
|
|
async def get_by_id(self, incident_id: str) -> Incident | None:
|
|
...
|
|
|
|
|
|
class Adr100RemediationService:
|
|
"""Read-only remediation preview and dry-run service."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
slo_service: Adr100SloStatusService | None = None,
|
|
incident_repository: _IncidentRepository | None = None,
|
|
auto_repair_service: AutoRepairService | None = None,
|
|
verifier: PostExecutionVerifier | None = None,
|
|
approval_service: Any | None = None,
|
|
timeline_service: Any | None = None,
|
|
alert_operation_log_repository: Any | None = None,
|
|
record_history: bool = True,
|
|
) -> None:
|
|
self._slo_service = slo_service or get_adr100_slo_status_service()
|
|
self._incident_repository = incident_repository or IncidentDBRepository()
|
|
self._auto_repair_service = auto_repair_service or AutoRepairService()
|
|
self._verifier = verifier or get_post_execution_verifier()
|
|
self._approval_service = approval_service
|
|
self._timeline_service = timeline_service
|
|
self._alert_operation_log_repository = alert_operation_log_repository
|
|
self._record_history_enabled = record_history
|
|
|
|
async def preview(self, work_item_id: str, mode: RemediationMode = "auto") -> dict[str, Any]:
|
|
"""Return the safe execution plan for a remediation queue item."""
|
|
|
|
item = await self._find_work_item(work_item_id)
|
|
selected_mode = _select_mode(item, mode)
|
|
checks = _base_checks(item)
|
|
allowed = all(check["passed"] for check in checks)
|
|
|
|
return {
|
|
"schema_version": "adr100_remediation_preview_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": selected_mode,
|
|
"allowed": allowed,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"checks": checks,
|
|
"plan": _plan_for_item(item, selected_mode),
|
|
"source": "adr100.verification_coverage.remediation_queue",
|
|
}
|
|
|
|
async def dry_run(self, work_item_id: str, mode: RemediationMode = "auto") -> dict[str, Any]:
|
|
"""Run a safe, read-only remediation dry-run for one queue item."""
|
|
|
|
item = await self._find_work_item(work_item_id)
|
|
selected_mode = _select_mode(item, mode)
|
|
checks = _base_checks(item)
|
|
incident = await self._load_incident(item)
|
|
checks.append({
|
|
"name": "incident_loaded",
|
|
"passed": incident is not None,
|
|
"detail": item.get("incident_id") or "missing incident_id",
|
|
})
|
|
|
|
if incident is None or not all(check["passed"] for check in checks):
|
|
payload = _dry_run_blocked_payload(item, selected_mode, checks)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
if selected_mode == "ticket":
|
|
return await self._dry_run_ticket_proposal(item, incident, checks)
|
|
if selected_mode == "replay":
|
|
return await self._dry_run_replay(item, incident, checks)
|
|
return await self._dry_run_reverify(item, incident, checks)
|
|
|
|
async def create_approval_request(
|
|
self,
|
|
work_item_id: str,
|
|
mode: RemediationMode = "approval",
|
|
) -> dict[str, Any]:
|
|
"""Create a record-only approval for PlayBook authoring remediation."""
|
|
|
|
item = await self._find_work_item(work_item_id)
|
|
selected_mode = _select_mode(item, mode)
|
|
checks = _base_checks(item)
|
|
checks.append({
|
|
"name": "playbook_authoring_ticket_required",
|
|
"passed": selected_mode in {"ticket", "approval"},
|
|
"detail": str(item.get("remediation_status") or "unknown"),
|
|
})
|
|
|
|
incident = await self._load_incident(item)
|
|
checks.append({
|
|
"name": "incident_loaded",
|
|
"passed": incident is not None,
|
|
"detail": item.get("incident_id") or "missing incident_id",
|
|
})
|
|
|
|
if incident is None or not all(check["passed"] for check in checks):
|
|
payload = _approval_blocked_payload(item, selected_mode, checks)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
approval_request = _approval_request_for_item(item, incident, checks)
|
|
approval_svc = self._approval_service
|
|
if approval_svc is None:
|
|
from src.services.approval_db import get_approval_service
|
|
|
|
approval_svc = get_approval_service()
|
|
|
|
fingerprint = _approval_fingerprint(item)
|
|
approval = None
|
|
if hasattr(approval_svc, "find_by_fingerprint"):
|
|
try:
|
|
approval = await approval_svc.find_by_fingerprint(fingerprint)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_approval_dedupe_lookup_failed",
|
|
fingerprint=fingerprint,
|
|
error=str(exc),
|
|
)
|
|
|
|
approval_created = approval is None
|
|
if approval is None and hasattr(approval_svc, "create_approval_with_fingerprint"):
|
|
approval = await approval_svc.create_approval_with_fingerprint(
|
|
approval_request,
|
|
fingerprint=fingerprint,
|
|
)
|
|
elif approval is None:
|
|
approval = await approval_svc.create_approval(approval_request)
|
|
|
|
payload = _approval_result_payload(
|
|
item=item,
|
|
incident=incident,
|
|
checks=checks,
|
|
approval=approval,
|
|
request=approval_request,
|
|
approval_created=approval_created,
|
|
fingerprint=fingerprint,
|
|
)
|
|
payload["history"] = await self._record_approval_history(item, payload)
|
|
return payload
|
|
|
|
async def history(
|
|
self,
|
|
*,
|
|
limit: int = 50,
|
|
incident_id: str | None = None,
|
|
work_item_id: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Return durable dry-run history written by this remediation service."""
|
|
|
|
safe_limit = max(1, min(limit, 200))
|
|
fetch_limit = min(max(safe_limit * 4, 50), 200)
|
|
rows: list[Any] = []
|
|
repo = self._alert_operation_log_repository
|
|
if repo is None:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
repo = get_alert_operation_log_repository()
|
|
|
|
for event_type in ("PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED", "APPROVAL_ESCALATED"):
|
|
try:
|
|
batch, _total = await repo.list_recent(
|
|
limit=fetch_limit,
|
|
event_type=event_type,
|
|
incident_id=incident_id,
|
|
)
|
|
rows.extend(batch)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_history_fetch_failed",
|
|
event_type=event_type,
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
rows.sort(key=_record_created_at, reverse=True)
|
|
|
|
items: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
context = getattr(row, "context", None) or {}
|
|
if context.get("schema_version") not in {
|
|
"adr100_remediation_dry_run_history_v1",
|
|
"adr100_remediation_approval_history_v1",
|
|
}:
|
|
continue
|
|
if work_item_id and context.get("work_item_id") != work_item_id:
|
|
continue
|
|
items.append(_history_item(row, context))
|
|
if len(items) >= safe_limit:
|
|
break
|
|
|
|
return {
|
|
"schema_version": "adr100_remediation_history_v1",
|
|
"total": len(items),
|
|
"limit": safe_limit,
|
|
"filters": {
|
|
"incident_id": incident_id,
|
|
"work_item_id": work_item_id,
|
|
},
|
|
"items": items,
|
|
"by_work_item": _summarize_history_by_work_item(items),
|
|
}
|
|
|
|
async def _find_work_item(self, work_item_id: str) -> dict[str, Any]:
|
|
report = await self._slo_service.fetch_report()
|
|
coverage = report.get("verification_coverage") or {}
|
|
queue = coverage.get("remediation_queue") or {}
|
|
|
|
for item in queue.get("items") or []:
|
|
if item.get("work_item_id") == work_item_id:
|
|
return dict(item)
|
|
|
|
raise RemediationNotFoundError(work_item_id)
|
|
|
|
async def _load_incident(self, item: dict[str, Any]) -> Incident | None:
|
|
incident_id = str(item.get("incident_id") or "")
|
|
if not incident_id:
|
|
return None
|
|
return await self._incident_repository.get_by_id(incident_id)
|
|
|
|
async def _dry_run_reverify(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
post_state = await self._collect_current_state(incident)
|
|
action_taken = f"dry_run_reverify:{item.get('playbook_id') or 'unknown'}"
|
|
result = _assess_recovery(None, post_state, action_taken)
|
|
|
|
payload = _dry_run_result_payload(
|
|
item=item,
|
|
mode="reverify",
|
|
checks=checks,
|
|
post_state=post_state,
|
|
verification_result_preview=result,
|
|
extra={
|
|
"promql": _promql_for_incident(incident),
|
|
"mcp_route": {
|
|
"agent_id": "post_execution_verifier",
|
|
"required_scope": "read",
|
|
"is_shadow": True,
|
|
"flywheel_node": "verify",
|
|
},
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
async def _dry_run_replay(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
diagnostic_command = _diagnostic_command_for_incident(incident)
|
|
route = self._auto_repair_service.preview_read_only_ssh_mcp_route(
|
|
incident,
|
|
diagnostic_command,
|
|
)
|
|
checks.append({
|
|
"name": "supported_executor_route",
|
|
"passed": route is not None,
|
|
"detail": "mcp:ssh_diagnose" if route else "missing host/container route",
|
|
})
|
|
|
|
post_state = await self._collect_current_state(incident)
|
|
action_taken = f"dry_run_replay:{item.get('playbook_id') or 'unknown'}"
|
|
result = _assess_recovery(None, post_state, action_taken)
|
|
|
|
payload = _dry_run_result_payload(
|
|
item=item,
|
|
mode="replay",
|
|
checks=checks,
|
|
post_state=post_state,
|
|
verification_result_preview=result,
|
|
extra={
|
|
"diagnostic_command_preview": diagnostic_command,
|
|
"mcp_route": route,
|
|
"promql": _promql_for_incident(incident),
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
async def _dry_run_ticket_proposal(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
ticket_preview = _ticket_preview_for_item(item, incident)
|
|
checks.append({
|
|
"name": "external_ticket_not_created",
|
|
"passed": True,
|
|
"detail": "dry_run_records_internal_history_only",
|
|
})
|
|
|
|
payload = _dry_run_result_payload(
|
|
item=item,
|
|
mode="ticket",
|
|
checks=checks,
|
|
post_state={},
|
|
verification_result_preview="ticket_proposal",
|
|
extra={
|
|
"ticket_preview": ticket_preview,
|
|
"writes_ticket": False,
|
|
"creates_external_ticket": False,
|
|
"plan": _plan_for_item(item, "ticket"),
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
async def _collect_current_state(self, incident: Incident) -> dict[str, Any]:
|
|
try:
|
|
return await asyncio.wait_for(
|
|
self._verifier._collect_post_state(incident),
|
|
timeout=12.0,
|
|
)
|
|
except asyncio.TimeoutError:
|
|
logger.warning(
|
|
"adr100_remediation_dry_run_timeout",
|
|
incident_id=incident.incident_id,
|
|
)
|
|
return {}
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_dry_run_collect_failed",
|
|
incident_id=incident.incident_id,
|
|
error=str(exc),
|
|
)
|
|
return {}
|
|
|
|
async def _record_dry_run_history(
|
|
self,
|
|
item: dict[str, Any],
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if not self._record_history_enabled:
|
|
return {"recorded": False, "reason": "disabled"}
|
|
|
|
incident_id = str(item.get("incident_id") or "")
|
|
if not incident_id:
|
|
return {"recorded": False, "reason": "missing_incident_id"}
|
|
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
context = _history_context(item, payload)
|
|
allowed = bool(payload.get("allowed"))
|
|
|
|
try:
|
|
repo = self._alert_operation_log_repository
|
|
if repo is None:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
repo = get_alert_operation_log_repository()
|
|
record = await repo.append(
|
|
"PRE_FLIGHT_PASSED" if allowed else "PRE_FLIGHT_FAILED",
|
|
incident_id=incident_id,
|
|
auto_repair_id=str(item.get("auto_repair_id") or "") or None,
|
|
actor="adr100_remediation_service",
|
|
action_detail=f"adr100_remediation_dry_run:{payload.get('mode')}"[:200],
|
|
success=allowed,
|
|
context=context,
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_alert_operation_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
try:
|
|
timeline = self._timeline_service
|
|
if timeline is None:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
timeline = get_timeline_service()
|
|
event = await timeline.add_event(
|
|
event_type="verifier",
|
|
status=_timeline_status(payload),
|
|
title="ADR-100 remediation dry-run",
|
|
description=_history_description(context),
|
|
actor="adr100_remediation_service",
|
|
actor_role=str(payload.get("mode") or "dry_run"),
|
|
incident_id=incident_id,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_timeline_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
history["recorded"] = bool(
|
|
history.get("alert_operation_id") or history.get("timeline_event_id")
|
|
)
|
|
return history
|
|
|
|
async def _record_approval_history(
|
|
self,
|
|
item: dict[str, Any],
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if not self._record_history_enabled:
|
|
return {"recorded": False, "reason": "disabled"}
|
|
|
|
incident_id = str(item.get("incident_id") or "")
|
|
approval_id = str(payload.get("approval_id") or "")
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
context = _approval_history_context(item, payload)
|
|
|
|
try:
|
|
repo = self._alert_operation_log_repository
|
|
if repo is None:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
repo = get_alert_operation_log_repository()
|
|
record = await repo.append(
|
|
"APPROVAL_ESCALATED",
|
|
incident_id=incident_id or None,
|
|
approval_id=approval_id or None,
|
|
auto_repair_id=str(item.get("auto_repair_id") or "") or None,
|
|
actor="adr100_remediation_service",
|
|
action_detail="adr100_playbook_authoring_approval_requested",
|
|
success=True,
|
|
context=context,
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_approval_history_failed",
|
|
incident_id=incident_id,
|
|
approval_id=approval_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
try:
|
|
timeline = self._timeline_service
|
|
if timeline is None:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
timeline = get_timeline_service()
|
|
event = await timeline.add_event(
|
|
event_type="human",
|
|
status="warning",
|
|
title="ADR-100 PlayBook authoring approval requested",
|
|
description=_approval_history_description(context),
|
|
actor="adr100_remediation_service",
|
|
actor_role="approval",
|
|
approval_id=approval_id or None,
|
|
incident_id=incident_id or None,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_approval_timeline_failed",
|
|
incident_id=incident_id,
|
|
approval_id=approval_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
history["recorded"] = bool(
|
|
history.get("alert_operation_id") or history.get("timeline_event_id")
|
|
)
|
|
return history
|
|
|
|
|
|
def _select_mode(item: dict[str, Any], requested: RemediationMode) -> Literal["reverify", "replay", "ticket"]:
|
|
if requested == "approval":
|
|
return "ticket"
|
|
if requested in ("reverify", "replay"):
|
|
return requested
|
|
if requested == "ticket":
|
|
return "ticket"
|
|
if item.get("remediation_status") in _TICKET_STATUSES:
|
|
return "ticket"
|
|
if item.get("remediation_action") in _TICKET_ACTIONS:
|
|
return "ticket"
|
|
if item.get("remediation_status") == "ready_for_reverify":
|
|
return "reverify"
|
|
if item.get("remediation_action") == "reverify_with_promql_template":
|
|
return "reverify"
|
|
return "replay"
|
|
|
|
|
|
def _base_checks(item: dict[str, Any]) -> list[dict[str, Any]]:
|
|
status = str(item.get("remediation_status") or "unknown")
|
|
action = str(item.get("remediation_action") or "unknown")
|
|
return [
|
|
{
|
|
"name": "queue_item_ready",
|
|
"passed": status in _READY_STATUSES or status in _TICKET_STATUSES,
|
|
"detail": status,
|
|
},
|
|
{
|
|
"name": "read_or_record_only_guardrail",
|
|
"passed": action in {
|
|
"replay_with_supported_executor",
|
|
"reverify_with_promql_template",
|
|
*_TICKET_ACTIONS,
|
|
},
|
|
"detail": action,
|
|
},
|
|
{
|
|
"name": "no_state_mutation",
|
|
"passed": True,
|
|
"detail": "dry_run_does_not_update_incident_or_auto_repair_rows",
|
|
},
|
|
]
|
|
|
|
|
|
def _plan_for_item(item: dict[str, Any], mode: str) -> dict[str, Any]:
|
|
if mode == "reverify":
|
|
return {
|
|
"step": "collect_current_state_and_assess",
|
|
"agent_id": "post_execution_verifier",
|
|
"required_scope": "read",
|
|
"writes": [],
|
|
}
|
|
if mode == "ticket":
|
|
return {
|
|
"step": "create_playbook_authoring_ticket_proposal",
|
|
"agent_id": "openclaw_playbook_planner",
|
|
"required_scope": "record_only",
|
|
"writes": ["alert_operation_log", "timeline"],
|
|
"target_action": item.get("remediation_action"),
|
|
}
|
|
if mode == "approval":
|
|
return {
|
|
"step": "request_playbook_authoring_approval",
|
|
"agent_id": "openclaw_playbook_planner",
|
|
"required_scope": "record_only",
|
|
"writes": ["approval_records", "alert_operation_log", "timeline"],
|
|
"target_action": item.get("remediation_action"),
|
|
}
|
|
return {
|
|
"step": "validate_supported_executor_route_then_collect_current_state",
|
|
"agent_id": "auto_repair_executor",
|
|
"required_scope": "read",
|
|
"writes": [],
|
|
"target_action": item.get("remediation_action"),
|
|
}
|
|
|
|
|
|
def _dry_run_blocked_payload(
|
|
item: dict[str, Any],
|
|
mode: str,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": mode,
|
|
"allowed": False,
|
|
"executed": False,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"creates_external_ticket": False,
|
|
"checks": checks,
|
|
"verification_result_preview": "blocked",
|
|
"post_state_summary": {},
|
|
}
|
|
|
|
|
|
def _approval_blocked_payload(
|
|
item: dict[str, Any],
|
|
mode: str,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_approval_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": "approval",
|
|
"requested_mode": mode,
|
|
"allowed": False,
|
|
"executed": False,
|
|
"safety_level": "approval_record_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"writes_approval_record": False,
|
|
"creates_external_ticket": False,
|
|
"checks": checks,
|
|
"verification_result_preview": "blocked",
|
|
"approval": None,
|
|
"approval_id": None,
|
|
"plan": _plan_for_item(item, "approval"),
|
|
}
|
|
|
|
|
|
def _dry_run_result_payload(
|
|
*,
|
|
item: dict[str, Any],
|
|
mode: str,
|
|
checks: list[dict[str, Any]],
|
|
post_state: dict[str, Any],
|
|
verification_result_preview: str,
|
|
extra: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": mode,
|
|
"allowed": all(check["passed"] for check in checks),
|
|
"executed": True,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": extra.get("writes_ticket", False),
|
|
"creates_external_ticket": extra.get("creates_external_ticket", False),
|
|
"checks": checks,
|
|
"verification_result_preview": verification_result_preview,
|
|
"post_state_summary": _summarize_post_state(post_state),
|
|
**extra,
|
|
}
|
|
|
|
|
|
def _approval_request_for_item(
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> ApprovalRequestCreate:
|
|
ticket_preview = _ticket_preview_for_item(item, incident)
|
|
services = [svc for svc in (incident.affected_services or []) if svc]
|
|
if not services:
|
|
services = [str(item.get("alertname") or "unknown_alert")]
|
|
playbook_id = str(item.get("playbook_id") or "unknown_playbook")
|
|
work_item_id = str(item.get("work_item_id") or "")
|
|
action = (
|
|
"PLAYBOOK_AUTHORING_RECORD_ONLY: "
|
|
f"ADR-100 promote diagnostic PlayBook {playbook_id}"
|
|
)
|
|
description = (
|
|
f"{ticket_preview.get('title')}\n\n"
|
|
f"{ticket_preview.get('body_preview')}\n\n"
|
|
"Approval scope: record-only PlayBook authoring. Signing this request "
|
|
"does not execute a runtime repair, does not resolve the incident, and "
|
|
"does not mark the old diagnostic run as verified_success."
|
|
)
|
|
return ApprovalRequestCreate(
|
|
action=action,
|
|
description=description[:4000],
|
|
risk_level=RiskLevel.MEDIUM,
|
|
blast_radius=BlastRadius(
|
|
affected_pods=0,
|
|
estimated_downtime="0",
|
|
related_services=services[:6],
|
|
data_impact=DataImpact.READ_ONLY,
|
|
),
|
|
dry_run_checks=[
|
|
DryRunCheck(
|
|
name=str(check.get("name") or "check"),
|
|
passed=bool(check.get("passed")),
|
|
message=str(check.get("detail") or ""),
|
|
)
|
|
for check in checks
|
|
],
|
|
requested_by="adr100_remediation_service",
|
|
expires_at=datetime.now(timezone.utc) + timedelta(hours=48),
|
|
metadata={
|
|
"schema_version": "adr100_playbook_authoring_approval_v1",
|
|
"approval_kind": "adr100_playbook_authoring",
|
|
"execution_kind": "playbook_authoring_record_only",
|
|
"execution_authorized": False,
|
|
"repair_attempted": False,
|
|
"repair_executed": False,
|
|
"work_item_id": work_item_id,
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"source": "adr100.verification_coverage.remediation_queue",
|
|
"ticket_preview": ticket_preview,
|
|
"target_action": item.get("remediation_action"),
|
|
"required_scope": "record_only",
|
|
"next_step": "author_mutating_repair_step",
|
|
"playbook_id": playbook_id,
|
|
"flywheel_node": "approval",
|
|
"agent_id": "openclaw_playbook_planner",
|
|
"mcp_gate": "not_required_record_only",
|
|
},
|
|
incident_id=str(item.get("incident_id") or incident.incident_id),
|
|
matched_playbook_id=playbook_id if playbook_id != "unknown_playbook" else None,
|
|
)
|
|
|
|
|
|
def _approval_fingerprint(item: dict[str, Any]) -> str:
|
|
work_item_id = str(item.get("work_item_id") or "")
|
|
playbook_id = str(item.get("playbook_id") or "")
|
|
incident_id = str(item.get("incident_id") or "")
|
|
basis = work_item_id or f"{incident_id}:{playbook_id}:{item.get('remediation_action') or ''}"
|
|
return hashlib.sha256(f"adr100_playbook_authoring:{basis}".encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _approval_result_payload(
|
|
*,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
approval: Any,
|
|
request: ApprovalRequestCreate,
|
|
approval_created: bool,
|
|
fingerprint: str,
|
|
) -> dict[str, Any]:
|
|
ticket_preview = (request.metadata or {}).get("ticket_preview") or _ticket_preview_for_item(
|
|
item,
|
|
incident,
|
|
)
|
|
approval_id = str(getattr(approval, "id", "") or "")
|
|
approval_status = getattr(getattr(approval, "status", None), "value", None) or getattr(
|
|
approval,
|
|
"status",
|
|
None,
|
|
)
|
|
risk_level = getattr(getattr(approval, "risk_level", None), "value", None) or getattr(
|
|
approval,
|
|
"risk_level",
|
|
None,
|
|
)
|
|
return {
|
|
"schema_version": "adr100_remediation_approval_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id") or incident.incident_id,
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": "approval",
|
|
"allowed": True,
|
|
"executed": False,
|
|
"safety_level": "approval_record_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"writes_approval_record": approval_created,
|
|
"creates_external_ticket": False,
|
|
"deduplicated": not approval_created,
|
|
"fingerprint": fingerprint,
|
|
"checks": checks,
|
|
"verification_result_preview": "approval_requested",
|
|
"approval_id": approval_id or None,
|
|
"approval": {
|
|
"id": approval_id or None,
|
|
"status": str(approval_status or ""),
|
|
"risk_level": str(risk_level or ""),
|
|
"required_signatures": getattr(approval, "required_signatures", None),
|
|
"current_signatures": getattr(approval, "current_signatures", None),
|
|
"requested_by": getattr(approval, "requested_by", None),
|
|
"incident_id": getattr(approval, "incident_id", None),
|
|
"matched_playbook_id": getattr(approval, "matched_playbook_id", None),
|
|
},
|
|
"ticket_preview": ticket_preview,
|
|
"plan": _plan_for_item(item, "approval"),
|
|
}
|
|
|
|
|
|
def _summarize_post_state(post_state: dict[str, Any]) -> dict[str, Any]:
|
|
keys = sorted(post_state.keys())
|
|
return {
|
|
"tool_count": len(keys),
|
|
"tools": keys[:8],
|
|
"has_state": bool(post_state),
|
|
}
|
|
|
|
|
|
def _history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_history_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"playbook_id": item.get("playbook_id"),
|
|
"alertname": item.get("alertname"),
|
|
"mode": payload.get("mode"),
|
|
"allowed": payload.get("allowed"),
|
|
"executed": payload.get("executed"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"writes_ticket": payload.get("writes_ticket"),
|
|
"creates_external_ticket": payload.get("creates_external_ticket"),
|
|
"ticket_preview": payload.get("ticket_preview"),
|
|
"plan": payload.get("plan"),
|
|
"verification_result_preview": payload.get("verification_result_preview"),
|
|
"post_state_summary": payload.get("post_state_summary"),
|
|
"mcp_route": payload.get("mcp_route"),
|
|
"checks": payload.get("checks"),
|
|
}
|
|
|
|
|
|
def _approval_history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_approval_history_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"playbook_id": item.get("playbook_id"),
|
|
"alertname": item.get("alertname"),
|
|
"mode": payload.get("mode"),
|
|
"allowed": payload.get("allowed"),
|
|
"executed": payload.get("executed"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"writes_ticket": payload.get("writes_ticket"),
|
|
"writes_approval_record": payload.get("writes_approval_record"),
|
|
"creates_external_ticket": payload.get("creates_external_ticket"),
|
|
"deduplicated": payload.get("deduplicated"),
|
|
"fingerprint": payload.get("fingerprint"),
|
|
"ticket_preview": payload.get("ticket_preview"),
|
|
"approval": payload.get("approval"),
|
|
"approval_id": payload.get("approval_id"),
|
|
"plan": payload.get("plan"),
|
|
"verification_result_preview": payload.get("verification_result_preview"),
|
|
"checks": payload.get("checks"),
|
|
}
|
|
|
|
|
|
def _timeline_status(payload: dict[str, Any]) -> str:
|
|
if not payload.get("allowed"):
|
|
return "warning"
|
|
if payload.get("verification_result_preview") == "success":
|
|
return "success"
|
|
return "warning"
|
|
|
|
|
|
def _history_description(context: dict[str, Any]) -> str:
|
|
tool_count = (context.get("post_state_summary") or {}).get("tool_count", 0)
|
|
route = context.get("mcp_route") or {}
|
|
agent = route.get("agent_id") or "unknown_agent"
|
|
tool = route.get("tool_name") or "current_state"
|
|
return (
|
|
f"mode={context.get('mode')} "
|
|
f"preview={context.get('verification_result_preview')} "
|
|
f"tools={tool_count} route={agent}/{tool} "
|
|
f"writes_incident={context.get('writes_incident_state')} "
|
|
f"writes_auto_repair={context.get('writes_auto_repair_result')}"
|
|
)[:500]
|
|
|
|
|
|
def _approval_history_description(context: dict[str, Any]) -> str:
|
|
approval = context.get("approval") or {}
|
|
return (
|
|
f"approval={approval.get('id') or context.get('approval_id') or 'unknown'} "
|
|
f"status={approval.get('status') or 'unknown'} "
|
|
f"preview={context.get('verification_result_preview')} "
|
|
f"writes_approval={context.get('writes_approval_record')} "
|
|
f"writes_incident={context.get('writes_incident_state')} "
|
|
f"writes_auto_repair={context.get('writes_auto_repair_result')}"
|
|
)[:500]
|
|
|
|
|
|
def _record_created_at(record: Any) -> str:
|
|
value = getattr(record, "created_at", None)
|
|
if hasattr(value, "isoformat"):
|
|
return value.isoformat()
|
|
return str(value or "")
|
|
|
|
|
|
def _history_item(record: Any, context: dict[str, Any]) -> dict[str, Any]:
|
|
route = context.get("mcp_route") or {}
|
|
post_state = context.get("post_state_summary") or {}
|
|
approval = context.get("approval") or {}
|
|
return {
|
|
"id": str(getattr(record, "id", "")),
|
|
"incident_id": getattr(record, "incident_id", None),
|
|
"auto_repair_id": getattr(record, "auto_repair_id", None)
|
|
or context.get("auto_repair_id"),
|
|
"event_type": str(getattr(record, "event_type", "")),
|
|
"actor": getattr(record, "actor", None),
|
|
"success": getattr(record, "success", None),
|
|
"created_at": _record_created_at(record),
|
|
"work_item_id": context.get("work_item_id"),
|
|
"playbook_id": context.get("playbook_id"),
|
|
"alertname": context.get("alertname"),
|
|
"mode": context.get("mode"),
|
|
"allowed": context.get("allowed"),
|
|
"executed": context.get("executed"),
|
|
"safety_level": context.get("safety_level"),
|
|
"verification_result_preview": context.get("verification_result_preview"),
|
|
"tool_count": post_state.get("tool_count", 0),
|
|
"tools": post_state.get("tools") or [],
|
|
"agent_id": route.get("agent_id"),
|
|
"tool_name": route.get("tool_name") or "current_state",
|
|
"required_scope": route.get("required_scope"),
|
|
"writes_incident_state": context.get("writes_incident_state"),
|
|
"writes_auto_repair_result": context.get("writes_auto_repair_result"),
|
|
"writes_ticket": context.get("writes_ticket"),
|
|
"writes_approval_record": context.get("writes_approval_record"),
|
|
"creates_external_ticket": context.get("creates_external_ticket"),
|
|
"approval_id": context.get("approval_id") or approval.get("id"),
|
|
"approval_status": approval.get("status"),
|
|
"approval_risk_level": approval.get("risk_level"),
|
|
"deduplicated": context.get("deduplicated"),
|
|
"fingerprint": context.get("fingerprint"),
|
|
"ticket_preview": context.get("ticket_preview"),
|
|
"plan": context.get("plan"),
|
|
"checks": context.get("checks") or [],
|
|
}
|
|
|
|
|
|
def _summarize_history_by_work_item(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
summary: dict[str, dict[str, Any]] = {}
|
|
for item in items:
|
|
key = str(item.get("work_item_id") or item.get("incident_id") or item.get("id"))
|
|
if key not in summary:
|
|
summary[key] = {
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"count": 0,
|
|
"latest_at": item.get("created_at"),
|
|
"latest_event_type": item.get("event_type"),
|
|
"latest_success": item.get("success"),
|
|
"latest_preview": item.get("verification_result_preview"),
|
|
"latest_mode": item.get("mode"),
|
|
"latest_agent_id": item.get("agent_id"),
|
|
"latest_tool_name": item.get("tool_name"),
|
|
"required_scope": item.get("required_scope"),
|
|
}
|
|
summary[key]["count"] += 1
|
|
return list(summary.values())
|
|
|
|
|
|
def _diagnostic_command_for_incident(incident: Incident) -> str:
|
|
labels = _labels_for_incident(incident)
|
|
host = str(labels.get("host") or labels.get("instance") or "{host}")
|
|
container = str(labels.get("container_name") or labels.get("container") or "")
|
|
if container:
|
|
return f"ssh {host} 'uptime; docker stats --no-stream {container}'"
|
|
return f"ssh {host} 'uptime; docker stats --no-stream'"
|
|
|
|
|
|
def _ticket_preview_for_item(item: dict[str, Any], incident: Incident) -> dict[str, Any]:
|
|
labels = _labels_for_incident(incident)
|
|
alertname = str(item.get("alertname") or labels.get("alertname") or "unknown_alert")
|
|
incident_id = str(item.get("incident_id") or incident.incident_id)
|
|
playbook_id = str(item.get("playbook_id") or "unknown_playbook")
|
|
host = str(labels.get("host") or labels.get("instance") or "unknown_host")
|
|
container = str(labels.get("container_name") or labels.get("container") or "")
|
|
target = f"host={host}" + (f" container={container}" if container else "")
|
|
title = f"[ADR-100] Promote diagnostic PlayBook to repair: {alertname}"
|
|
body = (
|
|
f"Incident: {incident_id}\n"
|
|
f"Auto repair: {item.get('auto_repair_id') or 'unknown'}\n"
|
|
f"PlayBook: {playbook_id}\n"
|
|
f"Target: {target}\n"
|
|
f"Failure class: {item.get('failure_class') or 'observe_only_playbook'}\n"
|
|
"Required change: add a gated mutating repair step such as docker restart, "
|
|
"Ansible check-mode/apply, or another approved executor action, then keep "
|
|
"post-execution verification tied to the same target.\n"
|
|
"Guardrail: do not mark the old diagnostic-only run as verified_success."
|
|
)
|
|
return {
|
|
"would_create": True,
|
|
"external_ticket_created": False,
|
|
"title": title,
|
|
"labels": [
|
|
"adr100",
|
|
"playbook-authoring",
|
|
"observe-only-playbook",
|
|
"needs-owner-review",
|
|
],
|
|
"body_preview": body[:1000],
|
|
"owner": item.get("remediation_owner") or "solver_or_operator",
|
|
"next_step": "author_mutating_repair_step",
|
|
"playbook_id": playbook_id,
|
|
"target": target,
|
|
}
|
|
|
|
|
|
def _promql_for_incident(incident: Incident) -> str:
|
|
labels = _labels_for_incident(incident)
|
|
alertname = ""
|
|
if incident.signals:
|
|
signal = incident.signals[0]
|
|
alertname = labels.get("alertname") or getattr(signal, "alert_name", "")
|
|
return _build_prometheus_query(alertname, labels)
|
|
|
|
|
|
def _labels_for_incident(incident: Incident) -> dict[str, Any]:
|
|
if incident.signals:
|
|
return incident.signals[0].labels or {}
|
|
return {}
|
|
|
|
|
|
_service: Adr100RemediationService | None = None
|
|
|
|
|
|
def get_adr100_remediation_service() -> Adr100RemediationService:
|
|
"""Return singleton ADR-100 remediation service."""
|
|
|
|
global _service
|
|
if _service is None:
|
|
_service = Adr100RemediationService()
|
|
return _service
|
|
|
|
|
|
def set_adr100_remediation_service(service: Adr100RemediationService | None) -> None:
|
|
"""Inject ADR-100 remediation service for tests."""
|
|
|
|
global _service
|
|
_service = service
|