607 lines
22 KiB
Python
607 lines
22 KiB
Python
"""
|
|
ADR-100 Remediation Service
|
|
===========================
|
|
Safe operator entrypoints for verification remediation work items.
|
|
|
|
T25: remediation queue items are now actionable without mutating incident state:
|
|
- preview: show the selected guardrail path
|
|
- dry-run: collect read-only current state and validate supported executor routing
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
from typing import Any, Literal, Protocol
|
|
|
|
import structlog
|
|
|
|
from src.models.incident import Incident
|
|
from src.repositories.incident_repository import IncidentDBRepository
|
|
from src.services.adr100_slo_status_service import (
|
|
Adr100SloStatusService,
|
|
get_adr100_slo_status_service,
|
|
)
|
|
from src.services.auto_repair_service import AutoRepairService
|
|
from src.services.post_execution_verifier import (
|
|
PostExecutionVerifier,
|
|
_assess_recovery,
|
|
_build_prometheus_query,
|
|
get_post_execution_verifier,
|
|
)
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
RemediationMode = Literal["auto", "reverify", "replay"]
|
|
|
|
_READY_STATUSES = {"ready_for_replay", "ready_for_reverify"}
|
|
|
|
|
|
class RemediationNotFoundError(LookupError):
|
|
"""Requested ADR-100 remediation work item is not in the current read model."""
|
|
|
|
|
|
class _IncidentRepository(Protocol):
|
|
async def get_by_id(self, incident_id: str) -> Incident | None:
|
|
...
|
|
|
|
|
|
class Adr100RemediationService:
|
|
"""Read-only remediation preview and dry-run service."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
slo_service: Adr100SloStatusService | None = None,
|
|
incident_repository: _IncidentRepository | None = None,
|
|
auto_repair_service: AutoRepairService | None = None,
|
|
verifier: PostExecutionVerifier | None = None,
|
|
timeline_service: Any | None = None,
|
|
alert_operation_log_repository: Any | None = None,
|
|
record_history: bool = True,
|
|
) -> None:
|
|
self._slo_service = slo_service or get_adr100_slo_status_service()
|
|
self._incident_repository = incident_repository or IncidentDBRepository()
|
|
self._auto_repair_service = auto_repair_service or AutoRepairService()
|
|
self._verifier = verifier or get_post_execution_verifier()
|
|
self._timeline_service = timeline_service
|
|
self._alert_operation_log_repository = alert_operation_log_repository
|
|
self._record_history_enabled = record_history
|
|
|
|
async def preview(self, work_item_id: str, mode: RemediationMode = "auto") -> dict[str, Any]:
|
|
"""Return the safe execution plan for a remediation queue item."""
|
|
|
|
item = await self._find_work_item(work_item_id)
|
|
selected_mode = _select_mode(item, mode)
|
|
checks = _base_checks(item)
|
|
allowed = all(check["passed"] for check in checks)
|
|
|
|
return {
|
|
"schema_version": "adr100_remediation_preview_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": selected_mode,
|
|
"allowed": allowed,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"checks": checks,
|
|
"plan": _plan_for_item(item, selected_mode),
|
|
"source": "adr100.verification_coverage.remediation_queue",
|
|
}
|
|
|
|
async def dry_run(self, work_item_id: str, mode: RemediationMode = "auto") -> dict[str, Any]:
|
|
"""Run a safe, read-only remediation dry-run for one queue item."""
|
|
|
|
item = await self._find_work_item(work_item_id)
|
|
selected_mode = _select_mode(item, mode)
|
|
checks = _base_checks(item)
|
|
incident = await self._load_incident(item)
|
|
checks.append({
|
|
"name": "incident_loaded",
|
|
"passed": incident is not None,
|
|
"detail": item.get("incident_id") or "missing incident_id",
|
|
})
|
|
|
|
if incident is None or not all(check["passed"] for check in checks):
|
|
payload = _dry_run_blocked_payload(item, selected_mode, checks)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
if selected_mode == "replay":
|
|
return await self._dry_run_replay(item, incident, checks)
|
|
return await self._dry_run_reverify(item, incident, checks)
|
|
|
|
async def history(
|
|
self,
|
|
*,
|
|
limit: int = 50,
|
|
incident_id: str | None = None,
|
|
work_item_id: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Return durable dry-run history written by this remediation service."""
|
|
|
|
safe_limit = max(1, min(limit, 200))
|
|
fetch_limit = min(max(safe_limit * 4, 50), 200)
|
|
rows: list[Any] = []
|
|
repo = self._alert_operation_log_repository
|
|
if repo is None:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
repo = get_alert_operation_log_repository()
|
|
|
|
for event_type in ("PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED"):
|
|
try:
|
|
batch, _total = await repo.list_recent(
|
|
limit=fetch_limit,
|
|
event_type=event_type,
|
|
incident_id=incident_id,
|
|
)
|
|
rows.extend(batch)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_history_fetch_failed",
|
|
event_type=event_type,
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
rows.sort(key=_record_created_at, reverse=True)
|
|
|
|
items: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
context = getattr(row, "context", None) or {}
|
|
if context.get("schema_version") != "adr100_remediation_dry_run_history_v1":
|
|
continue
|
|
if work_item_id and context.get("work_item_id") != work_item_id:
|
|
continue
|
|
items.append(_history_item(row, context))
|
|
if len(items) >= safe_limit:
|
|
break
|
|
|
|
return {
|
|
"schema_version": "adr100_remediation_history_v1",
|
|
"total": len(items),
|
|
"limit": safe_limit,
|
|
"filters": {
|
|
"incident_id": incident_id,
|
|
"work_item_id": work_item_id,
|
|
},
|
|
"items": items,
|
|
"by_work_item": _summarize_history_by_work_item(items),
|
|
}
|
|
|
|
async def _find_work_item(self, work_item_id: str) -> dict[str, Any]:
|
|
report = await self._slo_service.fetch_report()
|
|
coverage = report.get("verification_coverage") or {}
|
|
queue = coverage.get("remediation_queue") or {}
|
|
|
|
for item in queue.get("items") or []:
|
|
if item.get("work_item_id") == work_item_id:
|
|
return dict(item)
|
|
|
|
raise RemediationNotFoundError(work_item_id)
|
|
|
|
async def _load_incident(self, item: dict[str, Any]) -> Incident | None:
|
|
incident_id = str(item.get("incident_id") or "")
|
|
if not incident_id:
|
|
return None
|
|
return await self._incident_repository.get_by_id(incident_id)
|
|
|
|
async def _dry_run_reverify(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
post_state = await self._collect_current_state(incident)
|
|
action_taken = f"dry_run_reverify:{item.get('playbook_id') or 'unknown'}"
|
|
result = _assess_recovery(None, post_state, action_taken)
|
|
|
|
payload = _dry_run_result_payload(
|
|
item=item,
|
|
mode="reverify",
|
|
checks=checks,
|
|
post_state=post_state,
|
|
verification_result_preview=result,
|
|
extra={
|
|
"promql": _promql_for_incident(incident),
|
|
"mcp_route": {
|
|
"agent_id": "post_execution_verifier",
|
|
"required_scope": "read",
|
|
"is_shadow": True,
|
|
"flywheel_node": "verify",
|
|
},
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
async def _dry_run_replay(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
diagnostic_command = _diagnostic_command_for_incident(incident)
|
|
route = self._auto_repair_service.preview_read_only_ssh_mcp_route(
|
|
incident,
|
|
diagnostic_command,
|
|
)
|
|
checks.append({
|
|
"name": "supported_executor_route",
|
|
"passed": route is not None,
|
|
"detail": "mcp:ssh_diagnose" if route else "missing host/container route",
|
|
})
|
|
|
|
post_state = await self._collect_current_state(incident)
|
|
action_taken = f"dry_run_replay:{item.get('playbook_id') or 'unknown'}"
|
|
result = _assess_recovery(None, post_state, action_taken)
|
|
|
|
payload = _dry_run_result_payload(
|
|
item=item,
|
|
mode="replay",
|
|
checks=checks,
|
|
post_state=post_state,
|
|
verification_result_preview=result,
|
|
extra={
|
|
"diagnostic_command_preview": diagnostic_command,
|
|
"mcp_route": route,
|
|
"promql": _promql_for_incident(incident),
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
async def _collect_current_state(self, incident: Incident) -> dict[str, Any]:
|
|
try:
|
|
return await asyncio.wait_for(
|
|
self._verifier._collect_post_state(incident),
|
|
timeout=12.0,
|
|
)
|
|
except asyncio.TimeoutError:
|
|
logger.warning(
|
|
"adr100_remediation_dry_run_timeout",
|
|
incident_id=incident.incident_id,
|
|
)
|
|
return {}
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_dry_run_collect_failed",
|
|
incident_id=incident.incident_id,
|
|
error=str(exc),
|
|
)
|
|
return {}
|
|
|
|
async def _record_dry_run_history(
|
|
self,
|
|
item: dict[str, Any],
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if not self._record_history_enabled:
|
|
return {"recorded": False, "reason": "disabled"}
|
|
|
|
incident_id = str(item.get("incident_id") or "")
|
|
if not incident_id:
|
|
return {"recorded": False, "reason": "missing_incident_id"}
|
|
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
context = _history_context(item, payload)
|
|
allowed = bool(payload.get("allowed"))
|
|
|
|
try:
|
|
repo = self._alert_operation_log_repository
|
|
if repo is None:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
repo = get_alert_operation_log_repository()
|
|
record = await repo.append(
|
|
"PRE_FLIGHT_PASSED" if allowed else "PRE_FLIGHT_FAILED",
|
|
incident_id=incident_id,
|
|
auto_repair_id=str(item.get("auto_repair_id") or "") or None,
|
|
actor="adr100_remediation_service",
|
|
action_detail=f"adr100_remediation_dry_run:{payload.get('mode')}"[:200],
|
|
success=allowed,
|
|
context=context,
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_alert_operation_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
try:
|
|
timeline = self._timeline_service
|
|
if timeline is None:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
timeline = get_timeline_service()
|
|
event = await timeline.add_event(
|
|
event_type="verifier",
|
|
status=_timeline_status(payload),
|
|
title="ADR-100 remediation dry-run",
|
|
description=_history_description(context),
|
|
actor="adr100_remediation_service",
|
|
actor_role=str(payload.get("mode") or "dry_run"),
|
|
incident_id=incident_id,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_timeline_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
history["recorded"] = bool(
|
|
history.get("alert_operation_id") or history.get("timeline_event_id")
|
|
)
|
|
return history
|
|
|
|
|
|
def _select_mode(item: dict[str, Any], requested: RemediationMode) -> Literal["reverify", "replay"]:
|
|
if requested in ("reverify", "replay"):
|
|
return requested
|
|
if item.get("remediation_status") == "ready_for_reverify":
|
|
return "reverify"
|
|
if item.get("remediation_action") == "reverify_with_promql_template":
|
|
return "reverify"
|
|
return "replay"
|
|
|
|
|
|
def _base_checks(item: dict[str, Any]) -> list[dict[str, Any]]:
|
|
status = str(item.get("remediation_status") or "unknown")
|
|
action = str(item.get("remediation_action") or "unknown")
|
|
return [
|
|
{
|
|
"name": "queue_item_ready",
|
|
"passed": status in _READY_STATUSES,
|
|
"detail": status,
|
|
},
|
|
{
|
|
"name": "read_only_guardrail",
|
|
"passed": action in {
|
|
"replay_with_supported_executor",
|
|
"reverify_with_promql_template",
|
|
},
|
|
"detail": action,
|
|
},
|
|
{
|
|
"name": "no_state_mutation",
|
|
"passed": True,
|
|
"detail": "dry_run_does_not_update_incident_or_auto_repair_rows",
|
|
},
|
|
]
|
|
|
|
|
|
def _plan_for_item(item: dict[str, Any], mode: str) -> dict[str, Any]:
|
|
if mode == "reverify":
|
|
return {
|
|
"step": "collect_current_state_and_assess",
|
|
"agent_id": "post_execution_verifier",
|
|
"required_scope": "read",
|
|
"writes": [],
|
|
}
|
|
return {
|
|
"step": "validate_supported_executor_route_then_collect_current_state",
|
|
"agent_id": "auto_repair_executor",
|
|
"required_scope": "read",
|
|
"writes": [],
|
|
"target_action": item.get("remediation_action"),
|
|
}
|
|
|
|
|
|
def _dry_run_blocked_payload(
|
|
item: dict[str, Any],
|
|
mode: str,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": mode,
|
|
"allowed": False,
|
|
"executed": False,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"checks": checks,
|
|
"verification_result_preview": "blocked",
|
|
"post_state_summary": {},
|
|
}
|
|
|
|
|
|
def _dry_run_result_payload(
|
|
*,
|
|
item: dict[str, Any],
|
|
mode: str,
|
|
checks: list[dict[str, Any]],
|
|
post_state: dict[str, Any],
|
|
verification_result_preview: str,
|
|
extra: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": mode,
|
|
"allowed": all(check["passed"] for check in checks),
|
|
"executed": True,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"checks": checks,
|
|
"verification_result_preview": verification_result_preview,
|
|
"post_state_summary": _summarize_post_state(post_state),
|
|
**extra,
|
|
}
|
|
|
|
|
|
def _summarize_post_state(post_state: dict[str, Any]) -> dict[str, Any]:
|
|
keys = sorted(post_state.keys())
|
|
return {
|
|
"tool_count": len(keys),
|
|
"tools": keys[:8],
|
|
"has_state": bool(post_state),
|
|
}
|
|
|
|
|
|
def _history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_history_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"playbook_id": item.get("playbook_id"),
|
|
"alertname": item.get("alertname"),
|
|
"mode": payload.get("mode"),
|
|
"allowed": payload.get("allowed"),
|
|
"executed": payload.get("executed"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"verification_result_preview": payload.get("verification_result_preview"),
|
|
"post_state_summary": payload.get("post_state_summary"),
|
|
"mcp_route": payload.get("mcp_route"),
|
|
"checks": payload.get("checks"),
|
|
}
|
|
|
|
|
|
def _timeline_status(payload: dict[str, Any]) -> str:
|
|
if not payload.get("allowed"):
|
|
return "warning"
|
|
if payload.get("verification_result_preview") == "success":
|
|
return "success"
|
|
return "warning"
|
|
|
|
|
|
def _history_description(context: dict[str, Any]) -> str:
|
|
tool_count = (context.get("post_state_summary") or {}).get("tool_count", 0)
|
|
route = context.get("mcp_route") or {}
|
|
agent = route.get("agent_id") or "unknown_agent"
|
|
tool = route.get("tool_name") or "current_state"
|
|
return (
|
|
f"mode={context.get('mode')} "
|
|
f"preview={context.get('verification_result_preview')} "
|
|
f"tools={tool_count} route={agent}/{tool} "
|
|
f"writes_incident={context.get('writes_incident_state')} "
|
|
f"writes_auto_repair={context.get('writes_auto_repair_result')}"
|
|
)[:500]
|
|
|
|
|
|
def _record_created_at(record: Any) -> str:
|
|
value = getattr(record, "created_at", None)
|
|
if hasattr(value, "isoformat"):
|
|
return value.isoformat()
|
|
return str(value or "")
|
|
|
|
|
|
def _history_item(record: Any, context: dict[str, Any]) -> dict[str, Any]:
|
|
route = context.get("mcp_route") or {}
|
|
post_state = context.get("post_state_summary") or {}
|
|
return {
|
|
"id": str(getattr(record, "id", "")),
|
|
"incident_id": getattr(record, "incident_id", None),
|
|
"auto_repair_id": getattr(record, "auto_repair_id", None)
|
|
or context.get("auto_repair_id"),
|
|
"event_type": str(getattr(record, "event_type", "")),
|
|
"actor": getattr(record, "actor", None),
|
|
"success": getattr(record, "success", None),
|
|
"created_at": _record_created_at(record),
|
|
"work_item_id": context.get("work_item_id"),
|
|
"playbook_id": context.get("playbook_id"),
|
|
"alertname": context.get("alertname"),
|
|
"mode": context.get("mode"),
|
|
"allowed": context.get("allowed"),
|
|
"executed": context.get("executed"),
|
|
"safety_level": context.get("safety_level"),
|
|
"verification_result_preview": context.get("verification_result_preview"),
|
|
"tool_count": post_state.get("tool_count", 0),
|
|
"tools": post_state.get("tools") or [],
|
|
"agent_id": route.get("agent_id"),
|
|
"tool_name": route.get("tool_name") or "current_state",
|
|
"required_scope": route.get("required_scope"),
|
|
"writes_incident_state": context.get("writes_incident_state"),
|
|
"writes_auto_repair_result": context.get("writes_auto_repair_result"),
|
|
"checks": context.get("checks") or [],
|
|
}
|
|
|
|
|
|
def _summarize_history_by_work_item(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
summary: dict[str, dict[str, Any]] = {}
|
|
for item in items:
|
|
key = str(item.get("work_item_id") or item.get("incident_id") or item.get("id"))
|
|
if key not in summary:
|
|
summary[key] = {
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"count": 0,
|
|
"latest_at": item.get("created_at"),
|
|
"latest_event_type": item.get("event_type"),
|
|
"latest_success": item.get("success"),
|
|
"latest_preview": item.get("verification_result_preview"),
|
|
"latest_mode": item.get("mode"),
|
|
"latest_agent_id": item.get("agent_id"),
|
|
"latest_tool_name": item.get("tool_name"),
|
|
"required_scope": item.get("required_scope"),
|
|
}
|
|
summary[key]["count"] += 1
|
|
return list(summary.values())
|
|
|
|
|
|
def _diagnostic_command_for_incident(incident: Incident) -> str:
|
|
labels = _labels_for_incident(incident)
|
|
host = str(labels.get("host") or labels.get("instance") or "{host}")
|
|
container = str(labels.get("container_name") or labels.get("container") or "")
|
|
if container:
|
|
return f"ssh {host} 'uptime; docker stats --no-stream {container}'"
|
|
return f"ssh {host} 'uptime; docker stats --no-stream'"
|
|
|
|
|
|
def _promql_for_incident(incident: Incident) -> str:
|
|
labels = _labels_for_incident(incident)
|
|
alertname = ""
|
|
if incident.signals:
|
|
signal = incident.signals[0]
|
|
alertname = labels.get("alertname") or getattr(signal, "alert_name", "")
|
|
return _build_prometheus_query(alertname, labels)
|
|
|
|
|
|
def _labels_for_incident(incident: Incident) -> dict[str, Any]:
|
|
if incident.signals:
|
|
return incident.signals[0].labels or {}
|
|
return {}
|
|
|
|
|
|
_service: Adr100RemediationService | None = None
|
|
|
|
|
|
def get_adr100_remediation_service() -> Adr100RemediationService:
|
|
"""Return singleton ADR-100 remediation service."""
|
|
|
|
global _service
|
|
if _service is None:
|
|
_service = Adr100RemediationService()
|
|
return _service
|
|
|
|
|
|
def set_adr100_remediation_service(service: Adr100RemediationService | None) -> None:
|
|
"""Inject ADR-100 remediation service for tests."""
|
|
|
|
global _service
|
|
_service = service
|