1648 lines
63 KiB
Python
1648 lines
63 KiB
Python
"""
|
|
ADR-100 Remediation Service
|
|
===========================
|
|
Safe operator entrypoints for verification remediation work items.
|
|
|
|
T25: remediation queue items are now actionable without mutating incident state:
|
|
- preview: show the selected guardrail path
|
|
- dry-run: collect read-only current state and validate supported executor routing
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Any, Literal, Protocol
|
|
from uuid import NAMESPACE_URL, uuid5
|
|
|
|
import structlog
|
|
from sqlalchemy import select
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
|
|
from src.db.awooop_models import (
|
|
AwoooPRunIdempotency,
|
|
AwoooPRunState,
|
|
AwoooPRunStepJournal,
|
|
)
|
|
from src.db.base import get_db_context
|
|
from src.models.approval import (
|
|
ApprovalRequestCreate,
|
|
BlastRadius,
|
|
DataImpact,
|
|
DryRunCheck,
|
|
RiskLevel,
|
|
)
|
|
from src.models.incident import Incident
|
|
from src.repositories.incident_repository import IncidentDBRepository
|
|
from src.services.adr100_slo_status_service import (
|
|
Adr100SloStatusService,
|
|
get_adr100_slo_status_service,
|
|
)
|
|
from src.services.auto_repair_service import AutoRepairService
|
|
from src.services.post_execution_verifier import (
|
|
PostExecutionVerifier,
|
|
_assess_recovery,
|
|
_build_prometheus_query,
|
|
get_post_execution_verifier,
|
|
)
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
RemediationMode = Literal["auto", "reverify", "replay", "ticket", "approval"]
|
|
|
|
_READY_STATUSES = {"ready_for_replay", "ready_for_reverify"}
|
|
_TICKET_STATUSES = {"needs_playbook_ticket"}
|
|
_TICKET_ACTIONS = {"create_playbook_ticket", "promote_diagnostic_to_repair_playbook"}
|
|
_RUNTIME_REPLAY_STATUSES = {"ready_for_replay"}
|
|
_RUNTIME_REPLAY_ACTIONS = {"replay_with_supported_executor"}
|
|
_AWOOOP_GATE5_TRIGGER_TYPE = "adr100_runtime_replay_gate5"
|
|
_AWOOOP_GATE5_CHANNEL_TYPE = "adr100_gate5_approval"
|
|
_AWOOOP_GATE5_PROJECT_ID = "awoooi"
|
|
|
|
|
|
class RemediationNotFoundError(LookupError):
|
|
"""Requested ADR-100 remediation work item is not in the current read model."""
|
|
|
|
|
|
class _IncidentRepository(Protocol):
|
|
async def get_by_id(self, incident_id: str) -> Incident | None:
|
|
...
|
|
|
|
|
|
class Adr100RemediationService:
|
|
"""Read-only remediation preview and dry-run service."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
slo_service: Adr100SloStatusService | None = None,
|
|
incident_repository: _IncidentRepository | None = None,
|
|
auto_repair_service: AutoRepairService | None = None,
|
|
playbook_service: Any | None = None,
|
|
verifier: PostExecutionVerifier | None = None,
|
|
approval_service: Any | None = None,
|
|
awooop_approval_projector: Any | None = None,
|
|
timeline_service: Any | None = None,
|
|
alert_operation_log_repository: Any | None = None,
|
|
record_history: bool = True,
|
|
) -> None:
|
|
self._slo_service = slo_service or get_adr100_slo_status_service()
|
|
self._incident_repository = incident_repository or IncidentDBRepository()
|
|
self._auto_repair_service = auto_repair_service or AutoRepairService()
|
|
self._playbook_service = playbook_service
|
|
self._verifier = verifier or get_post_execution_verifier()
|
|
self._approval_service = approval_service
|
|
self._awooop_approval_projector = awooop_approval_projector
|
|
self._timeline_service = timeline_service
|
|
self._alert_operation_log_repository = alert_operation_log_repository
|
|
self._record_history_enabled = record_history
|
|
|
|
async def preview(self, work_item_id: str, mode: RemediationMode = "auto") -> dict[str, Any]:
|
|
"""Return the safe execution plan for a remediation queue item."""
|
|
|
|
item = await self._find_work_item(work_item_id)
|
|
selected_mode = _select_mode(item, mode)
|
|
checks = _base_checks(item)
|
|
allowed = all(check["passed"] for check in checks)
|
|
|
|
return {
|
|
"schema_version": "adr100_remediation_preview_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": selected_mode,
|
|
"allowed": allowed,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"checks": checks,
|
|
"plan": _plan_for_item(item, selected_mode),
|
|
"source": "adr100.verification_coverage.remediation_queue",
|
|
}
|
|
|
|
async def dry_run(self, work_item_id: str, mode: RemediationMode = "auto") -> dict[str, Any]:
|
|
"""Run a safe, read-only remediation dry-run for one queue item."""
|
|
|
|
item = await self._find_work_item(work_item_id)
|
|
selected_mode = _select_mode(item, mode)
|
|
checks = _base_checks(item)
|
|
incident = await self._load_incident(item)
|
|
checks.append({
|
|
"name": "incident_loaded",
|
|
"passed": incident is not None,
|
|
"detail": item.get("incident_id") or "missing incident_id",
|
|
})
|
|
|
|
if incident is None or not all(check["passed"] for check in checks):
|
|
payload = _dry_run_blocked_payload(item, selected_mode, checks)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
if selected_mode == "ticket":
|
|
return await self._dry_run_ticket_proposal(item, incident, checks)
|
|
if selected_mode == "replay":
|
|
return await self._dry_run_replay(item, incident, checks)
|
|
return await self._dry_run_reverify(item, incident, checks)
|
|
|
|
async def create_approval_request(
|
|
self,
|
|
work_item_id: str,
|
|
mode: RemediationMode = "approval",
|
|
) -> dict[str, Any]:
|
|
"""Create a record-only approval for PlayBook authoring or runtime replay."""
|
|
|
|
item = await self._find_work_item(work_item_id)
|
|
selected_mode = _select_mode(item, mode)
|
|
checks = _base_checks(item)
|
|
checks.append({
|
|
"name": "approval_request_supported",
|
|
"passed": selected_mode in {"ticket", "approval"},
|
|
"detail": str(item.get("remediation_status") or "unknown"),
|
|
})
|
|
|
|
incident = await self._load_incident(item)
|
|
checks.append({
|
|
"name": "incident_loaded",
|
|
"passed": incident is not None,
|
|
"detail": item.get("incident_id") or "missing incident_id",
|
|
})
|
|
|
|
if incident is None or not all(check["passed"] for check in checks):
|
|
payload = _approval_blocked_payload(item, selected_mode, checks)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
replay_gate: dict[str, Any] | None = None
|
|
if selected_mode == "approval":
|
|
replay_gate = await self._build_replay_gate(item, incident)
|
|
checks.append({
|
|
"name": "runtime_replay_gate_ready",
|
|
"passed": replay_gate.get("status") == "runtime_replay_ready",
|
|
"detail": str(replay_gate.get("status") or "unknown"),
|
|
})
|
|
if replay_gate.get("status") != "runtime_replay_ready":
|
|
payload = _approval_blocked_payload(
|
|
item,
|
|
selected_mode,
|
|
checks,
|
|
extra={
|
|
"replay_gate": replay_gate,
|
|
"verification_result_preview": "runtime_replay_gate_blocked",
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
approval_request = _runtime_replay_approval_request_for_item(
|
|
item,
|
|
incident,
|
|
checks,
|
|
replay_gate,
|
|
)
|
|
else:
|
|
approval_request = _approval_request_for_item(item, incident, checks)
|
|
|
|
approval_svc = self._approval_service
|
|
if approval_svc is None:
|
|
from src.services.approval_db import get_approval_service
|
|
|
|
approval_svc = get_approval_service()
|
|
|
|
approval_kind = str((approval_request.metadata or {}).get("approval_kind") or "")
|
|
fingerprint = _approval_fingerprint(item, approval_kind=approval_kind)
|
|
approval = None
|
|
if hasattr(approval_svc, "find_by_fingerprint"):
|
|
try:
|
|
approval = await approval_svc.find_by_fingerprint(fingerprint)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_approval_dedupe_lookup_failed",
|
|
fingerprint=fingerprint,
|
|
error=str(exc),
|
|
)
|
|
|
|
approval_created = approval is None
|
|
if approval is None and hasattr(approval_svc, "create_approval_with_fingerprint"):
|
|
approval = await approval_svc.create_approval_with_fingerprint(
|
|
approval_request,
|
|
fingerprint=fingerprint,
|
|
)
|
|
elif approval is None:
|
|
approval = await approval_svc.create_approval(approval_request)
|
|
|
|
payload = _approval_result_payload(
|
|
item=item,
|
|
incident=incident,
|
|
checks=checks,
|
|
approval=approval,
|
|
request=approval_request,
|
|
approval_created=approval_created,
|
|
fingerprint=fingerprint,
|
|
)
|
|
if payload.get("approval_kind") == _AWOOOP_GATE5_TRIGGER_TYPE:
|
|
payload["awooop_projection"] = await self._project_awooop_gate5_approval(
|
|
item=item,
|
|
incident=incident,
|
|
request=approval_request,
|
|
payload=payload,
|
|
)
|
|
payload["history"] = await self._record_approval_history(item, payload)
|
|
return payload
|
|
|
|
async def history(
|
|
self,
|
|
*,
|
|
limit: int = 50,
|
|
incident_id: str | None = None,
|
|
work_item_id: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Return durable dry-run history written by this remediation service."""
|
|
|
|
safe_limit = max(1, min(limit, 200))
|
|
fetch_limit = min(max(safe_limit * 4, 50), 200)
|
|
rows: list[Any] = []
|
|
repo = self._alert_operation_log_repository
|
|
if repo is None:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
repo = get_alert_operation_log_repository()
|
|
|
|
for event_type in ("PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED", "APPROVAL_ESCALATED"):
|
|
try:
|
|
batch, _total = await repo.list_recent(
|
|
limit=fetch_limit,
|
|
event_type=event_type,
|
|
incident_id=incident_id,
|
|
)
|
|
rows.extend(batch)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_history_fetch_failed",
|
|
event_type=event_type,
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
rows.sort(key=_record_created_at, reverse=True)
|
|
|
|
items: list[dict[str, Any]] = []
|
|
for row in rows:
|
|
context = getattr(row, "context", None) or {}
|
|
if context.get("schema_version") not in {
|
|
"adr100_remediation_dry_run_history_v1",
|
|
"adr100_remediation_approval_history_v1",
|
|
}:
|
|
continue
|
|
if work_item_id and context.get("work_item_id") != work_item_id:
|
|
continue
|
|
items.append(_history_item(row, context))
|
|
if len(items) >= safe_limit:
|
|
break
|
|
|
|
return {
|
|
"schema_version": "adr100_remediation_history_v1",
|
|
"total": len(items),
|
|
"limit": safe_limit,
|
|
"filters": {
|
|
"incident_id": incident_id,
|
|
"work_item_id": work_item_id,
|
|
},
|
|
"items": items,
|
|
"by_work_item": _summarize_history_by_work_item(items),
|
|
}
|
|
|
|
async def _find_work_item(self, work_item_id: str) -> dict[str, Any]:
|
|
report = await self._slo_service.fetch_report()
|
|
coverage = report.get("verification_coverage") or {}
|
|
queue = coverage.get("remediation_queue") or {}
|
|
|
|
for item in queue.get("items") or []:
|
|
if item.get("work_item_id") == work_item_id:
|
|
return dict(item)
|
|
|
|
raise RemediationNotFoundError(work_item_id)
|
|
|
|
async def _load_incident(self, item: dict[str, Any]) -> Incident | None:
|
|
incident_id = str(item.get("incident_id") or "")
|
|
if not incident_id:
|
|
return None
|
|
return await self._incident_repository.get_by_id(incident_id)
|
|
|
|
async def _dry_run_reverify(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
post_state = await self._collect_current_state(incident)
|
|
action_taken = f"dry_run_reverify:{item.get('playbook_id') or 'unknown'}"
|
|
result = _assess_recovery(None, post_state, action_taken)
|
|
|
|
payload = _dry_run_result_payload(
|
|
item=item,
|
|
mode="reverify",
|
|
checks=checks,
|
|
post_state=post_state,
|
|
verification_result_preview=result,
|
|
extra={
|
|
"promql": _promql_for_incident(incident),
|
|
"mcp_route": {
|
|
"agent_id": "post_execution_verifier",
|
|
"required_scope": "read",
|
|
"is_shadow": True,
|
|
"flywheel_node": "verify",
|
|
},
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
async def _dry_run_replay(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
diagnostic_command = _diagnostic_command_for_incident(incident)
|
|
route = self._auto_repair_service.preview_read_only_ssh_mcp_route(
|
|
incident,
|
|
diagnostic_command,
|
|
)
|
|
checks.append({
|
|
"name": "supported_executor_route",
|
|
"passed": route is not None,
|
|
"detail": "mcp:ssh_diagnose" if route else "missing host/container route",
|
|
})
|
|
|
|
post_state = await self._collect_current_state(incident)
|
|
replay_gate = await self._build_replay_gate(item, incident)
|
|
action_taken = f"dry_run_replay:{item.get('playbook_id') or 'unknown'}"
|
|
result = _assess_recovery(None, post_state, action_taken)
|
|
|
|
payload = _dry_run_result_payload(
|
|
item=item,
|
|
mode="replay",
|
|
checks=checks,
|
|
post_state=post_state,
|
|
verification_result_preview=result,
|
|
extra={
|
|
"diagnostic_command_preview": diagnostic_command,
|
|
"mcp_route": route,
|
|
"replay_gate": replay_gate,
|
|
"promql": _promql_for_incident(incident),
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
async def _build_replay_gate(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
) -> dict[str, Any]:
|
|
playbook_id = str(item.get("playbook_id") or "")
|
|
base: dict[str, Any] = {
|
|
"schema_version": "adr100_replay_gate_v1",
|
|
"playbook_id": playbook_id or None,
|
|
"playbook_loaded": False,
|
|
"status": "blocked_missing_playbook_id",
|
|
"execution_authorized": False,
|
|
"repair_executed": False,
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"can_runtime_replay": False,
|
|
"mutating_step_count": 0,
|
|
"supported_write_route_count": 0,
|
|
"unsupported_step_count": 0,
|
|
"approval_required_count": 0,
|
|
"next_step": "attach_playbook_to_work_item",
|
|
"steps": [],
|
|
}
|
|
if not playbook_id:
|
|
return base
|
|
|
|
playbook_svc = self._playbook_service
|
|
if playbook_svc is None:
|
|
from src.services.playbook_service import get_playbook_service
|
|
|
|
playbook_svc = get_playbook_service()
|
|
|
|
playbook = await playbook_svc.get_by_id(playbook_id)
|
|
if playbook is None:
|
|
return {
|
|
**base,
|
|
"status": "blocked_playbook_not_found",
|
|
"next_step": "reload_playbook_catalog_or_author_supported_executor",
|
|
}
|
|
|
|
steps: list[dict[str, Any]] = []
|
|
mutating_count = 0
|
|
supported_write_count = 0
|
|
unsupported_count = 0
|
|
approval_required_count = 0
|
|
|
|
for step in playbook.repair_steps:
|
|
command = str(getattr(step, "command", "") or "")
|
|
action_type = _step_action_type(step)
|
|
write_route = None
|
|
if action_type == "ssh_command":
|
|
write_route = self._auto_repair_service.preview_write_ssh_mcp_route(
|
|
incident,
|
|
command,
|
|
)
|
|
read_route = self._auto_repair_service.preview_read_only_ssh_mcp_route(
|
|
incident,
|
|
command,
|
|
) if action_type == "ssh_command" else None
|
|
is_mutating = bool(write_route) or _step_looks_mutating(step)
|
|
requires_approval = bool(getattr(step, "requires_approval", False))
|
|
if is_mutating:
|
|
mutating_count += 1
|
|
if write_route:
|
|
supported_write_count += 1
|
|
else:
|
|
unsupported_count += 1
|
|
if requires_approval:
|
|
approval_required_count += 1
|
|
|
|
steps.append({
|
|
"step_number": getattr(step, "step_number", None),
|
|
"action_type": action_type,
|
|
"risk_level": _step_risk_level(step),
|
|
"requires_approval": requires_approval,
|
|
"is_mutating": is_mutating,
|
|
"write_route": write_route,
|
|
"read_route": read_route,
|
|
"supported": bool(write_route) or not is_mutating,
|
|
"command_preview": _compact_command(command),
|
|
})
|
|
|
|
if mutating_count == 0:
|
|
status = "blocked_observe_only_playbook"
|
|
next_step = "author_mutating_repair_step"
|
|
can_runtime_replay = False
|
|
elif unsupported_count > 0:
|
|
status = "blocked_unsupported_write_route"
|
|
next_step = "author_supported_executor_step"
|
|
can_runtime_replay = False
|
|
elif approval_required_count > 0:
|
|
status = "approval_required"
|
|
next_step = "request_runtime_replay_approval"
|
|
can_runtime_replay = False
|
|
else:
|
|
status = "runtime_replay_ready"
|
|
next_step = "queue_runtime_replay_with_gate5_projection"
|
|
can_runtime_replay = True
|
|
|
|
return {
|
|
**base,
|
|
"playbook_loaded": True,
|
|
"status": status,
|
|
"can_runtime_replay": can_runtime_replay,
|
|
"mutating_step_count": mutating_count,
|
|
"supported_write_route_count": supported_write_count,
|
|
"unsupported_step_count": unsupported_count,
|
|
"approval_required_count": approval_required_count,
|
|
"next_step": next_step,
|
|
"steps": steps,
|
|
}
|
|
|
|
async def _dry_run_ticket_proposal(
|
|
self,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
ticket_preview = _ticket_preview_for_item(item, incident)
|
|
checks.append({
|
|
"name": "external_ticket_not_created",
|
|
"passed": True,
|
|
"detail": "dry_run_records_internal_history_only",
|
|
})
|
|
|
|
payload = _dry_run_result_payload(
|
|
item=item,
|
|
mode="ticket",
|
|
checks=checks,
|
|
post_state={},
|
|
verification_result_preview="ticket_proposal",
|
|
extra={
|
|
"ticket_preview": ticket_preview,
|
|
"writes_ticket": False,
|
|
"creates_external_ticket": False,
|
|
"plan": _plan_for_item(item, "ticket"),
|
|
},
|
|
)
|
|
payload["history"] = await self._record_dry_run_history(item, payload)
|
|
return payload
|
|
|
|
async def _collect_current_state(self, incident: Incident) -> dict[str, Any]:
|
|
try:
|
|
return await asyncio.wait_for(
|
|
self._verifier._collect_post_state(incident),
|
|
timeout=12.0,
|
|
)
|
|
except asyncio.TimeoutError:
|
|
logger.warning(
|
|
"adr100_remediation_dry_run_timeout",
|
|
incident_id=incident.incident_id,
|
|
)
|
|
return {}
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_dry_run_collect_failed",
|
|
incident_id=incident.incident_id,
|
|
error=str(exc),
|
|
)
|
|
return {}
|
|
|
|
async def _record_dry_run_history(
|
|
self,
|
|
item: dict[str, Any],
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if not self._record_history_enabled:
|
|
return {"recorded": False, "reason": "disabled"}
|
|
|
|
incident_id = str(item.get("incident_id") or "")
|
|
if not incident_id:
|
|
return {"recorded": False, "reason": "missing_incident_id"}
|
|
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
context = _history_context(item, payload)
|
|
allowed = bool(payload.get("allowed"))
|
|
|
|
try:
|
|
repo = self._alert_operation_log_repository
|
|
if repo is None:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
repo = get_alert_operation_log_repository()
|
|
record = await repo.append(
|
|
"PRE_FLIGHT_PASSED" if allowed else "PRE_FLIGHT_FAILED",
|
|
incident_id=incident_id,
|
|
auto_repair_id=str(item.get("auto_repair_id") or "") or None,
|
|
actor="adr100_remediation_service",
|
|
action_detail=f"adr100_remediation_dry_run:{payload.get('mode')}"[:200],
|
|
success=allowed,
|
|
context=context,
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_alert_operation_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
try:
|
|
timeline = self._timeline_service
|
|
if timeline is None:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
timeline = get_timeline_service()
|
|
event = await timeline.add_event(
|
|
event_type="verifier",
|
|
status=_timeline_status(payload),
|
|
title="ADR-100 remediation dry-run",
|
|
description=_history_description(context),
|
|
actor="adr100_remediation_service",
|
|
actor_role=str(payload.get("mode") or "dry_run"),
|
|
incident_id=incident_id,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_timeline_history_failed",
|
|
incident_id=incident_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
history["recorded"] = bool(
|
|
history.get("alert_operation_id") or history.get("timeline_event_id")
|
|
)
|
|
return history
|
|
|
|
async def _project_awooop_gate5_approval(
|
|
self,
|
|
*,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
request: ApprovalRequestCreate,
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
projector = self._awooop_approval_projector
|
|
if projector is not None:
|
|
return await projector.project_runtime_replay_approval(
|
|
item=item,
|
|
incident=incident,
|
|
request=request,
|
|
payload=payload,
|
|
)
|
|
|
|
try:
|
|
return await _project_runtime_replay_approval_to_awooop(
|
|
item=item,
|
|
incident=incident,
|
|
request=request,
|
|
payload=payload,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_gate5_awooop_projection_failed",
|
|
incident_id=item.get("incident_id"),
|
|
approval_id=payload.get("approval_id"),
|
|
error=str(exc),
|
|
)
|
|
return {
|
|
"schema_version": "adr100_runtime_replay_awooop_projection_v1",
|
|
"projected": False,
|
|
"projection_mode": "approval_projection_only",
|
|
"error": str(exc),
|
|
"execution_authorized": False,
|
|
"repair_executed": False,
|
|
}
|
|
|
|
async def _record_approval_history(
|
|
self,
|
|
item: dict[str, Any],
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
if not self._record_history_enabled:
|
|
return {"recorded": False, "reason": "disabled"}
|
|
|
|
incident_id = str(item.get("incident_id") or "")
|
|
approval_id = str(payload.get("approval_id") or "")
|
|
history: dict[str, Any] = {
|
|
"recorded": False,
|
|
"alert_operation_id": None,
|
|
"timeline_event_id": None,
|
|
}
|
|
context = _approval_history_context(item, payload)
|
|
approval_kind = str(payload.get("approval_kind") or "")
|
|
is_runtime_replay = approval_kind == "adr100_runtime_replay_gate5"
|
|
|
|
try:
|
|
repo = self._alert_operation_log_repository
|
|
if repo is None:
|
|
from src.repositories.alert_operation_log_repository import (
|
|
get_alert_operation_log_repository,
|
|
)
|
|
|
|
repo = get_alert_operation_log_repository()
|
|
record = await repo.append(
|
|
"APPROVAL_ESCALATED",
|
|
incident_id=incident_id or None,
|
|
approval_id=approval_id or None,
|
|
auto_repair_id=str(item.get("auto_repair_id") or "") or None,
|
|
actor="adr100_remediation_service",
|
|
action_detail=(
|
|
"adr100_runtime_replay_gate5_approval_requested"
|
|
if is_runtime_replay
|
|
else "adr100_playbook_authoring_approval_requested"
|
|
),
|
|
success=True,
|
|
context=context,
|
|
)
|
|
if record is not None:
|
|
history["alert_operation_id"] = getattr(record, "id", None)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_approval_history_failed",
|
|
incident_id=incident_id,
|
|
approval_id=approval_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
try:
|
|
timeline = self._timeline_service
|
|
if timeline is None:
|
|
from src.services.approval_db import get_timeline_service
|
|
|
|
timeline = get_timeline_service()
|
|
event = await timeline.add_event(
|
|
event_type="human",
|
|
status="warning",
|
|
title=(
|
|
"ADR-100 runtime replay Gate 5 approval requested"
|
|
if is_runtime_replay
|
|
else "ADR-100 PlayBook authoring approval requested"
|
|
),
|
|
description=_approval_history_description(context),
|
|
actor="adr100_remediation_service",
|
|
actor_role="approval",
|
|
approval_id=approval_id or None,
|
|
incident_id=incident_id or None,
|
|
)
|
|
if event:
|
|
history["timeline_event_id"] = event.get("id")
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"adr100_remediation_approval_timeline_failed",
|
|
incident_id=incident_id,
|
|
approval_id=approval_id,
|
|
error=str(exc),
|
|
)
|
|
|
|
history["recorded"] = bool(
|
|
history.get("alert_operation_id") or history.get("timeline_event_id")
|
|
)
|
|
return history
|
|
|
|
|
|
def _select_mode(
|
|
item: dict[str, Any],
|
|
requested: RemediationMode,
|
|
) -> Literal["reverify", "replay", "ticket", "approval"]:
|
|
if requested == "approval":
|
|
if item.get("remediation_status") in _TICKET_STATUSES:
|
|
return "ticket"
|
|
if item.get("remediation_action") in _TICKET_ACTIONS:
|
|
return "ticket"
|
|
if (
|
|
item.get("remediation_status") in _RUNTIME_REPLAY_STATUSES
|
|
or item.get("remediation_action") in _RUNTIME_REPLAY_ACTIONS
|
|
):
|
|
return "approval"
|
|
return "replay"
|
|
if requested in ("reverify", "replay"):
|
|
return requested
|
|
if requested == "ticket":
|
|
return "ticket"
|
|
if item.get("remediation_status") in _TICKET_STATUSES:
|
|
return "ticket"
|
|
if item.get("remediation_action") in _TICKET_ACTIONS:
|
|
return "ticket"
|
|
if item.get("remediation_status") == "ready_for_reverify":
|
|
return "reverify"
|
|
if item.get("remediation_action") == "reverify_with_promql_template":
|
|
return "reverify"
|
|
return "replay"
|
|
|
|
|
|
def _base_checks(item: dict[str, Any]) -> list[dict[str, Any]]:
|
|
status = str(item.get("remediation_status") or "unknown")
|
|
action = str(item.get("remediation_action") or "unknown")
|
|
return [
|
|
{
|
|
"name": "queue_item_ready",
|
|
"passed": status in _READY_STATUSES or status in _TICKET_STATUSES,
|
|
"detail": status,
|
|
},
|
|
{
|
|
"name": "read_or_record_only_guardrail",
|
|
"passed": action in {
|
|
"replay_with_supported_executor",
|
|
"reverify_with_promql_template",
|
|
*_TICKET_ACTIONS,
|
|
},
|
|
"detail": action,
|
|
},
|
|
{
|
|
"name": "no_state_mutation",
|
|
"passed": True,
|
|
"detail": "dry_run_does_not_update_incident_or_auto_repair_rows",
|
|
},
|
|
]
|
|
|
|
|
|
def _plan_for_item(item: dict[str, Any], mode: str) -> dict[str, Any]:
|
|
if mode == "reverify":
|
|
return {
|
|
"step": "collect_current_state_and_assess",
|
|
"agent_id": "post_execution_verifier",
|
|
"required_scope": "read",
|
|
"writes": [],
|
|
}
|
|
if mode == "ticket":
|
|
return {
|
|
"step": "create_playbook_authoring_ticket_proposal",
|
|
"agent_id": "openclaw_playbook_planner",
|
|
"required_scope": "record_only",
|
|
"writes": ["alert_operation_log", "timeline"],
|
|
"target_action": item.get("remediation_action"),
|
|
}
|
|
if mode == "approval":
|
|
if (
|
|
item.get("remediation_status") in _RUNTIME_REPLAY_STATUSES
|
|
or item.get("remediation_action") in _RUNTIME_REPLAY_ACTIONS
|
|
):
|
|
return {
|
|
"step": "request_runtime_replay_gate5_approval",
|
|
"agent_id": "auto_repair_executor",
|
|
"required_scope": "record_only_until_approved",
|
|
"writes": ["approval_records", "alert_operation_log", "timeline"],
|
|
"target_action": item.get("remediation_action"),
|
|
}
|
|
return {
|
|
"step": "request_playbook_authoring_approval",
|
|
"agent_id": "openclaw_playbook_planner",
|
|
"required_scope": "record_only",
|
|
"writes": ["approval_records", "alert_operation_log", "timeline"],
|
|
"target_action": item.get("remediation_action"),
|
|
}
|
|
return {
|
|
"step": "validate_supported_executor_route_then_collect_current_state",
|
|
"agent_id": "auto_repair_executor",
|
|
"required_scope": "read",
|
|
"writes": [],
|
|
"target_action": item.get("remediation_action"),
|
|
}
|
|
|
|
|
|
def _dry_run_blocked_payload(
|
|
item: dict[str, Any],
|
|
mode: str,
|
|
checks: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": mode,
|
|
"allowed": False,
|
|
"executed": False,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"creates_external_ticket": False,
|
|
"checks": checks,
|
|
"verification_result_preview": "blocked",
|
|
"post_state_summary": {},
|
|
}
|
|
|
|
|
|
def _approval_blocked_payload(
|
|
item: dict[str, Any],
|
|
mode: str,
|
|
checks: list[dict[str, Any]],
|
|
extra: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_approval_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": "approval",
|
|
"requested_mode": mode,
|
|
"allowed": False,
|
|
"executed": False,
|
|
"safety_level": "approval_record_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"writes_approval_record": False,
|
|
"creates_external_ticket": False,
|
|
"checks": checks,
|
|
"verification_result_preview": "blocked",
|
|
"approval": None,
|
|
"approval_id": None,
|
|
"plan": _plan_for_item(item, "approval"),
|
|
**(extra or {}),
|
|
}
|
|
|
|
|
|
def _dry_run_result_payload(
|
|
*,
|
|
item: dict[str, Any],
|
|
mode: str,
|
|
checks: list[dict[str, Any]],
|
|
post_state: dict[str, Any],
|
|
verification_result_preview: str,
|
|
extra: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": mode,
|
|
"allowed": all(check["passed"] for check in checks),
|
|
"executed": True,
|
|
"safety_level": "read_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": extra.get("writes_ticket", False),
|
|
"creates_external_ticket": extra.get("creates_external_ticket", False),
|
|
"checks": checks,
|
|
"verification_result_preview": verification_result_preview,
|
|
"post_state_summary": _summarize_post_state(post_state),
|
|
**extra,
|
|
}
|
|
|
|
|
|
def _approval_request_for_item(
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
) -> ApprovalRequestCreate:
|
|
ticket_preview = _ticket_preview_for_item(item, incident)
|
|
services = [svc for svc in (incident.affected_services or []) if svc]
|
|
if not services:
|
|
services = [str(item.get("alertname") or "unknown_alert")]
|
|
playbook_id = str(item.get("playbook_id") or "unknown_playbook")
|
|
work_item_id = str(item.get("work_item_id") or "")
|
|
action = (
|
|
"PLAYBOOK_AUTHORING_RECORD_ONLY: "
|
|
f"ADR-100 promote diagnostic PlayBook {playbook_id}"
|
|
)
|
|
description = (
|
|
f"{ticket_preview.get('title')}\n\n"
|
|
f"{ticket_preview.get('body_preview')}\n\n"
|
|
"Approval scope: record-only PlayBook authoring. Signing this request "
|
|
"does not execute a runtime repair, does not resolve the incident, and "
|
|
"does not mark the old diagnostic run as verified_success."
|
|
)
|
|
return ApprovalRequestCreate(
|
|
action=action,
|
|
description=description[:4000],
|
|
risk_level=RiskLevel.MEDIUM,
|
|
blast_radius=BlastRadius(
|
|
affected_pods=0,
|
|
estimated_downtime="0",
|
|
related_services=services[:6],
|
|
data_impact=DataImpact.READ_ONLY,
|
|
),
|
|
dry_run_checks=[
|
|
DryRunCheck(
|
|
name=str(check.get("name") or "check"),
|
|
passed=bool(check.get("passed")),
|
|
message=str(check.get("detail") or ""),
|
|
)
|
|
for check in checks
|
|
],
|
|
requested_by="adr100_remediation_service",
|
|
expires_at=datetime.now(timezone.utc) + timedelta(hours=48),
|
|
metadata={
|
|
"schema_version": "adr100_playbook_authoring_approval_v1",
|
|
"approval_kind": "adr100_playbook_authoring",
|
|
"execution_kind": "playbook_authoring_record_only",
|
|
"execution_authorized": False,
|
|
"repair_attempted": False,
|
|
"repair_executed": False,
|
|
"work_item_id": work_item_id,
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"source": "adr100.verification_coverage.remediation_queue",
|
|
"ticket_preview": ticket_preview,
|
|
"target_action": item.get("remediation_action"),
|
|
"required_scope": "record_only",
|
|
"next_step": "author_mutating_repair_step",
|
|
"playbook_id": playbook_id,
|
|
"flywheel_node": "approval",
|
|
"agent_id": "openclaw_playbook_planner",
|
|
"mcp_gate": "not_required_record_only",
|
|
},
|
|
incident_id=str(item.get("incident_id") or incident.incident_id),
|
|
matched_playbook_id=playbook_id if playbook_id != "unknown_playbook" else None,
|
|
)
|
|
|
|
|
|
def _runtime_replay_approval_request_for_item(
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
replay_gate: dict[str, Any],
|
|
) -> ApprovalRequestCreate:
|
|
services = [svc for svc in (incident.affected_services or []) if svc]
|
|
if not services:
|
|
services = [str(item.get("alertname") or "unknown_alert")]
|
|
playbook_id = str(item.get("playbook_id") or "unknown_playbook")
|
|
work_item_id = str(item.get("work_item_id") or "")
|
|
write_routes = [
|
|
step.get("write_route")
|
|
for step in replay_gate.get("steps") or []
|
|
if isinstance(step, dict) and step.get("write_route")
|
|
]
|
|
route_names = [
|
|
str(route.get("tool_name") or "unknown_write_route")
|
|
for route in write_routes
|
|
if isinstance(route, dict)
|
|
]
|
|
action = (
|
|
"RUNTIME_REPLAY_GATE5: "
|
|
f"ADR-100 replay {playbook_id} via {', '.join(route_names) or 'mcp_write'}"
|
|
)
|
|
description = (
|
|
f"Incident: {item.get('incident_id') or incident.incident_id}\n"
|
|
f"Work item: {work_item_id or '-'}\n"
|
|
f"PlayBook: {playbook_id}\n"
|
|
f"Replay gate: {replay_gate.get('status')}\n"
|
|
f"Write routes: {', '.join(route_names) or '-'}\n\n"
|
|
"Approval scope: Gate 5 authorization for a controlled runtime replay. "
|
|
"Creating this approval does not execute repair, does not restart a "
|
|
"container, does not update incident state, and does not write an "
|
|
"auto_repair_executions result. Execution must happen only after the "
|
|
"approval status reaches approved and the executor re-validates the gate."
|
|
)
|
|
return ApprovalRequestCreate(
|
|
action=action,
|
|
description=description[:4000],
|
|
risk_level=RiskLevel.MEDIUM,
|
|
blast_radius=BlastRadius(
|
|
affected_pods=max(1, int(replay_gate.get("supported_write_route_count") or 1)),
|
|
estimated_downtime="<1m",
|
|
related_services=services[:6],
|
|
data_impact=DataImpact.WRITE,
|
|
),
|
|
dry_run_checks=[
|
|
DryRunCheck(
|
|
name=str(check.get("name") or "check"),
|
|
passed=bool(check.get("passed")),
|
|
message=str(check.get("detail") or ""),
|
|
)
|
|
for check in checks
|
|
],
|
|
requested_by="adr100_remediation_service",
|
|
expires_at=datetime.now(timezone.utc) + timedelta(hours=2),
|
|
metadata={
|
|
"schema_version": "adr100_runtime_replay_gate5_approval_v1",
|
|
"approval_kind": "adr100_runtime_replay_gate5",
|
|
"execution_kind": "runtime_replay_gate5_pending",
|
|
"execution_authorized": False,
|
|
"repair_attempted": False,
|
|
"repair_executed": False,
|
|
"work_item_id": work_item_id,
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"source": "adr100.verification_coverage.remediation_queue",
|
|
"target_action": item.get("remediation_action"),
|
|
"required_scope": "write_after_approval",
|
|
"next_step": "approve_then_dispatch_auto_repair_executor",
|
|
"playbook_id": playbook_id,
|
|
"flywheel_node": "approval",
|
|
"agent_id": "auto_repair_executor",
|
|
"mcp_gate": "gate5_required",
|
|
"replay_gate": replay_gate,
|
|
"write_routes": write_routes,
|
|
},
|
|
incident_id=str(item.get("incident_id") or incident.incident_id),
|
|
matched_playbook_id=playbook_id if playbook_id != "unknown_playbook" else None,
|
|
)
|
|
|
|
|
|
def _approval_fingerprint(
|
|
item: dict[str, Any],
|
|
*,
|
|
approval_kind: str = "adr100_playbook_authoring",
|
|
) -> str:
|
|
work_item_id = str(item.get("work_item_id") or "")
|
|
playbook_id = str(item.get("playbook_id") or "")
|
|
incident_id = str(item.get("incident_id") or "")
|
|
basis = work_item_id or f"{incident_id}:{playbook_id}:{item.get('remediation_action') or ''}"
|
|
kind = approval_kind or "adr100_playbook_authoring"
|
|
return hashlib.sha256(f"{kind}:{basis}".encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _approval_result_payload(
|
|
*,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
checks: list[dict[str, Any]],
|
|
approval: Any,
|
|
request: ApprovalRequestCreate,
|
|
approval_created: bool,
|
|
fingerprint: str,
|
|
) -> dict[str, Any]:
|
|
metadata = request.metadata or {}
|
|
approval_kind = str(metadata.get("approval_kind") or "adr100_playbook_authoring")
|
|
ticket_preview = metadata.get("ticket_preview")
|
|
if ticket_preview is None and approval_kind == "adr100_playbook_authoring":
|
|
ticket_preview = _ticket_preview_for_item(item, incident)
|
|
replay_gate = metadata.get("replay_gate")
|
|
approval_id = str(getattr(approval, "id", "") or "")
|
|
approval_status = getattr(getattr(approval, "status", None), "value", None) or getattr(
|
|
approval,
|
|
"status",
|
|
None,
|
|
)
|
|
risk_level = getattr(getattr(approval, "risk_level", None), "value", None) or getattr(
|
|
approval,
|
|
"risk_level",
|
|
None,
|
|
)
|
|
return {
|
|
"schema_version": "adr100_remediation_approval_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id") or incident.incident_id,
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"mode": "approval",
|
|
"allowed": True,
|
|
"executed": False,
|
|
"safety_level": "approval_record_only",
|
|
"writes_incident_state": False,
|
|
"writes_auto_repair_result": False,
|
|
"writes_ticket": False,
|
|
"writes_approval_record": approval_created,
|
|
"creates_external_ticket": False,
|
|
"deduplicated": not approval_created,
|
|
"fingerprint": fingerprint,
|
|
"approval_kind": approval_kind,
|
|
"checks": checks,
|
|
"verification_result_preview": (
|
|
"runtime_replay_approval_requested"
|
|
if approval_kind == "adr100_runtime_replay_gate5"
|
|
else "approval_requested"
|
|
),
|
|
"approval_id": approval_id or None,
|
|
"approval": {
|
|
"id": approval_id or None,
|
|
"status": str(approval_status or ""),
|
|
"risk_level": str(risk_level or ""),
|
|
"required_signatures": getattr(approval, "required_signatures", None),
|
|
"current_signatures": getattr(approval, "current_signatures", None),
|
|
"requested_by": getattr(approval, "requested_by", None),
|
|
"incident_id": getattr(approval, "incident_id", None),
|
|
"matched_playbook_id": getattr(approval, "matched_playbook_id", None),
|
|
},
|
|
"ticket_preview": ticket_preview,
|
|
"replay_gate": replay_gate,
|
|
"plan": _plan_for_item(item, "approval"),
|
|
}
|
|
|
|
|
|
async def _project_runtime_replay_approval_to_awooop(
|
|
*,
|
|
item: dict[str, Any],
|
|
incident: Incident,
|
|
request: ApprovalRequestCreate,
|
|
payload: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
approval_id = str(payload.get("approval_id") or "")
|
|
if not approval_id:
|
|
return {
|
|
"schema_version": "adr100_runtime_replay_awooop_projection_v1",
|
|
"projected": False,
|
|
"projection_mode": "approval_projection_only",
|
|
"reason": "missing_approval_id",
|
|
"execution_authorized": False,
|
|
"repair_executed": False,
|
|
}
|
|
|
|
project_id = _AWOOOP_GATE5_PROJECT_ID
|
|
incident_id = str(item.get("incident_id") or incident.incident_id or "")
|
|
work_item_id = str(item.get("work_item_id") or "")
|
|
auto_repair_id = str(item.get("auto_repair_id") or "")
|
|
playbook_id = str(item.get("playbook_id") or "unknown_playbook")
|
|
run_id = uuid5(
|
|
NAMESPACE_URL,
|
|
f"awooop:{_AWOOOP_GATE5_TRIGGER_TYPE}:{project_id}:{approval_id}",
|
|
)
|
|
trigger_ref = f"adr100_gate5:{incident_id}:{approval_id}"[:256]
|
|
provider_event_id = f"adr100_gate5:{approval_id}"
|
|
now = datetime.now(timezone.utc).replace(tzinfo=None)
|
|
timeout_at = now + timedelta(hours=6)
|
|
projection_input = {
|
|
"schema_version": "adr100_runtime_replay_awooop_projection_input_v1",
|
|
"approval_id": approval_id,
|
|
"approval_kind": payload.get("approval_kind"),
|
|
"incident_id": incident_id,
|
|
"work_item_id": work_item_id,
|
|
"auto_repair_id": auto_repair_id,
|
|
"playbook_id": playbook_id,
|
|
"replay_gate_status": (payload.get("replay_gate") or {}).get("status"),
|
|
"write_route_tools": [
|
|
str(route.get("tool_name") or "")
|
|
for route in (request.metadata or {}).get("write_routes") or []
|
|
if isinstance(route, dict)
|
|
],
|
|
"execution_authorized": False,
|
|
"repair_executed": False,
|
|
"projection_mode": "approval_projection_only",
|
|
}
|
|
input_json = _stable_json(projection_input)
|
|
input_hash = hashlib.sha256(input_json.encode("utf-8")).hexdigest()
|
|
projection_metadata = {
|
|
"schema_version": "adr100_runtime_replay_awooop_projection_v1",
|
|
"approval_id": approval_id,
|
|
"legacy_approval_status": (payload.get("approval") or {}).get("status"),
|
|
"incident_id": incident_id,
|
|
"work_item_id": work_item_id,
|
|
"auto_repair_id": auto_repair_id,
|
|
"playbook_id": playbook_id,
|
|
"projection_mode": "approval_projection_only",
|
|
"execution_authorized": False,
|
|
"repair_attempted": False,
|
|
"repair_executed": False,
|
|
"required_handoff": "legacy_gate5_approval_to_auto_repair_executor",
|
|
}
|
|
|
|
async with get_db_context(project_id) as db:
|
|
run_insert = (
|
|
pg_insert(AwoooPRunState)
|
|
.values(
|
|
run_id=run_id,
|
|
project_id=project_id,
|
|
agent_id="auto_repair_executor",
|
|
state="waiting_approval",
|
|
attempt_count=0,
|
|
max_attempts=1,
|
|
trace_id=f"gate5:{approval_id}",
|
|
trigger_type=_AWOOOP_GATE5_TRIGGER_TYPE,
|
|
trigger_ref=trigger_ref,
|
|
is_shadow=True,
|
|
input_sha256=input_hash,
|
|
step_count=1,
|
|
error_code="E-ADR100-GATE5-PROJECTION",
|
|
error_detail=_stable_json(projection_metadata),
|
|
timeout_at=timeout_at,
|
|
)
|
|
.on_conflict_do_nothing(index_elements=[AwoooPRunState.run_id])
|
|
.returning(AwoooPRunState.run_id)
|
|
)
|
|
run_result = await db.execute(run_insert)
|
|
inserted = run_result.scalar_one_or_none() is not None
|
|
|
|
idempotency_insert = (
|
|
pg_insert(AwoooPRunIdempotency)
|
|
.values(
|
|
project_id=project_id,
|
|
channel_type=_AWOOOP_GATE5_CHANNEL_TYPE,
|
|
provider_event_id=provider_event_id,
|
|
run_id=run_id,
|
|
)
|
|
.on_conflict_do_nothing(constraint="uix_run_idempotency_key")
|
|
)
|
|
await db.execute(idempotency_insert)
|
|
|
|
step_insert = (
|
|
pg_insert(AwoooPRunStepJournal)
|
|
.values(
|
|
run_id=run_id,
|
|
project_id=project_id,
|
|
step_seq=1,
|
|
tool_name="adr100.runtime_replay_gate5.waiting_approval",
|
|
input_hash=input_hash,
|
|
compensation_json=projection_metadata,
|
|
result_status="pending",
|
|
error_code="E-ADR100-GATE5-PROJECTION",
|
|
was_blocked=True,
|
|
block_reason="approval_projection_only",
|
|
)
|
|
.on_conflict_do_nothing(constraint="uix_run_step_seq")
|
|
)
|
|
await db.execute(step_insert)
|
|
|
|
state_result = await db.execute(
|
|
select(AwoooPRunState.state, AwoooPRunState.timeout_at).where(
|
|
AwoooPRunState.run_id == run_id,
|
|
AwoooPRunState.project_id == project_id,
|
|
)
|
|
)
|
|
state_row = state_result.one_or_none()
|
|
|
|
state = str(state_row.state) if state_row else "unknown"
|
|
projected = state == "waiting_approval"
|
|
return {
|
|
"schema_version": "adr100_runtime_replay_awooop_projection_v1",
|
|
"projected": projected,
|
|
"inserted": inserted,
|
|
"deduplicated": not inserted,
|
|
"projection_mode": "approval_projection_only",
|
|
"run_id": str(run_id),
|
|
"project_id": project_id,
|
|
"state": state,
|
|
"timeout_at": state_row.timeout_at.isoformat() if state_row and state_row.timeout_at else None,
|
|
"trigger_type": _AWOOOP_GATE5_TRIGGER_TYPE,
|
|
"trigger_ref": trigger_ref,
|
|
"channel_type": _AWOOOP_GATE5_CHANNEL_TYPE,
|
|
"provider_event_id": provider_event_id,
|
|
"decision_endpoint_enabled": False,
|
|
"execution_authorized": False,
|
|
"repair_attempted": False,
|
|
"repair_executed": False,
|
|
"required_handoff": "legacy_gate5_approval_to_auto_repair_executor",
|
|
"step_journal": {
|
|
"step_seq": 1,
|
|
"tool_name": "adr100.runtime_replay_gate5.waiting_approval",
|
|
"result_status": "pending",
|
|
"was_blocked": True,
|
|
"block_reason": "approval_projection_only",
|
|
},
|
|
}
|
|
|
|
|
|
def _stable_json(value: dict[str, Any]) -> str:
|
|
return json.dumps(
|
|
value,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
default=str,
|
|
)
|
|
|
|
|
|
def _summarize_post_state(post_state: dict[str, Any]) -> dict[str, Any]:
|
|
keys = sorted(post_state.keys())
|
|
return {
|
|
"tool_count": len(keys),
|
|
"tools": keys[:8],
|
|
"has_state": bool(post_state),
|
|
}
|
|
|
|
|
|
def _step_action_type(step: Any) -> str:
|
|
action_type = getattr(step, "action_type", None)
|
|
return str(getattr(action_type, "value", action_type) or "unknown")
|
|
|
|
|
|
def _step_risk_level(step: Any) -> str:
|
|
risk_level = getattr(step, "risk_level", None)
|
|
return str(getattr(risk_level, "value", risk_level) or "unknown").lower()
|
|
|
|
|
|
def _step_looks_mutating(step: Any) -> bool:
|
|
command = str(getattr(step, "command", "") or "").strip().lower()
|
|
action_type = _step_action_type(step)
|
|
if not command or action_type == "manual":
|
|
return False
|
|
if action_type == "script":
|
|
return True
|
|
if action_type == "kubectl":
|
|
return not command.startswith((
|
|
"kubectl get ",
|
|
"kubectl describe ",
|
|
"kubectl logs ",
|
|
"kubectl top ",
|
|
"kubectl explain ",
|
|
))
|
|
if action_type == "ssh_command":
|
|
return any(token in command for token in (
|
|
"docker restart",
|
|
"docker start",
|
|
"docker stop",
|
|
"systemctl restart",
|
|
"systemctl start",
|
|
"certbot renew",
|
|
))
|
|
return False
|
|
|
|
|
|
def _compact_command(command: str) -> str:
|
|
compact = " ".join(str(command or "").split())
|
|
return compact[:240]
|
|
|
|
|
|
def _history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_dry_run_history_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"playbook_id": item.get("playbook_id"),
|
|
"alertname": item.get("alertname"),
|
|
"mode": payload.get("mode"),
|
|
"allowed": payload.get("allowed"),
|
|
"executed": payload.get("executed"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"writes_ticket": payload.get("writes_ticket"),
|
|
"creates_external_ticket": payload.get("creates_external_ticket"),
|
|
"ticket_preview": payload.get("ticket_preview"),
|
|
"plan": payload.get("plan"),
|
|
"verification_result_preview": payload.get("verification_result_preview"),
|
|
"post_state_summary": payload.get("post_state_summary"),
|
|
"mcp_route": payload.get("mcp_route"),
|
|
"replay_gate": payload.get("replay_gate"),
|
|
"checks": payload.get("checks"),
|
|
}
|
|
|
|
|
|
def _approval_history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": "adr100_remediation_approval_history_v1",
|
|
"work_item_id": item.get("work_item_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"playbook_id": item.get("playbook_id"),
|
|
"alertname": item.get("alertname"),
|
|
"mode": payload.get("mode"),
|
|
"allowed": payload.get("allowed"),
|
|
"executed": payload.get("executed"),
|
|
"safety_level": payload.get("safety_level"),
|
|
"writes_incident_state": payload.get("writes_incident_state"),
|
|
"writes_auto_repair_result": payload.get("writes_auto_repair_result"),
|
|
"writes_ticket": payload.get("writes_ticket"),
|
|
"writes_approval_record": payload.get("writes_approval_record"),
|
|
"creates_external_ticket": payload.get("creates_external_ticket"),
|
|
"deduplicated": payload.get("deduplicated"),
|
|
"fingerprint": payload.get("fingerprint"),
|
|
"approval_kind": payload.get("approval_kind"),
|
|
"ticket_preview": payload.get("ticket_preview"),
|
|
"replay_gate": payload.get("replay_gate"),
|
|
"awooop_projection": payload.get("awooop_projection"),
|
|
"approval": payload.get("approval"),
|
|
"approval_id": payload.get("approval_id"),
|
|
"plan": payload.get("plan"),
|
|
"verification_result_preview": payload.get("verification_result_preview"),
|
|
"checks": payload.get("checks"),
|
|
}
|
|
|
|
|
|
def _timeline_status(payload: dict[str, Any]) -> str:
|
|
if not payload.get("allowed"):
|
|
return "warning"
|
|
if payload.get("verification_result_preview") == "success":
|
|
return "success"
|
|
return "warning"
|
|
|
|
|
|
def _history_description(context: dict[str, Any]) -> str:
|
|
tool_count = (context.get("post_state_summary") or {}).get("tool_count", 0)
|
|
route = context.get("mcp_route") or {}
|
|
agent = route.get("agent_id") or "unknown_agent"
|
|
tool = route.get("tool_name") or "current_state"
|
|
return (
|
|
f"mode={context.get('mode')} "
|
|
f"preview={context.get('verification_result_preview')} "
|
|
f"tools={tool_count} route={agent}/{tool} "
|
|
f"writes_incident={context.get('writes_incident_state')} "
|
|
f"writes_auto_repair={context.get('writes_auto_repair_result')}"
|
|
)[:500]
|
|
|
|
|
|
def _approval_history_description(context: dict[str, Any]) -> str:
|
|
approval = context.get("approval") or {}
|
|
return (
|
|
f"kind={context.get('approval_kind') or 'unknown'} "
|
|
f"approval={approval.get('id') or context.get('approval_id') or 'unknown'} "
|
|
f"status={approval.get('status') or 'unknown'} "
|
|
f"preview={context.get('verification_result_preview')} "
|
|
f"writes_approval={context.get('writes_approval_record')} "
|
|
f"writes_incident={context.get('writes_incident_state')} "
|
|
f"writes_auto_repair={context.get('writes_auto_repair_result')}"
|
|
)[:500]
|
|
|
|
|
|
def _record_created_at(record: Any) -> str:
|
|
value = getattr(record, "created_at", None)
|
|
if hasattr(value, "isoformat"):
|
|
return value.isoformat()
|
|
return str(value or "")
|
|
|
|
|
|
def _history_item(record: Any, context: dict[str, Any]) -> dict[str, Any]:
|
|
route = context.get("mcp_route") or {}
|
|
post_state = context.get("post_state_summary") or {}
|
|
approval = context.get("approval") or {}
|
|
replay_gate = context.get("replay_gate") or {}
|
|
awooop_projection = context.get("awooop_projection") or {}
|
|
return {
|
|
"id": str(getattr(record, "id", "")),
|
|
"incident_id": getattr(record, "incident_id", None),
|
|
"auto_repair_id": getattr(record, "auto_repair_id", None)
|
|
or context.get("auto_repair_id"),
|
|
"event_type": str(getattr(record, "event_type", "")),
|
|
"actor": getattr(record, "actor", None),
|
|
"success": getattr(record, "success", None),
|
|
"created_at": _record_created_at(record),
|
|
"work_item_id": context.get("work_item_id"),
|
|
"playbook_id": context.get("playbook_id"),
|
|
"alertname": context.get("alertname"),
|
|
"mode": context.get("mode"),
|
|
"allowed": context.get("allowed"),
|
|
"executed": context.get("executed"),
|
|
"safety_level": context.get("safety_level"),
|
|
"verification_result_preview": context.get("verification_result_preview"),
|
|
"tool_count": post_state.get("tool_count", 0),
|
|
"tools": post_state.get("tools") or [],
|
|
"agent_id": route.get("agent_id"),
|
|
"tool_name": route.get("tool_name") or "current_state",
|
|
"required_scope": route.get("required_scope"),
|
|
"writes_incident_state": context.get("writes_incident_state"),
|
|
"writes_auto_repair_result": context.get("writes_auto_repair_result"),
|
|
"writes_ticket": context.get("writes_ticket"),
|
|
"writes_approval_record": context.get("writes_approval_record"),
|
|
"creates_external_ticket": context.get("creates_external_ticket"),
|
|
"approval_kind": context.get("approval_kind"),
|
|
"approval_id": context.get("approval_id") or approval.get("id"),
|
|
"approval_status": approval.get("status"),
|
|
"approval_risk_level": approval.get("risk_level"),
|
|
"deduplicated": context.get("deduplicated"),
|
|
"fingerprint": context.get("fingerprint"),
|
|
"ticket_preview": context.get("ticket_preview"),
|
|
"replay_gate": replay_gate or None,
|
|
"replay_gate_status": replay_gate.get("status"),
|
|
"replay_gate_next_step": replay_gate.get("next_step"),
|
|
"awooop_projection": awooop_projection or None,
|
|
"awooop_projection_run_id": awooop_projection.get("run_id"),
|
|
"awooop_projection_state": awooop_projection.get("state"),
|
|
"plan": context.get("plan"),
|
|
"checks": context.get("checks") or [],
|
|
}
|
|
|
|
|
|
def _summarize_history_by_work_item(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
summary: dict[str, dict[str, Any]] = {}
|
|
for item in items:
|
|
key = str(item.get("work_item_id") or item.get("incident_id") or item.get("id"))
|
|
if key not in summary:
|
|
summary[key] = {
|
|
"work_item_id": item.get("work_item_id"),
|
|
"incident_id": item.get("incident_id"),
|
|
"count": 0,
|
|
"latest_at": item.get("created_at"),
|
|
"latest_event_type": item.get("event_type"),
|
|
"latest_success": item.get("success"),
|
|
"latest_preview": item.get("verification_result_preview"),
|
|
"latest_mode": item.get("mode"),
|
|
"latest_agent_id": item.get("agent_id"),
|
|
"latest_tool_name": item.get("tool_name"),
|
|
"required_scope": item.get("required_scope"),
|
|
"latest_replay_gate_status": item.get("replay_gate_status"),
|
|
"latest_replay_gate_next_step": item.get("replay_gate_next_step"),
|
|
}
|
|
summary[key]["count"] += 1
|
|
return list(summary.values())
|
|
|
|
|
|
def _diagnostic_command_for_incident(incident: Incident) -> str:
|
|
labels = _labels_for_incident(incident)
|
|
host = str(labels.get("host") or labels.get("instance") or "{host}")
|
|
container = str(labels.get("container_name") or labels.get("container") or "")
|
|
if container:
|
|
return f"ssh {host} 'uptime; docker stats --no-stream {container}'"
|
|
return f"ssh {host} 'uptime; docker stats --no-stream'"
|
|
|
|
|
|
def _ticket_preview_for_item(item: dict[str, Any], incident: Incident) -> dict[str, Any]:
|
|
labels = _labels_for_incident(incident)
|
|
alertname = str(item.get("alertname") or labels.get("alertname") or "unknown_alert")
|
|
incident_id = str(item.get("incident_id") or incident.incident_id)
|
|
playbook_id = str(item.get("playbook_id") or "unknown_playbook")
|
|
host = str(labels.get("host") or labels.get("instance") or "unknown_host")
|
|
container = str(labels.get("container_name") or labels.get("container") or "")
|
|
target = f"host={host}" + (f" container={container}" if container else "")
|
|
title = f"[ADR-100] Promote diagnostic PlayBook to repair: {alertname}"
|
|
body = (
|
|
f"Incident: {incident_id}\n"
|
|
f"Auto repair: {item.get('auto_repair_id') or 'unknown'}\n"
|
|
f"PlayBook: {playbook_id}\n"
|
|
f"Target: {target}\n"
|
|
f"Failure class: {item.get('failure_class') or 'observe_only_playbook'}\n"
|
|
"Required change: add a gated mutating repair step such as docker restart, "
|
|
"Ansible check-mode/apply, or another approved executor action, then keep "
|
|
"post-execution verification tied to the same target.\n"
|
|
"Guardrail: do not mark the old diagnostic-only run as verified_success."
|
|
)
|
|
return {
|
|
"would_create": True,
|
|
"external_ticket_created": False,
|
|
"title": title,
|
|
"labels": [
|
|
"adr100",
|
|
"playbook-authoring",
|
|
"observe-only-playbook",
|
|
"needs-owner-review",
|
|
],
|
|
"body_preview": body[:1000],
|
|
"owner": item.get("remediation_owner") or "solver_or_operator",
|
|
"next_step": "author_mutating_repair_step",
|
|
"playbook_id": playbook_id,
|
|
"target": target,
|
|
}
|
|
|
|
|
|
def _promql_for_incident(incident: Incident) -> str:
|
|
labels = _labels_for_incident(incident)
|
|
alertname = ""
|
|
if incident.signals:
|
|
signal = incident.signals[0]
|
|
alertname = labels.get("alertname") or getattr(signal, "alert_name", "")
|
|
return _build_prometheus_query(alertname, labels)
|
|
|
|
|
|
def _labels_for_incident(incident: Incident) -> dict[str, Any]:
|
|
if incident.signals:
|
|
return incident.signals[0].labels or {}
|
|
return {}
|
|
|
|
|
|
_service: Adr100RemediationService | None = None
|
|
|
|
|
|
def get_adr100_remediation_service() -> Adr100RemediationService:
|
|
"""Return singleton ADR-100 remediation service."""
|
|
|
|
global _service
|
|
if _service is None:
|
|
_service = Adr100RemediationService()
|
|
return _service
|
|
|
|
|
|
def set_adr100_remediation_service(service: Adr100RemediationService | None) -> None:
|
|
"""Inject ADR-100 remediation service for tests."""
|
|
|
|
global _service
|
|
_service = service
|