"""Repair candidate generation from MCP evidence and PlayBook trust. This service is intentionally candidate-only. It never executes a repair; it builds an ApprovalRequestCreate or controlled-queue draft only when evidence, PlayBook status, trust and command safety gates are explicit enough for the AI automation flow to continue through no-write rehearsal / check-mode / verifier. """ from __future__ import annotations from dataclasses import dataclass, field from typing import Any from urllib.parse import urlencode import structlog from src.models.approval import ( ApprovalRequestCreate, BlastRadius, DataImpact, DryRunCheck, ) from src.models.approval import RiskLevel as ApprovalRiskLevel from src.models.incident import Incident from src.models.playbook import ( ActionType, Playbook, PlaybookStatus, ) from src.models.playbook import RiskLevel as PlaybookRiskLevel from src.repositories.playbook_repository import get_playbook_repository from src.services.action_parser import ActionKind, parse_kubectl_action from src.services.auto_repair_service import AutoRepairService from src.services.awooop_ansible_audit_service import ( build_ansible_decision_audit_payload, record_ansible_decision_audit, ) from src.services.awooop_deeplinks import work_item_url from src.services.evidence_snapshot import EvidenceSnapshot from src.services.incident_service import get_incident_service from src.services.playbook_match_resolver import resolve_playbook_id_for_alert from src.services.pre_decision_investigator import get_pre_decision_investigator logger = structlog.get_logger(__name__) MIN_REPAIR_CANDIDATE_TRUST = 0.30 @dataclass class RepairCandidateResult: """Candidate generation result used by webhook and Telegram handoff paths.""" approval_request: ApprovalRequestCreate | None = None evidence: EvidenceSnapshot | None = None playbook: Playbook | None = None blockers: list[str] = field(default_factory=list) metadata: dict[str, Any] = field(default_factory=dict) @property def candidate_found(self) -> bool: return self.approval_request is not None @property def draft_ready_for_owner_review(self) -> bool: return bool(self.metadata.get("repair_candidate_draft_ready")) class RepairCandidateService: """Build a repair approval or controlled-queue candidate without executing it.""" def __init__( self, *, incident_service: Any | None = None, investigator: Any | None = None, playbook_repository: Any | None = None, auto_repair_service: AutoRepairService | None = None, ansible_audit_builder: Any | None = None, ansible_audit_recorder: Any | None = None, ) -> None: self._incident_service = incident_service or get_incident_service() self._investigator = investigator or get_pre_decision_investigator() self._playbook_repository = playbook_repository or get_playbook_repository() self._auto_repair = auto_repair_service or AutoRepairService() self._ansible_audit_builder = ansible_audit_builder or build_ansible_decision_audit_payload self._ansible_audit_recorder = ansible_audit_recorder or record_ansible_decision_audit async def build_from_incident_id( self, *, incident_id: str, alertname: str, target_resource: str, namespace: str, message: str, fallback_action: str, matched_playbook_id: str | None = None, rule_id: str | None = None, severity: str | None = None, ) -> RepairCandidateResult: """Load an incident and produce a repair candidate or explicit blockers.""" incident = await self._incident_service.get_from_working_memory(incident_id) if incident is None: return RepairCandidateResult( blockers=["incident_not_found"], metadata={"repair_candidate_status": "blocked", "blockers": ["incident_not_found"]}, ) return await self.build_from_incident( incident=incident, alertname=alertname, target_resource=target_resource, namespace=namespace, message=message, fallback_action=fallback_action, matched_playbook_id=matched_playbook_id, rule_id=rule_id, severity=severity, ) async def build_from_incident( self, *, incident: Incident, alertname: str, target_resource: str, namespace: str, message: str, fallback_action: str, matched_playbook_id: str | None = None, rule_id: str | None = None, severity: str | None = None, ) -> RepairCandidateResult: evidence = await self._collect_evidence(incident) metadata = self._base_metadata(evidence=evidence) blockers: list[str] = [] if not evidence or evidence.sensors_succeeded <= 0: blockers.append("mcp_evidence_missing") playbook_id = matched_playbook_id or await resolve_playbook_id_for_alert( rule_id=rule_id, alertname=alertname, affected_services=[target_resource] if target_resource else [], severity=severity, ) if not playbook_id: blockers.append("playbook_not_matched") return await self._blocked_result_with_controlled_handoff( blockers=blockers, metadata=metadata, evidence=evidence, fallback_action=fallback_action, incident=incident, alertname=alertname, target_resource=target_resource, namespace=namespace, severity=severity, ) playbook = await self._playbook_repository.get_by_id(playbook_id) if not playbook: blockers.append("playbook_not_found") return await self._blocked_result_with_controlled_handoff( blockers=blockers, metadata=metadata, evidence=evidence, fallback_action=fallback_action, incident=incident, alertname=alertname, target_resource=target_resource, namespace=namespace, severity=severity, ) metadata["playbook_trust"] = { "playbook_id": playbook.playbook_id, "name": playbook.name, "status": playbook.status.value, "trust_score": float(playbook.trust_score), } if playbook.status != PlaybookStatus.APPROVED: blockers.append("playbook_not_approved") if float(playbook.trust_score) < MIN_REPAIR_CANDIDATE_TRUST: blockers.append("playbook_trust_below_gate") if self._is_generic_fallback_playbook(playbook): blockers.append("playbook_generic_fallback_not_repair") step, step_blockers = self._select_executable_step(incident, playbook) blockers.extend(step_blockers) if blockers or step is None: return await self._blocked_result_with_controlled_handoff( blockers=blockers, metadata=metadata, evidence=evidence, playbook=playbook, fallback_action=fallback_action, incident=incident, alertname=alertname, target_resource=target_resource, namespace=namespace, severity=severity, ) metadata["repair_candidate"] = { "source": "mcp_evidence_playbook_trust", "playbook_id": playbook.playbook_id, "playbook_name": playbook.name, "step_number": step.step_number, "command": step.command, "action_type": step.action_type.value, "risk_level": step.risk_level.value, "expected_result": step.expected_result, "rollback_command": step.rollback_command, "requires_approval": True, } metadata["verifier_plan"] = self._build_verifier_plan( command=step.command, namespace=namespace, target_resource=target_resource, evidence=evidence, ) metadata["repair_candidate_status"] = "candidate_ready_for_approval" metadata["fallback_replaced"] = True approval_request = ApprovalRequestCreate( action=step.command, description=self._build_description( message=message, playbook=playbook, step_command=step.command, evidence=evidence, verifier_plan=metadata["verifier_plan"], ), risk_level=self._approval_risk(step.risk_level), blast_radius=BlastRadius( affected_pods=1, estimated_downtime=f"{playbook.estimated_duration_minutes} min", related_services=[target_resource] if target_resource else [], data_impact=DataImpact.WRITE, ), dry_run_checks=self._build_dry_run_checks( evidence=evidence, playbook=playbook, command=step.command, ), requested_by="OpenClaw (MCP evidence + PlayBook trust)", incident_id=incident.incident_id, metadata=metadata, matched_playbook_id=playbook.playbook_id, ) logger.info( "repair_candidate_generated", incident_id=incident.incident_id, playbook_id=playbook.playbook_id, sensors_succeeded=evidence.sensors_succeeded if evidence else 0, command=step.command[:160], ) return RepairCandidateResult( approval_request=approval_request, evidence=evidence, playbook=playbook, metadata=metadata, ) async def _blocked_result_with_controlled_handoff( self, *, blockers: list[str], metadata: dict[str, Any], fallback_action: str, evidence: EvidenceSnapshot | None = None, playbook: Playbook | None = None, incident: Incident | None = None, alertname: str = "", target_resource: str = "", namespace: str = "", severity: str | None = None, ) -> RepairCandidateResult: result = self._blocked_result( blockers=blockers, metadata=metadata, evidence=evidence, playbook=playbook, fallback_action=fallback_action, incident=incident, alertname=alertname, target_resource=target_resource, namespace=namespace, ) await self._maybe_record_controlled_executor_handoff( result=result, incident=incident, severity=severity, ) return result async def _maybe_record_controlled_executor_handoff( self, *, result: RepairCandidateResult, incident: Incident | None, severity: str | None = None, ) -> None: if incident is None or not result.metadata.get("repair_candidate_draft_ready"): return draft_package = result.metadata.get("repair_candidate_draft_package") if not isinstance(draft_package, dict): return promotion_contract = draft_package.get("candidate_promotion_contract") if not isinstance(promotion_contract, dict): return route_id = str(promotion_contract.get("route_id") or "") if route_id.startswith("agent99_"): suggested_mode = "Recover" if route_id.startswith("agent99_recover") else "Status" handoff = { "schema_version": "repair_candidate_controlled_executor_handoff_v1", "status": "agent99_dispatch_required_on_actionable_card_send", "operation_type": "agent99_actionable_dispatch", "executor": "Agent99", "route_id": route_id, "suggested_mode": suggested_mode, "dispatch_receipt_required": True, "post_condition_verifier_required": True, "check_mode_queued": False, "runtime_execution_authorized": False, } result.metadata["controlled_executor_handoff"] = handoff draft_package["controlled_executor_handoff"] = handoff work_item = draft_package.get("awooop_work_item") if isinstance(work_item, dict): work_item["controlled_executor_handoff"] = handoff return proposal_data = { "source": "repair_candidate_controlled_queue", "risk_level": severity or getattr(getattr(incident, "severity", None), "value", ""), "action": promotion_contract.get("repair_command_template") or "", "matched_playbook_id": draft_package.get("matched_playbook_id"), "repair_candidate_status": result.metadata.get("repair_candidate_status"), "route_id": promotion_contract.get("route_id"), "controlled_playbook_queue": True, } payload = self._ansible_audit_builder( incident=incident, proposal_data=proposal_data, decision_path="repair_candidate_controlled_queue", not_used_reason=( "repair candidate controlled queue ready; " "Ansible check-mode worker should claim candidate" ), ) if payload is None: handoff = { "schema_version": "repair_candidate_controlled_executor_handoff_v1", "status": "no_ansible_catalog_candidate", "operation_type": "ansible_candidate_matched", "candidate_count": 0, "check_mode_worker": "awooop_ansible_check_mode_worker", "check_mode_queued": False, "runtime_execution_authorized": False, } else: written = await self._ansible_audit_recorder( incident=incident, proposal_data=proposal_data, decision_path="repair_candidate_controlled_queue", not_used_reason=( "repair candidate controlled queue ready; " "Ansible check-mode worker should claim candidate" ), ) candidates = payload.get("input", {}).get("executor_candidates") candidate_count = len(candidates) if isinstance(candidates, list) else 0 handoff = { "schema_version": "repair_candidate_controlled_executor_handoff_v1", "status": ( "ansible_candidate_matched_queued" if written else "ansible_candidate_matched_existing_or_write_skipped" ), "operation_type": payload.get("operation_type"), "decision_effect": payload.get("output", {}).get("decision_effect"), "next_required_step": payload.get("output", {}).get("next_required_step"), "candidate_count": candidate_count, "check_mode_worker": "awooop_ansible_check_mode_worker", "check_mode_queued": True, "runtime_execution_authorized": False, } result.metadata["controlled_executor_handoff"] = handoff draft_package["controlled_executor_handoff"] = handoff work_item = draft_package.get("awooop_work_item") if isinstance(work_item, dict): work_item["controlled_executor_handoff"] = handoff async def _collect_evidence(self, incident: Incident) -> EvidenceSnapshot | None: try: return await self._investigator.investigate(incident) except Exception as exc: logger.warning( "repair_candidate_evidence_collect_failed", incident_id=incident.incident_id, error=str(exc), ) return None def _select_executable_step( self, incident: Incident, playbook: Playbook, ) -> tuple[Any | None, list[str]]: blockers: list[str] = [] if not playbook.repair_steps: return None, ["playbook_has_no_repair_steps"] rejected_readonly = False rejected_unsafe = False for step in playbook.repair_steps: command = (step.command or "").strip() if not command: continue if step.action_type == ActionType.MANUAL: rejected_readonly = True continue if step.action_type == ActionType.KUBECTL or command.startswith("kubectl"): parsed = parse_kubectl_action(command) if not parsed.ok: rejected_unsafe = True continue if parsed.kind == ActionKind.READONLY: rejected_readonly = True continue return step, [] if step.action_type == ActionType.SSH_COMMAND or command.startswith("ssh "): if self._auto_repair.preview_write_ssh_mcp_route(incident, command): return step, [] if self._looks_like_diagnostic_command(command): rejected_readonly = True continue rejected_unsafe = True if rejected_unsafe: blockers.append("playbook_command_not_safely_routable") if rejected_readonly: blockers.append("playbook_observe_only") if not blockers: blockers.append("playbook_has_no_executable_step") return None, blockers def _base_metadata(self, *, evidence: EvidenceSnapshot | None) -> dict[str, Any]: if not evidence: return { "source": "mcp_playbook_repair_candidate", "mcp_evidence": {"status": "missing"}, } return { "source": "mcp_playbook_repair_candidate", "mcp_evidence": { "status": "collected" if evidence.sensors_succeeded > 0 else "degraded", "snapshot_id": evidence.snapshot_id, "sensors_attempted": evidence.sensors_attempted, "sensors_succeeded": evidence.sensors_succeeded, "mcp_health": evidence.mcp_health, "summary_excerpt": (evidence.evidence_summary or "")[:800], }, } def _blocked_result( self, *, blockers: list[str], metadata: dict[str, Any], fallback_action: str, evidence: EvidenceSnapshot | None = None, playbook: Playbook | None = None, incident: Incident | None = None, alertname: str = "", target_resource: str = "", namespace: str = "", ) -> RepairCandidateResult: metadata["repair_candidate_blockers"] = list(dict.fromkeys(blockers)) metadata["repair_candidate_blocker_summary"] = self._humanize_blockers( metadata["repair_candidate_blockers"] ) draft_package = self._build_draft_package( blockers=metadata["repair_candidate_blockers"], playbook=playbook, evidence=evidence, incident=incident, alertname=alertname, target_resource=target_resource, namespace=namespace, ) draft_ready = self._is_owner_review_ready_draft( draft_package=draft_package, blockers=metadata["repair_candidate_blockers"], ) metadata["repair_candidate_status"] = ( "controlled_playbook_queue_ready" if draft_ready else "blocked" ) metadata["repair_candidate_draft_ready"] = draft_ready metadata["repair_candidate_owner_review_required"] = not draft_ready metadata["repair_candidate_controlled_playbook_queue_ready"] = draft_ready metadata["repair_candidate_runtime_action_ready"] = False metadata["repair_candidate_runtime_execution_authorized"] = False if draft_ready: draft_package["status"] = "controlled_playbook_queue_ready" draft_package["owner_review_gate"] = "auto_waived_by_current_policy" draft_package["controlled_playbook_queue"] = True draft_package["next_action"] = "queue_check_mode_then_controlled_apply" draft_package["runtime_execution_authorized"] = False draft_package["writes_runtime_state"] = False work_item = draft_package.get("awooop_work_item") if isinstance(work_item, dict): work_item["status"] = "controlled_playbook_queue_ready" work_item["next_action"] = "queue_check_mode_then_controlled_apply" work_item["owner_review_required"] = False work_item["controlled_playbook_queue"] = True work_item["runtime_execution_authorized"] = False promotion_contract = draft_package.get("candidate_promotion_contract") if isinstance(promotion_contract, dict): metadata["repair_candidate_promotion_contract"] = promotion_contract metadata["repair_candidate_promotion_summary"] = ( self._promotion_summary_for_operator(promotion_contract) ) metadata["playbook_draft_required"] = True metadata["repair_candidate_prefilled_draft_summary"] = self._draft_summary_for_operator( draft_package.get("playbook_draft_template") or {} ) metadata["repair_candidate_next_step"] = self._next_step_with_draft_summary( str(draft_package["next_step"]), str(metadata["repair_candidate_prefilled_draft_summary"]), ) metadata["repair_candidate_draft_package"] = draft_package metadata["fallback_action"] = fallback_action return RepairCandidateResult( evidence=evidence, playbook=playbook, blockers=metadata["repair_candidate_blockers"], metadata=metadata, ) def _is_generic_fallback_playbook(self, playbook: Playbook) -> bool: alert_names = playbook.symptom_pattern.alert_names or [] if "*" in alert_names: return True return "通用兜底" in playbook.name or "generic fallback" in playbook.name.lower() def _looks_like_diagnostic_command(self, command: str) -> bool: command = command.lower() diagnostic_markers = ( "uptime", "ps aux", "docker stats", "journalctl", "systemctl status", "kubectl get", "kubectl describe", "tail ", "grep ", ) return any(marker in command for marker in diagnostic_markers) def _humanize_blockers(self, blockers: list[str]) -> str: labels = { "incident_not_found": "找不到 incident，無法綁定真相鏈", "mcp_evidence_missing": "MCP 證據未完成", "playbook_not_matched": "沒有命中專屬 PlayBook", "playbook_not_found": "命中的 PlayBook 不存在", "playbook_not_approved": "PlayBook 尚未批准", "playbook_trust_below_gate": "PlayBook trust score 低於候選門檻", "playbook_generic_fallback_not_repair": "只命中通用兜底 PlayBook，禁止當成修復命令", "playbook_has_no_repair_steps": "PlayBook 沒有修復步驟", "playbook_command_not_safely_routable": "PlayBook 命令未通過安全路由", "playbook_observe_only": "PlayBook 只有觀察或診斷步驟", "playbook_has_no_executable_step": "PlayBook 沒有可執行修復步驟", } return "；".join(labels.get(blocker, blocker) for blocker in blockers) def _build_draft_package( self, *, blockers: list[str], playbook: Playbook | None, evidence: EvidenceSnapshot | None, incident: Incident | None, alertname: str = "", target_resource: str = "", namespace: str = "", ) -> dict[str, Any]: """Describe the concrete owner-review package needed to unblock repair. The package is a handoff contract only. It must not be interpreted as approval to mutate runtime state or auto-create an approved PlayBook. """ blocker_set = set(blockers) if "incident_not_found" in blocker_set: lane = "restore_truth_chain_before_repair" next_step = "先修復 incident / approval 真相鏈綁定，再重跑 MCP evidence 與 PlayBook 匹配。" elif "mcp_evidence_missing" in blocker_set: lane = "rerun_mcp_evidence_collection" next_step = ( "先按重診收集 MCP evidence；成功後再建立服務專屬 PlayBook 草案，" "禁止只憑通用規則批准修復。" ) elif { "playbook_not_matched", "playbook_not_found", "playbook_generic_fallback_not_repair", } & blocker_set: lane = "create_service_specific_repair_playbook" next_step = ( "建立專屬 PlayBook 草案：綁定 alertname / target selector，補 MCP evidence refs、" "修復命令、rollback、verifier plan 與受控執行 route；通用兜底不可執行。" ) elif "playbook_observe_only" in blocker_set: lane = "promote_diagnostic_to_repair_playbook" next_step = ( "把診斷命令保留為 MCP evidence collector；另建獨立修復步驟、rollback " "與 verifier，排入 no-write rehearsal / check-mode 後才可進 controlled apply。" ) elif "playbook_command_not_safely_routable" in blocker_set: lane = "route_command_through_safe_mcp_or_ansible" next_step = ( "將命令改走 allowlisted MCP / Ansible route，補 blast radius、rollback " "與 verifier plan，再進受控自動化佇列。" ) elif { "playbook_not_approved", "playbook_trust_below_gate", } & blocker_set: lane = "ai_playbook_trust_repair_gate" next_step = ( "由 OpenClaw / NemoTron 重評 PlayBook 狀態與 trust score；補成功/失敗證據後再進受控修復候選。" ) else: lane = "repair_candidate_controlled_queue_repair" next_step = ( "AI 補 PlayBook 草案欄位、rollback、verifier 與 route，完成後自動重跑候選生成。" ) evidence_ref = None if evidence and evidence.snapshot_id: evidence_ref = evidence.snapshot_id required_fields = [ "alertname", "target_selector", "mcp_evidence_refs", "repair_command", "rollback_command", "verifier_plan", "owner_review", "script_or_ansible_ref", "schedule_or_monitoring_rule_ref", "km_update_plan", "automation_asset_record", ] blocked_operations = [ "auto_execute", "approve_no_action_as_repair", "generic_fallback_repair", ] automation_asset_requirements = [ { "asset_type": "KM", "required_record": "incident_root_cause_and_resolution_summary", "visibility": "knowledge_base", "owner_review_required": True, }, { "asset_type": "PlayBook", "required_record": "service_specific_repair_steps_with_trust_policy", "visibility": "awooop_work_items", "owner_review_required": True, }, { "asset_type": "ScriptOrAnsible", "required_record": "safe_route_command_or_ansible_playbook_ref", "visibility": "runs_and_work_items", "owner_review_required": True, }, { "asset_type": "ScheduleOrMonitoringRule", "required_record": "recurrence_detection_or_alert_rule_gap", "visibility": "observability_and_reports", "owner_review_required": True, }, { "asset_type": "Verifier", "required_record": "success_failure_and_rollback_verification_plan", "visibility": "incident_timeline", "owner_review_required": True, }, ] required_writebacks = [ "incident_timeline_stage_update", "execution_or_manual_handoff_result", "verifier_result", "km_update_draft", "playbook_trust_update", "automation_asset_inventory_record", ] coverage_gap = self._build_coverage_gap( blockers=blockers, lane=lane, alertname=alertname, target_resource=target_resource, namespace=namespace, evidence=evidence, playbook=playbook, ) playbook_draft_template = self._build_playbook_draft_template( coverage_gap=coverage_gap, blockers=blockers, lane=lane, alertname=alertname, target_resource=target_resource, namespace=namespace, evidence=evidence, playbook=playbook, ) incident_id = getattr(incident, "incident_id", "") if incident else "" project_id = str(getattr(incident, "project_id", "awoooi") or "awoooi") work_item: dict[str, Any] | None = None work_item_id = "" work_item_url_value = "" if incident_id: work_item_id = f"repair-candidate-draft:{project_id}:{incident_id}:{lane}" work_item_query = urlencode({ "project_id": project_id, "incident_id": incident_id, "work_item_id": work_item_id, }) run_query = urlencode({ "project_id": project_id, "incident_id": incident_id, }) work_item_url_value = work_item_url( work_item_id, incident_id=incident_id, project_id=project_id, ) work_item = { "schema_version": "awooop_repair_candidate_draft_work_item_v1", "work_item_id": work_item_id, "kind": "repair_candidate_playbook_draft", "status": "open", "needs_human": True, "project_id": project_id, "incident_id": incident_id, "alertname": alertname or None, "namespace": namespace or None, "target_resource": target_resource or None, "lane": lane, "reason": ",".join(blockers), "next_step": next_step, "required_fields": required_fields, "automation_asset_requirements": automation_asset_requirements, "required_writebacks": required_writebacks, "coverage_gap": coverage_gap, "playbook_draft_template": playbook_draft_template, "blocked_operations": blocked_operations, "target_href": f"/awooop/runs?{run_query}", "work_item_href": f"/awooop/work-items?{work_item_query}", "work_item_url": work_item_url_value, "decision_effect": "none", "safety_level": "read_only_work_item_projection", "writes_incident_state": False, "writes_auto_repair_result": False, "writes_runtime_state": False, } promotion_contract = self._build_candidate_promotion_contract( coverage_gap=coverage_gap, playbook_draft_template=playbook_draft_template, lane=lane, blockers=blockers, incident_id=incident_id, project_id=project_id, work_item_id=work_item_id, work_item_url_value=work_item_url_value, ) if work_item is not None: work_item["candidate_promotion_contract"] = promotion_contract package = { "schema_version": "repair_candidate_draft_package_v1", "status": "draft_required", "lane": lane, "next_step": next_step, "matched_playbook_id": playbook.playbook_id if playbook else None, "matched_playbook_name": playbook.name if playbook else None, "evidence_snapshot_id": evidence_ref, "required_fields": required_fields, "automation_asset_requirements": automation_asset_requirements, "required_writebacks": required_writebacks, "coverage_gap": coverage_gap, "playbook_draft_template": playbook_draft_template, "candidate_promotion_contract": promotion_contract, "blocked_operations": blocked_operations, } if work_item: package["awooop_work_item"] = work_item return package def _build_candidate_promotion_contract( self, *, coverage_gap: dict[str, Any], playbook_draft_template: dict[str, Any], lane: str, blockers: list[str], incident_id: str, project_id: str, work_item_id: str, work_item_url_value: str, ) -> dict[str, Any]: """Describe exactly what is needed to promote a draft into an apply gate. This contract is still fail-closed. It gives AwoooP / Telegram a machine-readable checklist so a draft does not collapse back into generic "manual review" text. """ route = str(playbook_draft_template.get("suggested_route") or "").strip() repair_template = str( playbook_draft_template.get("repair_command_template") or "" ).strip() rollback_template = str( playbook_draft_template.get("rollback_command_template") or "" ).strip() verifier_plan = playbook_draft_template.get("verifier_plan_template") if not isinstance(verifier_plan, list): verifier_plan = [] evidence_refs = playbook_draft_template.get("mcp_evidence_refs") if not isinstance(evidence_refs, list): evidence_refs = [] alert_selector = playbook_draft_template.get("alert_selector") if not isinstance(alert_selector, dict): alert_selector = {} promotion_ready = ( route.endswith("_after_owner_review") and bool(alert_selector.get("target_resource")) and bool(coverage_gap.get("mcp_evidence_ready") and evidence_refs) and bool(repair_template) and "<" not in repair_template and not repair_template.startswith("owner_supplied") and bool(rollback_template) and "<" not in rollback_template and not rollback_template.startswith("owner_supplied") and bool(verifier_plan) ) ai_policy_status = "ready" if promotion_ready else "blocked" ai_policy_source = ( "ai_agent_controlled_runtime_authorization" if promotion_ready else "missing_candidate_input" ) field_rows = [ self._promotion_contract_field( field="target_selector", label="Alert / target selector", status="ready" if alert_selector.get("target_resource") else "blocked", source="coverage_gap", value=coverage_gap.get("coverage_key"), ), self._promotion_contract_field( field="mcp_evidence_refs", label="MCP evidence refs", status=( "ready" if coverage_gap.get("mcp_evidence_ready") and evidence_refs else "blocked" ), source="mcp_evidence", value=evidence_refs[:8], ), self._promotion_contract_field( field="route_id", label="Controlled route candidate", status="ready" if route.endswith("_after_owner_review") else "blocked", source="playbook_draft_template", value=route or "--", ), self._promotion_contract_field( field="repair_command_template", label="Repair command template", status="ready" if repair_template and "<" not in repair_template and not repair_template.startswith("owner_supplied") else "blocked", source="playbook_draft_template", value=repair_template or "--", ), self._promotion_contract_field( field="rollback_command_template", label="Rollback command template", status="ready" if rollback_template and "<" not in rollback_template and not rollback_template.startswith("owner_supplied") else "blocked", source="playbook_draft_template", value=rollback_template or "--", ), self._promotion_contract_field( field="verifier_plan", label="Post-apply verifier plan", status="ready" if verifier_plan else "blocked", source="playbook_draft_template", value=verifier_plan[:8], ), self._promotion_contract_field( field="owner_review", label="Autonomous policy receipt", status=ai_policy_status, source=ai_policy_source, value=( "owner_policy_low_medium_high_auto_authorized" if promotion_ready else "blocked_until_route_ready" ), ), self._promotion_contract_field( field="maintenance_window", label="AI-selected maintenance window", status=ai_policy_status, source=ai_policy_source, value=( "standard_ai_window_with_verifier_and_rollback" if promotion_ready else "blocked_until_route_ready" ), ), self._promotion_contract_field( field="blast_radius", label="AI-prefilled blast radius", status=ai_policy_status, source=ai_policy_source, value=( f"target={alert_selector.get('target_resource') or '--'}; route={route or '--'}" if promotion_ready else "blocked_until_route_ready" ), ), self._promotion_contract_field( field="km_writeback_owner", label="KM writeback owner", status=ai_policy_status, source=ai_policy_source, value="Hermes" if promotion_ready else "blocked_until_route_ready", ), self._promotion_contract_field( field="playbook_trust_owner", label="PlayBook trust owner", status=ai_policy_status, source=ai_policy_source, value="OpenClaw/NemoTron" if promotion_ready else "blocked_until_route_ready", ), ] ready_fields = [row["field"] for row in field_rows if row["status"] == "ready"] blocked_fields = [row["field"] for row in field_rows if row["status"] != "ready"] status = ( "controlled_playbook_queue_ready" if { "target_selector", "mcp_evidence_refs", "route_id", "repair_command_template", "rollback_command_template", "verifier_plan", "owner_review", "maintenance_window", "blast_radius", "km_writeback_owner", "playbook_trust_owner", } <= set(ready_fields) else "blocked_missing_candidate_inputs" ) return { "schema_version": "repair_candidate_promotion_contract_v1", "status": status, "lane": lane, "incident_id": incident_id or None, "project_id": project_id, "source_work_item_id": work_item_id or None, "source_work_item_url": work_item_url_value or None, "route_id": route or "--", "repair_command_template": repair_template or "--", "rollback_command_template": rollback_template or "--", "verifier_plan_template": list(verifier_plan), "ready_count": len(ready_fields), "total_count": len(field_rows), "blocked_count": len(blocked_fields), "ready_fields": ready_fields, "blocked_fields": blocked_fields, "fields": field_rows, "blockers": list(dict.fromkeys(blockers)), "runtime_write_allowed": False, "runtime_execution_authorized": False, "controlled_playbook_queue": status == "controlled_playbook_queue_ready", "approval_required_before_execution": False if status == "controlled_playbook_queue_ready" else True, "owner_review_required": False if status == "controlled_playbook_queue_ready" else True, "forbidden_until_promoted": [ "unverified_auto_execute", "systemctl_restart", "ssh_write", "ansible_apply", "telegram_success_message", "km_writeback", "playbook_trust_writeback", ], "next_steps": [ "queue_no_write_rehearsal", "run_check_mode_or_equivalent_preflight", "promote_to_controlled_apply_when_preflight_passes", "run_post_apply_verifier", "write_km_and_playbook_trust_after_verified_execution", ], "controlled_automation_closure": { "schema_version": "repair_candidate_controlled_automation_closure_v1", "status": ( "controlled_playbook_queue_ready_no_write_rehearsal" if status == "controlled_playbook_queue_ready" else "blocked_missing_candidate_inputs" ), "runtime_write_gate": ( "controlled_queue_pending_check_mode" if status == "controlled_playbook_queue_ready" else "closed_until_candidate_ready" ), "no_write_rehearsal": { "rehearsal_id": ( f"repair-rehearsal:{project_id}:{incident_id}:{lane}" if incident_id else f"repair-rehearsal:{project_id}:unbound:{lane}" ), "status": ( "queued_by_ai" if status == "controlled_playbook_queue_ready" else "blocked_missing_candidate_inputs" ), "route_id": route or "--", "input_refs": { "source_work_item_id": work_item_id or None, "evidence_refs": list(evidence_refs), "coverage_key": coverage_gap.get("coverage_key"), }, "planned_steps": [ "render_repair_command_without_execution", "render_rollback_command_without_execution", "check_route_allowlist_and_target_selector", "produce_verifier_rehearsal_plan", "queue_check_mode_or_equivalent_preflight", ], "writes_runtime_state": False, "executes_command": False, }, "verifier_writeback_plan": { "verifier_id": ( f"verifier-plan:{project_id}:{incident_id}:{lane}" if incident_id else f"verifier-plan:{project_id}:unbound:{lane}" ), "status": "planned_not_executed", "checks": list(verifier_plan), "result_states": ["healthy", "degraded", "failed", "rollback_required"], "writes_after_execution": [ "incident_timeline", "auto_repair_execution_result", "km_draft", "playbook_trust_event", ], "writes_runtime_state": False, }, "automation_asset_ledger": [ { "asset_type": "KM", "asset_id": ( f"km-draft:{project_id}:{incident_id}:{lane}" if incident_id else f"km-draft:{project_id}:unbound:{lane}" ), "owner": "Hermes", "status": "draft_required_after_verifier", "next_action": "prepare_km_draft_from_incident_evidence", }, { "asset_type": "PlayBook", "asset_id": ( f"playbook-draft:{project_id}:{incident_id}:{lane}" if incident_id else f"playbook-draft:{project_id}:unbound:{lane}" ), "owner": "OpenClaw", "status": ( "controlled_playbook_queue_ready" if status == "controlled_playbook_queue_ready" else "draft_required" ), "next_action": "queue_check_mode_then_controlled_apply", }, { "asset_type": "ScriptOrAnsible", "asset_id": route or "--", "owner": "Executor lane", "status": ( "check_mode_queue_required" if status == "controlled_playbook_queue_ready" else "route_allowlist_review_required" ), "next_action": "run_check_mode_or_transport_preflight", }, { "asset_type": "Verifier", "asset_id": ( f"verifier-plan:{project_id}:{incident_id}:{lane}" if incident_id else f"verifier-plan:{project_id}:unbound:{lane}" ), "owner": "PostExecutionVerifier", "status": "planned_not_executed", "next_action": "run_post_apply_verifier_after_controlled_apply", }, ], "operator_next_actions": [ "queue_no_write_rehearsal_inputs", "run_check_mode_or_equivalent_preflight", "promote_to_controlled_apply_only_after_preflight_passes", "run_verifier_and_write_learning_receipt", ], }, } def _promotion_contract_field( self, *, field: str, label: str, status: str, source: str, value: Any, ) -> dict[str, Any]: return { "field": field, "label": label, "status": status, "source": source, "value": value, "runtime_execution_authorized": False, } def _promotion_summary_for_operator(self, contract: dict[str, Any]) -> str: route = str(contract.get("route_id") or "--") ready = int(contract.get("ready_count") or 0) total = int(contract.get("total_count") or 0) blocked = int(contract.get("blocked_count") or 0) status = str(contract.get("status") or "unknown") runtime_state = ( "controlled" if contract.get("runtime_execution_authorized") is True or contract.get("runtime_write_allowed") is True else "false" ) return ( f"route={route}; promotion={ready}/{total}; " f"blocked={blocked}; status={status}; runtime={runtime_state}" ) def _draft_summary_for_operator(self, template: dict[str, Any]) -> str: route = str(template.get("suggested_route") or "--") repair = str(template.get("repair_command_template") or "--") verifier = template.get("verifier_plan_template") or [] verifier_head = "--" if isinstance(verifier, list) and verifier: verifier_head = str(verifier[0]) return f"route={route}; repair_template={repair}; verifier={verifier_head}" def _next_step_with_draft_summary(self, next_step: str, draft_summary: str) -> str: if not draft_summary or draft_summary == "route=--; repair_template=--; verifier=--": return next_step return f"{next_step} 草案已預填：{draft_summary}" def _is_owner_review_ready_draft( self, *, draft_package: dict[str, Any], blockers: list[str], ) -> bool: """Return true when the handoff is a concrete controlled-queue draft. This still does not execute the action. It separates a useful prefilled route from a genuinely missing repair candidate so AI can queue no-write rehearsal / check-mode before controlled apply. """ coverage_gap = draft_package.get("coverage_gap") template = draft_package.get("playbook_draft_template") work_item = draft_package.get("awooop_work_item") if not isinstance(coverage_gap, dict) or not isinstance(template, dict): return False if not isinstance(work_item, dict): return False if "mcp_evidence_missing" in set(blockers): return False route = str(template.get("suggested_route") or "").strip() repair_template = str(template.get("repair_command_template") or "").strip() safe_agent99_route = bool( route in { "agent99_recover_after_owner_review", "agent99_status_after_owner_review", } and repair_template.startswith("agent99-mode ") ) if ( "playbook_command_not_safely_routable" in set(blockers) and not safe_agent99_route ): return False if not coverage_gap.get("mcp_evidence_ready"): return False if not repair_template or not route: return False if "<" in repair_template: return False if repair_template.startswith("owner_supplied"): return False if repair_template.startswith("service-specific repair"): return False return route.endswith("_after_owner_review") def _build_playbook_draft_template( self, *, coverage_gap: dict[str, Any], blockers: list[str], lane: str, alertname: str, target_resource: str, namespace: str, evidence: EvidenceSnapshot | None, playbook: Playbook | None, ) -> dict[str, Any]: """Prefill a service-specific PlayBook draft for the controlled queue.""" target_kind = str(coverage_gap.get("target_kind") or "unknown") target = (target_resource or "").strip() ns = (namespace or "").strip() command_template = "owner_supplied_service_specific_command_after_evidence_review" rollback_template = "owner_supplied_rollback_or_safe_stop_after_evidence_review" route = "owner_review_required" verifier_plan = [ "rerun_mcp_evidence_collection", "confirm_fingerprint_recurrence_stops_or_decreases", "write_execution_result_and_verifier_outcome", "update_km_and_playbook_trust_after_owner_review", "record_script_or_ansible_asset_ref", "record_schedule_or_monitoring_rule_gap", "publish_assets_to_runs_work_items_and_knowledge_base", ] if target_kind == "control_plane": normalized_target = target.lower() if any( marker in normalized_target for marker in ( "cold-start", "cold_start", "reboot-auto-recovery", "reboot_auto_recovery", ) ): command_template = "agent99-mode Recover controlledApply=true" rollback_template = "agent99-mode Status controlledApply=false" route = "agent99_recover_after_owner_review" verifier_plan.extend([ "agent99 Recover outcome verifier must resolve", "rerun reboot-auto-recovery-slo-scorecard", "confirm required hosts, Harbor, K3s, and public routes are green", ]) else: command_template = "agent99-mode Status controlledApply=false" rollback_template = "no runtime write; retain previous production deployment" route = "agent99_status_after_owner_review" verifier_plan.extend([ "compare failed commit with current production deploy marker", "confirm public routes and current production services remain green", "keep source CI failure open until a later successful run is observed", ]) elif target_kind == "k8s_workload": workload = target.replace("deployment/", "").replace("pod/", "").replace("svc/", "") workload_ref = workload or "" namespace_ref = ns or "" command_template = ( f"kubectl rollout restart deployment/{workload_ref} -n {namespace_ref}" ) rollback_template = ( f"kubectl rollout undo deployment/{workload_ref} -n {namespace_ref}" ) route = "k8s_rollout_after_owner_review" verifier_plan.extend([ f"kubectl rollout status deployment/{workload_ref} -n {namespace_ref}", f"kubectl get events -n {namespace_ref} --field-selector involvedObject.name={workload_ref}", ]) elif target_kind == "host_service": service_ref = target or "" command_template = f"systemctl restart {service_ref}" rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120" route = "host_service_route_after_owner_review" verifier_plan.extend([ f"systemctl is-active {service_ref}", f"journalctl -u {service_ref} -n 120 --no-pager", ]) elif target_kind == "database": db_ref = target or "postgres" command_template = ( "owner_supplied_query_or_index_fix_after_readonly_evidence_review" ) rollback_template = ( "owner_supplied_db_rollback_plan_with_restore_and_no_false_green_check" ) route = "database_slow_query_owner_review" verifier_plan.extend([ "read_only_pg_stat_activity_and_pg_locks_before_after", "read_only_pg_stat_statements_p95_or_top_query_delta", "confirm_api_latency_and_error_rate_recover", "confirm_no_new_lock_wait_or_connection_pool_saturation", f"confirm target database surface remains stable for {db_ref}", ]) elif target_kind == "service": service_ref = target or "" command_template = f"service-specific repair for {service_ref}" rollback_template = f"service-specific rollback for {service_ref}" route = "service_specific_route_after_owner_review" verifier_plan.append(f"confirm service health for {service_ref}") owner_review_checklist = [ "確認 alertname 與 target selector 沒有誤配", "確認 MCP evidence refs 足以支持修復假設", "確認 repair command 不是純診斷命令或通用兜底命令", "確認 rollback command 能在失敗時安全收斂", "確認 verifier plan 能判斷成功、失敗與是否升級人工", "確認 AI 已預填 blast radius、maintenance window 與 rollback owner", "確認 runtime_execution_authorized 仍為 false，直到 check-mode / verifier preflight 通過", ] return { "schema_version": "service_specific_playbook_draft_template_v1", "status": "prefilled_controlled_queue_candidate", "lane": lane, "coverage_key": coverage_gap.get("coverage_key"), "blocking_stage": coverage_gap.get("blocking_stage"), "blockers": list(dict.fromkeys(blockers)), "alert_selector": { "alertname": alertname or None, "namespace": ns or None, "target_resource": target or None, "target_kind": target_kind, }, "mcp_evidence_refs": coverage_gap.get("required_mcp_evidence_refs") or [], "matched_playbook": { "playbook_id": playbook.playbook_id if playbook else None, "name": playbook.name if playbook else None, "status": playbook.status.value if playbook else None, "trust_score": float(playbook.trust_score) if playbook else None, }, "suggested_route": route, "repair_command_template": command_template, "rollback_command_template": rollback_template, "verifier_plan_template": list(dict.fromkeys(verifier_plan)), "owner_review_checklist": owner_review_checklist, "evidence_snapshot_id": evidence.snapshot_id if evidence else None, "template_is_executable": False, "approval_required_before_execution": True, "runtime_execution_authorized": False, "writes_runtime_state": False, "telegram_send_authorized": False, } def _build_coverage_gap( self, *, blockers: list[str], lane: str, alertname: str, target_resource: str, namespace: str, evidence: EvidenceSnapshot | None, playbook: Playbook | None, ) -> dict[str, Any]: """Describe why this alert is not yet covered by an executable PlayBook.""" blocker_set = set(blockers) coverage_key = self._coverage_key(alertname=alertname, target_resource=target_resource) target_kind = self._infer_target_kind(target_resource=target_resource, namespace=namespace) if "mcp_evidence_missing" in blocker_set: blocking_stage = "mcp_evidence" elif { "playbook_not_matched", "playbook_not_found", "playbook_generic_fallback_not_repair", "playbook_observe_only", "playbook_has_no_repair_steps", "playbook_has_no_executable_step", } & blocker_set: blocking_stage = "service_playbook_coverage" elif {"playbook_not_approved", "playbook_trust_below_gate"} & blocker_set: blocking_stage = "playbook_trust_gate" elif "playbook_command_not_safely_routable" in blocker_set: blocking_stage = "safe_execution_route" else: blocking_stage = "owner_review" evidence_refs = [ "alertmanager_event", "incident_timeline", "mcp_health_snapshot", "k8s_or_host_status", "metrics_or_logs_window", "recurrence_fingerprint", ] if target_kind == "k8s_workload": evidence_refs.extend(["k8s_events", "rollout_status"]) elif target_kind == "host_service": evidence_refs.extend(["systemd_or_container_status", "host_resource_window"]) elif target_kind == "database": evidence_refs.extend([ "postgres_readonly_activity", "postgres_lock_waits", "pg_stat_statements_top_queries", "connection_pool_window", "recent_deploy_or_migration_refs", "backup_freshness_before_write_candidate", ]) return { "schema_version": "repair_candidate_coverage_gap_v1", "coverage_key": coverage_key, "target_kind": target_kind, "blocking_stage": blocking_stage, "next_owner_lane": lane, "alertname": alertname or None, "namespace": namespace or None, "target_resource": target_resource or None, "matched_playbook_id": playbook.playbook_id if playbook else None, "evidence_snapshot_id": evidence.snapshot_id if evidence else None, "mcp_evidence_ready": bool(evidence and evidence.sensors_succeeded > 0), "required_mcp_evidence_refs": list(dict.fromkeys(evidence_refs)), "playbook_template_required": True, "playbook_template_fields": [ "symptom_pattern.alert_names", "symptom_pattern.affected_services", "mcp_evidence_refs", "repair_steps.command_or_ansible_ref", "repair_steps.rollback_command", "verifier_plan", "owner_review_record", "trust_score_update_policy", "script_or_ansible_ref", "schedule_or_alert_rule_ref", "automation_asset_record", "dashboard_visibility_refs", ], "blocked_operations": [ "auto_execute", "mark_repaired_without_execution", "approve_no_action_as_repair", *( [ "terminate_backend_without_owner_review", "postgres_restart_without_maintenance_window", "migration_or_reindex_without_rollback", ] if target_kind == "database" else [] ), ], "runtime_execution_authorized": False, "writes_runtime_state": False, } def _coverage_key(self, *, alertname: str, target_resource: str) -> str: raw_alert = (alertname or "unknown-alert").strip().lower() raw_target = (target_resource or "unknown-target").strip().lower() safe_alert = "-".join(raw_alert.replace("_", "-").split()) safe_target = "-".join(raw_target.replace("_", "-").split()) return f"{safe_alert}:{safe_target}" def _infer_target_kind(self, *, target_resource: str, namespace: str) -> str: target = (target_resource or "").lower() control_plane_context = f"{target} {(namespace or '').lower()}" if any( marker in control_plane_context for marker in ( "cold-start", "cold_start", "reboot-auto-recovery", "reboot_auto_recovery", "provider_freshness", "alert-chain", "alert_chain", "ci/cd", "cicd", "build-and-deploy", "deployment failed", "部署失敗", ) ): return "control_plane" if any(marker in target for marker in ("postgres", "postgresql", "pgbouncer", "database", "db-")): return "database" if any(marker in target for marker in ("node-exporter", "host", "188", "111", "168")): return "host_service" if namespace or any(marker in target for marker in ("deployment/", "pod/", "svc/", "api")): return "k8s_workload" if target: return "service" return "unknown" def _build_description( self, *, message: str, playbook: Playbook, step_command: str, evidence: EvidenceSnapshot | None, verifier_plan: list[str], ) -> str: evidence_line = "MCP evidence missing" if evidence: evidence_line = ( f"MCP evidence {evidence.sensors_succeeded}/{evidence.sensors_attempted}; " f"snapshot={evidence.snapshot_id}" ) verifier_text = "; ".join(verifier_plan) return ( "LLM fallback 後由 MCP evidence + PlayBook trust 產生修復候選。\n" f"原始告警：{message[:500]}\n" f"PlayBook：{playbook.playbook_id} / {playbook.name} / trust={playbook.trust_score:.2f}\n" f"證據：{evidence_line}\n" f"候選命令：{step_command}\n" f"Verifier plan：{verifier_text}\n" "注意：這只是 approval candidate；未經批准不得執行。" ) def _build_dry_run_checks( self, *, evidence: EvidenceSnapshot | None, playbook: Playbook, command: str, ) -> list[DryRunCheck]: sensors_attempted = evidence.sensors_attempted if evidence else 0 sensors_succeeded = evidence.sensors_succeeded if evidence else 0 return [ DryRunCheck( name="MCP evidence collected", passed=sensors_succeeded > 0, message=f"{sensors_succeeded}/{sensors_attempted} sensors succeeded", ), DryRunCheck( name="PlayBook approved and trusted", passed=( playbook.status == PlaybookStatus.APPROVED and float(playbook.trust_score) >= MIN_REPAIR_CANDIDATE_TRUST ), message=f"{playbook.playbook_id} trust={playbook.trust_score:.2f}", ), DryRunCheck( name="Command safety gate", passed=True, message=command[:240], ), DryRunCheck( name="Post execution verifier planned", passed=True, message="verify health/evidence after execution before closing incident", ), ] def _build_verifier_plan( self, *, command: str, namespace: str, target_resource: str, evidence: EvidenceSnapshot | None, ) -> list[str]: plan = [ "rerun_pre_decision_evidence_after_execution", "compare_mcp_sensor_success_before_after", ] if command.startswith("kubectl"): plan.append(f"kubectl rollout status/get events in {namespace or 'awoooi-prod'}") if command.startswith("ssh "): plan.append("ssh_mcp_readonly_health_check_after_write_route") if target_resource: plan.append(f"confirm target_resource={target_resource} alert stops recurring") if evidence and evidence.snapshot_id: plan.append(f"baseline_snapshot={evidence.snapshot_id}") return plan def _approval_risk(self, risk: PlaybookRiskLevel) -> ApprovalRiskLevel: try: mapped = ApprovalRiskLevel(risk.value.lower()) except Exception: return ApprovalRiskLevel.MEDIUM if mapped == ApprovalRiskLevel.LOW: return ApprovalRiskLevel.MEDIUM return mapped _repair_candidate_service: RepairCandidateService | None = None def get_repair_candidate_service() -> RepairCandidateService: """Return singleton candidate builder.""" global _repair_candidate_service if _repair_candidate_service is None: _repair_candidate_service = RepairCandidateService() return _repair_candidate_service