From 4667a86c6d983e5a729f9988d6dc1b8f5cb90ffd Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 2 Jun 2026 10:04:25 +0800 Subject: [PATCH] feat(adr100): surface replay execution gate --- .../services/adr100_remediation_service.py | 165 ++++++++++++++++++ apps/api/src/services/auto_repair_service.py | 24 +++ .../tests/test_adr100_remediation_service.py | 72 +++++++- apps/web/messages/en.json | 7 + apps/web/messages/zh-TW.json | 7 + .../app/[locale]/awooop/work-items/page.tsx | 29 +++ docs/LOGBOOK.md | 59 +++++++ 7 files changed, 362 insertions(+), 1 deletion(-) diff --git a/apps/api/src/services/adr100_remediation_service.py b/apps/api/src/services/adr100_remediation_service.py index 82cf88339..a98094f58 100644 --- a/apps/api/src/services/adr100_remediation_service.py +++ b/apps/api/src/services/adr100_remediation_service.py @@ -65,6 +65,7 @@ class Adr100RemediationService: slo_service: Adr100SloStatusService | None = None, incident_repository: _IncidentRepository | None = None, auto_repair_service: AutoRepairService | None = None, + playbook_service: Any | None = None, verifier: PostExecutionVerifier | None = None, approval_service: Any | None = None, timeline_service: Any | None = None, @@ -74,6 +75,7 @@ class Adr100RemediationService: self._slo_service = slo_service or get_adr100_slo_status_service() self._incident_repository = incident_repository or IncidentDBRepository() self._auto_repair_service = auto_repair_service or AutoRepairService() + self._playbook_service = playbook_service self._verifier = verifier or get_post_execution_verifier() self._approval_service = approval_service self._timeline_service = timeline_service @@ -323,6 +325,7 @@ class Adr100RemediationService: }) post_state = await self._collect_current_state(incident) + replay_gate = await self._build_replay_gate(item, incident) action_taken = f"dry_run_replay:{item.get('playbook_id') or 'unknown'}" result = _assess_recovery(None, post_state, action_taken) @@ -335,12 +338,125 @@ class Adr100RemediationService: extra={ "diagnostic_command_preview": diagnostic_command, "mcp_route": route, + "replay_gate": replay_gate, "promql": _promql_for_incident(incident), }, ) payload["history"] = await self._record_dry_run_history(item, payload) return payload + async def _build_replay_gate( + self, + item: dict[str, Any], + incident: Incident, + ) -> dict[str, Any]: + playbook_id = str(item.get("playbook_id") or "") + base: dict[str, Any] = { + "schema_version": "adr100_replay_gate_v1", + "playbook_id": playbook_id or None, + "playbook_loaded": False, + "status": "blocked_missing_playbook_id", + "execution_authorized": False, + "repair_executed": False, + "writes_incident_state": False, + "writes_auto_repair_result": False, + "can_runtime_replay": False, + "mutating_step_count": 0, + "supported_write_route_count": 0, + "unsupported_step_count": 0, + "approval_required_count": 0, + "next_step": "attach_playbook_to_work_item", + "steps": [], + } + if not playbook_id: + return base + + playbook_svc = self._playbook_service + if playbook_svc is None: + from src.services.playbook_service import get_playbook_service + + playbook_svc = get_playbook_service() + + playbook = await playbook_svc.get_by_id(playbook_id) + if playbook is None: + return { + **base, + "status": "blocked_playbook_not_found", + "next_step": "reload_playbook_catalog_or_author_supported_executor", + } + + steps: list[dict[str, Any]] = [] + mutating_count = 0 + supported_write_count = 0 + unsupported_count = 0 + approval_required_count = 0 + + for step in playbook.repair_steps: + command = str(getattr(step, "command", "") or "") + action_type = _step_action_type(step) + write_route = None + if action_type == "ssh_command": + write_route = self._auto_repair_service.preview_write_ssh_mcp_route( + incident, + command, + ) + read_route = self._auto_repair_service.preview_read_only_ssh_mcp_route( + incident, + command, + ) if action_type == "ssh_command" else None + is_mutating = bool(write_route) or _step_looks_mutating(step) + requires_approval = bool(getattr(step, "requires_approval", False)) + if is_mutating: + mutating_count += 1 + if write_route: + supported_write_count += 1 + else: + unsupported_count += 1 + if requires_approval: + approval_required_count += 1 + + steps.append({ + "step_number": getattr(step, "step_number", None), + "action_type": action_type, + "risk_level": _step_risk_level(step), + "requires_approval": requires_approval, + "is_mutating": is_mutating, + "write_route": write_route, + "read_route": read_route, + "supported": bool(write_route) or not is_mutating, + "command_preview": _compact_command(command), + }) + + if mutating_count == 0: + status = "blocked_observe_only_playbook" + next_step = "author_mutating_repair_step" + can_runtime_replay = False + elif unsupported_count > 0: + status = "blocked_unsupported_write_route" + next_step = "author_supported_executor_step" + can_runtime_replay = False + elif approval_required_count > 0: + status = "approval_required" + next_step = "request_runtime_replay_approval" + can_runtime_replay = False + else: + status = "runtime_replay_ready" + next_step = "queue_runtime_replay_with_gate5_projection" + can_runtime_replay = True + + return { + **base, + "playbook_loaded": True, + "status": status, + "can_runtime_replay": can_runtime_replay, + "mutating_step_count": mutating_count, + "supported_write_route_count": supported_write_count, + "unsupported_step_count": unsupported_count, + "approval_required_count": approval_required_count, + "next_step": next_step, + "steps": steps, + } + async def _dry_run_ticket_proposal( self, item: dict[str, Any], @@ -842,6 +958,48 @@ def _summarize_post_state(post_state: dict[str, Any]) -> dict[str, Any]: } +def _step_action_type(step: Any) -> str: + action_type = getattr(step, "action_type", None) + return str(getattr(action_type, "value", action_type) or "unknown") + + +def _step_risk_level(step: Any) -> str: + risk_level = getattr(step, "risk_level", None) + return str(getattr(risk_level, "value", risk_level) or "unknown").lower() + + +def _step_looks_mutating(step: Any) -> bool: + command = str(getattr(step, "command", "") or "").strip().lower() + action_type = _step_action_type(step) + if not command or action_type == "manual": + return False + if action_type == "script": + return True + if action_type == "kubectl": + return not command.startswith(( + "kubectl get ", + "kubectl describe ", + "kubectl logs ", + "kubectl top ", + "kubectl explain ", + )) + if action_type == "ssh_command": + return any(token in command for token in ( + "docker restart", + "docker start", + "docker stop", + "systemctl restart", + "systemctl start", + "certbot renew", + )) + return False + + +def _compact_command(command: str) -> str: + compact = " ".join(str(command or "").split()) + return compact[:240] + + def _history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]: return { "schema_version": "adr100_remediation_dry_run_history_v1", @@ -862,6 +1020,7 @@ def _history_context(item: dict[str, Any], payload: dict[str, Any]) -> dict[str, "verification_result_preview": payload.get("verification_result_preview"), "post_state_summary": payload.get("post_state_summary"), "mcp_route": payload.get("mcp_route"), + "replay_gate": payload.get("replay_gate"), "checks": payload.get("checks"), } @@ -938,6 +1097,7 @@ def _history_item(record: Any, context: dict[str, Any]) -> dict[str, Any]: route = context.get("mcp_route") or {} post_state = context.get("post_state_summary") or {} approval = context.get("approval") or {} + replay_gate = context.get("replay_gate") or {} return { "id": str(getattr(record, "id", "")), "incident_id": getattr(record, "incident_id", None), @@ -971,6 +1131,9 @@ def _history_item(record: Any, context: dict[str, Any]) -> dict[str, Any]: "deduplicated": context.get("deduplicated"), "fingerprint": context.get("fingerprint"), "ticket_preview": context.get("ticket_preview"), + "replay_gate": replay_gate or None, + "replay_gate_status": replay_gate.get("status"), + "replay_gate_next_step": replay_gate.get("next_step"), "plan": context.get("plan"), "checks": context.get("checks") or [], } @@ -993,6 +1156,8 @@ def _summarize_history_by_work_item(items: list[dict[str, Any]]) -> list[dict[st "latest_agent_id": item.get("agent_id"), "latest_tool_name": item.get("tool_name"), "required_scope": item.get("required_scope"), + "latest_replay_gate_status": item.get("replay_gate_status"), + "latest_replay_gate_next_step": item.get("replay_gate_next_step"), } summary[key]["count"] += 1 return list(summary.values()) diff --git a/apps/api/src/services/auto_repair_service.py b/apps/api/src/services/auto_repair_service.py index 35dfa9bc0..e6fa6a318 100644 --- a/apps/api/src/services/auto_repair_service.py +++ b/apps/api/src/services/auto_repair_service.py @@ -1237,6 +1237,30 @@ class AutoRepairService: "flywheel_node": "execute", } + def preview_write_ssh_mcp_route( + self, + incident: Incident, + command: str, + ) -> dict[str, Any] | None: + """Preview whether a legacy SSH repair can use the write MCP Gateway. + + This mirrors the executor's narrow Docker restart routing without + projecting a Gate 5 approval or mutating any runtime state. It lets + ADR-100 Work Items show whether ``ready_for_replay`` is actually ready + for a governed write executor. + """ + + route = self._route_legacy_ssh_write_command_to_mcp(incident, command) + if route is None: + return None + return { + "tool_name": route.tool_name, + "params": route.params, + "agent_id": "auto_repair_executor", + "required_scope": route.required_scope, + "flywheel_node": "execute", + } + def _resolve_ssh_host_for_incident(self, incident: Incident, command: str) -> str: """Resolve ``{host}``, short host labels, and exporter instance ports.""" diff --git a/apps/api/tests/test_adr100_remediation_service.py b/apps/api/tests/test_adr100_remediation_service.py index 7741cdb2f..910116d13 100644 --- a/apps/api/tests/test_adr100_remediation_service.py +++ b/apps/api/tests/test_adr100_remediation_service.py @@ -11,7 +11,12 @@ from fastapi.testclient import TestClient from src.api.v1.ai_slo import router from src.models.approval import ApprovalRequest, ApprovalStatus, RiskLevel from src.models.incident import Incident, IncidentStatus, Severity, Signal -from src.models.playbook import Playbook +from src.models.playbook import ( + ActionType, + Playbook, + RepairStep, + RiskLevel as PlaybookRiskLevel, +) from src.services.adr100_remediation_service import ( Adr100RemediationService, RemediationNotFoundError, @@ -152,6 +157,16 @@ class _NoopPlaybookService: return True +class _FakePlaybookService(_NoopPlaybookService): + def __init__(self, playbook: Playbook | None) -> None: + self.playbook = playbook + + async def get_by_id(self, playbook_id: str) -> Playbook | None: + if self.playbook and self.playbook.playbook_id == playbook_id: + return self.playbook + return None + + async def _no_cooldown(*_args, **_kwargs) -> tuple[bool, str]: # noqa: ANN002, ANN003 return True, "test" @@ -201,6 +216,7 @@ def _service( item: dict[str, Any], incident: Incident | None = None, state: dict[str, Any] | None = None, + playbook_service: Any | None = None, approval_service: Any | None = None, timeline_service: Any | None = None, alert_operation_log_repository: Any | None = None, @@ -213,6 +229,7 @@ def _service( playbook_service=_NoopPlaybookService(), cooldown_checker=_no_cooldown, ), + playbook_service=playbook_service, verifier=_FakeVerifier(state or {"k8s_get_pod_status": {"phase": "Running"}}), approval_service=approval_service, timeline_service=timeline_service, @@ -221,6 +238,24 @@ def _service( ) +def _runtime_replay_playbook() -> Playbook: + return Playbook( + playbook_id="PB-1", + name="Docker container restart", + description="Replay Docker restart through governed MCP write route.", + repair_steps=[ + RepairStep( + step_number=1, + action_type=ActionType.SSH_COMMAND, + command="ssh {host} 'docker restart {container}'", + expected_result="container restarted", + requires_approval=False, + risk_level=PlaybookRiskLevel.MEDIUM, + ) + ], + ) + + @pytest.mark.asyncio async def test_preview_marks_replay_work_item_read_only(): svc = _service(item=_queue_item()) @@ -407,6 +442,41 @@ async def test_dry_run_replay_validates_supported_executor_route(): assert result["mcp_route"]["params"]["host"] == "192.168.0.110" assert result["mcp_route"]["params"]["container_name"] == "momo-scheduler" assert result["diagnostic_command_preview"].startswith("ssh 110") + assert result["replay_gate"]["status"] == "blocked_playbook_not_found" + assert result["replay_gate"]["can_runtime_replay"] is False + + +@pytest.mark.asyncio +async def test_dry_run_replay_surfaces_runtime_write_gate_without_executing(): + alert_repo = _FakeAlertOperationLogRepository() + svc = _service( + item=_queue_item(), + playbook_service=_FakePlaybookService(_runtime_replay_playbook()), + alert_operation_log_repository=alert_repo, + record_history=True, + ) + + result = await svc.dry_run("verification:INC-20260514-TEST01:are-1") + + assert result["allowed"] is True + assert result["mode"] == "replay" + assert result["writes_incident_state"] is False + assert result["writes_auto_repair_result"] is False + gate = result["replay_gate"] + assert gate["schema_version"] == "adr100_replay_gate_v1" + assert gate["status"] == "runtime_replay_ready" + assert gate["can_runtime_replay"] is True + assert gate["execution_authorized"] is False + assert gate["repair_executed"] is False + assert gate["mutating_step_count"] == 1 + assert gate["supported_write_route_count"] == 1 + assert gate["unsupported_step_count"] == 0 + assert gate["next_step"] == "queue_runtime_replay_with_gate5_projection" + assert gate["steps"][0]["write_route"]["tool_name"] == "ssh_docker_restart" + assert gate["steps"][0]["write_route"]["required_scope"] == "write" + assert gate["steps"][0]["write_route"]["params"]["host"] == "192.168.0.110" + assert gate["steps"][0]["write_route"]["params"]["container_name"] == "momo-scheduler" + assert alert_repo.calls[0]["context"]["replay_gate"]["status"] == "runtime_replay_ready" @pytest.mark.asyncio diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index d8c0e2627..e1ae54bc8 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -2993,6 +2993,13 @@ "approvalWrite": "已建立審批紀錄", "deduplicated": "已收斂既有審批", "approval": "審批 {id} / {status} / {risk}" + }, + "replayGate": { + "title": "Replay Gate", + "status": "狀態={value}", + "next": "下一步={value}", + "routes": "write route={write} / unsupported={unsupported}", + "auth": "authorized={authorized} / executed={executed}" } }, "callbackTraceRecoveryActions": { diff --git a/apps/web/messages/zh-TW.json b/apps/web/messages/zh-TW.json index d8c0e2627..e1ae54bc8 100644 --- a/apps/web/messages/zh-TW.json +++ b/apps/web/messages/zh-TW.json @@ -2993,6 +2993,13 @@ "approvalWrite": "已建立審批紀錄", "deduplicated": "已收斂既有審批", "approval": "審批 {id} / {status} / {risk}" + }, + "replayGate": { + "title": "Replay Gate", + "status": "狀態={value}", + "next": "下一步={value}", + "routes": "write route={write} / unsupported={unsupported}", + "auth": "authorized={authorized} / executed={executed}" } }, "callbackTraceRecoveryActions": { diff --git a/apps/web/src/app/[locale]/awooop/work-items/page.tsx b/apps/web/src/app/[locale]/awooop/work-items/page.tsx index bf6c93045..779d42753 100644 --- a/apps/web/src/app/[locale]/awooop/work-items/page.tsx +++ b/apps/web/src/app/[locale]/awooop/work-items/page.tsx @@ -217,6 +217,17 @@ type RecurrenceWorkItemActionResult = { labels?: string[] | null; body_preview?: string | null; } | null; + replay_gate?: { + status?: string | null; + next_step?: string | null; + can_runtime_replay?: boolean | null; + execution_authorized?: boolean | null; + repair_executed?: boolean | null; + mutating_step_count?: number | null; + supported_write_route_count?: number | null; + unsupported_step_count?: number | null; + approval_required_count?: number | null; + } | null; approval_id?: string | null; approval?: { id?: string | null; @@ -2765,6 +2776,7 @@ function Adr100RemediationQueuePanel({ const result = state?.result ?? null; const ticketPreview = result?.ticket_preview ?? null; const approval = result?.approval ?? null; + const replayGate = result?.replay_gate ?? null; const canCreateApproval = item.remediation_status === "needs_playbook_ticket" || item.remediation_action === "promote_diagnostic_to_repair_playbook"; return ( @@ -2876,6 +2888,23 @@ function Adr100RemediationQueuePanel({

) : null} + {replayGate ? ( +
+

{t("replayGate.title")}

+
+ {t("replayGate.status", { value: replayGate.status ?? "--" })} + {t("replayGate.next", { value: replayGate.next_step ?? "--" })} + {t("replayGate.routes", { + write: replayGate.supported_write_route_count ?? 0, + unsupported: replayGate.unsupported_step_count ?? 0, + })} + {t("replayGate.auth", { + authorized: String(replayGate.execution_authorized ?? false), + executed: String(replayGate.repair_executed ?? false), + })} +
+
+ ) : null} ) : null} diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index c85c4f012..2d53769d8 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,62 @@ +## 2026-06-02｜ADR-100 ready-for-replay Replay Gate 可視化 + +**背景**： + +- Production `/api/v1/ai/slo` 目前仍有 ADR-100 補救工作 `8` 筆，其中 `ready_for_replay` 有 `2` 筆：`INC-20260601-1B3388` / `GiteaDown` 與 `INC-20260601-B51DFD` / `DockerContainerUnhealthy`。 +- 既有 Work Items 只能做 read-only dry-run，operator 看得到 `mcp:ssh_diagnose`，但看不到下一步是否已具備 write executor、是否要 approval、是否真的會寫 `auto_repair_executions`。 +- 本階段不直接執行 runtime repair、不更新 incident state、不回填舊 auto-repair result，避免把 replay preflight 誤宣稱為 `verified_success`。 + +**Production 現況驗證（變更前）**： + +- `verification:INC-20260601-1B3388:7d73ac5f-6224-4fa4-8ec8-96622b51f739` dry-run： + - `mode=replay`、`allowed=true`、`executed=true`、`verification_result_preview=degraded` + - `mcp_route=auto_repair_executor/ssh_diagnose/read` + - `writes_incident_state=false`、`writes_auto_repair_result=false` + - history recorded：`alert_operation_id=df52f951-a68c-49e5-844d-72c64ec80999`、`timeline_event_id=583e25be-a9b8-469a-b171-a92d822bef6b` +- `verification:INC-20260601-B51DFD:c9635db3-ec54-405f-a909-7e6371775676` dry-run： + - `mode=replay`、`allowed=true`、`executed=true`、`verification_result_preview=failed` + - `mcp_route=auto_repair_executor/ssh_diagnose/read` + - current-state tools：`prometheus_query`、`prometheus_query_range`、`ssh_diagnose`、`ssh_get_container_status`、`ssh_get_top_processes` + - history recorded：`alert_operation_id=ad847006-d2d2-4d0a-b68c-0c13e09d4927`、`timeline_event_id=882030aa-7b00-45e8-96b5-9d16a5e51640` + +**本次調整**： + +- `apps/api/src/services/auto_repair_service.py`： + - 新增 `preview_write_ssh_mcp_route()`，只預覽 legacy SSH repair step 是否可走 `ssh_docker_restart/write` MCP Gateway。 + - 不投影 Gate 5 approval、不呼叫 MCP、不執行 Docker restart、不寫 DB。 +- `apps/api/src/services/adr100_remediation_service.py`： + - replay dry-run 新增 `replay_gate`，讀取 PlayBook repair steps 並判斷： + - `runtime_replay_ready` + - `approval_required` + - `blocked_playbook_not_found` + - `blocked_observe_only_playbook` + - `blocked_unsupported_write_route` + - `replay_gate` 會寫入 dry-run history context，讓 AwoooP history/API/frontend 可追同一個 gate 判斷。 + - payload 明確保留 `execution_authorized=false`、`repair_executed=false`、`writes_incident_state=false`、`writes_auto_repair_result=false`。 +- `apps/web/src/app/[locale]/awooop/work-items/page.tsx`： + - ADR-100 補救工作結果新增 `Replay Gate` 區塊，顯示 status、next step、write route/unsupported 數量，以及 authorized/executed 狀態。 +- `apps/web/messages/zh-TW.json` / `en.json`： + - 新增 `replayGate` 文案；英文語系維持繁中文案鏡像。 + +**驗證**： + +- `python3 -m py_compile apps/api/src/services/adr100_remediation_service.py apps/api/src/services/auto_repair_service.py apps/api/tests/test_adr100_remediation_service.py` +- `DATABASE_URL=postgresql://test:test@localhost:5432/test PYTHONPATH=apps/api /Users/ogt/.pyenv/shims/pytest apps/api/tests/test_adr100_remediation_service.py -q` → `13 passed` +- `DATABASE_URL=postgresql://test:test@localhost:5432/test PYTHONPATH=apps/api /Users/ogt/.pyenv/shims/pytest apps/api/tests/test_auto_repair_service.py -q` → `29 passed` +- `python3 -m json.tool apps/web/messages/zh-TW.json` / `apps/web/messages/en.json` / `cmp -s` +- `pnpm install --offline --frozen-lockfile`（clean worktree dependency setup only） +- `pnpm --dir apps/web exec tsc --noEmit --tsBuildInfoFile /tmp/awoooi-ready-replay-gate.tsbuildinfo` +- `NEXT_PUBLIC_API_URL=https://awoooi.wooo.work NEXT_PRIVATE_BUILD_WORKER_COUNT=1 pnpm --dir apps/web run build` +- `git diff --check` +- `python3 scripts/security/security-mirror-progress-guard.py --root .` → `SECURITY_MIRROR_PROGRESS_GUARD_OK` + +**目前整體進度（本階段完成後）**： + +- ADR-100 非成功驗證補救工作項：約 `95%`；`ready_for_replay` 已從只看 read-only dry-run，提升到可判斷 write replay gate / blocked reason / next step。 +- Approval / execution 誠實度：約 `95%`；前後端會明確顯示 replay 尚未 authorized/executed，也不寫 incident 或 auto-repair result。 +- 真正 verified auto-repair 成功樣本：仍約 `3-4%`；本階段沒有執行 runtime repair，不能上修這項。 +- 完整 AI 自動化飛輪總進度：維持 `61%`；下一個上調條件是 `ready_for_replay` 進入受控 write execution，並產生 production `verification_result=success` + KM / learning 回寫。 + ## 2026-06-01｜IwoooS S4.9 補件草稿詳情層落地 **背景**：