diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 0a9192b4d..3427d12f7 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -112,6 +112,34 @@ _WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION = ( "build_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_" "then_rerun_no_secret_collector_no_vm_power_change" ) +_WINDOWS99_REQUIRED_VM_ALIASES = ["111", "188", "120", "121", "112"] +_WINDOWS99_EXPECTED_VMX_CANDIDATES = { + "111": [ + r"D:\Documents\Virtual Machines\192.168.0.111_Ubuntu_64-bit\192.168.0.111_Ubuntu_64-bit.vmx", + ], + "188": [ + r"D:\Documents\Virtual Machines\Ollama_Ubuntu_64-bit\Ollama_Ubuntu_64-bit.vmx", + ], + "120": [ + r"D:\Documents\Virtual Machines\192.168.0.120_Ubuntu_64-bit\192.168.0.120_Ubuntu_64-bit.vmx", + ], + "121": [ + r"D:\Documents\Virtual Machines\192.168.0.121_Ubuntu_64-bit\192.168.0.121_Ubuntu_64-bit.vmx", + ], + "112": [ + r"D:\Downloads\kali-linux-2025.4-vmware-amd64\kali-linux-2025.4-vmware-amd64.vmwarevm\kali-linux-2025.4-vmware-amd64.vmx", + ], +} +_WINDOWS99_VMX_REPAIR_FORBIDDEN_ACTIONS = [ + "read_windows_password_or_secret", + "vm_power_change", + "vmrun_start_or_stop", + "host_reboot", + "windows_service_restart", + "windows_registry_apply", + "scheduled_task_register_or_modify", + "docker_nginx_k3s_db_firewall_restart", +] def load_latest_reboot_auto_recovery_slo_scorecard( @@ -771,6 +799,12 @@ def _apply_prometheus_windows99_vmware_readback( rollups["windows99_verify_collection_blocker_count"] = len( _strings(collection.get("collection_blockers")) ) + _sync_windows99_vmx_source_repair_package( + payload, + generated_at=str( + payload.get("runtime_readback_generated_at") or _taipei_now_iso() + ), + ) def _reconcile_prometheus_metric_active_blockers_with_source_controls( @@ -1643,6 +1677,12 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: windows99_management=windows99_management, windows99_collector=windows99_collector, ) + windows99_vmx_source_repair_package = ( + _build_windows99_vmx_source_repair_package( + windows99, + generated_at=runtime_readback_generated_at, + ) + ) blocked_by_fresh_reboot_window_only = active_blockers == [ "host_boot_observation_older_than_target_window" ] @@ -1769,6 +1809,22 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "windows99_vmware_powered_off_count": len( _strings(windows99.get("powered_off_aliases")) ), + "windows99_vmx_source_repair_package_ready": ( + windows99_vmx_source_repair_package["package_ready"] is True + ), + "windows99_vmx_source_repair_package_status": str( + windows99_vmx_source_repair_package["status"] + ), + "windows99_vmx_source_repair_package_safe_next_step": str( + windows99_vmx_source_repair_package["safe_next_step"] + ), + "windows99_vmx_source_repair_package_apply_allowed": ( + windows99_vmx_source_repair_package["apply_allowed_by_this_package"] + is True + ), + "windows99_vmx_source_repair_missing_vmx_aliases": _strings( + windows99_vmx_source_repair_package.get("missing_vmx_aliases") + ), "windows99_verify_collection_status": windows99_verify_collection["status"], "windows99_verify_collection_can_collect_no_secret": ( windows99_verify_collection["can_collect_no_secret_verify"] is True @@ -1960,6 +2016,21 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "windows99_update_no_auto_reboot_ready": rollups[ "windows99_update_no_auto_reboot_ready" ], + "windows99_vmx_source_repair_package_ready": rollups[ + "windows99_vmx_source_repair_package_ready" + ], + "windows99_vmx_source_repair_package_status": rollups[ + "windows99_vmx_source_repair_package_status" + ], + "windows99_vmx_source_repair_package_safe_next_step": rollups[ + "windows99_vmx_source_repair_package_safe_next_step" + ], + "windows99_vmx_source_repair_package_apply_allowed": rollups[ + "windows99_vmx_source_repair_package_apply_allowed" + ], + "windows99_vmx_source_repair_missing_vmx_aliases": rollups[ + "windows99_vmx_source_repair_missing_vmx_aliases" + ], "windows99_verify_collection_status": rollups[ "windows99_verify_collection_status" ], @@ -2015,6 +2086,7 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "public_maintenance_fallback": public_maintenance, "windows99_vmware_autostart": windows99, "windows99_vmware_verify_collector": windows99_collector, + "windows99_vmx_source_repair_package": windows99_vmx_source_repair_package, "windows99_verify_collection": windows99_verify_collection, "windows99_management_channel": windows99_management, "source_controls": source_controls, @@ -2357,6 +2429,132 @@ def _build_windows99_verify_collection_packet( } +def _powershell_test_path(path: str) -> str: + escaped = path.replace("'", "''") + return f"Test-Path -LiteralPath '{escaped}'" + + +def _build_windows99_vmx_source_repair_package( + windows99: dict[str, Any], + *, + generated_at: str, +) -> dict[str, Any]: + missing_vmx_aliases = _unique_strings( + _strings(windows99.get("missing_vmx_aliases")) + ) + powered_off_aliases = _unique_strings( + _strings(windows99.get("powered_off_aliases")) + ) + required_aliases = _unique_strings( + _strings(windows99.get("required_vm_aliases")) + or _WINDOWS99_REQUIRED_VM_ALIASES + ) + package_ready = bool(missing_vmx_aliases or powered_off_aliases) + if missing_vmx_aliases: + status = "check_mode_package_ready_windows99_vmx_source_repair_required" + safe_next_step = _WINDOWS99_MISSING_VMX_NEXT_SAFE_ACTION + elif powered_off_aliases: + status = "check_mode_package_ready_windows99_power_check_required" + safe_next_step = _WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION + else: + status = "ready_no_windows99_vmx_source_repair_required" + safe_next_step = "rerun_no_secret_collector_and_reboot_slo_scorecard_verify_only" + + rows: list[dict[str, Any]] = [] + for alias in required_aliases: + candidates = _WINDOWS99_EXPECTED_VMX_CANDIDATES.get(alias, []) + rows.append( + { + "alias": alias, + "missing_vmx_source": alias in missing_vmx_aliases, + "powered_off": alias in powered_off_aliases, + "expected_vmx_candidates": candidates, + "check_mode_probe_commands": [ + _powershell_test_path(candidate) for candidate in candidates + ], + "controlled_restore_required": alias in missing_vmx_aliases, + "controlled_restore_hint": ( + "restore_or_relink_existing_vmx_source_to_expected_path_then_verify" + if alias in missing_vmx_aliases + else "" + ), + } + ) + + return { + "schema_version": "windows99_vmx_source_repair_check_mode_package_v1", + "generated_at": generated_at, + "status": status, + "package_ready": package_ready, + "target_host_alias": "99", + "missing_vmx_aliases": missing_vmx_aliases, + "powered_off_aliases": powered_off_aliases, + "required_vm_aliases": required_aliases, + "vm_rows": rows, + "safe_next_step": safe_next_step, + "controlled_apply_mode": "check_mode_package_only_no_windows_or_vm_write", + "apply_allowed_by_this_package": False, + "post_verifier": [ + "bash scripts/reboot-recovery/collect-windows99-vmware-verify.sh --collect", + "bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh --verify-only", + "GET /api/v1/agents/reboot-auto-recovery-slo-scorecard", + ], + "forbidden_actions": _WINDOWS99_VMX_REPAIR_FORBIDDEN_ACTIONS, + "operation_boundaries": { + "secret_value_read": False, + "remote_write_performed": False, + "vm_power_change_performed": False, + "windows_service_restart_performed": False, + "windows_registry_apply_performed": False, + "host_reboot_performed": False, + }, + } + + +def _sync_windows99_vmx_source_repair_package( + payload: dict[str, Any], + *, + generated_at: str, +) -> dict[str, Any]: + windows99 = _dict(payload.get("windows99_vmware_autostart")) + package = _build_windows99_vmx_source_repair_package( + windows99, + generated_at=generated_at, + ) + payload["windows99_vmx_source_repair_package"] = package + + readback = _dict(payload.setdefault("readback", {})) + readback["windows99_vmx_source_repair_package_ready"] = ( + package["package_ready"] is True + ) + readback["windows99_vmx_source_repair_package_status"] = str(package["status"]) + readback["windows99_vmx_source_repair_package_safe_next_step"] = str( + package["safe_next_step"] + ) + readback["windows99_vmx_source_repair_package_apply_allowed"] = ( + package["apply_allowed_by_this_package"] is True + ) + readback["windows99_vmx_source_repair_missing_vmx_aliases"] = _strings( + package.get("missing_vmx_aliases") + ) + + rollups = _dict(payload.setdefault("rollups", {})) + rollups["windows99_vmx_source_repair_package_ready"] = ( + package["package_ready"] is True + ) + rollups["windows99_vmx_source_repair_package_status"] = str(package["status"]) + rollups["windows99_vmx_source_repair_package_safe_next_step"] = str( + package["safe_next_step"] + ) + rollups["windows99_vmx_source_repair_package_apply_allowed"] = ( + package["apply_allowed_by_this_package"] is True + ) + rollups["windows99_vmx_source_repair_missing_vmx_aliases"] = _strings( + package.get("missing_vmx_aliases") + ) + return package + + def _host_row_by_alias( host_boot_detection: dict[str, Any], alias: str, diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 869ad10f1..60c545c63 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -440,6 +440,30 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( == WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS ) assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 2 + package = payload["windows99_vmx_source_repair_package"] + assert package["schema_version"] == ( + "windows99_vmx_source_repair_check_mode_package_v1" + ) + assert package["package_ready"] is True + assert package["apply_allowed_by_this_package"] is False + assert package["missing_vmx_aliases"] == ["111"] + assert package["safe_next_step"] == EXPECTED_WINDOWS99_MISSING_VMX_NEXT_STEP + vm_rows = {row["alias"]: row for row in package["vm_rows"]} + assert vm_rows["111"]["controlled_restore_required"] is True + assert vm_rows["111"]["check_mode_probe_commands"] == [ + ( + "Test-Path -LiteralPath 'D:\\Documents\\Virtual Machines\\" + "192.168.0.111_Ubuntu_64-bit\\192.168.0.111_Ubuntu_64-bit.vmx'" + ) + ] + assert payload["readback"]["windows99_vmx_source_repair_package_ready"] is True + assert ( + payload["readback"]["windows99_vmx_source_repair_package_apply_allowed"] + is False + ) + assert payload["rollups"]["windows99_vmx_source_repair_missing_vmx_aliases"] == [ + "111" + ] assert payload["service_green"] is True assert payload["backup_core_green"] is True assert payload["product_data_green"] is True diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index e841f14dc..608cb3e27 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,18 @@ +## 2026-07-03 — 10:58 Windows99 VMX repair package 接入 API scorecard + +**完成內容**： +- `reboot-auto-recovery-slo-scorecard` API readback 新增 `windows99_vmx_source_repair_package`，直接把目前 Windows99 `missing_vmx_aliases` / `powered_off_aliases` 轉成 machine-readable no-secret check-mode package。 +- package 同步投影到 `readback` 與 `rollups`：`windows99_vmx_source_repair_package_ready`、`windows99_vmx_source_repair_package_status`、`windows99_vmx_source_repair_package_safe_next_step`、`windows99_vmx_source_repair_package_apply_allowed`、`windows99_vmx_source_repair_missing_vmx_aliases`。 +- Prometheus/runtime Windows99 overlay 後會重建 package，避免 committed snapshot 與 live metric 不一致；目前 111 VMX missing 會輸出 expected VMX path 與 `Test-Path -LiteralPath ...` probe command。 +- package 固定 `apply_allowed_by_this_package=false`，只授權 source / verifier / check-mode package；不授權 VM power change、Windows service restart、registry apply、scheduled task modify、host reboot 或 secret/password 讀取。 + +**已跑驗證**： +- `python3.11 -m py_compile apps/api/src/services/reboot_auto_recovery_slo_scorecard.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py scripts/reboot-recovery/build-windows99-vmx-source-repair-package.py scripts/reboot-recovery/tests/test_windows99_vmx_source_repair_package.py`：通過。 +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py scripts/reboot-recovery/tests/test_windows99_vmx_source_repair_package.py -q -p no:cacheprovider`：`22 passed`。 + +**仍維持**： +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未啟動或關閉 VM；未重啟 host / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 10:53 Reboot SLO current readback anchor 校正 **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 124e9cf28..60fcaa7f1 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.112 +> Version: v1.113 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -56,6 +56,8 @@ v1.111 Gitea CD Docker step bounded-timeout rule：public queue 顯示 `cd.yaml` v1.112 current production SLO readback anchor：2026-07-03 10:50 production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 的目前事實是 `status=blocked_reboot_auto_recovery_slo_not_ready`、`active_blocker_count=10`、`readiness_percent=60`、`can_claim_all_services_recovered_within_target=false`、`primary_blocker=reboot_event_required_host_unreachable`。active blockers 固定為 `all_required_hosts_not_in_10_minute_reboot_window`、`fresh_all_host_reboot_event_missing`、`host_boot_observation_older_than_target_window`、`host_unreachable_after_reboot`、`host_uptime_unknown`、`reboot_event_required_host_unreachable`、`windows99_vmware_guest_power_not_ready`、`windows99_vmware_vmx_missing`、`windows99_vmware_autostart_config_not_ready`、`windows99_update_no_auto_reboot_policy_not_ready`；`runtime_metric_runtime_readback_added_blockers` 固定含 Windows99 config / policy 兩項，`windows99_update_no_auto_reboot_ready=false`。下一步固定為 `restore_windows99_missing_vmx_source_for_aliases_then_rerun_no_secret_collector_and_scorecard_no_vm_power_change`。不得再用 08:23 的 `8 / 67%` 當目前狀態；它只能保留為歷史 readback。此錨點仍只授權 source / verifier / check-mode package，不授權 VM power change、Windows service restart、registry apply、host reboot、Docker / Nginx / K3s / DB / firewall restart、restore、prune、delete 或 secret 讀取。 +v1.113 Windows99 VMX source repair package API rule：production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須直接輸出 `windows99_vmx_source_repair_package`，並在 `readback` / `rollups` 同步 `windows99_vmx_source_repair_package_ready`、`windows99_vmx_source_repair_package_status`、`windows99_vmx_source_repair_package_safe_next_step`、`windows99_vmx_source_repair_package_apply_allowed` 與 `windows99_vmx_source_repair_missing_vmx_aliases`。package 只從已讀回的 Windows99 verifier / Prometheus runtime overlay 建立，Prometheus overlay 後必須重建，避免 committed snapshot 與 live metric 不一致。111 VMX missing 時，package 必須列出 expected path `D:\Documents\Virtual Machines\192.168.0.111_Ubuntu_64-bit\192.168.0.111_Ubuntu_64-bit.vmx` 與 `Test-Path -LiteralPath ...` check-mode probe；`apply_allowed_by_this_package=false` 固定不變。此 package 是 work item / Telegram / UI 的 next-action input，不是 runtime apply 授權；不得從此 package 直接做 VM power change、Windows service restart、registry apply、scheduled task modify、host reboot 或 secret/password 讀取。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 35e594b5f..03a4d78c0 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.5 +> **版本**: v5.6 > **最後更新**: 2026-07-03 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-03 10:50 production SLO scorecard readback：Windows99 VMX / guest power / service config / Windows Update policy blocker 已上卷，10 分鐘 SLO 仍 blocked +> **觸發事件**: 2026-07-03 10:58 production SLO scorecard：Windows99 VMX source repair check-mode package 已接入 API readback，10 分鐘 SLO 仍 blocked --- @@ -44,7 +44,7 @@ - 188 可達，但 `systemd_state=degraded` 且 `awoooi-startup.service failed`。 - `reboot_detected=false`、`fresh_boot_hosts=[]`，因此 10 分鐘 SLO 尚未證明。 -固定下一步：`restore_windows99_missing_vmx_source_for_aliases_then_rerun_no_secret_collector_and_scorecard_no_vm_power_change`。先恢復 111 VMX source 與 99 no-secret management / console Verify stdout，使 `windows99-vmware-autostart.ps1 -Mode Verify` 產生 normalized artifact；同時補 111 reachability，再重跑 host probe、reboot-event detector、SLO scorecard。禁止讀 Windows 密碼、啟動 / 關閉 VM、host reboot、service restart、Docker / Nginx / K3s / DB / firewall restart、restore、prune、delete。 +固定下一步：`restore_windows99_missing_vmx_source_for_aliases_then_rerun_no_secret_collector_and_scorecard_no_vm_power_change`。先讀 production scorecard 的 `windows99_vmx_source_repair_package`，確認 111 expected VMX path 與 `Test-Path -LiteralPath ...` check-mode probe；再恢復 111 VMX source 與 99 no-secret management / console Verify stdout，使 `windows99-vmware-autostart.ps1 -Mode Verify` 產生 normalized artifact；同時補 111 reachability，再重跑 host probe、reboot-event detector、SLO scorecard。禁止讀 Windows 密碼、啟動 / 關閉 VM、host reboot、service restart、Docker / Nginx / K3s / DB / firewall restart、restore、prune、delete。 ### 五主機全貌 diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index c297756e8..78155cc05 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -59,7 +59,7 @@ | --- | --- | --- | --- | --- | --- | --- | | 1 | CIR-P0-RBT-001 | P0 | 「主機重啟後 10 分鐘內全部恢復，且要自動判斷所有主機被重啟」 | 建立 99/110/111/112/120/121/188 reboot event detector + 10 分鐘 SLO scorecard + fixed triage order | 2026-07-03 10:50 production scorecard：readiness `60%`、active blockers `10`、`can_claim_all_services_recovered_within_target=false`；blockers 已含 reboot event / host unreachable / uptime unknown / Windows99 VMX / guest power / VMware config / Windows Update policy；artifact `/tmp/awoooi-reboot-continue-20260703-085205`：111 不可達、99 uptime unknown、188 startup failed/degraded；09:22 已把 unreliable console artifact 投影成 machine-readable blocker | 優先收斂 111 VMX source、99 local console Verify output / no-secret management channel 與 111 reachability；不可宣稱 10 分鐘 SLA 已證明，不得用 RDP clipboard 片段當完成證據 | | 2 | CIR-P0-RBT-002 | P0 | 「沒有偵測到主機重啟」 | 修正 host reboot/shutdown/up detection：boot_id / uptime / node exporter / Windows exporter / VMware VM power state 都要進同一事件 | Scorecard 已接 collection packet + management probe；08:23 host probe 有 7 列但 99 只有 ping/RDP reachable、uptime unknown，111 unreachable，112/120/121/188 uptime 都已超過 10 分鐘窗口 | 讓 99 verifier / Windows exporter 或等效 no-secret readback 進入 host boot event，並補 111 reachability 證據 | -| 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；10:15 source/API patch 已把 nested `service_blockers=["VMAuthdService","VMnetDHCP"]` 與 VMX / power evidence 提升到 active blocker matrix，不再只顯示 guest power not ready；09:53 management probe 讀回 99 host reachable、RDP / Hyper-V VMConnect reachable，但 SSH BatchMode `permission_denied`、WinRM timeout，`console_artifact_status=blocked_clipboard_unreliable` | 先修 111 VMX source 與 VMware service / autostart task check-mode package，重跑 no-secret collector 與 scorecard；不得從 scorecard 直接啟動 / 關閉 VM、重啟 Windows service、讀密碼或 host reboot | +| 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；10:58 API scorecard 已接入 `windows99_vmx_source_repair_package`，會把 111 VMX missing 轉成 expected path / `Test-Path` check-mode probe；10:15 source/API patch 已把 nested `service_blockers=["VMAuthdService","VMnetDHCP"]` 與 VMX / power evidence 提升到 active blocker matrix；09:53 management probe 讀回 99 host reachable、RDP / Hyper-V VMConnect reachable，但 SSH BatchMode `permission_denied`、WinRM timeout，`console_artifact_status=blocked_clipboard_unreliable` | 先依 API package 修 111 VMX source，再處理 VMware service / autostart task check-mode package，重跑 no-secret collector 與 scorecard；不得從 scorecard 直接啟動 / 關閉 VM、重啟 Windows service、讀密碼或 host reboot | | 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | 10:15 source/API patch：若 nested verifier 回 `policy_blockers=["windows_update_policy_readback_missing"]`，API 必須 fail-closed 重新提升 `windows99_update_no_auto_reboot_policy_not_ready`；不得只信舊 Prometheus `windows99_update_no_auto_reboot_ready=true` | 補 no-secret policy readback / drift alert；若 policy 不綠，另走 controlled apply package + rollback + post-verifier，禁止要求或記錄 Windows 密碼，禁止由 scorecard 直接 apply registry | | 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；Gitea CD `#4459` Success、deploy marker `8d7a6faaf`，production API scorecard 已讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、unreachable without L1=`0`，P0 blockers `11`、readiness `47`；仍不可宣稱整體 10 分鐘 SLO 完成 | 若未來 public route 出現 raw 5xx，先走 L0 Nginx intercept / header verifier；若 edge unreachable，才產生 L1 external static origin / CDN decision record + rollback | | 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | HostDown / HostRebootEventDetected / RebootAutoRecoverySLOMissed 已存在；per-blocker reboot alerts 與 backup receipt rules 已 deploy/readback。Backup receipt 缺段已從 100 條 stage 噪音收斂為 110 / 188 兩條 host-level pending；仍需完整 shutdown/up E2E receipt | 補 Prometheus / Alertmanager active/resolved 與 outbound receipt；backup alert 先補 `/backup/alert-receipts/*.last_success` 脫敏 marker，不送測試 secret、不重啟主機 | @@ -136,7 +136,7 @@ | AI 專業 UI / 非文字牆 cockpit | 已列為 P2 UX 驗收 | 將長文字區塊收斂成 first-viewport cockpit、cards、flow rows 與 expandable details | | 10 分鐘 reboot auto-recovery SLA | 2026-07-03 10:50 production scorecard 固定 `10` blockers / readiness `60%`，但 `can_claim_all_services_recovered_within_target=false`；artifact `/tmp/awoooi-reboot-verify-only-20260703-082310` 保留 host probe / reboot detector / scorecard，production readback 已把 Windows99 config / policy blockers 上卷 | 收斂 111 VMX source、99 no-secret Verify、111 reachability、188 startup failed/degraded，再 rerun host probe + reboot-event detector | | AWOOOI latest source production deploy readback | Gitea `main=bc8d3af3f`，deploy marker `dacdd90`；public CD `#4558` log job succeeded / production deploy readback matched；production scorecard 已讀回 latest API 行為並把 Windows99 config / policy blockers 上卷 | 下一步轉回 Windows99 VMX / service / policy / power 與 host reboot-event blockers；不得因 deploy/readback 成功宣稱 10 分鐘 SLO 完成 | -| 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 尚未收集；08:23 management probe 顯示 RDP / Hyper-V VMConnect reachable，但 SSH BatchMode / WinRM blocked，collector `blocked_ssh_publickey_auth_missing` | 收集 99 no-secret Verify output，確認 VM 111/188/120/121/112 running、scheduled task / services / Windows Update policy 全綠 | +| 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；10:58 API scorecard 已接入 `windows99_vmx_source_repair_package`，111 VMX missing 會輸出 expected path 與 check-mode probe；live 99 Verify output 尚未收集；08:23 management probe 顯示 RDP / Hyper-V VMConnect reachable，但 SSH BatchMode / WinRM blocked，collector `blocked_ssh_publickey_auth_missing` | 依 API package 修 111 VMX source，收集 99 no-secret Verify output，確認 VM 111/188/120/121/112 running、scheduled task / services / Windows Update policy 全綠 | | Reboot SLO blocker 收斂 | production scorecard 已固定 `10` blockers / readiness `60%`，source / runtime 已補 per-blocker metric/alert；剩餘主 blocker 是 host boot detection + Windows99 VMware VMX / guest power / autostart config / Windows Update policy | 依具名 blocker 收斂 99/111/188；不得用 route green 或 RDP 可見宣稱 SLA 完成 | | 全備份監控告警 coverage | exporter/rule 已有 host/DB/site/service/package/tool/log coverage 與 backup alert receipt requirement；runtime rules 已部署，缺段 alert 聚合成 110 / 188 host-level pending；production receipt markers / full Gitea dump / DB/settings/issues/packages/LFS 尚未全綠 | 補 `/backup/alert-receipts/*.last_success`、Gitea full dump 與 restore drill verifier | | Stock/Postgres hot pressure | 110 live 已導向 Stock/Postgres playbook；尚未完成 hot query / backup export playbook closure | 下一步執行 read-only Stock/Postgres evidence 與 source freshness / query attribution |