diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 85b40b58d..d36286084 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -75,12 +75,17 @@ _PROMETHEUS_RUNTIME_COMBINED_QUERY = ( + "|".join(_PROMETHEUS_RUNTIME_QUERIES.values()) + '"}' ) -_PUBLIC_MAINTENANCE_BLOCKERS = { - "edge_public_maintenance_live_config_drift", +_PUBLIC_MAINTENANCE_ROUTE_BLOCKERS = { "public_maintenance_fallback_runtime_readback_missing", "public_route_raw_5xx_without_maintenance_fallback", "public_route_unreachable_without_external_l1_fallback", } +_PUBLIC_MAINTENANCE_EDGE_BLOCKERS = { + "edge_public_maintenance_live_config_drift", +} +_PUBLIC_MAINTENANCE_BLOCKERS = ( + _PUBLIC_MAINTENANCE_ROUTE_BLOCKERS | _PUBLIC_MAINTENANCE_EDGE_BLOCKERS +) _RUNTIME_BOOLEAN_READY_BLOCKERS = { "backup_core_green_not_1": "backup_core_green", "host_188_service_green_not_1": "host_188_service_green", @@ -895,17 +900,9 @@ def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks( ) -> list[str]: runtime_ready_blockers: set[str] = set() public_maintenance = _dict(payload.get("public_maintenance_fallback")) - public_maintenance_ready = ( - public_maintenance.get("runtime_readback_present") is True - and public_maintenance.get("ready") is True - and _int(public_maintenance.get("raw_5xx_without_fallback_count")) == 0 - and _int( - public_maintenance.get("route_unreachable_without_external_fallback_count") - ) - == 0 + runtime_ready_blockers.update( + _public_maintenance_reconciled_metric_blockers(public_maintenance) ) - if public_maintenance_ready: - runtime_ready_blockers.update(_PUBLIC_MAINTENANCE_BLOCKERS) service_backup = _dict(payload.get("controlled_service_data_backup_readback")) rollups = _dict(payload.get("rollups")) @@ -949,6 +946,37 @@ def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks( return _unique_strings(reconciled) +def _public_maintenance_reconciled_metric_blockers( + public_maintenance: dict[str, Any], +) -> set[str]: + reconciled: set[str] = set() + if public_maintenance.get("runtime_readback_present") is not True: + return reconciled + + reconciled.add("public_maintenance_fallback_runtime_readback_missing") + public_route_ready = ( + _int(public_maintenance.get("raw_5xx_without_fallback_count")) == 0 + and _int( + public_maintenance.get("route_unreachable_without_external_fallback_count") + ) + == 0 + ) + if public_route_ready: + reconciled.update( + blocker + for blocker in _PUBLIC_MAINTENANCE_ROUTE_BLOCKERS + if blocker != "public_maintenance_fallback_runtime_readback_missing" + ) + + edge_fallback_ready = ( + public_maintenance.get("edge_fallback_runtime_readback_present") is True + and public_maintenance.get("edge_fallback_ready") is True + ) + if edge_fallback_ready: + reconciled.update(_PUBLIC_MAINTENANCE_EDGE_BLOCKERS) + return reconciled + + def _add_runtime_readback_active_blockers_missing_from_metric( payload: dict[str, Any], active_blockers: list[str], @@ -1273,10 +1301,13 @@ def apply_public_maintenance_runtime_readback( public_maintenance = _build_public_maintenance_fallback_readback(runtime_readback) payload["public_maintenance_fallback"] = public_maintenance + reconciled_blockers = _public_maintenance_reconciled_metric_blockers( + public_maintenance + ) active_blockers = [ blocker for blocker in _strings(payload.get("active_blockers")) - if blocker not in _PUBLIC_MAINTENANCE_BLOCKERS + if blocker not in reconciled_blockers ] active_blockers.extend(_strings(public_maintenance.get("blockers"))) active_blockers = _unique_strings(active_blockers) diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 404a51c5b..1595f5a27 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -229,6 +229,43 @@ def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_ready_public_mainten ] == 1 +def test_reboot_auto_recovery_slo_scorecard_preserves_edge_metric_blocker_until_edge_readback_ready( + tmp_path, +): + runtime_path = tmp_path / "public-maintenance-fallback.json" + runtime_path.write_text( + json.dumps(PUBLIC_MAINTENANCE_RUNTIME_READY), + encoding="utf-8", + ) + metric_readback = dict(PROMETHEUS_RUNTIME_READBACK) + metric_readback["active_blockers"] = [ + *PROMETHEUS_RUNTIME_BLOCKERS, + "public_route_raw_5xx_without_maintenance_fallback", + "edge_public_maintenance_live_config_drift", + ] + metric_readback["active_blocker_count"] = len(metric_readback["active_blockers"]) + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + public_maintenance_runtime_path=runtime_path, + prometheus_metric_readback=metric_readback, + ) + + assert "public_route_raw_5xx_without_maintenance_fallback" not in payload[ + "active_blockers" + ] + assert "edge_public_maintenance_live_config_drift" in payload["active_blockers"] + assert ( + payload["public_maintenance_fallback"][ + "edge_fallback_runtime_readback_present" + ] + is False + ) + assert payload["readback"][ + "runtime_metric_runtime_readback_reconciled_blockers" + ] == ["public_route_raw_5xx_without_maintenance_fallback"] + assert payload["active_blocker_count"] == 9 + + def test_reboot_auto_recovery_slo_scorecard_overlays_edge_fallback_live_drift( tmp_path, ): diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index e110b5658..f5f6b43cd 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,25 @@ +## 2026-07-03 — 15:35 Public maintenance edge API overlay 修正 + +**完成內容**： +- 修正 production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 的 public maintenance runtime overlay：route fallback ready 與 edge Nginx fallback ready 已拆成不同 blocker group。 +- route runtime readback present 只清 `public_maintenance_fallback_runtime_readback_missing`；route 0 raw 5xx / 0 unreachable 只清 route-level blockers；只有 `edge_fallback_runtime_readback_present=true` 且 `edge_fallback_ready=true` 才能清 `edge_public_maintenance_live_config_drift`。 +- 新增 regression test：當 public route probe ready、但 Prometheus active label 含 `edge_public_maintenance_live_config_drift` 且缺 edge ready readback 時，API active list 必須保留 edge drift。 +- `FULL-STACK-COLD-START-SOP` 更新至 v1.124，`REBOOT-RECOVERY-SOP` 更新至 v5.16；P0 workplan 的 P0-2 狀態改為 `API_OVERLAY_FIX_PENDING_DEPLOY_READBACK`。 + +**已跑驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py -q -p no:cacheprovider -k 'public_maintenance or prometheus_runtime_metrics'`：`3 passed, 19 deselected`。 +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py -q -p no:cacheprovider`：`22 passed`。 +- `python3.11 -m py_compile apps/api/src/services/reboot_auto_recovery_slo_scorecard.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py`：通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`：`GITEA_RUNNER_PRESSURE_GUARD_OK workflow_files=12 scheduled_workflows=4 auto_branch_events_on_110=0 generic_runner_labels=0`。 +- `git diff --check`：通過。 + +**仍維持**： +- 這只修 API / scorecard readback，不代表 10 分鐘重啟自動恢復 SLO 完成。 +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有使用 GitHub / gh；沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart / reload，沒有 VM power change，沒有 DROP / TRUNCATE / restore / prune / delete / force push。 + +**下一步**： +- commit / push 到 Gitea main；等待 CD deploy marker matched 後讀回 production SLO，確認 `active_blockers` 含 `edge_public_maintenance_live_config_drift` 且 `runtime_metric_active_blocker_count=8`。 + ## 2026-07-03 — 15:13 Public maintenance edge SLO runtime readback 收斂 **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index b7656eda6..55b789b7e 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.123 +> Version: v1.124 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -78,6 +78,8 @@ v1.122 public maintenance edge fallback drift metric rule：deploy / reboot wind v1.123 SLO installer edge-source packaging rule：110 SLO installer payload 必須包含 `ops/nginx`、`ops/maintenance` 與 `ops/monitoring`。若 110 runtime `public-maintenance-edge-fallback-apply.sh --check` 先回 `local_nginx_conf_missing`，那是 verifier source package 不完整，必須修 installer payload 並重新套用 verifier；不得把它當 live edge 已驗證或直接跳到 Nginx apply。2026-07-03 15:13 controlled install 已證明修正後 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-151219` 正確讀到 188 live edge：`REMOTE_CONF_READABLE=1`、VIP `192.168.0.125` API/Web upstream 存在，但 `proxy_intercept_errors` / `error_page` / fallback header / internal maintenance location / maintenance file 均缺，`SUDO_N_READY=0`，textfile 輸出 `awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0` 與 `edge_public_maintenance_live_config_drift`，blocker count `8`。這仍只授權 no-write verifier 與告警，不授權讀密碼、reload Nginx、重啟服務或 host。 +v1.124 public maintenance route / edge blocker split rule：production API overlay 不得把 route fallback ready 與 edge Nginx fallback ready 視為同一件事。`public_maintenance_fallback.runtime_readback_present=true` 只能清除 `public_maintenance_fallback_runtime_readback_missing`；route 0 raw 5xx / 0 unreachable 只能清除 route-level blockers；只有 `edge_fallback_runtime_readback_present=true` 且 `edge_fallback_ready=true` 才能清除 `edge_public_maintenance_live_config_drift`。若 Prometheus textfile active label 已有 edge drift，但 public route probe 是 200 / ready 且缺 edge readback，`/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須保留 edge drift；否則會把下次 reboot / deploy raw 502 風險誤報成正常。此規則只修 scorecard/API readback，不授權 Nginx reload、app restart、Docker / host restart 或讀密碼。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 33deebec8..dc9fdbd2c 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.15 +> **版本**: v5.16 > **最後更新**: 2026-07-03 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-03 15:13 110 SLO runtime 已正確上卷 public maintenance edge live drift，installer payload 補入 `ops/nginx` +> **觸發事件**: 2026-07-03 15:35 production API overlay 分離 route ready 與 edge ready，避免清掉 `edge_public_maintenance_live_config_drift` --- @@ -21,6 +21,12 @@ ## 架構概覽與依賴圖 +### 2026-07-03 15:35 P0-2 API Readback 覆蓋 + +110 textfile / Prometheus 已有 `awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0` 與 active blocker `edge_public_maintenance_live_config_drift` 時，production API 必須把此 blocker 合併進 `active_blockers`。route probe 200 / route fallback ready 不能替代 edge Nginx fallback ready；只有 `edge_fallback_runtime_readback_present=true` 且 `edge_fallback_ready=true` 才能清掉 edge drift。 + +本輪 source fix 將 public maintenance blockers 拆成 route-level 與 edge-level 兩組：route readback present / route 0 raw 5xx 只清 route blockers；edge blocker 必須由 edge readback green 才能清。這是 readback overlay 修正，不授權讀密碼、Nginx reload、app restart、Docker / host restart、VM power change 或 DB / firewall 操作。 + ### 2026-07-03 15:13 P0-2 Runtime 覆蓋 110 SLO controlled install 已完成，`awoooi-reboot-auto-recovery-slo.timer` 為 `enabled/active`，service `Result=success`、`ExecMainStatus=0`。本輪只 stage SLO verifier source、啟動 verifier oneshot 與寫 textfile metric；未重啟 Docker / Nginx / K3s / DB / firewall，未讀 secret / token / `.env` / raw sessions / SQLite / auth。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 7366efc36..69f628d77 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -20,7 +20,7 @@ | 優先 | 狀態 | 工作項 | 最新證據 | 下一步 / 完成條件 | |------|------|--------|----------|-------------------| | P0-1 | BLOCKED_HOST_WINDOWS | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 10:44 production scorecard：`status=blocked_reboot_auto_recovery_slo_not_ready`、`active_blocker_count=10`、`readiness_percent=60`、active blockers 仍含 reboot event / host unreachable / 99+111+Windows99 VMX / guest power，且新增 `windows99_vmware_autostart_config_not_ready`、`windows99_update_no_auto_reboot_policy_not_ready`。no-write artifact `/tmp/awoooi-reboot-continue-20260703-085205`：111 socket probe 仍 timeout / unreachable；99 RDP / VMConnect reachable 但 uptime unknown；188 socket probe reachable，前次 host probe 仍為 `systemd_state=degraded` / `startup_active=failed`。 | 先收斂 99 / Windows99 / VMware 與 111：恢復 no-secret management channel 或取得 validator 可接受的完整 console Verify stdout，讀回 VMX / VM power / host uptime，再 rerun host probe + reboot-event detector；不得 reboot、不得 VM power change、不得讀 Windows 密碼，不得用 RDP clipboard 片段當完成證據。 | -| P0-2 | RUNTIME_EDGE_DRIFT_ACTIVE_PROD_API_READBACK_PENDING | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 2026-07-03 15:13 110 SLO controlled install 已套用新版 exporter，並修正 installer payload 缺 `ops/nginx` 導致的 `local_nginx_conf_missing` 誤判。最新 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-151219` 正確讀到 188 live edge：`REMOTE_CONF_READABLE=1`、`LIVE_CONF_HAS_INTERCEPT=0`、`LIVE_CONF_HAS_ERROR_PAGE_503=0`、`LIVE_CONF_HAS_FALLBACK_HEADER=0`、`LIVE_CONF_HAS_INTERNAL_LOCATION=0`、`LIVE_CONF_HAS_API_VIP=1`、`LIVE_CONF_HAS_WEB_VIP=1`、`MAINTENANCE_FILE_PRESENT=0`、`SUDO_N_READY=0`、`EDGE_FALLBACK_READY=0`。110 textfile 已輸出 `awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0`、`awoooi_reboot_auto_recovery_slo_blocker_count=8` 與 active blocker `edge_public_maintenance_live_config_drift`。 | 等最新 CD deploy marker readback matched 後，production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須上卷 edge drift；具備 privileged channel 後才可受控 `public-maintenance-edge-fallback-apply.sh --apply`，要求 backup、`nginx -t`、reload、public header verifier 全綠；或完成 L1 external cloud/CDN fallback。不得讀密碼、不得用 app / Docker / host restart 掩蓋 edge fallback drift。 | +| P0-2 | API_OVERLAY_FIX_PENDING_DEPLOY_READBACK | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 2026-07-03 15:35 source fix 已把 production API overlay 的 public maintenance blockers 拆成 route-level 與 edge-level：route probe 200 / ready 只能清 route blockers，不能清 `edge_public_maintenance_live_config_drift`；只有 `edge_fallback_runtime_readback_present=true` 且 `edge_fallback_ready=true` 才能清 edge drift。110 SLO artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-151219` 與 Prometheus/textfile 已證明 edge 未 ready：`awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0`、`awoooi_reboot_auto_recovery_slo_blocker_count=8`、active blocker `edge_public_maintenance_live_config_drift`。本地 regression：route ready + edge metric blocker 時 API active list 必須保留 edge，22 個 scorecard API 測試通過。 | commit / push 後等待 Gitea CD deploy marker matched；production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須讀回 `active_blockers` 含 `edge_public_maintenance_live_config_drift` 且 `runtime_metric_active_blocker_count=8`。後續才進 privileged edge apply 或 L1 cloud/CDN fallback decision；不得讀密碼、不得用 app / Docker / host restart 掩蓋 edge fallback drift。 | | P0-3 | PARTIAL_GREEN_DEPLOYED_SOURCE | 所有產品 / 網站版本與資料最新性 | Gitea `main=bc8d3af3f chore(cd): deploy dacdd90 [skip ci]`；CD `#4558` log 顯示 API/Web build/push wrapper 全部 `rc=0`、deploy marker `dacdd90`、production deploy readback matched。Production SLO route 已讀回最新 API 行為：`runtime_metric_runtime_readback_added_blockers=["windows99_vmware_autostart_config_not_ready","windows99_update_no_auto_reboot_policy_not_ready"]`、`windows99_update_no_auto_reboot_ready=false`。Stock freshness 仍為 `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。 | 版本層已補到 AWOOOI production；完成整體 P0 仍需收斂 Windows99 VMX / service / policy / power 與 host reboot-event blockers。後續每個 public product 仍要維持 source SHA、deploy marker、runtime endpoint、public route watch、freshness 四層一致。 | | P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | 10:44 production readback 已把 nested Windows99 evidence 上卷到 active blockers：`windows99_vmware_vmx_missing`、`windows99_vmware_guest_power_not_ready`、`windows99_vmware_autostart_config_not_ready`、`windows99_update_no_auto_reboot_policy_not_ready`；`runtime_metric_runtime_readback_added_blockers` 含 config / policy 兩項，`windows99_update_no_auto_reboot_ready=false`。09:53 no-secret management probe：99 reachable、RDP / Hyper-V VMConnect reachable，WinRM timeout，Mac vantage collector 仍為 `ssh_batchmode_auth_ready=0` / `blocked_ssh_publickey_auth_missing`；completion 仍取決於 VMX / service / policy / power 全綠。 | 先修 111 VMX source 與 VMware service / policy readback，使 check-mode package 能解釋 powered-off aliases；不得從 scorecard 直接重啟 Windows service、改 registry、啟動 / 關閉 VM 或 host reboot。完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable，且 active blocker matrix 顯示 service / policy / VMX / power 分開收斂。 | | P0-5 | RUNTIME_READY_BACKUP_RECEIPT_GAP | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。2026-07-03 source / runtime 已部署 `awoooi_backup_alert_receipt_*` 指標與 Prometheus rules；110 exporter 讀回 88 個 stage requirement、188 讀回 12 個 stage requirement，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 已修成每 `host / receipt_channel` 聚合 pending：110 一條、188 一條。 | 補 `/backup/alert-receipts/*.last_success` 脫敏 marker；下一層仍要補 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控。 |