From 33c391808297f87b43dafaad5d2af4970c7ac830 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 14:10:04 +0800 Subject: [PATCH] fix(reboot): prefer windows99 runtime metrics --- .../reboot_auto_recovery_slo_scorecard.py | 52 +++++++++++--- ..._reboot_auto_recovery_slo_scorecard_api.py | 68 +++++++++++++++---- docs/LOGBOOK.md | 25 +++++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 4 +- docs/runbooks/REBOOT-RECOVERY-SOP.md | 23 +++---- .../reboot-auto-recovery-slo-exporter.sh | 6 ++ ...test_reboot_auto_recovery_slo_installer.py | 1 + 7 files changed, 142 insertions(+), 37 deletions(-) diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 821b869fe..a1aa13595 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -46,6 +46,9 @@ _PROMETHEUS_RUNTIME_QUERIES = { "last_run_timestamp": "awoooi_reboot_auto_recovery_slo_last_run_timestamp", "windows99_vmware_readback_present": "awoooi_windows99_vmware_readback_present", "windows99_vmware_verify_ready": "awoooi_windows99_vmware_verify_ready", + "windows99_vmware_autostart_config_ready": ( + "awoooi_windows99_vmware_autostart_config_ready" + ), "windows99_windows_update_ready": ( "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready" ), @@ -401,12 +404,19 @@ def _load_reboot_slo_prometheus_metric_readback( ) == 1 ) - windows99_update_ready = ( - _prometheus_first_int( - scalar_results.get("windows99_windows_update_ready") or [], - default=0, - ) - == 1 + windows99_config_metric = _prometheus_first_int( + scalar_results.get("windows99_vmware_autostart_config_ready") or [], + default=-1, + ) + windows99_config_ready: bool | None = ( + None if windows99_config_metric < 0 else windows99_config_metric == 1 + ) + windows99_update_metric = _prometheus_first_int( + scalar_results.get("windows99_windows_update_ready") or [], + default=-1, + ) + windows99_update_ready: bool | None = ( + None if windows99_update_metric < 0 else windows99_update_metric == 1 ) windows99_missing_vmx_aliases = _prometheus_label_values( missing_vmx_results, @@ -440,6 +450,7 @@ def _load_reboot_slo_prometheus_metric_readback( "last_run_timestamp": last_run_timestamp, "windows99_vmware_readback_present": windows99_readback_present, "windows99_vmware_verify_ready": windows99_verify_ready, + "windows99_vmware_autostart_config_ready": windows99_config_ready, "windows99_windows_update_no_auto_reboot_ready": windows99_update_ready, "windows99_missing_vmx_aliases": windows99_missing_vmx_aliases, "windows99_powered_off_aliases": windows99_powered_off_aliases, @@ -714,9 +725,15 @@ def _apply_prometheus_windows99_vmware_readback( blocker for blocker in active_blockers if blocker.startswith("windows99_") ] verify_ready = metric_readback.get("windows99_vmware_verify_ready") is True - update_ready = ( - metric_readback.get("windows99_windows_update_no_auto_reboot_ready") is True + metric_config_ready = metric_readback.get( + "windows99_vmware_autostart_config_ready" ) + metric_has_config_ready = isinstance(metric_config_ready, bool) + metric_update_ready = metric_readback.get( + "windows99_windows_update_no_auto_reboot_ready" + ) + metric_has_update_ready = isinstance(metric_update_ready, bool) + update_ready = metric_update_ready is True windows99 = _dict(payload.setdefault("windows99_vmware_autostart", {})) source_service_blockers = _strings(windows99.get("service_blockers")) source_policy_blockers = _strings(windows99.get("policy_blockers")) @@ -724,13 +741,26 @@ def _apply_prometheus_windows99_vmware_readback( source_config_blocked = bool(source_service_blockers) or ( bool(source_task) and source_task.get("ok") is not True ) - if source_config_blocked: + if metric_has_config_ready: + if metric_config_ready is not True: + windows99_blockers.append("windows99_vmware_autostart_config_not_ready") + elif source_config_blocked: windows99_blockers.append("windows99_vmware_autostart_config_not_ready") - if source_policy_blockers: + if metric_has_update_ready: + if metric_update_ready is not True: + windows99_blockers.append( + "windows99_update_no_auto_reboot_policy_not_ready" + ) + elif source_policy_blockers: windows99_blockers.append("windows99_update_no_auto_reboot_policy_not_ready") update_ready = False windows99_blockers = _unique_strings(windows99_blockers) - config_ready = not missing_vmx_aliases and not source_config_blocked + metric_or_source_config_ready = ( + metric_config_ready is True + if metric_has_config_ready + else not source_config_blocked + ) + config_ready = not missing_vmx_aliases and metric_or_source_config_ready windows99.update( { "readback_present": True, diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 24aa49c28..ec33ee59b 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -103,7 +103,6 @@ PROMETHEUS_SOURCE_RECONCILED_BLOCKER = ( ) WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS = [ "windows99_vmware_autostart_config_not_ready", - "windows99_update_no_auto_reboot_policy_not_ready", ] PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS = [ blocker @@ -426,7 +425,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( ) assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1 assert payload["active_blockers"] == expected_active_blockers - assert payload["active_blocker_count"] == 9 + assert payload["active_blocker_count"] == 8 assert payload["readiness_percent"] == 47 assert payload["primary_blocker"] == "reboot_event_required_host_unreachable" assert payload["next_safe_action"] == ( @@ -439,7 +438,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( payload["runtime_metric_runtime_readback_added_blockers"] == WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS ) - assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 2 + assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 1 package = payload["windows99_vmx_source_repair_package"] assert package["schema_version"] == ( "windows99_vmx_source_repair_check_mode_package_v1" @@ -481,7 +480,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( is True ) assert payload["controlled_service_data_backup_readback"]["blocking_fields"] == [] - assert payload["active_blocker_action_matrix"]["item_count"] == 9 + assert payload["active_blocker_action_matrix"]["item_count"] == 8 assert payload["windows99_vmware_autostart"]["readback_present"] is True assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"] assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [ @@ -494,13 +493,13 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert payload["windows99_vmware_autostart"]["blockers"] == [ "windows99_vmware_guest_power_not_ready", "windows99_vmware_vmx_missing", - *WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS, + "windows99_vmware_autostart_config_not_ready", ] assert payload["readback"]["windows99_vmware_readback_present"] is True assert payload["readback"]["windows99_vmrun_present"] is True assert payload["readback"]["windows99_vmware_config_ready"] is False assert payload["readback"]["windows99_vmware_power_ready"] is False - assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is False + assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is True assert payload["readback"]["windows99_runtime_metric_overlay_applied"] is True assert ( payload["windows99_verify_collection"]["status"] @@ -515,7 +514,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert payload["rollups"]["windows99_vmrun_present"] is True assert payload["rollups"]["windows99_vmware_config_ready"] is False assert payload["rollups"]["windows99_vmware_power_ready"] is False - assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is False + assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True assert payload["rollups"]["windows99_vmware_missing_vmx_count"] == 1 assert payload["rollups"]["windows99_vmware_powered_off_count"] == 5 assert payload["rollups"]["windows99_missing_vmx_alias_count"] == 1 @@ -560,7 +559,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( ] == 5 assert payload["active_blocker_action_matrix"]["category_counts"][ "windows99_vmware_autostart" - ] == 4 + ] == 3 assert payload["active_blocker_action_matrix"]["category_counts"].get( "post_reboot_service_readiness", 0, @@ -573,7 +572,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( "security_observability", 0, ) == 0 - assert payload["readback"]["active_blocker_count"] == 9 + assert payload["readback"]["active_blocker_count"] == 8 assert payload["readback"]["runtime_metric_readback_present"] is True assert payload["readback"][ "runtime_metric_source_control_reconciled_blocker_count" @@ -583,20 +582,63 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( ] == WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS assert payload["readback"][ "runtime_metric_runtime_readback_added_blocker_count" - ] == 2 - assert payload["rollups"]["active_blocker_count"] == 9 + ] == 1 + assert payload["rollups"]["active_blocker_count"] == 8 assert payload["rollups"]["runtime_metric_readback_present"] is True assert payload["rollups"][ "runtime_metric_source_control_reconciled_blocker_count" ] == 1 assert payload["rollups"][ "runtime_metric_runtime_readback_added_blocker_count" - ] == 2 + ] == 1 assert payload["rollups"]["primary_blocker_owner_lane"] == ( "reboot_event_detector_and_host_probe" ) +def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_windows99_config_when_runtime_metric_is_ready(): + metric_readback = { + **PROMETHEUS_RUNTIME_READBACK, + "active_blockers": [ + "all_required_hosts_not_in_10_minute_reboot_window", + "fresh_all_host_reboot_event_missing", + "host_boot_observation_older_than_target_window", + "host_unreachable_after_reboot", + "host_uptime_unknown", + "reboot_event_required_host_unreachable", + "windows99_vmware_guest_power_not_ready", + ], + "active_blocker_count": 7, + "windows99_vmware_autostart_config_ready": True, + "windows99_windows_update_no_auto_reboot_ready": True, + "windows99_missing_vmx_aliases": [], + "windows99_powered_off_aliases": ["110", "112", "120", "121", "188"], + } + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + prometheus_metric_readback=metric_readback, + ) + + assert payload["active_blockers"] == metric_readback["active_blockers"] + assert payload["active_blocker_count"] == 7 + assert payload["runtime_metric_runtime_readback_added_blockers"] == [] + assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 0 + assert "windows99_vmware_autostart_config_not_ready" not in payload[ + "active_blockers" + ] + assert "windows99_update_no_auto_reboot_policy_not_ready" not in payload[ + "active_blockers" + ] + assert payload["windows99_vmware_autostart"]["config_ready"] is True + assert payload["windows99_vmware_autostart"][ + "windows_update_no_auto_reboot_ready" + ] is True + assert payload["readback"]["windows99_vmware_config_ready"] is True + assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is True + assert payload["rollups"]["windows99_vmware_config_ready"] is True + assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True + + def test_reboot_auto_recovery_slo_scorecard_surfaces_windows99_console_artifact_blocker( tmp_path, ): @@ -710,7 +752,7 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when *PROMETHEUS_RUNTIME_BLOCKERS, *WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS, ] - assert payload["active_blocker_count"] == 10 + assert payload["active_blocker_count"] == 9 assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0 action_by_blocker = { item["blocker"]: item diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index a5a002202..9a5f932ca 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,28 @@ +## 2026-07-03 — 14:10 Windows99 runtime metric overlay 防 stale snapshot 假 blocker + +**完成內容**： +- Gitea CD `#4587` 已完成 source `30c89a0` 部署，deploy marker `4da6dda chore(cd): deploy 30c89a0 [skip ci]`；110 `awoooi-reboot-auto-recovery-slo.timer` 已 `enabled/active`，每輪 artifact 約 2 分鐘內正常完成。 +- Controlled install 已把新版 verifier / validator / scorecard staged 到 110；latest runtime artifact 顯示 `missing_vmx_aliases=[]`、`powered_off_aliases=110,112,120,121,188`、`VMWARE_AUTOSTART_CONFIG_READY=1`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1`、`VMWARE_AUTOSTART_VERIFY_READY=0`。 +- 110 textfile metric 已清掉 `windows99_vmware_vmx_missing`：`awoooi_reboot_auto_recovery_slo_blocker_count=7`，active blockers 只剩 host reboot event / fresh 10-minute window / unreachable / unknown uptime / `windows99_vmware_guest_power_not_ready`。 +- Production API readback 發現 source/API overlay 仍因 committed snapshot 反加 `windows99_vmware_autostart_config_not_ready` 與 `windows99_update_no_auto_reboot_policy_not_ready`，造成 API `active_blocker_count=9` 但 runtime metric `active_blocker_count=7`。 +- `reboot-auto-recovery-slo-exporter.sh` 新增 `awoooi_windows99_vmware_autostart_config_ready` gauge；API Prometheus readback parser 新增三態 config-ready 欄位，只有 metric 缺欄位或明確 not ready 時才允許 fallback 到 snapshot blocker。 +- `reboot_auto_recovery_slo_scorecard` overlay 改成 runtime metric precedence：當 metric 已證明 config ready / Windows Update no-auto-reboot ready 時，不再把 stale `service_blockers` / `policy_blockers` 反加成 active blockers。 +- `FULL-STACK-COLD-START-SOP` 升到 v1.120，`REBOOT-RECOVERY-SOP` 升到 v5.12，固定記錄 runtime metric overlay precedence 與目前 7-blocker truth。 + +**已跑驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py scripts/reboot-recovery/tests/test_windows99_vmx_source_repair_package.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_windows99_vmx_source_repair_package_api.py apps/api/tests/test_windows99_vmx_source_locator_readback_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py -q -p no:cacheprovider`：`92 passed`。 +- `python3.11 -m py_compile apps/api/src/services/reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py`：通過。 +- `bash -n scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh scripts/reboot-recovery/collect-windows99-vmware-verify.sh`：通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py`：`GITEA_RUNNER_PRESSURE_GUARD_OK workflow_files=12 scheduled_workflows=4 auto_branch_events_on_110=0 generic_runner_labels=0`。 +- `git diff --check`：通過。 + +**仍維持**： +- P0-006 尚未完成：不得宣稱全主機重啟後 10 分鐘內全部恢復；剩餘真 blocker 是 fresh all-host reboot event / host reachability / 10-minute window evidence，以及 Windows99 powered-off VM guest readback。 +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未啟動或關閉 VM；未重啟 host / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + +**下一步**： +- commit / push 到 Gitea main；等待 CD deploy marker 後讀回 production `/api/v1/agents/reboot-auto-recovery-slo-scorecard`，確認 API `active_blocker_count` 與 runtime metric 同步為 7，且不再出現 Windows99 config / Windows Update policy 假 blocker。 + ## 2026-07-03 — 13:40 Windows99 VM target source-of-truth 校正 **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 812de1b30..85b99e2cd 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.119 +> Version: v1.120 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -70,6 +70,8 @@ v1.118 Windows99 VM target source-of-truth reconciliation rule：99 VMware autos v1.119 Windows99 no-secret verifier via-host / validator rule：本工作站直連 99 SSH 缺 public-key auth 時，`collect-windows99-vmware-verify.sh` 必須支援 `--via-host wooo@192.168.0.110`，沿 110→99 已驗證 no-secret path 執行 Verify；collector 必須輸出 `via_host=`，仍固定 BatchMode/publickey、in-memory stdin scriptblock、`remote_write_performed=false`、`vm_power_change_performed=false`。`validate-windows99-console-verify-artifact.py` default required aliases 必須與 v1.118 一致為 `110 / 188 / 120 / 121 / 112`；若未來要驗 explicit `111` artifact，必須用 `--required-aliases 111,188,120,121,112` 顯式指定。2026-07-03 via-host live artifact 已證明新版 Verify readback：VMX missing 為空、config ready、Windows Update no-auto-reboot ready，剩餘 blocker 是 `windows99_vmware_guest_power_not_ready`；不得再把 validator 舊預設造成的 `111` missing 當真 blocker。 +v1.120 Windows99 runtime metric overlay precedence rule：v1.110 的 nested blocker promotion 只適用於 runtime metric 缺少 Windows99 config / policy 欄位，或 metric 明確回 not ready 的情況。`reboot-auto-recovery-slo-exporter.sh` 必須從同一份 latest `scorecard.json` 輸出 `awoooi_windows99_vmware_autostart_config_ready`、`awoooi_windows99_vmware_windows_update_no_auto_reboot_ready`、`awoooi_windows99_vmware_missing_vmx_alias` 與 `awoooi_windows99_vmware_powered_off_alias`。production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 讀到 runtime metric `config_ready=1`、`windows_update_no_auto_reboot_ready=1`、`missing_vmx_aliases=[]` 時，不得再用 committed snapshot 的舊 `service_blockers` / `policy_blockers` 反加 `windows99_vmware_autostart_config_not_ready` 或 `windows99_update_no_auto_reboot_policy_not_ready`。若 `runtime_metric_active_blocker_count=7` 但 API `active_blocker_count` 被 stale snapshot 放大，優先修 API overlay / exporter metric，不得把假 blocker 轉成 Windows service restart、registry apply、VM power change、host reboot、Docker / Nginx / K3s / DB / firewall restart 或 secret 讀取。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index d95327f53..3a6cf3b17 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.11 +> **版本**: v5.12 > **最後更新**: 2026-07-03 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-03 13:50 no-secret via-host Verify readback：新版 99 verifier 已證明 default VMX source 不缺，剩餘 blocker 是 VM power/readback；validator / collector source 已同步 +> **觸發事件**: 2026-07-03 14:10 Windows99 runtime metric overlay：110 SLO textfile 已證明 VMX missing 清除、config / Windows Update ready；API 不得用 stale snapshot 反加假 blocker --- @@ -21,9 +21,9 @@ ## 架構概覽與依賴圖 -### 2026-07-03 12:34 P0 現況覆蓋 +### 2026-07-03 14:10 P0 現況覆蓋 -目前不得宣稱「全主機重啟後 10 分鐘內全部恢復」。deploy marker `4e7f10cfd chore(cd): deploy ed419da [skip ci]` 後，production SLO scorecard 仍為 `blocked_reboot_auto_recovery_slo_not_ready`，`active_blocker_count=10`、`readiness_percent=60`、primary blocker `reboot_event_required_host_unreachable`，`can_claim_all_services_recovered_within_target=false`。10 個 active blockers 固定為： +目前仍不得宣稱「全主機重啟後 10 分鐘內全部恢復」。Gitea CD `#4587` 已完成 source `30c89a0` 部署並推 deploy marker `4da6dda`；110 上 `awoooi-reboot-auto-recovery-slo.timer` 已 `enabled/active`，最新 textfile runtime metric 為 `active_blocker_count=7`，primary blocker `reboot_event_required_host_unreachable`，`can_claim_all_services_recovered_within_target=false`。7 個 runtime active blockers 固定為： - `all_required_hosts_not_in_10_minute_reboot_window` - `fresh_all_host_reboot_event_missing` @@ -32,21 +32,18 @@ - `host_uptime_unknown` - `reboot_event_required_host_unreachable` - `windows99_vmware_guest_power_not_ready` -- `windows99_vmware_vmx_missing` -- `windows99_vmware_autostart_config_not_ready` -- `windows99_update_no_auto_reboot_policy_not_ready` -最新 no-write artifact 以 production API readback、110 上最新 Windows99 collector readback 與本輪 VMX locate collector 為來源： +最新 no-write artifact 以 110 runtime textfile、latest SLO artifact、production API readback 與本輪 Windows99 collector readback 為來源： -- 99 no-secret VMware verify stdout 已能透過 110 / public-key SSH 收集，但 deploy 後 production scorecard 仍上卷 `windows99_vmware_autostart_config_not_ready` 與 `windows99_update_no_auto_reboot_policy_not_ready`；未清除前不得把服務 / policy 視為綠燈。 -- 111 VMX expected path 不存在；`windows99_vmware_vmx_missing` 仍是 Windows99 主要 source blocker。 -- 111 / 188 / 120 / 121 / 112 VM power readback 仍為 not running；`windows99_vmware_guest_power_not_ready` 仍 blocked，且此 SOP 不授權直接 power change。 +- 99 no-secret VMware verify stdout 已能透過 110 / public-key SSH 收集；latest artifact 顯示 `missing_vmx_aliases=[]`、`VMWARE_AUTOSTART_CONFIG_READY=1`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1`、`VMWARE_AUTOSTART_VERIFY_READY=0`。 +- 110 / 188 / 120 / 121 / 112 VMX source 已存在，`windows99_vmware_vmx_missing` 不得再當目前 blocker。 +- 110 / 188 / 120 / 121 / 112 VM power readback 仍為 not running；`windows99_vmware_guest_power_not_ready` 仍 blocked，且此 SOP 不授權直接 power change。 - 111 host 仍 unreachable。 - 112 / 120 / 121 可達，但 uptime 已超過 10 分鐘目標窗口。 - 188 可達，但 `systemd_state=degraded` 且 `awoooi-startup.service failed`。 - `reboot_event_required_host_unreachable` 仍為 primary blocker，因此 10 分鐘 SLO 尚未證明。 -固定下一步：`locate_or_relink_missing_vmx_source_check_mode`。production `/api/v1/agents/windows99-vmx-source-locator-readback` 已回 `status=check_mode_locator_ready_windows99_vmx_source_required`、`missing_vmx_aliases=["111"]`、`safe_next_step=bash scripts/reboot-recovery/locate-windows99-vmx-source.sh --collect`、`secret_value_read=false`、`remote_write_performed=false`、`vm_power_change_allowed=false`。目前工作站必須加 `--via-host wooo@192.168.0.110 --users wooo --max-auth-users 1` 走已驗證的 110 no-secret path；live 結果為 `locator_collection_status=collected_windows99_vmx_source_locator_stdout`、`expected_vmx_path_present_count=0`、`candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`、`raw_path_output=false`、`path_fingerprint_only=true`。下一步只能取得既有 VMX source 或 external verified backup path，再產生 scoped relink / restore dry-run 與 post-verifier。若 public API response 顯示 `public_lan_topology_redacted=true` 或 `check_mode_probe_commands_executable=false`，其中 `host:...` VMX path 只能當 public display evidence，不得直接貼到 Windows console 執行；要執行 probe 時必須使用授權內部 console / no-secret package 保留的原始 Windows path。禁止讀 Windows 密碼、啟動 / 關閉 VM、host reboot、service restart、Docker / Nginx / K3s / DB / firewall restart、restore、prune、delete。 +固定下一步：保留 110 timer / textfile，每輪讀最新 `scorecard.json` 與 Prometheus metric；Windows99 lane 只建立 powered-off aliases 的 check-mode / post-verifier package，不能從 scorecard 直接 power on。API overlay 必須以 runtime metric 的 `awoooi_windows99_vmware_autostart_config_ready` 與 `awoooi_windows99_vmware_windows_update_no_auto_reboot_ready` 為準；當這些 metric 為 ready 時，不得用 committed snapshot 的舊 `service_blockers` / `policy_blockers` 反加 `windows99_vmware_autostart_config_not_ready` 或 `windows99_update_no_auto_reboot_policy_not_ready`。同時繼續補 fresh all-host reboot event / approved drill proof；禁止讀 Windows 密碼、啟動 / 關閉 VM、host reboot、service restart、Docker / Nginx / K3s / DB / firewall restart、restore、prune、delete。 2026-07-03 13:10 最新 overlay：Gitea CD `#4582` 已推 deploy marker `558b9e9`，source `ba0849f70`，production deploy readback matched=true 且 job_succeeded=true；Gitea UI 若仍顯示 Running，只能視為 run-level lag。production `/api/v1/agents/windows99-vmx-source-locator-readback` 已上卷 redacted locator receipt：`candidate_source_count=6`、`alias_hint_candidate_count=0`、`unassigned_ubuntu_candidate_count=5`、`identity_unassigned_ubuntu_candidate_count=1`、`candidate_confirmation_required=true`，狀態為 `collector_readback_single_identity_candidate_confirmation_required`。新的 `candidate_confirmation_gate` 固定輸出 `candidate_identity_evidence_sufficient=false`、`candidate_relink_dry_run_ready=false`、`apply_allowed_by_candidate_confirmation_gate=false`；缺少 evidence 為 `target_alias_identity_hint`、`verified_backup_or_authorized_console_candidate_identity`、`pre_apply_backup_plan`、`scoped_relink_dry_run`。因此下一步已從「重跑 locator」收斂為：取得授權 console candidate identity evidence 或 verified backup path，重跑 confirmation gate，再產生 scoped relink dry-run + rollback + post-verifier；仍不得猜測 generic Ubuntu candidate、不得 relink、不得 restore、不得 power on VM。 @@ -54,6 +51,8 @@ 2026-07-03 13:50 no-secret verifier overlay：本工作站直連 99 仍回 `blocked_ssh_publickey_auth_missing`，但 `collect-windows99-vmware-verify.sh --collect --via-host wooo@192.168.0.110 --users wooo --max-auth-users 1` 成功取得 Verify stdout。validator default 已改成 `110 / 188 / 120 / 121 / 112`，同輪 artifact 驗證為 `valid_console_verify_artifact_vmware_not_ready`、`missing_vmx_aliases=[]`、`powered_off_aliases=110,188,120,121,112`、`VMWARE_AUTOSTART_CONFIG_READY=1`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1`、`VMWARE_AUTOSTART_VERIFY_READY=0`。因此目前 Windows99 lane 不再是 111 VMX source repair；下一步是部署新版 SLO exporter/collector 到 110、重跑 verifier service，讓 Prometheus / production scorecard 清掉舊 `windows99_vmware_vmx_missing`，再處理 VM power/readback blocker。仍不得啟動 VM、relink、restore、重啟 host 或讀密碼。 +2026-07-03 14:10 runtime metric overlay：110 最新 SLO textfile 已輸出 `active_blocker_count=7`，且 `awoooi_windows99_vmware_missing_vmx_alias` 沒有 alias gauge、`awoooi_windows99_vmware_powered_off_alias` 只列 `110 / 112 / 120 / 121 / 188`。production API 在 source 修正前仍可能因 stale committed snapshot 反加 Windows99 config / Windows Update policy 假 blocker；新版 API 必須讀 `awoooi_windows99_vmware_autostart_config_ready` gauge，並以 runtime metric overlay 清掉這兩個假 blocker。若 production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 仍顯示 `runtime_metric_active_blocker_count=7` 但 `active_blocker_count>7` 且多出的 blocker 是 Windows99 config / policy，先部署 / 驗證 API overlay source，不得回頭做 VMX locate、relink、power change 或 Windows policy apply。 + ### 五主機全貌 ``` diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 7448aa5ed..be1be5072 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -387,6 +387,10 @@ gauge( "awoooi_windows99_vmware_verify_ready", windows99.get("verify_ready") is True, ) +gauge( + "awoooi_windows99_vmware_autostart_config_ready", + windows99.get("config_ready") is True, +) gauge( "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready", windows99.get("windows_update_no_auto_reboot_ready") is True, @@ -514,6 +518,8 @@ awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_route # TYPE awoooi_windows99_vmware_readback_present gauge # HELP awoooi_windows99_vmware_verify_ready Whether Windows99 VMware autostart verifier is fully green. # TYPE awoooi_windows99_vmware_verify_ready gauge +# HELP awoooi_windows99_vmware_autostart_config_ready Whether Windows99 VMware autostart config is verified. +# TYPE awoooi_windows99_vmware_autostart_config_ready gauge # HELP awoooi_windows99_vmware_windows_update_no_auto_reboot_ready Whether Windows Update no-auto-reboot policy is verified. # TYPE awoooi_windows99_vmware_windows_update_no_auto_reboot_ready gauge # HELP awoooi_windows99_vmware_missing_vmx_alias Missing required Windows99 VMX aliases. diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py index 935583aa1..be680a36c 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py @@ -127,6 +127,7 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None: assert "POST_REBOOT_READINESS_SUMMARY_FALLBACK_MAX_AGE_SECONDS" in text assert "awoooi_windows99_vmware_missing_vmx_alias" in text assert "awoooi_windows99_vmware_powered_off_alias" in text + assert "awoooi_windows99_vmware_autostart_config_ready" in text assert "active_blocker_metrics" in text assert 'for blocker in payload.get("active_blockers") or []' in text assert "awoooi_reboot_auto_recovery_slo_active_blocker" in text