From c2dd14acba4951ddba58865dbc55163c8f45e96f Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 16:42:14 +0800 Subject: [PATCH] fix(reboot): use host111 as windows99 vm target --- .../reboot_auto_recovery_slo_scorecard.py | 2 +- ..._reboot_auto_recovery_slo_scorecard_api.py | 12 +++++------ ...vmware_autostart_check_mode_package_api.py | 12 +++++------ docs/LOGBOOK.md | 20 +++++++++++++++++++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 8 ++++---- .../PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md | 4 ++-- docs/runbooks/REBOOT-RECOVERY-SOP.md | 10 +++++----- ...oot-cold-start-backup-recovery-workplan.md | 2 +- .../reboot-auto-recovery-slo-scorecard.py | 2 +- .../test_reboot_p0_operational_contract.py | 3 +-- ...ows99_console_verify_artifact_validator.py | 14 ++++++------- ...idate-windows99-console-verify-artifact.py | 2 +- .../windows99-vmware-autostart.ps1 | 5 +---- 13 files changed, 56 insertions(+), 40 deletions(-) diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 4a171047c..08ced1e53 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -130,7 +130,7 @@ _WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION = ( "run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_" "then_rerun_no_secret_collector_no_vm_power_change" ) -_WINDOWS99_REQUIRED_VM_ALIASES = ["110", "188", "120", "121", "112"] +_WINDOWS99_REQUIRED_VM_ALIASES = ["111", "188", "120", "121", "112"] _WINDOWS99_EXPECTED_VMX_CANDIDATES = { "110": [ r"D:\Documents\Virtual Machines\Ubuntu 64-bit (3)\Ubuntu 64-bit (3).vmx", diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index cab667750..eb6d8843a 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -716,7 +716,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_windows99_task_run_state(): "windows99_vmware_autostart_config_ready": True, "windows99_windows_update_no_auto_reboot_ready": True, "windows99_missing_vmx_aliases": [], - "windows99_powered_off_aliases": ["110", "112", "120", "121", "188"], + "windows99_powered_off_aliases": ["111", "112", "120", "121", "188"], "windows99_autostart_task_start_script_present": True, "windows99_autostart_task_last_result": 267011, "windows99_autostart_task_last_result_hex": "0x41303", @@ -736,7 +736,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_windows99_task_run_state(): assert task["never_ran"] is True assert payload["windows99_vmware_autostart"]["config_ready"] is True assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [ - "110", + "111", "112", "120", "121", @@ -786,7 +786,7 @@ def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_windows99_config_whe "windows99_vmware_autostart_config_ready": True, "windows99_windows_update_no_auto_reboot_ready": True, "windows99_missing_vmx_aliases": [], - "windows99_powered_off_aliases": ["110", "112", "120", "121", "188"], + "windows99_powered_off_aliases": ["111", "112", "120", "121", "188"], } payload = load_latest_reboot_auto_recovery_slo_scorecard( @@ -817,14 +817,14 @@ def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_windows99_config_whe ) assert autostart_package["package_ready"] is True assert autostart_package["powered_off_aliases"] == [ - "110", + "111", "112", "120", "121", "188", ] assert autostart_package["work_order_id"] == ( - "windows99-vmware-autostart-check-mode-110" + "windows99-vmware-autostart-check-mode-111" ) assert autostart_package["action_stage"] == "power_readback_check_mode_ready" assert autostart_package["apply_allowed_by_this_package"] is False @@ -1661,7 +1661,7 @@ def _assert_reboot_slo_payload(payload: dict): assert "manual_db_update" in service_data_backup["forbidden_actions"] windows99 = payload["windows99_vmware_autostart"] assert windows99["readback_present"] is False - assert windows99["required_vm_aliases"] == ["110", "188", "120", "121", "112"] + assert windows99["required_vm_aliases"] == ["111", "188", "120", "121", "112"] assert windows99["blockers"] == ["windows99_vmware_autostart_readback_missing"] collection = payload["windows99_verify_collection"] assert collection["schema_version"] == ( diff --git a/apps/api/tests/test_windows99_vmware_autostart_check_mode_package_api.py b/apps/api/tests/test_windows99_vmware_autostart_check_mode_package_api.py index 7b5374c32..3ef149b5f 100644 --- a/apps/api/tests/test_windows99_vmware_autostart_check_mode_package_api.py +++ b/apps/api/tests/test_windows99_vmware_autostart_check_mode_package_api.py @@ -16,9 +16,9 @@ def _scorecard() -> dict: "status": "blocked_reboot_auto_recovery_slo_not_ready", "generated_at": "2026-07-03T14:26:00+08:00", "windows99_vmware_autostart": { - "required_vm_aliases": ["110", "188", "120", "121", "112"], + "required_vm_aliases": ["111", "188", "120", "121", "112"], "missing_vmx_aliases": [], - "powered_off_aliases": ["110", "112", "120", "121", "188"], + "powered_off_aliases": ["111", "112", "120", "121", "188"], "config_ready": True, "windows_update_no_auto_reboot_ready": True, }, @@ -63,7 +63,7 @@ def test_windows99_vmware_autostart_check_mode_package_builder_is_no_power_write ) assert payload["package_ready"] is True assert payload["target_host_alias"] == "99" - assert payload["powered_off_aliases"] == ["110", "112", "120", "121", "188"] + assert payload["powered_off_aliases"] == ["111", "112", "120", "121", "188"] assert payload["powered_off_count"] == 5 assert payload["safe_next_step"] == ( "run_windows99_vmware_autostart_check_mode_package_for_powered_" @@ -74,13 +74,13 @@ def test_windows99_vmware_autostart_check_mode_package_builder_is_no_power_write ) assert payload["apply_allowed_by_this_package"] is False assert payload["work_order_id"] == ( - "windows99-vmware-autostart-check-mode-110" + "windows99-vmware-autostart-check-mode-111" ) assert payload["action_stage"] == "power_readback_check_mode_ready" assert payload["next_executable_step"] == ( "build_vmware_autostart_power_readback_check_mode" ) - assert payload["target_vm_aliases"] == ["110", "112", "120", "121", "188"] + assert payload["target_vm_aliases"] == ["111", "112", "120", "121", "188"] assert payload["check_mode_allowed_by_this_package"] is True assert payload["check_mode_probe_command_count"] == 2 assert payload["rollback_required"] is True @@ -143,7 +143,7 @@ def test_windows99_vmware_autostart_check_mode_package_endpoint_returns_readback "windows99_vmware_autostart_check_mode_package_v1" ) assert payload["work_order_id"] == ( - "windows99-vmware-autostart-check-mode-110" + "windows99-vmware-autostart-check-mode-111" ) assert payload["public_display_only"] is True assert payload["check_mode_package_execution_authorized_by_this_endpoint"] is False diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 6899f20d1..b6fdcbef5 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,23 @@ +## 2026-07-03 — 16:33 Windows99 VMware default target 修正為 111 + +**完成內容**： +- production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 已讀回新版 task run-state：`active_blocker_count=9`、`runtime_metric_active_blocker_count=9`、`windows99_vmware_autostart_task_never_ran` 已進 active blockers，scheduled task `last_task_result=267011` / `0x41303`、`last_run_time=1999-11-30T00:00:00`。 +- 110 no-secret artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-163131` 證明 collector 未讀 secret、未 remote write、未 VM power change；但 host probe 同時顯示 `111 reachable=0`，而舊 Windows99 default target 卻是 `110 / 188 / 120 / 121 / 112`。 +- 修正 source-of-truth：`windows99-vmware-autostart.ps1`、CLI scorecard、API scorecard fallback、console validator default、check-mode package tests 全部改為 `111 / 188 / 120 / 121 / 112`；110 回到 P0 host / control-plane truth，不再當 99 VMware guest default。 +- `FULL-STACK-COLD-START-SOP`、`REBOOT-RECOVERY-SOP`、`PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK` 與 P0 workplan 已整合這次經驗，避免舊 `110` default 再遮蔽 `111 unreachable`。 + +**已跑驗證**： +- `python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py -q -p no:cacheprovider`：`40 passed`。 +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_windows99_vmware_autostart_check_mode_package_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py -q -p no:cacheprovider`：`44 passed`。 +- `python3.11 -m py_compile ...`、`bash -n scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh scripts/reboot-recovery/collect-windows99-vmware-verify.sh`、`python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`、`git diff --check`：通過。 + +**仍維持**： +- 這不代表 10 分鐘重啟自動恢復 SLO 完成；目前 primary blocker 仍是 `reboot_event_required_host_unreachable`，且 `111` 仍需新版 Windows99 collector / production readback 確認。 +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有使用 GitHub / gh；沒有啟動或關閉 VM，沒有重啟 Windows host / VMware service，沒有 Docker / Nginx / K3s / DB / firewall restart / reload，沒有 DROP / TRUNCATE / restore / prune / delete / force push。 + +**下一步**： +- commit / push 到 Gitea main；CD deploy 後受控安裝新版 110 SLO exporter 並重跑 no-secret collector，讀回 production `required_vm_aliases` / `powered_off_aliases` / check-mode package 均以 111 為 Windows99 VM target。 + ## 2026-07-03 — 15:57 Windows99 VMware autostart task run-state 上卷 **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 184c7b99c..3dbe0585c 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -20,7 +20,7 @@ v1.80 / v1.81 credential escrow intake scorecard rule：同一輪 owner response v1.93 reboot SLA readback fixed fields rule：`scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py`、`docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json` 與 `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須固定輸出 `current_phase`、`eta_or_wait_reason`、`primary_blocker`、`active_blockers`、`next_safe_action` 與 `reboot_sop_progress.fixed_triage_order`。回報時先讀這些欄位，不得重新臨場發明排查順序；若 `current_phase=host_boot_detection_blocked` 且 `eta_or_wait_reason=target_window_elapsed_eta_unavailable_until_fresh_all_host_reboot_event_and_probe`，只能宣稱「10 分鐘 SLO 尚未證明」，下一步固定補 fresh all-host reboot event / probe 與 99 VMware/Windows readback，不得把部分 Linux host green 當全主機 green。 -v1.94 Windows 99 / VMware verifier rule：99 主機與 guest VM 自動啟動不得再只靠「ping 到 99」或「VMX source 存在」宣稱。`scripts/reboot-recovery/windows99-vmware-autostart.ps1 -Mode Verify` 必須輸出 no-secret 固定欄位：`VMRUN_PRESENT`、`VMX alias=... present=...`、`VMWARE_SERVICE ... ok=...`、`VMWARE_AUTOSTART_TASK ... ok=...`、`WINDOWS_UPDATE_POLICY ... ok=...`、`VM_POWER alias=... running=...`、`VMWARE_AUTOSTART_CONFIG_READY`、`VMWARE_AUTOSTART_POWER_READY`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`、`VMWARE_AUTOSTART_VERIFY_READY`。`reboot-auto-recovery-slo-scorecard.py --windows99-vmware-file ` 必須解析這些欄位；缺 readback 時 active blocker 固定為 `windows99_vmware_autostart_readback_missing`，next action 固定為 `collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot`。預設 required guest VM alias 以 99 runtime apply truth 為準：`110 / 188 / 120 / 121 / 112`；`111` 保留為 required host / service reachability 與 explicit `Host111Vmx` override，不得在缺少 verified VMX source 時被當成 99 VMware 預設 VM。此 verifier 不讀 Windows 密碼、不讀 secret、不啟動 VM、不重啟 host；`Apply` 仍需獨立 controlled apply 與 post-verifier。 +v1.94 Windows 99 / VMware verifier rule：99 主機與 guest VM 自動啟動不得再只靠「ping 到 99」或「VMX source 存在」宣稱。`scripts/reboot-recovery/windows99-vmware-autostart.ps1 -Mode Verify` 必須輸出 no-secret 固定欄位：`VMRUN_PRESENT`、`VMX alias=... present=...`、`VMWARE_SERVICE ... ok=...`、`VMWARE_AUTOSTART_TASK ... ok=...`、`WINDOWS_UPDATE_POLICY ... ok=...`、`VM_POWER alias=... running=...`、`VMWARE_AUTOSTART_CONFIG_READY`、`VMWARE_AUTOSTART_POWER_READY`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`、`VMWARE_AUTOSTART_VERIFY_READY`。`reboot-auto-recovery-slo-scorecard.py --windows99-vmware-file ` 必須解析這些欄位；缺 readback 時 active blocker 固定為 `windows99_vmware_autostart_readback_missing`，next action 固定為 `collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot`。預設 required guest VM alias 以本輪 reboot 事故 source-of-truth 為準：`111 / 188 / 120 / 121 / 112`；`110` 保留為 required host / control-plane reachability，不得被當成 99 VMware 預設 VM。此 verifier 不讀 Windows 密碼、不讀 secret、不啟動 VM、不重啟 host；`Apply` 仍需獨立 controlled apply 與 post-verifier。 v1.95 Windows 99 management-channel readback rule：若 99 可 ping / RDP / TCP，但 `windows99-vmware-autostart.ps1 -Mode Verify` 尚未收回，不得只寫成「等 verifier」。必須先跑 `scripts/reboot-recovery/windows99-management-channel-probe.py --output `，並把結果用 `reboot-auto-recovery-slo-scorecard.py --windows99-management-file ` 接入同一份 SLO scorecard。此 probe 只做 no-secret readback：TCP `22 / 135 / 445 / 2179 / 3389 / 5985 / 5986`、SSH BatchMode publickey、WinRM port、RDP console reachability 與 VMConnect / local console reachability；不得讀 Windows 密碼、不得啟動 VM、不得重啟、不得改 Windows Update。2026-07-02 18:28 live readback：`host_reachable=true`、`rdp_console_reachable=true`、`hyperv_vmconnect_open=true`、`local_console_channel_reachable=true`、`console_collection_channels=["rdp_console","hyperv_vmconnect"]`、`ssh_batch.status=permission_denied`、`winrm_http_open=false`、`winrm_https_open=false`、`remote_execution_channel_ready=false`；因此 active blockers 必須包含 `windows99_remote_execution_channel_unavailable` 與 `windows99_vmware_autostart_readback_missing`，next action 固定為 `restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot`。 @@ -66,9 +66,9 @@ v1.116 Windows99 VMX locator via-host execution rule：若目前工作站直連 v1.117 Windows99 VMX candidate confirmation gate rule：locator receipt 顯示 `single_identity_unassigned_ubuntu_candidate_requires_confirmation` 時，不能把「唯一 generic Ubuntu candidate」直接升級成 111 VMX source。`/api/v1/agents/windows99-vmx-source-locator-readback` 必須輸出 `candidate_confirmation_gate`，並同步到 priority readback / Work Items；目前 gate 固定 `candidate_identity_evidence_sufficient=false`、`candidate_relink_dry_run_ready=false`、`apply_allowed_by_candidate_confirmation_gate=false`，missing evidence 至少包含 `target_alias_identity_hint`、`verified_backup_or_authorized_console_candidate_identity`、`pre_apply_backup_plan`、`scoped_relink_dry_run`。只有取得 target alias / MAC / UUID / display-name 或 verified backup path 證據，且產生 rollback + post-verifier 的 scoped relink dry-run 後，才可進下一階段；scorecard / locator / work item 均不得授權 VM power change、Windows write、service restart、host reboot、secret/password 讀取或 production restore。 -v1.118 Windows99 VM target source-of-truth reconciliation rule：99 VMware autostart 的預設 target 不得與 2026-07-01 runtime apply truth 分叉。repo source、API scorecard fallback、CLI scorecard fallback 必須一致使用 `110 / 188 / 120 / 121 / 112`；`111` 仍是全主機 reboot SLO 的 required host，但不是 99 VMware 預設 VMX source。若 verifier stdout 已列出 `VMX alias=...` / `VM_POWER alias=...`，scorecard 必須以 verifier 實際 alias 集合為準，讓舊的 explicit `111` artifact 或未來 `Host111Vmx` override 仍可走 `111` repair lane。`windows99_vmx_source_repair_package.vm_rows` 必須是 default required aliases 與 runtime missing / powered-off aliases 的 union，避免 runtime 明確送進來的 alias 被 UI/API row 漏掉。此規則只修 source-of-truth / readback；不授權改 Windows task、relink、restore、啟動 VM 或重啟 host。 +v1.118 Windows99 VM target source-of-truth reconciliation rule：99 VMware autostart 的預設 target 不得與目前 reboot 事故目標分叉。repo source、API scorecard fallback、CLI scorecard fallback、validator default 必須一致使用 `111 / 188 / 120 / 121 / 112`；`110` 仍是全主機 reboot SLO 的 required host 與 control-plane host，但不是 99 VMware 預設 VMX source。若 verifier stdout 已列出 `VMX alias=...` / `VM_POWER alias=...`，scorecard 必須以 verifier 實際 alias 集合為準。`windows99_vmx_source_repair_package.vm_rows` 必須是 default required aliases 與 runtime missing / powered-off aliases 的 union，避免 runtime 明確送進來的 alias 被 UI/API row 漏掉。此規則只修 source-of-truth / readback；不授權改 Windows task、relink、restore、啟動 VM 或重啟 host。 -v1.119 Windows99 no-secret verifier via-host / validator rule：本工作站直連 99 SSH 缺 public-key auth 時，`collect-windows99-vmware-verify.sh` 必須支援 `--via-host wooo@192.168.0.110`，沿 110→99 已驗證 no-secret path 執行 Verify；collector 必須輸出 `via_host=`，仍固定 BatchMode/publickey、in-memory stdin scriptblock、`remote_write_performed=false`、`vm_power_change_performed=false`。`validate-windows99-console-verify-artifact.py` default required aliases 必須與 v1.118 一致為 `110 / 188 / 120 / 121 / 112`；若未來要驗 explicit `111` artifact，必須用 `--required-aliases 111,188,120,121,112` 顯式指定。2026-07-03 via-host live artifact 已證明新版 Verify readback：VMX missing 為空、config ready、Windows Update no-auto-reboot ready，剩餘 blocker 是 `windows99_vmware_guest_power_not_ready`；不得再把 validator 舊預設造成的 `111` missing 當真 blocker。 +v1.119 Windows99 no-secret verifier via-host / validator rule：本工作站直連 99 SSH 缺 public-key auth 時，`collect-windows99-vmware-verify.sh` 必須支援 `--via-host wooo@192.168.0.110`，沿 110→99 已驗證 no-secret path 執行 Verify；collector 必須輸出 `via_host=`，仍固定 BatchMode/publickey、in-memory stdin scriptblock、`remote_write_performed=false`、`vm_power_change_performed=false`。`validate-windows99-console-verify-artifact.py` default required aliases 必須與 v1.118 一致為 `111 / 188 / 120 / 121 / 112`。2026-07-03 16:33 已證明舊 `110` default 會遮蔽 111 unreachable；不得再把舊 validator default 造成的 111 漏檢當真 green。 v1.120 Windows99 runtime metric overlay precedence rule：v1.110 的 nested blocker promotion 只適用於 runtime metric 缺少 Windows99 config / policy 欄位，或 metric 明確回 not ready 的情況。`reboot-auto-recovery-slo-exporter.sh` 必須從同一份 latest `scorecard.json` 輸出 `awoooi_windows99_vmware_autostart_config_ready`、`awoooi_windows99_vmware_windows_update_no_auto_reboot_ready`、`awoooi_windows99_vmware_missing_vmx_alias` 與 `awoooi_windows99_vmware_powered_off_alias`。production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 讀到 runtime metric `config_ready=1`、`windows_update_no_auto_reboot_ready=1`、`missing_vmx_aliases=[]` 時，不得再用 committed snapshot 的舊 `service_blockers` / `policy_blockers` 反加 `windows99_vmware_autostart_config_not_ready` 或 `windows99_update_no_auto_reboot_policy_not_ready`。若 `runtime_metric_active_blocker_count=7` 但 API `active_blocker_count` 被 stale snapshot 放大，優先修 API overlay / exporter metric，不得把假 blocker 轉成 Windows service restart、registry apply、VM power change、host reboot、Docker / Nginx / K3s / DB / firewall restart 或 secret 讀取。 @@ -80,7 +80,7 @@ v1.123 SLO installer edge-source packaging rule：110 SLO installer payload 必 v1.124 public maintenance route / edge blocker split rule：production API overlay 不得把 route fallback ready 與 edge Nginx fallback ready 視為同一件事。`public_maintenance_fallback.runtime_readback_present=true` 只能清除 `public_maintenance_fallback_runtime_readback_missing`；route 0 raw 5xx / 0 unreachable 只能清除 route-level blockers；只有 `edge_fallback_runtime_readback_present=true` 且 `edge_fallback_ready=true` 才能清除 `edge_public_maintenance_live_config_drift`。若 Prometheus textfile active label 已有 edge drift，但 public route probe 是 200 / ready 且缺 edge readback，`/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須保留 edge drift；否則會把下次 reboot / deploy raw 502 風險誤報成正常。此規則只修 scorecard/API readback，不授權 Nginx reload、app restart、Docker / host restart 或讀密碼。 -v1.125 Windows99 autostart task run-state rule：`windows99-vmware-autostart.ps1 -Mode Verify` 必須在既有 `VMWARE_AUTOSTART_TASK ... ok=...` 之外輸出 `VMWARE_AUTOSTART_TASK_DETAIL` 與 `VMWARE_AUTOSTART_TASK_ACTION`，至少包含 principal、run level、logon type、target script present、last run time、last task result、next run time、action execute / argument。`last_task_result=267011` / `0x41303` 或 `last_run_time=1999-11-30T00:00:00` 代表 task exists/enabled 但尚未跑過，若同時 `powered_off_aliases` 非空，scorecard 必須新增 active blocker `windows99_vmware_autostart_task_never_ran`，textfile exporter 必須輸出 `awoooi_windows99_vmware_autostart_task_*` metrics，API overlay / action matrix 必須把 next action 指到 `run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_then_rerun_no_secret_collector_no_vm_power_change`。2026-07-03 live no-secret via-110 readback 證明 `AWOOOI-Start-VMware-VMs` present/enabled/Ready，但 `last_task_result=267011`、`last_run_time=1999-11-30T00:00:00`，且 `110 / 188 / 120 / 121 / 112` 全部 `running=0`；因此 P0-4 不是「缺 task」，而是「task 未觸發 / 未跑」。此規則只授權 readback、metrics、alert routing 與 check-mode package；不得直接啟動 VM、重啟 Windows host / VMware service、改 scheduled task、改 registry 或讀密碼 / secret。 +v1.125 Windows99 autostart task run-state rule：`windows99-vmware-autostart.ps1 -Mode Verify` 必須在既有 `VMWARE_AUTOSTART_TASK ... ok=...` 之外輸出 `VMWARE_AUTOSTART_TASK_DETAIL` 與 `VMWARE_AUTOSTART_TASK_ACTION`，至少包含 principal、run level、logon type、target script present、last run time、last task result、next run time、action execute / argument。`last_task_result=267011` / `0x41303` 或 `last_run_time=1999-11-30T00:00:00` 代表 task exists/enabled 但尚未跑過，若同時 `powered_off_aliases` 非空，scorecard 必須新增 active blocker `windows99_vmware_autostart_task_never_ran`，textfile exporter 必須輸出 `awoooi_windows99_vmware_autostart_task_*` metrics，API overlay / action matrix 必須把 next action 指到 `run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_then_rerun_no_secret_collector_no_vm_power_change`。2026-07-03 live no-secret via-110 readback 證明 `AWOOOI-Start-VMware-VMs` present/enabled/Ready，但 `last_task_result=267011`、`last_run_time=1999-11-30T00:00:00`；P0-4 真 target set 必須是 `111 / 188 / 120 / 121 / 112`，部署後重新收集 powered-off aliases，不得再沿用舊 `110` default。此規則只授權 readback、metrics、alert routing 與 check-mode package；不得直接啟動 VM、重啟 Windows host / VMware service、改 scheduled task、改 registry 或讀密碼 / secret。 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 diff --git a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md index b7997a0a1..e95893abc 100644 --- a/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md +++ b/docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md @@ -33,12 +33,12 @@ L1 active 條件： - `probe_success` 對主要 public routes 連續 2 分鐘失敗，或 - `awoooi_reboot_event_detected == 1` 且 `awoooi_reboot_auto_recovery_slo_ready == 0` 超過 10 分鐘，或 -- 99 / 110 / 188 / 120 / 121 / 112 任一 P0 host down 且 public route 同時 502 / timeout。 +- 99 / 110 / 111 / 188 / 120 / 121 / 112 任一 P0 host down 且 public route 同時 502 / timeout。 ## 驗證 - L0：對測試 vhost 讀回 `503` 的維護頁 HTML，且 header `X-AWOOOI-Fallback=local-maintenance` 與 `Retry-After` 存在；probe 若讀到 `502` / `503` / `504` 但同時有 fallback header/body，必須分類為 maintenance fallback，不得算 raw 5xx。 -- L1：從外部網路讀回靜態維護頁，且不經過 99 / 110 / 188 / 120 / 121 / 112。 +- L1：從外部網路讀回靜態維護頁，且不經過 99 / 110 / 111 / 188 / 120 / 121 / 112。 - Recovery：主要 upstream 連續 2 分鐘健康後切回正式 origin；Telegram 發送 recovery 通知，並在 cold-start scorecard 留下 readback。 ## 2026-07-02 scorecard gate diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 9e29d879d..be9d2bce8 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -23,7 +23,7 @@ ### 2026-07-03 15:57 P0-4 Windows99 Task Run-State 覆蓋 -Windows99 no-secret via-110 collector 已讀回 `AWOOOI-Start-VMware-VMs` scheduled task：`present=1`、`enabled=1`、`state=Ready`、`trigger_count=1`、`action_count=1`，VMware service 與 Windows Update no-auto-reboot policy 也為 ready；但新增 task detail 顯示 `last_run_time=1999-11-30T00:00:00`、`last_task_result=267011`，等價於 `0x41303`，表示 task 尚未跑過 / 未在本輪 boot 觸發。五台 guest `110 / 188 / 120 / 121 / 112` 仍全部 `running=0`，因此目前 P0-4 真 blocker 是 `windows99_vmware_guest_power_not_ready` 加 `windows99_vmware_autostart_task_never_ran`。 +Windows99 no-secret via-110 collector 已讀回 `AWOOOI-Start-VMware-VMs` scheduled task：`present=1`、`enabled=1`、`state=Ready`、`trigger_count=1`、`action_count=1`，VMware service 與 Windows Update no-auto-reboot policy 也為 ready；但新增 task detail 顯示 `last_run_time=1999-11-30T00:00:00`、`last_task_result=267011`，等價於 `0x41303`，表示 task 尚未跑過 / 未在本輪 boot 觸發。2026-07-03 16:33 進一步確認舊 verifier default 把 `110` 誤列為 Windows99 guest；最新 source-of-truth 改為 `111 / 188 / 120 / 121 / 112`，以對齊本輪 host probe `111 unreachable` 與 99 VMware guest 需求。因此目前 P0-4 真 blocker 是 `windows99_vmware_guest_power_not_ready` 加 `windows99_vmware_autostart_task_never_ran`，且 deployment 後必須用新版 collector 重新讀回 111。 SOP v5.17 起，`windows99-vmware-autostart.ps1 -Mode Verify` 必須輸出 `VMWARE_AUTOSTART_TASK_DETAIL` 與 `VMWARE_AUTOSTART_TASK_ACTION`；110 textfile exporter 必須輸出 `awoooi_windows99_vmware_autostart_task_start_script_present`、`awoooi_windows99_vmware_autostart_task_last_result`、`awoooi_windows99_vmware_autostart_task_never_ran`；production scorecard / API 必須把 `windows99_vmware_autostart_task_never_ran` 放進 active blockers 與 action matrix。固定下一步是執行已建立的 Windows99 VMware autostart check-mode package，然後 rerun no-secret collector 與 reboot SLO scorecard；不得直接 power on VM、重啟 Windows host / VMware service、修改 scheduled task、修改 registry、讀 Windows 密碼或讀 secret。 @@ -84,8 +84,8 @@ Production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 已與 runtime me 最新 no-write artifact 以 110 runtime textfile、latest SLO artifact、production API readback 與本輪 Windows99 collector readback 為來源： - 99 no-secret VMware verify stdout 已能透過 110 / public-key SSH 收集；latest artifact 顯示 `missing_vmx_aliases=[]`、`VMWARE_AUTOSTART_CONFIG_READY=1`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1`、`VMWARE_AUTOSTART_VERIFY_READY=0`。 -- 110 / 188 / 120 / 121 / 112 VMX source 已存在，`windows99_vmware_vmx_missing` 不得再當目前 blocker。 -- 110 / 188 / 120 / 121 / 112 VM power readback 仍為 not running；`windows99_vmware_guest_power_not_ready` 仍 blocked，且此 SOP 不授權直接 power change。 +- 111 / 188 / 120 / 121 / 112 才是 Windows99 VMware default guest set；110 是 P0 host / control-plane truth，不得再被 validator 或 scorecard fallback 當成 99 VMware guest default。 +- 111 / 188 / 120 / 121 / 112 VM power readback 仍需用新版 no-secret collector 確認；`windows99_vmware_guest_power_not_ready` 仍 blocked，且此 SOP 不授權直接 power change。 - 111 host 仍 unreachable。 - 112 / 120 / 121 可達，但 uptime 已超過 10 分鐘目標窗口。 - 188 可達，但 `systemd_state=degraded` 且 `awoooi-startup.service failed`。 @@ -95,9 +95,9 @@ Production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 已與 runtime me 2026-07-03 13:10 最新 overlay：Gitea CD `#4582` 已推 deploy marker `558b9e9`，source `ba0849f70`，production deploy readback matched=true 且 job_succeeded=true；Gitea UI 若仍顯示 Running，只能視為 run-level lag。production `/api/v1/agents/windows99-vmx-source-locator-readback` 已上卷 redacted locator receipt：`candidate_source_count=6`、`alias_hint_candidate_count=0`、`unassigned_ubuntu_candidate_count=5`、`identity_unassigned_ubuntu_candidate_count=1`、`candidate_confirmation_required=true`，狀態為 `collector_readback_single_identity_candidate_confirmation_required`。新的 `candidate_confirmation_gate` 固定輸出 `candidate_identity_evidence_sufficient=false`、`candidate_relink_dry_run_ready=false`、`apply_allowed_by_candidate_confirmation_gate=false`；缺少 evidence 為 `target_alias_identity_hint`、`verified_backup_or_authorized_console_candidate_identity`、`pre_apply_backup_plan`、`scoped_relink_dry_run`。因此下一步已從「重跑 locator」收斂為：取得授權 console candidate identity evidence 或 verified backup path，重跑 confirmation gate，再產生 scoped relink dry-run + rollback + post-verifier；仍不得猜測 generic Ubuntu candidate、不得 relink、不得 restore、不得 power on VM。 -2026-07-03 13:40 source-of-truth overlay：7/1 runtime apply evidence 顯示 99 實際 VMware layout 是 `110 -> 188 -> 120 -> 121 -> 112`，且當時已標記 `host111_vmware_target=missing_no_vmx_found_on_99_ping_only`。因此 repo source、API scorecard fallback、CLI scorecard fallback 必須使用 `110 / 188 / 120 / 121 / 112` 作為 Windows99 default VM target；`111` 仍是 reboot SLO required host，但不是 99 VMware 預設 VM。若未來取得 verified `Host111Vmx` 或 verifier stdout 明確列出 `VMX alias=111` / `VM_POWER alias=111`，scorecard 仍會用 verifier 實際 alias 集合把 `111` 納入 repair package。禁止用 generic Ubuntu candidate 推測成 111、禁止為了清 blocker 修改 Windows task、relink、restore、啟動 VM 或重啟 host。 +2026-07-03 16:33 source-of-truth correction：本輪全主機 reboot 需求與 host probe 均指向 `111` 是 Windows99 guest target，舊的 `110 -> 188 -> 120 -> 121 -> 112` default 會讓 111 unreachable 被錯誤排除。因此 repo source、API scorecard fallback、CLI scorecard fallback、validator default 必須使用 `111 / 188 / 120 / 121 / 112` 作為 Windows99 default VM target；`110` 仍是 reboot SLO required host 與 control-plane host，但不是 99 VMware 預設 VM。禁止用舊 110 default 清掉 111 blocker，禁止為了清 blocker 修改 Windows task、relink、restore、啟動 VM 或重啟 host。 -2026-07-03 13:50 no-secret verifier overlay：本工作站直連 99 仍回 `blocked_ssh_publickey_auth_missing`，但 `collect-windows99-vmware-verify.sh --collect --via-host wooo@192.168.0.110 --users wooo --max-auth-users 1` 成功取得 Verify stdout。validator default 已改成 `110 / 188 / 120 / 121 / 112`，同輪 artifact 驗證為 `valid_console_verify_artifact_vmware_not_ready`、`missing_vmx_aliases=[]`、`powered_off_aliases=110,188,120,121,112`、`VMWARE_AUTOSTART_CONFIG_READY=1`、`WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1`、`VMWARE_AUTOSTART_VERIFY_READY=0`。因此目前 Windows99 lane 不再是 111 VMX source repair；下一步是部署新版 SLO exporter/collector 到 110、重跑 verifier service，讓 Prometheus / production scorecard 清掉舊 `windows99_vmware_vmx_missing`，再處理 VM power/readback blocker。仍不得啟動 VM、relink、restore、重啟 host 或讀密碼。 +2026-07-03 13:50 no-secret verifier overlay 曾以舊 default `110 / 188 / 120 / 121 / 112` 讀回 artifact；2026-07-03 16:33 已判定該 default 與目前事故目標不一致。下一步必須部署新版 SLO exporter/collector 到 110、重跑 verifier service，確認 validator default、Prometheus textfile、production scorecard 全部改讀 `111 / 188 / 120 / 121 / 112`。仍不得啟動 VM、relink、restore、重啟 host 或讀密碼。 2026-07-03 14:10 runtime metric overlay：110 最新 SLO textfile 已輸出 `active_blocker_count=7`，且 `awoooi_windows99_vmware_missing_vmx_alias` 沒有 alias gauge、`awoooi_windows99_vmware_powered_off_alias` 只列 `110 / 112 / 120 / 121 / 188`。production API 在 source 修正前仍可能因 stale committed snapshot 反加 Windows99 config / Windows Update policy 假 blocker；新版 API 必須讀 `awoooi_windows99_vmware_autostart_config_ready` gauge，並以 runtime metric overlay 清掉這兩個假 blocker。若 production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 仍顯示 `runtime_metric_active_blocker_count=7` 但 `active_blocker_count>7` 且多出的 blocker 是 Windows99 config / policy，先部署 / 驗證 API overlay source，不得回頭做 VMX locate、relink、power change 或 Windows policy apply。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 6ac9decaa..74ad3e76a 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -22,7 +22,7 @@ | P0-1 | BLOCKED_REBOOT_EVENT_AND_HOST_REACHABILITY | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 15:49 verify-only artifact `/tmp/awoooi-reboot-event-verify-20260703-154907`：`reboot_detected=false`、`all_required_hosts_observed=false`、`all_required_hosts_in_reboot_window=false`、`recovery_deadline_status=target_window_elapsed`、`unreachable_hosts=["111"]`、99 reachable 但 `uptime_seconds=unknown`、188 reachable 但 `systemd_state=degraded` / `startup_active=failed`。production SLO 仍為 `blocked_reboot_auto_recovery_slo_not_ready`，不能宣稱 10 分鐘自動恢復。 | 固定先 rerun host probe + reboot-event detector verify-only，並收斂 111 reachability、99 uptime / VMware readback、188 degraded/startup failed；不得 reboot、不得 VM power change、不得讀 Windows 密碼，不得用 ping-only 或舊 boot evidence 當完成證據。 | | P0-2 | DEPLOYED_RUNTIME_EDGE_DRIFT_BLOCKED | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 2026-07-03 15:35 source fix 已部署讀回：production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 保留 `edge_public_maintenance_live_config_drift`，`runtime_metric_active_blocker_count=8`，且 route fallback ready 不再清掉 edge Nginx drift。110 SLO artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-151219` 與 textfile 證明 edge 未 ready：`awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0`、active blocker `edge_public_maintenance_live_config_drift`。 | 後續才進 privileged edge apply 或 L1 cloud/CDN fallback decision；完成條件是 edge fallback live config / maintenance file / `nginx -t` / reload / public header verifier 或外部 L1 fallback 全綠。不得讀密碼、不得用 app / Docker / host restart 掩蓋 edge fallback drift。 | | P0-3 | PARTIAL_GREEN_DEPLOYED_SOURCE | 所有產品 / 網站版本與資料最新性 | Gitea `main=bc8d3af3f chore(cd): deploy dacdd90 [skip ci]`；CD `#4558` log 顯示 API/Web build/push wrapper 全部 `rc=0`、deploy marker `dacdd90`、production deploy readback matched。Production SLO route 已讀回最新 API 行為：`runtime_metric_runtime_readback_added_blockers=["windows99_vmware_autostart_config_not_ready","windows99_update_no_auto_reboot_policy_not_ready"]`、`windows99_update_no_auto_reboot_ready=false`。Stock freshness 仍為 `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。 | 版本層已補到 AWOOOI production；完成整體 P0 仍需收斂 Windows99 VMX / service / policy / power 與 host reboot-event blockers。後續每個 public product 仍要維持 source SHA、deploy marker、runtime endpoint、public route watch、freshness 四層一致。 | -| P0-4 | BLOCKED_WINDOWS99_TASK_NOT_TRIGGERED | 192.168.0.99 VMware 自動啟動與 VM guest 110 / 188 / 120 / 121 / 112 | 2026-07-03 15:57 no-secret via-110 collector artifact `/tmp/awoooi-windows99-autostart-detail-20260703-155710`：VMX `110 / 188 / 120 / 121 / 112` 全存在，VMware services ok，Windows Update no-auto-reboot ready，scheduled task `AWOOOI-Start-VMware-VMs` present/enabled/Ready；但 `VMWARE_AUTOSTART_TASK_DETAIL last_run_time=1999-11-30T00:00:00 last_task_result=267011` (`0x41303`) 且五台 `VM_POWER running=0`。 | 將 `windows99_vmware_autostart_task_never_ran` 納入 active blocker / metrics / Telegram routing，執行已建立的 autostart check-mode package，再 rerun no-secret collector + SLO scorecard。不得從 scorecard 直接 power on VM、重啟 Windows host / VMware service、改 scheduled task / registry 或讀 Windows 密碼。 | +| P0-4 | BLOCKED_WINDOWS99_TASK_NOT_TRIGGERED | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | 2026-07-03 16:33 production readback 已證明 source/API/textfile 都會輸出 `windows99_vmware_autostart_task_never_ran`，但同時發現舊 source-of-truth 把 Windows99 default VM target 寫成 `110 / 188 / 120 / 121 / 112`，與本輪事故需求及 host probe `111 unreachable` 不一致。最新 source 修正為 `111 / 188 / 120 / 121 / 112`，110 回到 P0 host / control-plane truth，不再當 99 VMware guest default。 | 部署修正後重新跑 no-secret via-110 collector + SLO scorecard，確認 `required_vm_aliases`、`powered_off_aliases`、check-mode package、validator 與 API readback 都以 111 為 Windows99 VM target。仍不得從 scorecard 直接 power on VM、重啟 Windows host / VMware service、改 scheduled task / registry 或讀 Windows 密碼。 | | P0-5 | RUNTIME_READY_BACKUP_RECEIPT_GAP | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。2026-07-03 source / runtime 已部署 `awoooi_backup_alert_receipt_*` 指標與 Prometheus rules；110 exporter 讀回 88 個 stage requirement、188 讀回 12 個 stage requirement，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 已修成每 `host / receipt_channel` 聚合 pending：110 一條、188 一條。 | 補 `/backup/alert-receipts/*.last_success` 脫敏 marker；下一層仍要補 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控。 | | P0-6 | RUNTIME_READY_ALERT_RECEIPT_GAP | 主機關機 / 重啟 / SLO miss / backup failure Telegram 告警 | Reboot per-blocker alert 與 backup receipt alert rules 已 deploy/readback；backup receipt 缺段不再產生 100 條 stage 噪音，現在聚合成 110 / 188 兩條 host-level pending。scorecard 目前有 10 個 reboot active blockers，其中 Windows99 config / policy blocker 已可供 per-blocker alert routing 使用；尚未完成 shutdown / reboot / backup alert 的 production 脫敏 delivery receipt 全矩陣。 | 補 alert receipt readback：host down、host up、SLO miss、Windows99 blocker、backup stale/failed、deploy 502、freshness stale；完成條件是每類告警都有 sent / received / dedup / escalation evidence。 | | P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase`、`eta_or_wait_reason`、`primary_blocker`、fixed triage order 與 action matrix；2026-07-03 新增 `windows99_vmware_autostart_task_never_ran`，其 next action 固定為 `run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_then_rerun_no_secret_collector_no_vm_power_change`，不再退回泛用「再收一次 verify」。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard；完成條件是重啟後自動判斷、主動告警、主動 rerun verifier，不再人工臨場猜流程。 | diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index bf3553475..ffb5db885 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -16,7 +16,7 @@ from typing import Any ROOT = Path(__file__).resolve().parents[2] SCHEMA_VERSION = "awoooi_reboot_auto_recovery_slo_scorecard_v1" REQUIRED_HOSTS = {"99", "110", "111", "112", "120", "121", "188"} -WINDOWS99_REQUIRED_VM_ALIASES = {"110", "112", "120", "121", "188"} +WINDOWS99_REQUIRED_VM_ALIASES = {"111", "112", "120", "121", "188"} def parse_args() -> argparse.Namespace: diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 21b06b4ea..b713cc067 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -87,8 +87,7 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "Host112Vmx" in windows99 assert "D:\\Documents\\Virtual Machines" in windows99 assert "D:\\Downloads" in windows99 - assert '[string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112")' in windows99 - assert 'if ($Host111Vmx -and -not ($VmOrder -contains "111"))' in windows99 + assert '[string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112")' in windows99 assert "HOST111_VMWARE_TARGET=explicit_or_required" in windows99 assert "VMWARE_AUTOSTART_VERIFY_READY" in windows99 assert "WINDOWS_UPDATE_POLICY" in windows99 diff --git a/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py b/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py index 2b4b375f3..8360afccf 100644 --- a/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py +++ b/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py @@ -18,8 +18,8 @@ VALID_RED_VERIFY = """\ AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1 MODE=Verify VMRUN_PRESENT=1 -HOST111_VMWARE_TARGET=not_required_by_default_missing_no_vmx_found_on_99 -VMX alias=110 path=D:\\Documents\\Virtual Machines\\110\\110.vmx present=1 +HOST111_VMWARE_TARGET=explicit_or_required +VMX alias=111 path=D:\\Documents\\Virtual Machines\\111\\111.vmx present=1 VMX alias=188 path=D:\\Documents\\Virtual Machines\\188\\188.vmx present=1 VMX alias=120 path=D:\\Documents\\Virtual Machines\\120\\120.vmx present=1 VMX alias=121 path=D:\\Documents\\Virtual Machines\\121\\121.vmx present=1 @@ -28,7 +28,7 @@ VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Auto ok VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Auto ok=1 VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1 WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1 -VM_POWER alias=110 vmx_present=1 running=0 +VM_POWER alias=111 vmx_present=1 running=0 VM_POWER alias=188 vmx_present=1 running=0 VM_POWER alias=120 vmx_present=1 running=0 VM_POWER alias=121 vmx_present=1 running=0 @@ -94,7 +94,7 @@ def test_valid_console_verify_red_artifact_is_accepted_and_normalized( assert values["status"] == "valid_console_verify_artifact_vmware_not_ready" assert values["missing_required_field_count"] == "0" assert values["missing_vmx_aliases"] == "" - assert values["powered_off_aliases"] == "110,188,120,121,112" + assert values["powered_off_aliases"] == "111,188,120,121,112" assert values["secret_value_read"] == "false" assert normalized.read_text(encoding="utf-8").startswith( "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1\n" @@ -130,11 +130,11 @@ def test_explicit_host111_artifact_can_be_validated_with_required_alias_override raw = tmp_path / "console-111.txt" raw.write_text( VALID_RED_VERIFY.replace( - "HOST111_VMWARE_TARGET=not_required_by_default_missing_no_vmx_found_on_99", + "HOST111_VMWARE_TARGET=explicit_or_required", "HOST111_VMWARE_TARGET=explicit_or_required", ) - .replace("VMX alias=110 path=D:\\Documents\\Virtual Machines\\110\\110.vmx present=1", "VMX alias=111 path= present=0") - .replace("VM_POWER alias=110 vmx_present=1 running=0", "VM_POWER alias=111 vmx_present=0 running=0"), + .replace("VMX alias=111 path=D:\\Documents\\Virtual Machines\\111\\111.vmx present=1", "VMX alias=111 path= present=0") + .replace("VM_POWER alias=111 vmx_present=1 running=0", "VM_POWER alias=111 vmx_present=0 running=0"), encoding="utf-8", ) diff --git a/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py b/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py index f90f63a0c..16ee21437 100755 --- a/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py +++ b/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py @@ -10,7 +10,7 @@ from pathlib import Path SCHEMA_VERSION = "windows99_console_verify_artifact_validator_v1" -DEFAULT_REQUIRED_ALIASES = ("110", "188", "120", "121", "112") +DEFAULT_REQUIRED_ALIASES = ("111", "188", "120", "121", "112") SENSITIVE_RE = re.compile( r"(?i)(password|passwd|secret|token|authorization|cookie|private[_ -]?key)\s*[=:]" ) diff --git a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 index 1ab7e11c5..906590bdc 100644 --- a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 +++ b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 @@ -8,7 +8,7 @@ param( [string]$Host120Vmx = "", [string]$Host121Vmx = "", [string]$Host112Vmx = "", - [string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112"), + [string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112"), [string[]]$DiscoveryRoot = @( "D:\Documents\Virtual Machines", "D:\Downloads", @@ -25,9 +25,6 @@ $TaskName = "AWOOOI-Start-VMware-VMs" $ProgramDataDir = "C:\ProgramData\AWOOOI" $StartScript = Join-Path $ProgramDataDir "Start-AWOOOI-VMs.ps1" $VmOrder = @($RequiredVmAliases) -if ($Host111Vmx -and -not ($VmOrder -contains "111")) { - $VmOrder = @("111") + $VmOrder -} $SuppliedVmx = @{ "110" = $Host110Vmx "111" = $Host111Vmx