diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index d36286084..4a171047c 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -52,6 +52,15 @@ _PROMETHEUS_RUNTIME_QUERIES = { "windows99_windows_update_ready": ( "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready" ), + "windows99_autostart_task_start_script_present": ( + "awoooi_windows99_vmware_autostart_task_start_script_present" + ), + "windows99_autostart_task_last_result": ( + "awoooi_windows99_vmware_autostart_task_last_result" + ), + "windows99_autostart_task_never_ran": ( + "awoooi_windows99_vmware_autostart_task_never_ran" + ), "windows99_missing_vmx_alias": "awoooi_windows99_vmware_missing_vmx_alias", "windows99_powered_off_alias": "awoooi_windows99_vmware_powered_off_alias", "windows99_no_secret_collector_readback_present": ( @@ -434,6 +443,54 @@ def _load_reboot_slo_prometheus_metric_readback( windows99_update_ready: bool | None = ( None if windows99_update_metric < 0 else windows99_update_metric == 1 ) + windows99_task_start_script_metric = _prometheus_first_int( + scalar_results.get("windows99_autostart_task_start_script_present") or [], + default=-1, + ) + windows99_task_start_script_present: bool | None = ( + None + if windows99_task_start_script_metric < 0 + else windows99_task_start_script_metric == 1 + ) + windows99_task_last_result_results = results_by_name.get( + _PROMETHEUS_RUNTIME_QUERIES["windows99_autostart_task_last_result"], + [], + ) + windows99_task_last_result = _prometheus_first_int( + windows99_task_last_result_results, + default=-1, + ) + windows99_task_last_result_hex = _prometheus_first_label_value( + windows99_task_last_result_results, + "result_hex", + ) + windows99_task_last_run_time = _prometheus_first_label_value( + windows99_task_last_result_results, + "last_run_time", + ) + windows99_task_never_ran_results = results_by_name.get( + _PROMETHEUS_RUNTIME_QUERIES["windows99_autostart_task_never_ran"], + [], + ) + windows99_task_never_ran_metric = _prometheus_first_int( + windows99_task_never_ran_results, + default=-1, + ) + windows99_task_never_ran: bool | None = ( + None + if windows99_task_never_ran_metric < 0 + else windows99_task_never_ran_metric == 1 + ) + if not windows99_task_last_result_hex: + windows99_task_last_result_hex = _prometheus_first_label_value( + windows99_task_never_ran_results, + "result_hex", + ) + if not windows99_task_last_run_time: + windows99_task_last_run_time = _prometheus_first_label_value( + windows99_task_never_ran_results, + "last_run_time", + ) windows99_missing_vmx_aliases = _prometheus_label_values( missing_vmx_results, "alias", @@ -468,6 +525,13 @@ def _load_reboot_slo_prometheus_metric_readback( "windows99_vmware_verify_ready": windows99_verify_ready, "windows99_vmware_autostart_config_ready": windows99_config_ready, "windows99_windows_update_no_auto_reboot_ready": windows99_update_ready, + "windows99_autostart_task_start_script_present": ( + windows99_task_start_script_present + ), + "windows99_autostart_task_last_result": windows99_task_last_result, + "windows99_autostart_task_last_result_hex": windows99_task_last_result_hex, + "windows99_autostart_task_last_run_time": windows99_task_last_run_time, + "windows99_autostart_task_never_ran": windows99_task_never_ran, "windows99_missing_vmx_aliases": windows99_missing_vmx_aliases, "windows99_powered_off_aliases": windows99_powered_off_aliases, "windows99_no_secret_collector_readback_present": ( @@ -791,6 +855,31 @@ def _apply_prometheus_windows99_vmware_readback( "runtime_metric_overlay_applied": True, } ) + scheduled_task = _dict(windows99.setdefault("scheduled_task", {})) + task_start_script_present = metric_readback.get( + "windows99_autostart_task_start_script_present" + ) + if isinstance(task_start_script_present, bool): + scheduled_task["start_script_present"] = task_start_script_present + raw_task_last_result = metric_readback.get("windows99_autostart_task_last_result") + task_last_result = ( + int(raw_task_last_result) + if isinstance(raw_task_last_result, int | float) + else -1 + ) + if task_last_result >= 0: + scheduled_task["last_task_result"] = str(task_last_result) + scheduled_task["last_task_result_hex"] = str( + metric_readback.get("windows99_autostart_task_last_result_hex") or "" + ) + task_last_run_time = str( + metric_readback.get("windows99_autostart_task_last_run_time") or "" + ) + if task_last_run_time: + scheduled_task["last_run_time"] = task_last_run_time + task_never_ran = metric_readback.get("windows99_autostart_task_never_ran") + if isinstance(task_never_ran, bool): + scheduled_task["never_ran"] = task_never_ran payload["windows99_vmware_verify_ready"] = verify_ready payload["windows99_update_no_auto_reboot_ready"] = update_ready @@ -810,6 +899,19 @@ def _apply_prometheus_windows99_vmware_readback( readback["windows99_runtime_metric_overlay_applied"] = True readback["windows99_missing_vmx_aliases"] = missing_vmx_aliases readback["windows99_powered_off_aliases"] = powered_off_aliases + readback["windows99_autostart_task_start_script_present"] = scheduled_task.get( + "start_script_present" + ) + readback["windows99_autostart_task_last_result"] = scheduled_task.get( + "last_task_result" + ) + readback["windows99_autostart_task_last_result_hex"] = scheduled_task.get( + "last_task_result_hex" + ) + readback["windows99_autostart_task_last_run_time"] = scheduled_task.get( + "last_run_time" + ) + readback["windows99_autostart_task_never_ran"] = scheduled_task.get("never_ran") collection = _dict(payload.setdefault("windows99_verify_collection", {})) preserved_collection_blockers = [ @@ -844,6 +946,16 @@ def _apply_prometheus_windows99_vmware_readback( rollups["windows99_vmware_powered_off_count"] = len(powered_off_aliases) rollups["windows99_missing_vmx_alias_count"] = len(missing_vmx_aliases) rollups["windows99_powered_off_alias_count"] = len(powered_off_aliases) + rollups["windows99_autostart_task_start_script_present"] = scheduled_task.get( + "start_script_present" + ) + rollups["windows99_autostart_task_last_result"] = scheduled_task.get( + "last_task_result" + ) + rollups["windows99_autostart_task_last_result_hex"] = scheduled_task.get( + "last_task_result_hex" + ) + rollups["windows99_autostart_task_never_ran"] = scheduled_task.get("never_ran") rollups["windows99_verify_collection_status"] = collection["status"] rollups["windows99_verify_collection_blocker_count"] = len( _strings(collection.get("collection_blockers")) @@ -2567,6 +2679,8 @@ def _build_windows99_verify_collection_packet( "VMX alias= present=<0|1>", "VMWARE_SERVICE name= ok=<0|1>", "VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs ok=<0|1>", + "VMWARE_AUTOSTART_TASK_DETAIL last_task_result=", + "VMWARE_AUTOSTART_TASK_ACTION execute=", "WINDOWS_UPDATE_POLICY name= ok=<0|1>", "VM_POWER alias= running=<0|1>", "VMWARE_AUTOSTART_CONFIG_READY", @@ -3322,6 +3436,7 @@ def _reboot_sop_current_phase(active_blockers: list[str], can_claim_slo: bool) - "windows99_vmrun_missing", "windows99_vmware_vmx_missing", "windows99_vmware_autostart_config_not_ready", + "windows99_vmware_autostart_task_never_ran", "windows99_vmware_guest_power_not_ready", "windows99_update_no_auto_reboot_policy_not_ready", "windows99_console_clipboard_unreliable", @@ -3376,6 +3491,7 @@ def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "windows99_vmrun_missing", "windows99_vmware_vmx_missing", "windows99_vmware_autostart_config_not_ready", + "windows99_vmware_autostart_task_never_ran", "windows99_vmware_guest_power_not_ready", "windows99_update_no_auto_reboot_policy_not_ready", "windows99_console_clipboard_unreliable", @@ -3505,7 +3621,10 @@ def _active_blocker_action_row( ) elif blocker == "windows99_vmware_vmx_missing": next_safe_action = _WINDOWS99_MISSING_VMX_NEXT_SAFE_ACTION - elif blocker == "windows99_vmware_guest_power_not_ready": + elif blocker in { + "windows99_vmware_guest_power_not_ready", + "windows99_vmware_autostart_task_never_ran", + }: next_safe_action = _WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION elif blocker in { "windows99_console_clipboard_unreliable", diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 1595f5a27..cab667750 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -699,6 +699,77 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( ) +def test_reboot_auto_recovery_slo_scorecard_overlays_windows99_task_run_state(): + metric_readback = { + **PROMETHEUS_RUNTIME_READBACK, + "active_blockers": [ + "all_required_hosts_not_in_10_minute_reboot_window", + "fresh_all_host_reboot_event_missing", + "host_boot_observation_older_than_target_window", + "host_unreachable_after_reboot", + "host_uptime_unknown", + "reboot_event_required_host_unreachable", + "windows99_vmware_guest_power_not_ready", + "windows99_vmware_autostart_task_never_ran", + ], + "active_blocker_count": 8, + "windows99_vmware_autostart_config_ready": True, + "windows99_windows_update_no_auto_reboot_ready": True, + "windows99_missing_vmx_aliases": [], + "windows99_powered_off_aliases": ["110", "112", "120", "121", "188"], + "windows99_autostart_task_start_script_present": True, + "windows99_autostart_task_last_result": 267011, + "windows99_autostart_task_last_result_hex": "0x41303", + "windows99_autostart_task_last_run_time": "1999-11-30T00:00:00", + "windows99_autostart_task_never_ran": True, + } + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + prometheus_metric_readback=metric_readback, + ) + + task = payload["windows99_vmware_autostart"]["scheduled_task"] + assert task["start_script_present"] is True + assert task["last_task_result"] == "267011" + assert task["last_task_result_hex"] == "0x41303" + assert task["last_run_time"] == "1999-11-30T00:00:00" + assert task["never_ran"] is True + assert payload["windows99_vmware_autostart"]["config_ready"] is True + assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [ + "110", + "112", + "120", + "121", + "188", + ] + assert "windows99_vmware_autostart_task_never_ran" in payload[ + "active_blockers" + ] + assert payload["readback"]["windows99_autostart_task_last_result"] == "267011" + assert payload["readback"]["windows99_autostart_task_last_result_hex"] == ( + "0x41303" + ) + assert payload["readback"]["windows99_autostart_task_last_run_time"] == ( + "1999-11-30T00:00:00" + ) + assert payload["readback"]["windows99_autostart_task_never_ran"] is True + assert payload["rollups"]["windows99_autostart_task_last_result"] == "267011" + assert payload["rollups"]["windows99_autostart_task_last_result_hex"] == ( + "0x41303" + ) + assert payload["rollups"]["windows99_autostart_task_never_ran"] is True + action_by_blocker = { + item["blocker"]: item + for item in payload["active_blocker_action_matrix"]["items"] + } + assert action_by_blocker["windows99_vmware_autostart_task_never_ran"][ + "category" + ] == "windows99_vmware_autostart" + assert action_by_blocker["windows99_vmware_autostart_task_never_ran"][ + "next_safe_action" + ] == EXPECTED_WINDOWS99_POWERED_OFF_NEXT_STEP + + def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_windows99_config_when_runtime_metric_is_ready(): metric_readback = { **PROMETHEUS_RUNTIME_READBACK, @@ -959,6 +1030,35 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch): }, "value": [1783010479, "1"], }, + { + "metric": { + "__name__": ( + "awoooi_windows99_vmware_autostart_task_" + "start_script_present" + ), + }, + "value": [1783010479, "1"], + }, + { + "metric": { + "__name__": ( + "awoooi_windows99_vmware_autostart_task_last_result" + ), + "result_hex": "0x41303", + "last_run_time": "1999-11-30T00:00:00", + }, + "value": [1783010479, "267011"], + }, + { + "metric": { + "__name__": ( + "awoooi_windows99_vmware_autostart_task_never_ran" + ), + "result_hex": "0x41303", + "last_run_time": "1999-11-30T00:00:00", + }, + "value": [1783010479, "1"], + }, { "metric": { "__name__": "awoooi_windows99_vmware_missing_vmx_alias", @@ -1042,6 +1142,13 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch): assert payload["primary_blocker"] == "reboot_event_required_host_unreachable" assert payload["ready"] is False assert payload["last_run_timestamp"] == 1783010479 + assert payload["windows99_autostart_task_start_script_present"] is True + assert payload["windows99_autostart_task_last_result"] == 267011 + assert payload["windows99_autostart_task_last_result_hex"] == "0x41303" + assert payload["windows99_autostart_task_last_run_time"] == ( + "1999-11-30T00:00:00" + ) + assert payload["windows99_autostart_task_never_ran"] is True assert payload["windows99_missing_vmx_aliases"] == ["111"] assert payload["windows99_powered_off_aliases"] == [ "111", diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index f5f6b43cd..6899f20d1 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,26 @@ +## 2026-07-03 — 15:57 Windows99 VMware autostart task run-state 上卷 + +**完成內容**: +- `windows99-vmware-autostart.ps1 -Mode Verify` 新增 `VMWARE_AUTOSTART_TASK_DETAIL` 與 `VMWARE_AUTOSTART_TASK_ACTION`,把 scheduled task principal、run level、logon type、target script present、last run time、last task result、next run time 與 action command 納入 no-secret readback。 +- CLI scorecard 新增 `windows99_vmware_autostart_task_never_ran`:當 `last_task_result=267011` / `0x41303` 或 `last_run_time=1999-11-30...` 且 powered-off aliases 非空時,主動列為 active blocker。 +- 110 textfile exporter 新增 `awoooi_windows99_vmware_autostart_task_start_script_present`、`awoooi_windows99_vmware_autostart_task_last_result`、`awoooi_windows99_vmware_autostart_task_never_ran` metrics;API Prometheus overlay 會讀回 task run-state 到 `windows99_vmware_autostart.scheduled_task`、readback 與 rollups。 +- CLI / API action matrix 已把 `windows99_vmware_autostart_task_never_ran` 指到 `run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_then_rerun_no_secret_collector_no_vm_power_change`,不再退回泛用收集 verify。 +- Live no-secret via-110 artifact `/tmp/awoooi-windows99-autostart-detail-20260703-155710` 顯示 task present/enabled/Ready,但 `last_run_time=1999-11-30T00:00:00`、`last_task_result=267011`,且 `110 / 188 / 120 / 121 / 112` 全部 `running=0`。 +- `FULL-STACK-COLD-START-SOP` 更新至 v1.125,`REBOOT-RECOVERY-SOP` 更新至 v5.17;P0 workplan 的 P0-4 改為 `BLOCKED_WINDOWS99_TASK_NOT_TRIGGERED`。 + +**已跑驗證**: +- `python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py -q -p no:cacheprovider`:`40 passed`。 +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_windows99_vmware_autostart_check_mode_package_api.py -q -p no:cacheprovider`:`25 passed`。 +- `python3.11 -m py_compile scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py apps/api/src/services/reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py`:通過。 +- `bash -n scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh scripts/reboot-recovery/collect-windows99-vmware-verify.sh`:通過。 + +**仍維持**: +- 這不代表 10 分鐘重啟自動恢復 SLO 完成;目前只是把「task 存在但沒有跑」轉成可觀測、可告警、可路由的真 blocker。 +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth;沒有使用 GitHub / gh;沒有啟動或關閉 VM,沒有重啟 Windows host / VMware service,沒有 Docker / Nginx / K3s / DB / firewall restart / reload,沒有 DROP / TRUNCATE / restore / prune / delete / force push。 + +**下一步**: +- commit / push 到 Gitea main;CD deploy 後讀回 production SLO 與 110 textfile,確認 task run-state metrics 與 `windows99_vmware_autostart_task_never_ran` active blocker 上線。 + ## 2026-07-03 — 15:35 Public maintenance edge API overlay 修正 **完成內容**: diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 55b789b7e..184c7b99c 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.124 +> Version: v1.125 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號,但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal;此納入不代表授權任何破壞性 runtime apply。 @@ -80,6 +80,8 @@ v1.123 SLO installer edge-source packaging rule:110 SLO installer payload 必 v1.124 public maintenance route / edge blocker split rule:production API overlay 不得把 route fallback ready 與 edge Nginx fallback ready 視為同一件事。`public_maintenance_fallback.runtime_readback_present=true` 只能清除 `public_maintenance_fallback_runtime_readback_missing`;route 0 raw 5xx / 0 unreachable 只能清除 route-level blockers;只有 `edge_fallback_runtime_readback_present=true` 且 `edge_fallback_ready=true` 才能清除 `edge_public_maintenance_live_config_drift`。若 Prometheus textfile active label 已有 edge drift,但 public route probe 是 200 / ready 且缺 edge readback,`/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須保留 edge drift;否則會把下次 reboot / deploy raw 502 風險誤報成正常。此規則只修 scorecard/API readback,不授權 Nginx reload、app restart、Docker / host restart 或讀密碼。 +v1.125 Windows99 autostart task run-state rule:`windows99-vmware-autostart.ps1 -Mode Verify` 必須在既有 `VMWARE_AUTOSTART_TASK ... ok=...` 之外輸出 `VMWARE_AUTOSTART_TASK_DETAIL` 與 `VMWARE_AUTOSTART_TASK_ACTION`,至少包含 principal、run level、logon type、target script present、last run time、last task result、next run time、action execute / argument。`last_task_result=267011` / `0x41303` 或 `last_run_time=1999-11-30T00:00:00` 代表 task exists/enabled 但尚未跑過,若同時 `powered_off_aliases` 非空,scorecard 必須新增 active blocker `windows99_vmware_autostart_task_never_ran`,textfile exporter 必須輸出 `awoooi_windows99_vmware_autostart_task_*` metrics,API overlay / action matrix 必須把 next action 指到 `run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_then_rerun_no_secret_collector_no_vm_power_change`。2026-07-03 live no-secret via-110 readback 證明 `AWOOOI-Start-VMware-VMs` present/enabled/Ready,但 `last_task_result=267011`、`last_run_time=1999-11-30T00:00:00`,且 `110 / 188 / 120 / 121 / 112` 全部 `running=0`;因此 P0-4 不是「缺 task」,而是「task 未觸發 / 未跑」。此規則只授權 readback、metrics、alert routing 與 check-mode package;不得直接啟動 VM、重啟 Windows host / VMware service、改 scheduled task、改 registry 或讀密碼 / secret。 + 2026-07-02 110 control-path / Harbor recovery receipt rule:若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure,但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`,則該 Gitea Harbor repair 失敗只能列為 historical queue metadata,不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier,並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示:public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`;active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure,以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker,non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker;non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule:110 runner 壓力事故後,legacy / generic runner 仍必須 fail-closed;但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail,且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時,才可保留 drain config / binary / unit,並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service;若 registration 缺失,readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走,優先修 source enforcer / unit guardrail,不要手工反覆補同一組 artifact。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index dc9fdbd2c..9e29d879d 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.16 +> **版本**: v5.17 > **最後更新**: 2026-07-03 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-03 15:35 production API overlay 分離 route ready 與 edge ready,避免清掉 `edge_public_maintenance_live_config_drift` +> **觸發事件**: 2026-07-03 15:57 Windows99 VMware autostart task exists/enabled 但 `0x41303` 顯示未觸發,五台 VM guest 仍 powered off --- @@ -21,6 +21,12 @@ ## 架構概覽與依賴圖 +### 2026-07-03 15:57 P0-4 Windows99 Task Run-State 覆蓋 + +Windows99 no-secret via-110 collector 已讀回 `AWOOOI-Start-VMware-VMs` scheduled task:`present=1`、`enabled=1`、`state=Ready`、`trigger_count=1`、`action_count=1`,VMware service 與 Windows Update no-auto-reboot policy 也為 ready;但新增 task detail 顯示 `last_run_time=1999-11-30T00:00:00`、`last_task_result=267011`,等價於 `0x41303`,表示 task 尚未跑過 / 未在本輪 boot 觸發。五台 guest `110 / 188 / 120 / 121 / 112` 仍全部 `running=0`,因此目前 P0-4 真 blocker 是 `windows99_vmware_guest_power_not_ready` 加 `windows99_vmware_autostart_task_never_ran`。 + +SOP v5.17 起,`windows99-vmware-autostart.ps1 -Mode Verify` 必須輸出 `VMWARE_AUTOSTART_TASK_DETAIL` 與 `VMWARE_AUTOSTART_TASK_ACTION`;110 textfile exporter 必須輸出 `awoooi_windows99_vmware_autostart_task_start_script_present`、`awoooi_windows99_vmware_autostart_task_last_result`、`awoooi_windows99_vmware_autostart_task_never_ran`;production scorecard / API 必須把 `windows99_vmware_autostart_task_never_ran` 放進 active blockers 與 action matrix。固定下一步是執行已建立的 Windows99 VMware autostart check-mode package,然後 rerun no-secret collector 與 reboot SLO scorecard;不得直接 power on VM、重啟 Windows host / VMware service、修改 scheduled task、修改 registry、讀 Windows 密碼或讀 secret。 + ### 2026-07-03 15:35 P0-2 API Readback 覆蓋 110 textfile / Prometheus 已有 `awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0` 與 active blocker `edge_public_maintenance_live_config_drift` 時,production API 必須把此 blocker 合併進 `active_blockers`。route probe 200 / route fallback ready 不能替代 edge Nginx fallback ready;只有 `edge_fallback_runtime_readback_present=true` 且 `edge_fallback_ready=true` 才能清掉 edge drift。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 69f628d77..6ac9decaa 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -19,13 +19,13 @@ | 優先 | 狀態 | 工作項 | 最新證據 | 下一步 / 完成條件 | |------|------|--------|----------|-------------------| -| P0-1 | BLOCKED_HOST_WINDOWS | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 10:44 production scorecard:`status=blocked_reboot_auto_recovery_slo_not_ready`、`active_blocker_count=10`、`readiness_percent=60`、active blockers 仍含 reboot event / host unreachable / 99+111+Windows99 VMX / guest power,且新增 `windows99_vmware_autostart_config_not_ready`、`windows99_update_no_auto_reboot_policy_not_ready`。no-write artifact `/tmp/awoooi-reboot-continue-20260703-085205`:111 socket probe 仍 timeout / unreachable;99 RDP / VMConnect reachable 但 uptime unknown;188 socket probe reachable,前次 host probe 仍為 `systemd_state=degraded` / `startup_active=failed`。 | 先收斂 99 / Windows99 / VMware 與 111:恢復 no-secret management channel 或取得 validator 可接受的完整 console Verify stdout,讀回 VMX / VM power / host uptime,再 rerun host probe + reboot-event detector;不得 reboot、不得 VM power change、不得讀 Windows 密碼,不得用 RDP clipboard 片段當完成證據。 | -| P0-2 | API_OVERLAY_FIX_PENDING_DEPLOY_READBACK | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 2026-07-03 15:35 source fix 已把 production API overlay 的 public maintenance blockers 拆成 route-level 與 edge-level:route probe 200 / ready 只能清 route blockers,不能清 `edge_public_maintenance_live_config_drift`;只有 `edge_fallback_runtime_readback_present=true` 且 `edge_fallback_ready=true` 才能清 edge drift。110 SLO artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-151219` 與 Prometheus/textfile 已證明 edge 未 ready:`awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0`、`awoooi_reboot_auto_recovery_slo_blocker_count=8`、active blocker `edge_public_maintenance_live_config_drift`。本地 regression:route ready + edge metric blocker 時 API active list 必須保留 edge,22 個 scorecard API 測試通過。 | commit / push 後等待 Gitea CD deploy marker matched;production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 必須讀回 `active_blockers` 含 `edge_public_maintenance_live_config_drift` 且 `runtime_metric_active_blocker_count=8`。後續才進 privileged edge apply 或 L1 cloud/CDN fallback decision;不得讀密碼、不得用 app / Docker / host restart 掩蓋 edge fallback drift。 | +| P0-1 | BLOCKED_REBOOT_EVENT_AND_HOST_REACHABILITY | 全主機 reboot auto-detection / auto-trigger / 10 分鐘恢復 SLO | 2026-07-03 15:49 verify-only artifact `/tmp/awoooi-reboot-event-verify-20260703-154907`:`reboot_detected=false`、`all_required_hosts_observed=false`、`all_required_hosts_in_reboot_window=false`、`recovery_deadline_status=target_window_elapsed`、`unreachable_hosts=["111"]`、99 reachable 但 `uptime_seconds=unknown`、188 reachable 但 `systemd_state=degraded` / `startup_active=failed`。production SLO 仍為 `blocked_reboot_auto_recovery_slo_not_ready`,不能宣稱 10 分鐘自動恢復。 | 固定先 rerun host probe + reboot-event detector verify-only,並收斂 111 reachability、99 uptime / VMware readback、188 degraded/startup failed;不得 reboot、不得 VM power change、不得讀 Windows 密碼,不得用 ping-only 或舊 boot evidence 當完成證據。 | +| P0-2 | DEPLOYED_RUNTIME_EDGE_DRIFT_BLOCKED | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 2026-07-03 15:35 source fix 已部署讀回:production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 保留 `edge_public_maintenance_live_config_drift`,`runtime_metric_active_blocker_count=8`,且 route fallback ready 不再清掉 edge Nginx drift。110 SLO artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-151219` 與 textfile 證明 edge 未 ready:`awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0`、active blocker `edge_public_maintenance_live_config_drift`。 | 後續才進 privileged edge apply 或 L1 cloud/CDN fallback decision;完成條件是 edge fallback live config / maintenance file / `nginx -t` / reload / public header verifier 或外部 L1 fallback 全綠。不得讀密碼、不得用 app / Docker / host restart 掩蓋 edge fallback drift。 | | P0-3 | PARTIAL_GREEN_DEPLOYED_SOURCE | 所有產品 / 網站版本與資料最新性 | Gitea `main=bc8d3af3f chore(cd): deploy dacdd90 [skip ci]`;CD `#4558` log 顯示 API/Web build/push wrapper 全部 `rc=0`、deploy marker `dacdd90`、production deploy readback matched。Production SLO route 已讀回最新 API 行為:`runtime_metric_runtime_readback_added_blockers=["windows99_vmware_autostart_config_not_ready","windows99_update_no_auto_reboot_policy_not_ready"]`、`windows99_update_no_auto_reboot_ready=false`。Stock freshness 仍為 `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。 | 版本層已補到 AWOOOI production;完成整體 P0 仍需收斂 Windows99 VMX / service / policy / power 與 host reboot-event blockers。後續每個 public product 仍要維持 source SHA、deploy marker、runtime endpoint、public route watch、freshness 四層一致。 | -| P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | 10:44 production readback 已把 nested Windows99 evidence 上卷到 active blockers:`windows99_vmware_vmx_missing`、`windows99_vmware_guest_power_not_ready`、`windows99_vmware_autostart_config_not_ready`、`windows99_update_no_auto_reboot_policy_not_ready`;`runtime_metric_runtime_readback_added_blockers` 含 config / policy 兩項,`windows99_update_no_auto_reboot_ready=false`。09:53 no-secret management probe:99 reachable、RDP / Hyper-V VMConnect reachable,WinRM timeout,Mac vantage collector 仍為 `ssh_batchmode_auth_ready=0` / `blocked_ssh_publickey_auth_missing`;completion 仍取決於 VMX / service / policy / power 全綠。 | 先修 111 VMX source 與 VMware service / policy readback,使 check-mode package 能解釋 powered-off aliases;不得從 scorecard 直接重啟 Windows service、改 registry、啟動 / 關閉 VM 或 host reboot。完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable,且 active blocker matrix 顯示 service / policy / VMX / power 分開收斂。 | +| P0-4 | BLOCKED_WINDOWS99_TASK_NOT_TRIGGERED | 192.168.0.99 VMware 自動啟動與 VM guest 110 / 188 / 120 / 121 / 112 | 2026-07-03 15:57 no-secret via-110 collector artifact `/tmp/awoooi-windows99-autostart-detail-20260703-155710`:VMX `110 / 188 / 120 / 121 / 112` 全存在,VMware services ok,Windows Update no-auto-reboot ready,scheduled task `AWOOOI-Start-VMware-VMs` present/enabled/Ready;但 `VMWARE_AUTOSTART_TASK_DETAIL last_run_time=1999-11-30T00:00:00 last_task_result=267011` (`0x41303`) 且五台 `VM_POWER running=0`。 | 將 `windows99_vmware_autostart_task_never_ran` 納入 active blocker / metrics / Telegram routing,執行已建立的 autostart check-mode package,再 rerun no-secret collector + SLO scorecard。不得從 scorecard 直接 power on VM、重啟 Windows host / VMware service、改 scheduled task / registry 或讀 Windows 密碼。 | | P0-5 | RUNTIME_READY_BACKUP_RECEIPT_GAP | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready:expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok;backup core green。2026-07-03 source / runtime 已部署 `awoooi_backup_alert_receipt_*` 指標與 Prometheus rules;110 exporter 讀回 88 個 stage requirement、188 讀回 12 個 stage requirement,`BackupAlertReceiptMetricMissing*` inactive,`BackupAlertReceiptStageMissing` 已修成每 `host / receipt_channel` 聚合 pending:110 一條、188 一條。 | 補 `/backup/alert-receipts/*.last_success` 脫敏 marker;下一層仍要補 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控。 | | P0-6 | RUNTIME_READY_ALERT_RECEIPT_GAP | 主機關機 / 重啟 / SLO miss / backup failure Telegram 告警 | Reboot per-blocker alert 與 backup receipt alert rules 已 deploy/readback;backup receipt 缺段不再產生 100 條 stage 噪音,現在聚合成 110 / 188 兩條 host-level pending。scorecard 目前有 10 個 reboot active blockers,其中 Windows99 config / policy blocker 已可供 per-blocker alert routing 使用;尚未完成 shutdown / reboot / backup alert 的 production 脫敏 delivery receipt 全矩陣。 | 補 alert receipt readback:host down、host up、SLO miss、Windows99 blocker、backup stale/failed、deploy 502、freshness stale;完成條件是每類告警都有 sent / received / dedup / escalation evidence。 | -| P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase=host_boot_detection_blocked`、`eta_or_wait_reason=reboot_event_readback_missing_eta_unavailable`、`primary_blocker=reboot_event_required_host_unreachable`、fixed triage order 與 next safe action;08:23 artifact 固定下一步為 no-secret Windows99 verify / host probe rerun。09:22 `active_blocker_action_matrix` 已能把 unreliable console artifact 指向 `windows99_console_or_no_secret_management_channel` 與固定 next safe action。10:30 `read-public-gitea-actions-queue.py` 已能把 `BLOCKER cd_docker_step_timeout` 上卷成 `blocked_cd_docker_step_timeout` / `latest_visible_cd_docker_step_timeout*` / fixed safe-next-action;10 分鐘內自動恢復仍未達標。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard;完成條件是重啟後自動判斷、主動告警、主動 rerun verifier,不再人工臨場猜流程。 | +| P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase`、`eta_or_wait_reason`、`primary_blocker`、fixed triage order 與 action matrix;2026-07-03 新增 `windows99_vmware_autostart_task_never_ran`,其 next action 固定為 `run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_then_rerun_no_secret_collector_no_vm_power_change`,不再退回泛用「再收一次 verify」。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard;完成條件是重啟後自動判斷、主動告警、主動 rerun verifier,不再人工臨場猜流程。 | | P0-8 | BLOCKED_POLICY_READBACK | Windows99 禁止 Windows Update 無預警重啟 | 10:44 production readback 已 fail-closed:`windows99_update_no_auto_reboot_ready=false`,active blockers 含 `windows99_update_no_auto_reboot_policy_not_ready`,來源是 nested verifier 的 `policy_blockers=["windows_update_policy_readback_missing"]`。 | 保留 no-secret verifier,補週期性 policy readback 與 Telegram drift alert;完成條件是 Windows Update policy drift / missing readback 會自動告警且不需讀 secret,不得從 scorecard 直接 apply registry。 | | 優先 | 狀態 | 工作項 | 2026-06-30 證據 | 下一步 / 完成條件 | @@ -48,6 +48,7 @@ - 2026-06-30 實測證明:source 上有 reboot detector / alert / VMware autostart / maintenance fallback 並不等於 runtime 已達標;必須同時讀回 all-host probe、SLO metric、public `/v2`、Stock freshness、backup-status、Telegram delivery。 - 2026-07-03 實測證明:host probe 本身若沒有 bounded TCP fallback,會從 verifier 層製造 partial evidence,讓 10 分鐘 SLO 無法判斷。`reboot-auto-recovery-host-probe.sh` 的 `nc -z` 必須由 `run_with_timeout` 包住;partial probe / missing host rows 一律 fail-closed,不可宣稱 all-host observed。 - 2026-07-03 08:59 實測證明:RDP console 可見不等於 console artifact 可自動收集。若 Mac / RDP clipboard 只貼入 Windows 端舊內容,必須標記 `windows99_console_clipboard_unreliable` 類 blocker,回到 no-secret management channel 或人工 console stdout capture,不可用截斷 / 非 Verify marker 的畫面輸出餵 scorecard。 +- 2026-07-03 15:57 實測證明:Windows scheduled task present/enabled/Ready 不等於 VMware guest 會自動啟動。`last_task_result=267011` / `0x41303` 與 `1999-11-30` placeholder 必須列為 `windows99_vmware_autostart_task_never_ran`,並與 powered-off aliases 一起進 alert / action matrix;下一步只能走 check-mode package + no-secret collector rerun,不得直接 power on VM 或改 Windows task。 | Area | Status | Completion | Evidence | |------|--------|------------|----------| diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 197461ee6..6d908bde6 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -461,6 +461,13 @@ def gauge(name: str, value: bool) -> None: print(f'{name}{{scope="windows99"}} {1 if value else 0}') +def int_value(value: object, default: int = -1) -> int: + try: + return int(str(value), 0) + except (TypeError, ValueError): + return default + + gauge( "awoooi_windows99_vmware_readback_present", windows99.get("readback_present") is True, @@ -477,6 +484,28 @@ gauge( "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready", windows99.get("windows_update_no_auto_reboot_ready") is True, ) +task = windows99.get("scheduled_task") or windows99.get("autostart_task") or {} +if task: + start_script_present = task.get("start_script_present") is True + task_last_result = int_value(task.get("last_task_result")) + task_last_result_hex = hex(task_last_result) if task_last_result >= 0 else "unknown" + task_last_run_time = str(task.get("last_run_time") or "unknown") + task_never_ran = task.get("never_ran") is True + gauge( + "awoooi_windows99_vmware_autostart_task_start_script_present", + start_script_present, + ) + if task_last_result >= 0: + print( + "awoooi_windows99_vmware_autostart_task_last_result" + f'{{scope="windows99",result_hex="{label(task_last_result_hex)}",' + f'last_run_time="{label(task_last_run_time)}"}} {task_last_result}' + ) + print( + "awoooi_windows99_vmware_autostart_task_never_ran" + f'{{scope="windows99",result_hex="{label(task_last_result_hex)}",' + f'last_run_time="{label(task_last_run_time)}"}} {1 if task_never_ran else 0}' + ) for alias in windows99.get("missing_vmx_aliases") or []: if alias: print( @@ -607,6 +636,12 @@ awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} $public_edg # TYPE awoooi_windows99_vmware_autostart_config_ready gauge # HELP awoooi_windows99_vmware_windows_update_no_auto_reboot_ready Whether Windows Update no-auto-reboot policy is verified. # TYPE awoooi_windows99_vmware_windows_update_no_auto_reboot_ready gauge +# HELP awoooi_windows99_vmware_autostart_task_start_script_present Whether the Windows99 VMware autostart scheduled task target script exists. +# TYPE awoooi_windows99_vmware_autostart_task_start_script_present gauge +# HELP awoooi_windows99_vmware_autostart_task_last_result Windows Scheduled Task last task result for VMware autostart. +# TYPE awoooi_windows99_vmware_autostart_task_last_result gauge +# HELP awoooi_windows99_vmware_autostart_task_never_ran Whether the VMware autostart task appears never to have run. +# TYPE awoooi_windows99_vmware_autostart_task_never_ran gauge # HELP awoooi_windows99_vmware_missing_vmx_alias Missing required Windows99 VMX aliases. # TYPE awoooi_windows99_vmware_missing_vmx_alias gauge # HELP awoooi_windows99_vmware_powered_off_alias Required Windows99 VMware guests observed powered off. diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index a0e8031f4..bf3553475 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -202,6 +202,31 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: "ok": match.group(7) == "1", } continue + match = re.match( + r"^VMWARE_AUTOSTART_TASK_DETAIL name=(\S+) principal_user=(\S+) " + r"run_level=(\S+) logon_type=(\S+) start_script_present=([01]) " + r"last_run_time=(\S+) last_task_result=(\S+) next_run_time=(\S+)$", + line, + ) + if match: + task.setdefault("name", match.group(1)) + task["principal_user"] = match.group(2) + task["run_level"] = match.group(3) + task["logon_type"] = match.group(4) + task["start_script_present"] = match.group(5) == "1" + task["last_run_time"] = match.group(6) + task["last_task_result"] = match.group(7) + task["next_run_time"] = match.group(8) + continue + match = re.match( + r"^VMWARE_AUTOSTART_TASK_ACTION name=(\S+) execute=(\S+) argument=(\S+)$", + line, + ) + if match: + task.setdefault("name", match.group(1)) + task["action_execute"] = match.group(2) + task["action_argument"] = match.group(3) + continue match = re.match( r"^WINDOWS_UPDATE_POLICY name=(\S+) value=(\S+) expected=(\S+) ok=([01])$", line, @@ -259,6 +284,13 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: power_ready = not powered_off_aliases if "VMWARE_AUTOSTART_VERIFY_READY" not in kv: verify_ready = config_ready and power_ready + task_last_result = str(task.get("last_task_result") or "") + task_last_run_time = str(task.get("last_run_time") or "") + task_never_ran = task_last_result in {"267011", "0x41303"} or ( + task_last_run_time.startswith("1999-11-30") + ) + if task: + task["never_ran"] = task_never_ran blockers: list[str] = [] if not vmrun_present: @@ -269,6 +301,8 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: blockers.append("windows99_vmware_autostart_config_not_ready") if powered_off_aliases: blockers.append("windows99_vmware_guest_power_not_ready") + if powered_off_aliases and task_never_ran: + blockers.append("windows99_vmware_autostart_task_never_ran") if not update_ready: blockers.append("windows99_update_no_auto_reboot_policy_not_ready") if not verify_ready and not blockers: @@ -1213,6 +1247,8 @@ def build_windows99_verify_collection_packet( "VMX alias= present=<0|1>", "VMWARE_SERVICE name= ok=<0|1>", "VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs ok=<0|1>", + "VMWARE_AUTOSTART_TASK_DETAIL last_task_result=", + "VMWARE_AUTOSTART_TASK_ACTION execute=", "WINDOWS_UPDATE_POLICY name= ok=<0|1>", "VM_POWER alias= running=<0|1>", "VMWARE_AUTOSTART_CONFIG_READY", @@ -1460,6 +1496,7 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str "windows99_vmrun_missing", "windows99_vmware_vmx_missing", "windows99_vmware_autostart_config_not_ready", + "windows99_vmware_autostart_task_never_ran", "windows99_vmware_guest_power_not_ready", "windows99_update_no_auto_reboot_policy_not_ready", "windows99_console_clipboard_unreliable", @@ -1514,6 +1551,7 @@ def reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "windows99_vmrun_missing", "windows99_vmware_vmx_missing", "windows99_vmware_autostart_config_not_ready", + "windows99_vmware_autostart_task_never_ran", "windows99_vmware_guest_power_not_ready", "windows99_update_no_auto_reboot_policy_not_ready", "windows99_console_clipboard_unreliable", @@ -1671,7 +1709,10 @@ def active_blocker_action_row( "restore_windows99_missing_vmx_source_for_aliases_then_rerun_" "no_secret_collector_and_scorecard_no_vm_power_change" ) - elif blocker == "windows99_vmware_guest_power_not_ready": + elif blocker in { + "windows99_vmware_guest_power_not_ready", + "windows99_vmware_autostart_task_never_ran", + }: next_safe_action = ( "run_windows99_vmware_autostart_check_mode_package_for_powered_" "off_aliases_then_rerun_no_secret_collector_no_vm_power_change" diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py index 2fde4a549..3b0180a56 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py @@ -73,6 +73,8 @@ VMX alias=112 path=D:\\VMs\\112.vmx present=1 VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Auto ok=1 VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Auto ok=1 VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1 +VMWARE_AUTOSTART_TASK_DETAIL name=AWOOOI-Start-VMware-VMs principal_user=SYSTEM run_level=Highest logon_type=ServiceAccount start_script_present=1 last_run_time=2026-07-03T15:10:00 last_task_result=0 next_run_time=never +VMWARE_AUTOSTART_TASK_ACTION name=AWOOOI-Start-VMware-VMs execute=powershell.exe argument=-NoProfile_-ExecutionPolicy_Bypass_-File_C:\\ProgramData\\AWOOOI\\Start-AWOOOI-VMs.ps1 WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1 WINDOWS_UPDATE_POLICY name=AlwaysAutoRebootAtScheduledTime value=0 expected=0 ok=1 WINDOWS_UPDATE_POLICY name=AUOptions value=3 expected=3 ok=1 @@ -527,6 +529,12 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - "188", ] assert payload["windows99_vmware_autostart"]["verify_ready"] is True + task = payload["windows99_vmware_autostart"]["scheduled_task"] + assert task["principal_user"] == "SYSTEM" + assert task["run_level"] == "Highest" + assert task["start_script_present"] is True + assert task["last_task_result"] == "0" + assert task["action_execute"] == "powershell.exe" assert payload["windows99_verify_collection"]["collection_blockers"] == [] assert payload["active_blocker_action_matrix"]["item_count"] == 0 assert payload["active_blocker_action_matrix"]["items"] == [] @@ -540,6 +548,52 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - ] is True +def test_windows99_autostart_task_never_ran_blocks_powered_off_guests( + tmp_path: Path, +) -> None: + windows99 = WINDOWS99_VMWARE_GREEN + for alias in ["111", "188", "120", "121", "112"]: + windows99 = windows99.replace( + f"VM_POWER alias={alias} vmx_present=1 running=1", + f"VM_POWER alias={alias} vmx_present=1 running=0", + ) + windows99 = ( + windows99.replace( + "last_run_time=2026-07-03T15:10:00 last_task_result=0", + "last_run_time=1999-11-30T00:00:00 last_task_result=267011", + ) + .replace("VMWARE_AUTOSTART_POWER_READY=1", "VMWARE_AUTOSTART_POWER_READY=0") + .replace("VMWARE_AUTOSTART_VERIFY_READY=1", "VMWARE_AUTOSTART_VERIFY_READY=0") + ) + + payload = run_scorecard(tmp_path, GREEN_SUMMARY, windows99=windows99) + + assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready" + assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [ + "111", + "112", + "120", + "121", + "188", + ] + task = payload["windows99_vmware_autostart"]["scheduled_task"] + assert task["last_task_result"] == "267011" + assert task["last_run_time"] == "1999-11-30T00:00:00" + assert task["never_ran"] is True + assert "windows99_vmware_guest_power_not_ready" in payload["active_blockers"] + assert "windows99_vmware_autostart_task_never_ran" in payload["active_blockers"] + action_by_blocker = { + item["blocker"]: item + for item in payload["active_blocker_action_matrix"]["items"] + } + assert action_by_blocker["windows99_vmware_autostart_task_never_ran"][ + "next_safe_action" + ] == ( + "run_windows99_vmware_autostart_check_mode_package_for_powered_off_" + "aliases_then_rerun_no_secret_collector_no_vm_power_change" + ) + + def test_missing_public_maintenance_runtime_readback_fails_closed(tmp_path: Path) -> None: payload = run_scorecard( tmp_path, diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 71d0d1516..21b06b4ea 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -94,6 +94,10 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "WINDOWS_UPDATE_POLICY" in windows99 assert "VM_POWER alias=$alias" in windows99 assert "VM_ALREADY_RUNNING" in windows99 + assert "VMWARE_AUTOSTART_TASK_DETAIL" in windows99 + assert "last_task_result" in windows99 + assert "start_script_present" in windows99 + assert "VMWARE_AUTOSTART_TASK_ACTION" in windows99 def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: diff --git a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 index 996929cc1..1ab7e11c5 100644 --- a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 +++ b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 @@ -191,26 +191,72 @@ function Get-VmwareServiceReadback { } } +function Format-ReadbackValue { + param([object]$Value) + + if ($null -eq $Value) { + return "unknown" + } + $text = "$Value" + if ([string]::IsNullOrWhiteSpace($text)) { + return "unknown" + } + return ($text -replace "\s+", "_") +} + function Get-AutostartTaskReadback { $task = $null + $taskInfo = $null try { $task = Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue } catch { $task = $null } + try { + $taskInfo = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction SilentlyContinue + } catch { + $taskInfo = $null + } $present = [int][bool]$task $enabled = 0 $state = "missing" $triggerCount = 0 $actionCount = 0 + $principalUser = "missing" + $runLevel = "missing" + $logonType = "missing" + $lastRunTime = "never" + $lastTaskResult = "unknown" + $nextRunTime = "unknown" + $actionExecute = "missing" + $actionArgument = "missing" if ($task) { $enabled = [int]($task.State -ne "Disabled") $state = "$($task.State)" $triggerCount = @($task.Triggers).Count $actionCount = @($task.Actions).Count + if ($task.Principal) { + $principalUser = "$($task.Principal.UserId)" + $runLevel = "$($task.Principal.RunLevel)" + $logonType = "$($task.Principal.LogonType)" + } + $actionExecute = (@($task.Actions) | ForEach-Object { "$($_.Execute)" }) -join "|" + $actionArgument = (@($task.Actions) | ForEach-Object { "$($_.Arguments)" }) -join "|" } + if ($taskInfo) { + if ($taskInfo.LastRunTime) { + $lastRunTime = $taskInfo.LastRunTime.ToString("s") + } + $lastTaskResult = "$($taskInfo.LastTaskResult)" + if ($taskInfo.NextRunTime) { + $nextRunTime = $taskInfo.NextRunTime.ToString("s") + } + } + $startScriptPresent = [int](Test-Path -LiteralPath $StartScript) $ok = [int]($task -and $enabled -eq 1 -and $triggerCount -gt 0 -and $actionCount -gt 0) Write-Output "VMWARE_AUTOSTART_TASK name=$TaskName present=$present enabled=$enabled state=$state trigger_count=$triggerCount action_count=$actionCount ok=$ok" + Write-Output "VMWARE_AUTOSTART_TASK_DETAIL name=$TaskName principal_user=$(Format-ReadbackValue $principalUser) run_level=$(Format-ReadbackValue $runLevel) logon_type=$(Format-ReadbackValue $logonType) start_script_present=$startScriptPresent last_run_time=$(Format-ReadbackValue $lastRunTime) last_task_result=$(Format-ReadbackValue $lastTaskResult) next_run_time=$(Format-ReadbackValue $nextRunTime)" + Write-Output "VMWARE_AUTOSTART_TASK_ACTION name=$TaskName execute=$(Format-ReadbackValue $actionExecute) argument=$(Format-ReadbackValue $actionArgument)" } function Get-WindowsUpdatePolicyReadback {