From b91c6e8a40c051bc6bbd12fa141b1125c3b9ae7d Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 10:06:01 +0800 Subject: [PATCH] fix(reboot): route windows99 collected verifier blockers --- docs/LOGBOOK.md | 17 +++ .../reboot-auto-recovery-slo-scorecard.py | 32 +++++ ...test_reboot_auto_recovery_slo_scorecard.py | 109 ++++++++++++++++++ 3 files changed, 158 insertions(+) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index a40ac8439..c8fe08169 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -54027,3 +54027,20 @@ production browser smoke: **下一步**： - commit / push 到 Gitea main；等 deploy marker 後讀回 `/api/v1/agents/telegram-alert-monitoring-coverage-readback`，並以 Playwright smoke 驗證 `/zh-TW/awooop`、Runs、Work Items、Alerts、Approvals 顯示 completion / pipeline / queue / tag dimensions。 + +## 2026-07-03 — P0-006 Windows99 no-secret collector readback 與 next-action 修正 + +**完成內容**： +- 對 110 執行 reboot SLO controlled install / verifier，未重啟主機、未重啟 Docker / Nginx / K3s / DB / firewall、未讀 secret；production readback 已從 `windows99_no_secret_collector_status=unknown` 推進為 `collected_windows99_vmware_verify_stdout`。 +- `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-095909/scorecard.json` 讀回 collector `readback_present=true`、`ssh_batchmode_auth_ready=true`、`remote_verify_attempted=true`，Windows Update no-auto-reboot policy `ready=true`。 +- 剩餘 P0 blocker 維持 `8` 個：主 blocker 仍是 `reboot_event_required_host_unreachable`，Windows99 具體剩 `windows99_vmware_vmx_missing`（111）與 `windows99_vmware_guest_power_not_ready`（111/112/120/121/188）。 +- `reboot-auto-recovery-slo-scorecard.py` 修正 Windows99 next-action：collector 已成功時不再重複要求收 Verify，而是導向 `restore_windows99_missing_vmx_source_for_aliases...` 或 `build_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases...`，並保留 no VM power change 邊界。 + +**本地驗證結果**： +- `python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py -q -p no:cacheprovider`：`15 passed`。 +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py -q -p no:cacheprovider`：`37 passed`。 +- `python3.11 -m pytest scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py -q -p no:cacheprovider`：`9 passed`。 +- `python3.11 -m py_compile scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py`、`git diff --check`：通過。 + +**下一步**： +- commit / push 到 Gitea main；等 CD deploy marker 後重新 stage verifier 到 110，讀回 production API 的 `safe_next_step` 是否改為 VMX/source/autostart check-mode，而不是重複 collect Verify。 diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index 018523495..cbdf738db 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -1187,7 +1187,11 @@ def choose_safe_next_step( stockplatform: dict[str, Any], host_pressure: dict[str, Any], public_maintenance: dict[str, Any], + windows99: dict[str, Any] | None = None, + windows99_collector: dict[str, Any] | None = None, ) -> str: + windows99 = windows99 or {} + windows99_collector = windows99_collector or {} freshness_status = str(stockplatform.get("freshness_status") or "unknown") eod_window = stockplatform.get("eod_window") if isinstance(stockplatform.get("eod_window"), dict) else {} eod_next_action = str(eod_window.get("next_action") or "") @@ -1231,6 +1235,22 @@ def choose_safe_next_step( "verify_readback_then_rerun_reboot_scorecard_no_reboot" ) if any(blocker.startswith("windows99_") for blocker in blockers): + collector_collected = ( + windows99_collector.get("status") + == "collected_windows99_vmware_verify_stdout" + ) + missing_vmx_aliases = strings(windows99.get("missing_vmx_aliases")) + powered_off_aliases = strings(windows99.get("powered_off_aliases")) + if collector_collected and missing_vmx_aliases: + return ( + "restore_windows99_missing_vmx_source_for_aliases_then_rerun_" + "no_secret_collector_and_scorecard_no_vm_power_change" + ) + if collector_collected and powered_off_aliases: + return ( + "build_windows99_vmware_autostart_check_mode_package_for_powered_" + "off_aliases_then_rerun_no_secret_collector_no_vm_power_change" + ) return ( "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_" "reboot_scorecard_no_secret_no_reboot" @@ -1559,6 +1579,16 @@ def active_blocker_action_row( "restore_windows99_no_secret_management_channel_or_collect_local_" "console_verify_readback_then_rerun_reboot_scorecard_no_reboot" ) + elif blocker == "windows99_vmware_vmx_missing": + next_safe_action = ( + "restore_windows99_missing_vmx_source_for_aliases_then_rerun_" + "no_secret_collector_and_scorecard_no_vm_power_change" + ) + elif blocker == "windows99_vmware_guest_power_not_ready": + next_safe_action = ( + "build_windows99_vmware_autostart_check_mode_package_for_powered_" + "off_aliases_then_rerun_no_secret_collector_no_vm_power_change" + ) elif blocker in { "windows99_console_clipboard_unreliable", "windows99_console_focus_unreliable", @@ -2265,6 +2295,8 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: stockplatform=stockplatform, host_pressure=host_pressure, public_maintenance=public_maintenance, + windows99=windows99, + windows99_collector=windows99_collector, ) payload = { "schema_version": SCHEMA_VERSION, diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py index a75167832..bf02cfa1a 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py @@ -209,6 +209,73 @@ windows_update_policy_apply_performed=false """ +WINDOWS99_COLLECTOR_COLLECTED_WITH_111_VMX_MISSING = """\ +schema_version=windows99_vmware_verify_collector_v1 +dry_run=false +target_host=192.168.0.99 +target_host_alias=99 +connect_timeout_seconds=3 +ssh_timeout_seconds=3 +remote_verify_timeout_seconds=45 +port_timeout_wrapper=timeout +ssh_auth_probe_user_limit=5 +ssh_timeout_wrapper=timeout +port_22_open=1 +port_3389_open=1 +port_5985_open=0 +port_5986_open=0 +port_9182_open=0 +ssh_candidate_users=ogt,wooo,ooo,administrator,Administrator +ssh_auth_probed_users=2 +ssh_batchmode_auth_ready=1 +ssh_authenticated_user=wooo +ssh_auth_probe_exit_status=0 +ssh_auth_probe_stdout_present=1 +remote_verify_mode=in_memory_stdin_scriptblock +local_verify_script_present=1 +remote_verify_attempted=1 +remote_verify_exit_status=0 +verify_collection_status=collected_windows99_vmware_verify_stdout +safe_next_step=commit_no_secret_verify_artifact_then_rerun_reboot_auto_recovery_slo_scorecard +secret_value_read=false +password_prompt_allowed=false +remote_write_performed=false +host_reboot_performed=false +vm_power_change_performed=false +windows_update_policy_apply_performed=false +remote_verify_output_begin +AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1 +MODE=Verify +VMRUN_PRESENT=1 +HOST111_VMWARE_TARGET=explicit_or_required +VMX alias=111 path= present=0 +VMX alias=188 path=D:\\VMs\\188.vmx present=1 +VMX alias=120 path=D:\\VMs\\120.vmx present=1 +VMX alias=121 path=D:\\VMs\\121.vmx present=1 +VMX alias=112 path=D:\\VMs\\112.vmx present=1 +VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Auto ok=1 +VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Auto ok=1 +VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1 +WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1 +WINDOWS_UPDATE_POLICY name=AlwaysAutoRebootAtScheduledTime value=0 expected=0 ok=1 +WINDOWS_UPDATE_POLICY name=AUOptions value=3 expected=3 ok=1 +WINDOWS_UPDATE_POLICY name=ScheduledInstallDay value=0 expected=0 ok=1 +WINDOWS_UPDATE_POLICY name=ScheduledInstallTime value=3 expected=3 ok=1 +WINDOWS_UPDATE_POLICY name=AUPowerManagement value=0 expected=0 ok=1 +VM_POWER alias=111 vmx_present=0 running=0 +VM_POWER alias=188 vmx_present=1 running=0 +VM_POWER alias=120 vmx_present=1 running=0 +VM_POWER alias=121 vmx_present=1 running=0 +VM_POWER alias=112 vmx_present=1 running=0 +VMWARE_AUTOSTART_CONFIG_READY=0 +VMWARE_AUTOSTART_POWER_READY=0 +WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1 +VMWARE_AUTOSTART_VERIFY_READY=0 +MISSING_VMX_ALIASES=111 +remote_verify_output_end +""" + + def run_scorecard( tmp_path: Path, summary: str, @@ -697,6 +764,48 @@ def test_windows99_no_secret_collector_publickey_blocker_is_visible( ) +def test_collected_windows99_verifier_routes_to_vmx_source_repair( + tmp_path: Path, +) -> None: + payload = run_scorecard( + tmp_path, + GREEN_SUMMARY, + windows99=WINDOWS99_COLLECTOR_COLLECTED_WITH_111_VMX_MISSING, + windows99_management=json.dumps(WINDOWS99_MANAGEMENT_BLOCKED), + windows99_collector=WINDOWS99_COLLECTOR_COLLECTED_WITH_111_VMX_MISSING, + ) + + assert payload["windows99_vmware_verify_collector"]["status"] == ( + "collected_windows99_vmware_verify_stdout" + ) + assert ( + payload["windows99_verify_collection"][ + "no_secret_collector_readback_present" + ] + is True + ) + assert payload["windows99_verify_collection"][ + "no_secret_collector_ssh_batchmode_auth_ready" + ] is True + assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"] + assert payload["safe_next_step"] == ( + "restore_windows99_missing_vmx_source_for_aliases_then_rerun_" + "no_secret_collector_and_scorecard_no_vm_power_change" + ) + action_by_blocker = { + item["blocker"]: item["next_safe_action"] + for item in payload["active_blocker_action_matrix"]["items"] + } + assert action_by_blocker["windows99_vmware_vmx_missing"] == ( + "restore_windows99_missing_vmx_source_for_aliases_then_rerun_" + "no_secret_collector_and_scorecard_no_vm_power_change" + ) + assert action_by_blocker["windows99_vmware_guest_power_not_ready"] == ( + "build_windows99_vmware_autostart_check_mode_package_for_powered_" + "off_aliases_then_rerun_no_secret_collector_no_vm_power_change" + ) + + def test_degraded_wazuh_and_old_boot_observation_block_slo(tmp_path: Path) -> None: summary = GREEN_SUMMARY.replace("WAZUH_DASHBOARD_DEGRADED=0", "WAZUH_DASHBOARD_DEGRADED=1") probe = HOST_PROBE_GREEN.replace("uptime_seconds=150", "uptime_seconds=900")