diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 6b1d800b9..c537fdf2f 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -54675,3 +54675,27 @@ production browser smoke: **下一步**： - commit / push 到 Gitea main；deploy 後讀回 production locator endpoint 與 priority readback。接續 P0 是提供 / 找到 111 verified VMX source 或 backup path，再產生 scoped dry-run，不直接 relink 或 power on。 + +## 2026-07-11 - P0-008 112 Kali 真正開機完成與自動恢復閉環 + +**事故判讀**： +- Windows99 的 SYSTEM scheduled task 以 headless 方式啟動 112 VMX；Administrator VMware Workstation session 因 principal/session 隔離顯示 `Powered off`，但 VMX process、Ping、SSH 與 guest boot identity 都證明 VM 正在執行。 +- 112 Guest 當時預設為 `multi-user.target`，LightDM 未啟動；`wazuh-indexer.service` 在開機競爭期間觸及 3 分鐘 timeout。舊 SOP 只看 VM power、Ping/SSH/node-exporter，錯誤把「可達」視為「完整開機」。 + +**受控修復**： +- live 恢復 `graphical.target`、LightDM/Xorg、open-vm-tools 與 Wazuh Indexer，systemd 回 `running`。 +- 新增 `host112-guest-readiness.sh`、systemd service/timer 與 installer；timer 開機 20 秒後啟動、每 60 秒檢查，Index retry 具 210 秒 boot delay 與 600 秒 cooldown。 +- 建立 110 到 112 的來源限制 forced-command 公鑰讀回與 exact-command sudo executor；任何 VM power、host reboot、firewall、DB 或 secret 操作都不在 allowlist。 +- Host probe / scorecard / exporter 新增 Console、Wazuh、recovery timer 與 guest-ready gates；Prometheus 新增 `Host112GuestReadbackMissing` 與 `Host112GuestNotReady`。 +- Windows99 VMware verifier新增 VMX owner/session readback，112 明確輸出 `runtime_running_gui_session_isolated`，避免依錯誤 VMware UI 再次 power on。 + +**驗證結果**： +- 112：`systemd=running`、`console_ready=1`、`services_ready=1`、`recovery_timer_ready=1`、`guest_ready=1`。 +- 110 artifact：`/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-194901`；`host112_* blocker=0`。 +- Prometheus：五項 112 gauges `5/5`；兩條專屬規則 health `ok`、state `inactive`；source/canonical/container SHA 一致，drift guard `repaired=0`。 +- Windows99：112 VMX PID `22288`、session `0`、owner `NT AUTHORITY\\SYSTEM`；VMware verify ready。 +- 測試：`pytest -q scripts/reboot-recovery/tests` 為 `142 passed`；JSON/YAML parse 與 bash syntax 通過。 +- Receipt：`docs/operations/host112-guest-recovery-20260711.snapshot.json`；runtime rollback installer 位於 `/usr/local/sbin/awoooi-install-host112-guest-recovery`。 + +**剩餘驗收**： +- 112 runtime scope 已恢復並驗證；全主機 program 仍保留歷史 10 分鐘逾時，不以事後綠燈覆蓋。下一次真實全主機 reboot 必須在同一 600 秒 window 重新通過 VMX、112 Console/Wazuh/timer、七主機、服務、資料、備份與 public fallback。 diff --git a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json index de2289272..b2938466b 100644 --- a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json +++ b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json @@ -461,24 +461,24 @@ "order": 8, "priority": "P0", "status": "in_progress", - "title": "Windows 99 VMware autostart, VMX, lock and update policy", + "title": "Windows 99 VMware autostart, VMX, 112 guest readiness, lock and update policy", "owner_lane": "windows99_vmware", "dependencies": ["AG99-P0-003", "AG99-P0-005"], "scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188"], - "problem": "The drill showed an 18-minute-45-second delay between Windows boot and the VMware autostart task, a principal visibility mismatch for the Kali VM and two later VIX SoftReboot actions without an attributable trace. Current VM power is green, but the next startup must prove timing and every reset/stop/suspend must be audited.", + "problem": "The drill showed an 18-minute-45-second delay between Windows boot and the VMware autostart task, a principal visibility mismatch for the Kali VM and two later VIX SoftReboot actions without an attributable trace. The 2026-07-11 follow-up proved that VMX, Ping, SSH and node-exporter were insufficient for host 112: Kali booted to multi-user.target without LightDM, Wazuh Indexer timed out during boot, and the Administrator VMware UI falsely displayed Powered off while a SYSTEM-owned VMX process was running.", "trigger": "Windows startup, VMware exclusive-lock error, guest power mismatch, VMX drift or Windows Update policy drift.", "ai_role": "Windows and VMware control agent", "action_mode": "check_mode_then_controlled_apply", - "evidence_sources": ["VMX inventory", "vmrun list", "VMware services", "scheduled tasks", "lock owners", "Windows Update policy"], - "controlled_actions": ["repair stale lock after owner proof", "configure autostart order", "verify guest tools and network", "apply reversible update policy"], + "evidence_sources": ["VMX inventory and process principal", "vmrun list", "guest boot identity and uptime", "112 forced-command readiness", "graphical target and LightDM", "VMware Tools", "Wazuh services", "scheduled tasks", "lock owners", "Windows Update policy"], + "controlled_actions": ["repair stale lock after owner proof", "configure autostart order", "verify guest tools and network", "start host 112 graphical target, LightDM and VMware Tools", "retry Wazuh Indexer after boot with cooldown", "apply reversible update policy"], "prohibited_actions": ["delete unknown lock", "guess VMX path", "power off VM", "reset stop or suspend a VM without trace and verifier", "reboot Windows without break-glass"], "rollback": "Restore previous task, service and policy exports; keep guest power unchanged during check mode.", - "verifier": "All required VMX paths exist, guests report running and reachable, tasks survive reboot, update policy drift is zero.", + "verifier": "All required VMX paths exist; guests report running and reachable; host 112 returns systemd running, graphical target, LightDM, Xorg, VMware Tools, Wazuh stack and recovery timer green through a forced-command readback; tasks survive reboot; update policy drift is zero.", "callback": "Update reboot SLO VMware and Windows policy gates.", - "acceptance": "99 startup automatically launches VMware guests 110/188/112/120/121 in order, physical host 111 is verified separately, no exclusive-lock failure occurs and Windows Update cannot surprise-reboot the host.", - "implementation_progress": "The controlled apply receipt windows99-vmware-autostart-apply-20260711-173505.json completed DryRun, backup, Apply and independent Verify. VMX process count remained 5, required guests 110/188/120/121/112 are running, the generated startup script has a process-aware duplicate-start guard, the task and Windows Update policy are green, and no reboot, VM power change or secret read occurred. VMware logs prove the 120/121 secondary reboots were VIX SoftReboot actions outside Agent99; actor attribution was unavailable because process auditing was not enabled.", - "evidence_refs": ["C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json", "C:\\Wooo\\Agent99\\deploy\\vmware-autostart-backup-20260711-173505", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623/windows99-vmware-verify.txt"], - "next_action": "Keep guest power unchanged, add trace-correlated VMware control auditing, and on the next approved drill verify that the startup task runs promptly after Windows boot and all five guests plus physical host 111 recover inside the same 600-second window." + "acceptance": "99 startup automatically launches VMware guests 110/188/112/120/121 in order; physical host 111 is verified separately; 112 must pass Console-ready and Wazuh service gates rather than only Ping or SSH; no exclusive-lock failure occurs; Windows Update cannot surprise-reboot the host.", + "implementation_progress": "The controlled VMware apply remains green with five running VMX processes. The 112 follow-up restored graphical.target, LightDM and Wazuh Indexer, deployed a bounded guest recovery timer, restricted 110 forced-command readback, exact-command sudo executor and durable rollback installer. The 110 scorecard now reads systemd running plus Console/Wazuh/timer/guest 1/1/1/1, all five host112 Prometheus gauges are 1, host112 blockers are zero, both dedicated alert rules are healthy/inactive, and Windows99 records the SYSTEM session VMX as runtime_running_gui_session_isolated. Reboot-recovery tests pass 142/142.", + "evidence_refs": ["C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json", "C:\\Wooo\\Agent99\\deploy\\vmware-autostart-backup-20260711-173505", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-194901", "/var/lib/awoooi-host112-recovery/backups/host112-guest-recovery-20260711T195449+0800-43505", "docs/operations/host112-guest-recovery-20260711.snapshot.json", "reboot-recovery suite 142 passed"], + "next_action": "Keep guest power unchanged, route any host112 state change to Agent99 controlled apply and lifecycle callback, then prove the same Console/Wazuh/timer gates inside the next real all-host 600-second drill." }, { "id": "AG99-P0-009", @@ -658,9 +658,13 @@ "dependencies": ["AG99-P0-006"], "scope_refs": ["surface:Telegram", "all:incidents"], "action_mode": "controlled_apply", + "problem": "Current Agent99 Telegram output exposes verifier field names, Windows paths, truncated summaries and duplicated screenshot text. It does not give operators a concise impact statement, lifecycle state, actual AI action, verified result, duration or next update, and duplicate same-state messages create alert fatigue.", + "controlled_actions": ["render detected, investigating, remediating and recovered lifecycle cards", "use Traditional Chinese operator summary", "attach real trend chart, topology or screenshot only when it adds evidence", "edit or thread the same incident message", "suppress duplicate fingerprints until state changes"], + "prohibited_actions": ["send raw verifier dumps", "repeat the same text as an image", "expose internal paths or secrets", "declare resolved from send success", "send periodic same-state noise"], "verifier": "Message snapshot tests, duplicate suppression, state-change delivery and evidence-link checks.", - "acceptance": "Every card explains impact, AI action, verified result and next step in Traditional Chinese; charts or screenshots are attached when they improve understanding.", - "next_action": "Replace raw technical completion messages with one lifecycle card per incident state change." + "acceptance": "Every card explains severity, affected service or host, user impact, lifecycle state, AI diagnosis, controlled action, verifier result, duration, recurrence count and next update in Traditional Chinese. Charts or real screenshots are attached only when they improve understanding; one fingerprint produces one lifecycle thread and no same-state spam.", + "implementation_progress": "The 2026-07-11 Agent99 incident-card screenshot was captured as a failing UX fixture and added to the P1 acceptance contract; runtime renderer replacement remains pending behind the current host 112 P0 closure.", + "next_action": "Replace the raw automation-result renderer with detected, progress and recovered card templates; add fingerprint edit/thread dedup, redaction, chart-or-screenshot evidence rules and Telegram delivery snapshot tests." }, { "id": "AG99-P1-004", diff --git a/docs/operations/host112-guest-recovery-20260711.snapshot.json b/docs/operations/host112-guest-recovery-20260711.snapshot.json new file mode 100644 index 000000000..7cf6034c2 --- /dev/null +++ b/docs/operations/host112-guest-recovery-20260711.snapshot.json @@ -0,0 +1,140 @@ +{ + "schema_version": "host112_guest_recovery_receipt_v1", + "generated_at": "2026-07-11T19:52:31+08:00", + "work_item_id": "AG99-P0-008", + "trace_id": "host112-guest-recovery-20260711T193137+0800-35698", + "incident_fingerprint": "host112_vmware_ui_false_powered_off_guest_headless_wazuh_boot_timeout", + "asset": { + "canonical_id": "host:112", + "address": "192.168.0.112", + "guest_role": "Kali and Wazuh security node", + "vm_owner_host": "host:99", + "vmx_alias": "112" + }, + "sensor_receipt": { + "vmware_ui_observation": "Powered off", + "vmx_process_present": true, + "vmx_process_id": 22288, + "vmx_process_owner": "NT AUTHORITY\\SYSTEM", + "vmx_process_session_id": 0, + "guest_boot_id": "f265ad96-a262-4285-9836-d2dab7eefb97", + "guest_network_reachable": true, + "guest_ssh_reachable": true, + "pre_recovery_default_target": "multi-user.target", + "pre_recovery_lightdm": "inactive", + "pre_recovery_wazuh_indexer": "failed_timeout", + "wazuh_indexer_boot_timeout": "3min", + "wazuh_indexer_boot_memory_peak": "1.2G" + }, + "normalized_diagnosis": { + "vm_power_state": "running", + "vmware_ui_state": "untrusted_cross_principal_false_powered_off", + "guest_boot_state": "reachable_but_not_console_or_service_ready", + "root_causes": [ + "SYSTEM-owned headless VMX is isolated from the Administrator VMware UI session", + "Kali defaulted to multi-user.target and LightDM was inactive", + "Wazuh Indexer timed out during boot contention and recovered on a delayed retry" + ] + }, + "source_diff": { + "new_assets": [ + "scripts/reboot-recovery/host112-guest-readiness.sh", + "scripts/reboot-recovery/awoooi-host112-guest-recovery.service", + "scripts/reboot-recovery/awoooi-host112-guest-recovery.timer", + "scripts/reboot-recovery/install-host112-guest-recovery.sh" + ], + "updated_assets": [ + "scripts/reboot-recovery/windows99-vmware-autostart.ps1", + "scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh", + "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py", + "scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh", + "ops/monitoring/alerts-unified.yml", + "docs/runbooks/REBOOT-RECOVERY-SOP.md" + ], + "source_sha256": { + "host112_readiness": "dc874bf9defe655cf2cae7a678783a847b6653b4255ef5798903afbc974e3e2d", + "host112_installer": "c0ea2d8c10aebef6079f4f73f8614cd9c3fb875903b55824f6b44acf23ab40db", + "windows99_vmware_verifier": "437cc7766f06d9fd543cc472569b238149a79f2dff454dbb9ed704d9e3be7648", + "host_probe": "31dce6f90dc89e2fd38bcde91049aa79711b3e5e11a1ae2d1053e27bbbc1f01f", + "scorecard": "835abf7bf6cf587d40467351974f5eba2b84f731c78c2e6c7aaa254f0eaaf33d", + "exporter": "85f16a3d190cdca866ad44f03c4c3e4decfbdf0d3af873c8e4af4d66780d5214", + "alert_rules": "2f606acd9d35b3708f57f313fcfe1f75c7b274316de0b27f494636a132e8ba4d" + } + }, + "decision": { + "risk": "medium", + "controlled_apply_allowed": true, + "candidate_action": "Install a bounded guest recovery timer, restricted forced-command readback, exact-command sudo executor, scorecard gates and dedicated alerts.", + "forbidden_actions": [ + "host reboot", + "VM power off, reset or duplicate power on", + "firewall change", + "database restore or write", + "secret or private key value read" + ] + }, + "check_receipt": { + "staged_source_hashes_matched": true, + "control_public_key_fingerprint_verified": true, + "focused_contract_tests": "31 passed", + "json_yaml_parse": "passed" + }, + "execution_receipt": { + "initial_recovery_run_id": "host112-guest-recovery-20260711T193137+0800-35698", + "durable_installer_run_id": "host112-guest-recovery-20260711T195449+0800-43505", + "backup_dirs": [ + "/var/lib/awoooi-host112-recovery/backups/host112-guest-recovery-20260711T193137+0800-35698", + "/var/lib/awoooi-host112-recovery/backups/host112-guest-recovery-20260711T195449+0800-43505" + ], + "durable_installer_path": "/usr/local/sbin/awoooi-install-host112-guest-recovery", + "timer_enabled": true, + "timer_active": true, + "restricted_control_key_installed": true, + "sudo_exact_command_installed": true, + "host_reboot_performed": false, + "vm_power_change_performed": false, + "secret_read": false + }, + "post_verifier": { + "host112_guest_ready": true, + "host112_console_ready": true, + "host112_services_ready": true, + "host112_recovery_timer_ready": true, + "host112_systemd_state": "running", + "host112_default_target": "graphical.target", + "host112_lightdm": "active", + "host112_vmware_tools": "active", + "host112_wazuh_indexer": "active", + "host112_wazuh_manager": "active", + "host112_wazuh_dashboard": "active", + "host112_filebeat": "active", + "forced_command_from_110": "passed", + "sudo_n_controlled_apply": "passed_no_action_needed", + "scorecard_artifact": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-194901", + "host112_active_blocker_count": 0, + "prometheus_host112_metrics_green": "5/5", + "prometheus_rule_health": "2/2 ok inactive", + "prometheus_rule_sha_source_canonical_runtime_matched": true, + "prometheus_rule_drift_guard_repaired": false, + "windows99_vmware_verify_ready": true, + "windows99_operator_power_truth": "trust_vmx_process_and_guest_readiness_not_interactive_vmware_ui", + "reboot_recovery_tests": "142 passed" + }, + "rollback": { + "command": "sudo /usr/local/sbin/awoooi-install-host112-guest-recovery --rollback /var/lib/awoooi-host112-recovery/backups/host112-guest-recovery-20260711T195449+0800-43505", + "tested": false, + "reason_not_executed": "Post-verifier passed; destructive rollback rehearsal was not required on the recovered production guest." + }, + "closure": { + "host112_runtime_scope": "recovered_and_verified", + "full_host_reboot_program": "in_progress_next_600_second_drill_required", + "remaining_reboot_scorecard_blockers": [ + "all_required_hosts_not_in_10_minute_reboot_window", + "fresh_all_host_reboot_event_missing", + "host_boot_observation_older_than_target_window" + ], + "telegram_receipt": "not_sent_healthy_state_no_duplicate_notification", + "km_writeback": "docs/runbooks/REBOOT-RECOVERY-SOP.md", + "work_item_writeback": "docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json" + } +} diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 943fb7fb2..76d62d445 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -69,6 +69,16 @@ L0 source contract 是 `scripts/reboot-recovery/external-l0-recovery-supervisor. Windows 99 VMware task 已經由 `scripts/reboot-recovery/windows99-vmware-autostart-controlled-apply.ps1` 正式受控套用：DryRun、Apply、Verify 同一 receipt；task XML、舊 startup script、Windows Update policy 與 service start modes 都有 rollback。Evidence `C:\Wooo\Agent99\evidence\windows99-vmware-autostart-apply-20260711-173505.json` 回報 pre/post VMX process 都是 `5`、running guest rows `5`、process-aware duplicate-start guard true、沒有 host reboot、沒有 VM power change、沒有 secret read。這才是 Agent99 與重啟 SOP 的 runtime 整合證據；仍不取代下一次 600 秒 E2E drill。 +### 112 Kali 真正開機完成 Gate + +112 一直存在於 Windows99 required VM inventory、host probe 與 reboot event required host set；本次事故暴露的缺陷是舊流程只用 VMX process、Ping、SSH port 或 node-exporter 判定「已開機」，沒有驗證 Kali 的圖形登入與安全服務。這會讓 112 實際以 `multi-user.target` 無 GUI 啟動、LightDM 未執行且 Wazuh Indexer 開機逾時時仍被錯誤判成綠燈。另一個獨立現象是 SYSTEM principal 以 `vmrun ... nogui` 啟動的 VMX 不會出現在 Administrator 的 VMware Workstation session，GUI 可能顯示 `Powered off`；在 VMX process 與 guest boot identity 已存在時，禁止因此再次按 Power on，避免 exclusive lock 與重複啟動。 + +112 的固定完成鏈改為：`VMX process -> guest boot_id/uptime -> SSH forced-command readback -> systemd running -> graphical.target -> LightDM/display-manager -> Xorg -> open-vm-tools -> Wazuh Indexer/Manager/Dashboard/Filebeat -> recovery timer -> Console-ready`。`scripts/reboot-recovery/host112-guest-readiness.sh` 提供 no-secret `--check` 與 allowlisted `--apply`；apply 只可補起 graphical target、LightDM、VMware Tools，以及在開機 210 秒後、600 秒 cooldown 內重試一次 Wazuh Indexer。`awoooi-host112-guest-recovery.timer` 於開機 20 秒後開始、每 60 秒檢查；不得 reboot host、reset/power cycle VM、修改 firewall、讀取 secret 或執行資料庫寫入。 + +110 到 112 使用來源限制、`restrict` 與 forced command 的既有公鑰，只能執行 `/usr/local/sbin/awoooi-host112-guest-readiness --check`，不能取得 shell 或 forwarding。Host probe 必須輸出 `console_ready=1`、`services_ready=1`、`recovery_timer_ready=1`、`guest_ready=1`；任何一項缺失都建立具名 `host112_*` blocker，阻止 full-host recovery 與 maintenance withdrawal。Prometheus 固定輸出 `awoooi_host112_guest_readback_present`、`awoooi_host112_console_ready`、`awoooi_host112_services_ready`、`awoooi_host112_recovery_timer_ready`、`awoooi_host112_guest_ready`，Agent99 收到狀態變更後執行受控修復並以同一 incident 回報 verifier 結果。 + +2026-07-11 runtime closure：installer runs `host112-guest-recovery-20260711T193137+0800-35698` 與 `host112-guest-recovery-20260711T195449+0800-43505` 已完成備份、受限公鑰、exact-command sudo、timer 與 durable rollback installer，未 reboot、未改 VM power、未讀 secret。110 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-194901` 讀回 112 `systemd=running`、Console/Wazuh/timer/guest `1/1/1/1`、五項 Prometheus gauges `5/5`、`host112_* blocker=0`。兩條 112 告警規則 health `2/2 ok` 且 healthy state 為 inactive；source/canonical/container SHA 一致，drift guard `repaired=0`。Windows99 verifier 同輪讀到 `VM_PROCESS alias=112 process_id=22288 session_id=0 owner=NT_AUTHORITY\SYSTEM ui_truth=runtime_running_gui_session_isolated`，因此 operator power truth 固定信任 VMX process + guest readiness，不信任跨 principal 的互動式 VMware UI。完整 receipt 見 `docs/operations/host112-guest-recovery-20260711.snapshot.json`；reboot-recovery tests `142 passed`。下一個未關閉驗收只剩下一次真實全主機 600 秒 drill，不能用這次事後恢復覆蓋歷史逾時。 + ### 2026-07-10 P0 Agent99 Transport / Verified Remediation 覆蓋 99 的 Agent99 是全主機恢復控制節點。Windows 開機後由既有 Agent99 scheduled tasks 啟動；`Status` 讀到 boot identity 變更時，只能用 single-flight queue 觸發一次 `Recover`，並以 10 分鐘 cooldown 避免併發恢復。每次執行必須寫入 `bootState`、`recoveryTrigger`、`recoveryCoordinator` 與 `recoverySlo` evidence。只有實際全主機重啟事件、七主機 reachability、必要服務、public routes、資料 freshness、備份、edge fallback、VMware/Windows policy 與 188 hygiene 全部在同一個 10 分鐘 window 通過，才可宣稱 reboot recovery SLO 達標。 diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 12c43984d..217bd2f1a 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -245,6 +245,42 @@ groups: description: "awoooi_reboot_auto_recovery_slo_blocker_count > 0，但沒有 awoooi_reboot_auto_recovery_slo_active_blocker；Telegram 只能看到總數，無法自動指出真問題。" runbook: "部署新版 reboot-auto-recovery-slo-exporter.sh，確認 node-exporter textfile 中有 awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}，再重跑 Prometheus rule drift guard。" + - alert: Host112GuestReadbackMissing + expr: awoooi_host112_guest_readback_present{scope="host112"} == 0 + for: 2m + labels: + severity: warning + layer: guest-112 + component: host112-guest-recovery + host: "112" + team: ops + alert_category: reboot_recovery + notification_type: TYPE-1 + auto_repair: "true" + automation_mode: controlled_apply + annotations: + summary: "112 開機完成讀回中斷" + description: "110 無法取得 112 的 forced-command readiness；Ping 或 SSH port 可達不代表 Kali 圖形登入、VMware Tools 與 Wazuh 已恢復。" + runbook: "Agent99 先驗證 112 VMX process 與 SSH，再恢復限定的 110→112 forced-command 公鑰通道；不得因 UI 顯示 Powered off 就重複 power on。" + + - alert: Host112GuestNotReady + expr: awoooi_host112_guest_ready{scope="host112"} == 0 + for: 2m + labels: + severity: critical + layer: guest-112 + component: host112-guest-recovery + host: "112" + team: ops + alert_category: reboot_recovery + notification_type: TYPE-3 + auto_repair: "true" + automation_mode: controlled_apply + annotations: + summary: "112 尚未真正完成開機恢復" + description: "112 的 Console、VMware Tools、Wazuh 服務或 recovery timer 至少一項未通過；Agent99 應執行 allowlisted guest recovery 並以 forced-command readback 驗證。" + runbook: "執行 sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply；僅允許補起 graphical.target、LightDM、open-vm-tools 與 cooldown-guarded Wazuh Indexer，不得 reboot、reset VM 或讀取 secret。" + - alert: RebootAutoRecoverySLOExporterStale expr: time() - awoooi_reboot_auto_recovery_slo_last_run_timestamp{scope="99_110_111_112_120_121_188"} > 600 for: 10m diff --git a/scripts/reboot-recovery/awoooi-host112-guest-recovery.service b/scripts/reboot-recovery/awoooi-host112-guest-recovery.service new file mode 100644 index 000000000..9d8fb1209 --- /dev/null +++ b/scripts/reboot-recovery/awoooi-host112-guest-recovery.service @@ -0,0 +1,19 @@ +[Unit] +Description=AWOOOI host 112 guest readiness and bounded recovery +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply +TimeoutStartSec=300 +User=root +Group=root +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectControlGroups=true +RestrictSUIDSGID=true +LockPersonality=true diff --git a/scripts/reboot-recovery/awoooi-host112-guest-recovery.timer b/scripts/reboot-recovery/awoooi-host112-guest-recovery.timer new file mode 100644 index 000000000..8a69a95ad --- /dev/null +++ b/scripts/reboot-recovery/awoooi-host112-guest-recovery.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run AWOOOI host 112 guest recovery after boot and continuously + +[Timer] +OnBootSec=20s +OnUnitInactiveSec=60s +AccuracySec=5s +Persistent=true +Unit=awoooi-host112-guest-recovery.service + +[Install] +WantedBy=timers.target diff --git a/scripts/reboot-recovery/host112-guest-readiness.sh b/scripts/reboot-recovery/host112-guest-readiness.sh new file mode 100755 index 000000000..1269b4c89 --- /dev/null +++ b/scripts/reboot-recovery/host112-guest-readiness.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# Host 112 guest readiness and bounded recovery executor. + +set -uo pipefail + +MODE="check" +INDEXER_RETRY_AFTER_SECONDS="${HOST112_INDEXER_RETRY_AFTER_SECONDS:-210}" +INDEXER_RETRY_COOLDOWN_SECONDS="${HOST112_INDEXER_RETRY_COOLDOWN_SECONDS:-600}" +INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-240}" +STATE_DIR="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}" +RETRY_MARKER="${HOST112_INDEXER_RETRY_MARKER:-/run/awoooi-host112-indexer-retry.epoch}" + +usage() { + cat <<'USAGE' +Usage: host112-guest-readiness.sh [--check|--apply] + +--check emits a bounded no-secret guest readiness row. +--apply performs only allowlisted idempotent recovery for graphical.target, +LightDM, open-vm-tools and a cooldown-guarded Wazuh Indexer start retry. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --check) MODE="check" ;; + --apply) MODE="apply" ;; + -h|--help) usage; exit 0 ;; + *) echo "BLOCKER unknown_argument=$1" >&2; exit 64 ;; + esac + shift +done + +actions=() + +unit_active() { + local unit="$1" state + state="$(systemctl is-active "$unit" 2>/dev/null || true)" + printf '%s' "${state:-unknown}" +} + +unit_enabled() { + local unit="$1" state + state="$(systemctl is-enabled "$unit" 2>/dev/null || true)" + printf '%s' "${state:-unknown}" +} + +unit_loaded() { + [ "$(systemctl show "$1" -p LoadState --value 2>/dev/null || true)" = "loaded" ] +} + +record_action() { + actions+=("$1") +} + +start_if_inactive() { + local unit="$1" action="$2" state + state="$(unit_active "$unit")" + if [ "$state" = "active" ] || [ "$state" = "activating" ]; then + return 0 + fi + if systemctl start "$unit"; then + record_action "$action" + else + record_action "${action}_failed" + fi +} + +if [ "$MODE" = "apply" ]; then + if [ "${EUID:-$(id -u)}" -ne 0 ]; then + echo "BLOCKER host112_apply_requires_root" >&2 + exit 77 + fi + + current_default="$(systemctl get-default 2>/dev/null || true)" + if [ "$current_default" != "graphical.target" ]; then + if systemctl set-default graphical.target >/dev/null; then + record_action "set_default_graphical_target" + else + record_action "set_default_graphical_target_failed" + fi + fi + + start_if_inactive graphical.target start_graphical_target + start_if_inactive open-vm-tools.service start_vmware_tools + start_if_inactive lightdm.service start_lightdm + + uptime_for_retry="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo 0)" + indexer_state="$(unit_active wazuh-indexer.service)" + if unit_loaded wazuh-indexer.service \ + && [ "$indexer_state" != "active" ] \ + && [ "$indexer_state" != "activating" ] \ + && [ "$uptime_for_retry" -ge "$INDEXER_RETRY_AFTER_SECONDS" ]; then + now_epoch="$(date +%s)" + last_retry_epoch=0 + if [ -r "$RETRY_MARKER" ]; then + read -r last_retry_epoch <"$RETRY_MARKER" || last_retry_epoch=0 + fi + if ! [[ "$last_retry_epoch" =~ ^[0-9]+$ ]]; then + last_retry_epoch=0 + fi + if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then + printf '%s\n' "$now_epoch" >"$RETRY_MARKER" + systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true + if timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then + record_action "retry_wazuh_indexer" + else + record_action "retry_wazuh_indexer_failed" + fi + else + record_action "wazuh_indexer_retry_cooldown" + fi + fi +fi + +boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)" +uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo unknown)" +systemd_state="$(systemctl is-system-running 2>/dev/null || true)" +systemd_state="${systemd_state:-unknown}" +default_target="$(systemctl get-default 2>/dev/null || true)" +default_target="${default_target:-unknown}" +graphical_target="$(unit_active graphical.target)" +display_manager="$(unit_active display-manager.service)" +lightdm="$(unit_active lightdm.service)" +vmware_tools="$(unit_active open-vm-tools.service)" +wazuh_indexer="$(unit_active wazuh-indexer.service)" +wazuh_manager="$(unit_active wazuh-manager.service)" +wazuh_dashboard="$(unit_active wazuh-dashboard.service)" +filebeat="$(unit_active filebeat.service)" +startup_enabled="$(unit_enabled awoooi-host112-guest-recovery.timer)" +startup_active="$(unit_active awoooi-host112-guest-recovery.timer)" + +xorg="inactive" +if pgrep -x Xorg >/dev/null 2>&1 || pgrep -x Xwayland >/dev/null 2>&1; then + xorg="active" +fi + +console_ready=0 +if [ "$default_target" = "graphical.target" ] \ + && [ "$graphical_target" = "active" ] \ + && [ "$display_manager" = "active" ] \ + && [ "$lightdm" = "active" ] \ + && [ "$vmware_tools" = "active" ] \ + && [ "$xorg" = "active" ]; then + console_ready=1 +fi + +services_ready=0 +if [ "$wazuh_indexer" = "active" ] \ + && [ "$wazuh_manager" = "active" ] \ + && [ "$wazuh_dashboard" = "active" ] \ + && [ "$filebeat" = "active" ]; then + services_ready=1 +fi + +recovery_timer_ready=0 +if [ "$startup_enabled" = "enabled" ] && [ "$startup_active" = "active" ]; then + recovery_timer_ready=1 +fi + +guest_ready=0 +if [ "$console_ready" -eq 1 ] \ + && [ "$services_ready" -eq 1 ] \ + && [ "$recovery_timer_ready" -eq 1 ] \ + && [ "$systemd_state" = "running" ]; then + guest_ready=1 +fi + +action_text="none" +if [ "${#actions[@]}" -gt 0 ]; then + action_text="$(IFS=,; printf '%s' "${actions[*]}")" +fi + +readback="boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready action_count=${#actions[@]} actions=$action_text" +printf '%s\n' "$readback" + +if [ "$MODE" = "apply" ]; then + install -d -m 0750 "$STATE_DIR" + tmp_readback="${STATE_DIR}/last-readback.txt.tmp.$$" + printf 'generated_at=%s mode=%s %s\n' "$(date --iso-8601=seconds)" "$MODE" "$readback" >"$tmp_readback" + chmod 0640 "$tmp_readback" + mv "$tmp_readback" "${STATE_DIR}/last-readback.txt" +fi + +[ "$guest_ready" -eq 1 ] diff --git a/scripts/reboot-recovery/install-host112-guest-recovery.sh b/scripts/reboot-recovery/install-host112-guest-recovery.sh new file mode 100755 index 000000000..c23504007 --- /dev/null +++ b/scripts/reboot-recovery/install-host112-guest-recovery.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODE="check" +ROLLBACK_DIR="" +TARGET_USER="${HOST112_TARGET_USER:-kali}" +CONTROL_SOURCE_ADDRESS="${HOST112_CONTROL_SOURCE_ADDRESS:-192.168.0.110}" +CONTROL_PUBLIC_KEY_PATH="${HOST112_CONTROL_PUBLIC_KEY_PATH:-${SCRIPT_DIR}/control.pub}" +READINESS_SOURCE="${SCRIPT_DIR}/host112-guest-readiness.sh" +SERVICE_SOURCE="${SCRIPT_DIR}/awoooi-host112-guest-recovery.service" +TIMER_SOURCE="${SCRIPT_DIR}/awoooi-host112-guest-recovery.timer" +READINESS_TARGET="/usr/local/sbin/awoooi-host112-guest-readiness" +INSTALLER_TARGET="/usr/local/sbin/awoooi-install-host112-guest-recovery" +SERVICE_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.service" +TIMER_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.timer" +SUDOERS_TARGET="/etc/sudoers.d/awoooi-host112-agent99" +STATE_ROOT="/var/lib/awoooi-host112-recovery" + +usage() { + cat <<'USAGE' +Usage: + install-host112-guest-recovery.sh --check + install-host112-guest-recovery.sh --apply + install-host112-guest-recovery.sh --rollback BACKUP_DIR + +Apply installs the bounded host-112 recovery timer, an exact-command sudo rule, +and a forced-command readback key for host 110. It never reboots the host. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --check) MODE="check" ;; + --apply) MODE="apply" ;; + --rollback) + MODE="rollback" + shift + ROLLBACK_DIR="${1:-}" + ;; + -h|--help) usage; exit 0 ;; + *) echo "UNKNOWN_ARG=$1" >&2; usage >&2; exit 64 ;; + esac + shift +done + +check() { + echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" + echo "MODE=check" + for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET"; do + if [ -f "$path" ]; then + echo "TARGET path=$path present=1 sha256=$(sha256sum "$path" | awk '{print $1}')" + else + echo "TARGET path=$path present=0 sha256=missing" + fi + done + echo "TIMER_ENABLED=$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)" + echo "TIMER_ACTIVE=$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)" + if [ -x "$READINESS_TARGET" ]; then + "$READINESS_TARGET" --check || true + fi +} + +require_root() { + if [ "${EUID:-$(id -u)}" -ne 0 ]; then + echo "BLOCKER host112_installer_requires_root" >&2 + exit 77 + fi +} + +backup_path() { + local path="$1" backup_dir="$2" relative + if [ -e "$path" ]; then + relative="${path#/}" + install -d "${backup_dir}/rootfs/$(dirname "$relative")" + cp -a "$path" "${backup_dir}/rootfs/${relative}" + printf 'present\t%s\n' "$path" >>"${backup_dir}/manifest.tsv" + else + printf 'missing\t%s\n' "$path" >>"${backup_dir}/manifest.tsv" + fi +} + +if [ "$MODE" = "check" ]; then + check + exit 0 +fi + +require_root + +if [ "$MODE" = "rollback" ]; then + case "$ROLLBACK_DIR" in + "$STATE_ROOT"/backups/*) ;; + *) echo "BLOCKER invalid_host112_rollback_dir" >&2; exit 64 ;; + esac + [ -f "$ROLLBACK_DIR/manifest.tsv" ] || { echo "BLOCKER rollback_manifest_missing" >&2; exit 66; } + systemctl disable --now awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || true + while IFS=$'\t' read -r state path; do + [ -n "$path" ] || continue + if [ "$state" = "present" ]; then + install -d "$(dirname "$path")" + cp -a "$ROLLBACK_DIR/rootfs/${path#/}" "$path" + else + rm -f "$path" + fi + done <"$ROLLBACK_DIR/manifest.tsv" + systemctl daemon-reload + if [ -f "$ROLLBACK_DIR/timer_was_enabled" ]; then + systemctl enable awoooi-host112-guest-recovery.timer >/dev/null + fi + if [ -f "$ROLLBACK_DIR/timer_was_active" ]; then + systemctl start awoooi-host112-guest-recovery.timer + fi + echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" + echo "MODE=rollback" + echo "ROLLBACK_APPLIED=1" + check + exit 0 +fi + +for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH"; do + [ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; exit 66; } +done + +user_home="$(getent passwd "$TARGET_USER" | cut -d: -f6)" +user_group="$(id -gn "$TARGET_USER")" +[ -n "$user_home" ] || { echo "BLOCKER target_user_home_missing" >&2; exit 66; } +auth_dir="${user_home}/.ssh" +auth_file="${auth_dir}/authorized_keys" + +read -r key_type key_blob _ <"$CONTROL_PUBLIC_KEY_PATH" +case "$key_type" in + ssh-ed25519|ssh-rsa|ecdsa-sha2-*) ;; + *) echo "BLOCKER unsupported_control_public_key_type" >&2; exit 65 ;; +esac +[[ "$key_blob" =~ ^[A-Za-z0-9+/=]+$ ]] || { echo "BLOCKER invalid_control_public_key_blob" >&2; exit 65; } +key_fingerprint="$(ssh-keygen -lf "$CONTROL_PUBLIC_KEY_PATH" | awk '{print $2}')" + +run_id="host112-guest-recovery-$(date +%Y%m%dT%H%M%S%z)-$$" +backup_dir="${STATE_ROOT}/backups/${run_id}" +install -d -m 0750 "$backup_dir" +: >"${backup_dir}/manifest.tsv" +for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET" "$auth_file"; do + backup_path "$path" "$backup_dir" +done +systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 && touch "$backup_dir/timer_was_enabled" || true +systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 && touch "$backup_dir/timer_was_active" || true + +install -m 0755 "$READINESS_SOURCE" "$READINESS_TARGET" +install -m 0755 "${SCRIPT_DIR}/install-host112-guest-recovery.sh" "$INSTALLER_TARGET" +install -m 0644 "$SERVICE_SOURCE" "$SERVICE_TARGET" +install -m 0644 "$TIMER_SOURCE" "$TIMER_TARGET" + +sudoers_tmp="$(mktemp)" +trap 'rm -f "$sudoers_tmp"' EXIT +printf '%s ALL=(root) NOPASSWD: %s --apply\n' "$TARGET_USER" "$READINESS_TARGET" >"$sudoers_tmp" +visudo -cf "$sudoers_tmp" >/dev/null +install -m 0440 "$sudoers_tmp" "$SUDOERS_TARGET" + +install -d -m 0700 -o "$TARGET_USER" -g "$user_group" "$auth_dir" +if [ ! -f "$auth_file" ]; then + install -m 0600 -o "$TARGET_USER" -g "$user_group" /dev/null "$auth_file" +fi +auth_tmp="$(mktemp "${auth_file}.tmp.XXXXXX")" +awk -v blob="$key_blob" 'index($0, blob) == 0 && index($0, "awoooi-host112-probe-110") == 0' "$auth_file" >"$auth_tmp" +printf 'from="%s",restrict,command="%s --check" %s %s awoooi-host112-probe-110\n' \ + "$CONTROL_SOURCE_ADDRESS" "$READINESS_TARGET" "$key_type" "$key_blob" >>"$auth_tmp" +chown "$TARGET_USER:$user_group" "$auth_tmp" +chmod 0600 "$auth_tmp" +mv "$auth_tmp" "$auth_file" + +systemctl daemon-reload +systemctl enable --now awoooi-host112-guest-recovery.timer >/dev/null +systemctl start awoooi-host112-guest-recovery.service + +echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" +echo "MODE=apply" +echo "RUN_ID=$run_id" +echo "BACKUP_DIR=$backup_dir" +echo "CONTROL_KEY_FINGERPRINT=$key_fingerprint" +echo "REBOOT_PERFORMED=0" +echo "SECRET_READ=0" +echo "RESTRICTED_CONTROL_KEY_INSTALLED=1" +check diff --git a/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh b/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh index 9b65a9dae..dc87f4053 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh @@ -22,7 +22,7 @@ HOST_SPECS=( "99=192.168.0.99:vmware-host-autostart" "110=wooo@192.168.0.110:awoooi-startup-110.service" "111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy" - "112=192.168.0.112:vm-host-boot" + "112=kali@192.168.0.112:awoooi-host112-guest-recovery.timer" "120=wooo@192.168.0.120:k3s.service" "121=wooo@192.168.0.121:k3s.service" "188=ollama@192.168.0.188:awoooi-startup.service" @@ -94,14 +94,19 @@ emit_boot_row() { local systemd_state="$7" local enabled="$8" local active="$9" + local details="${10:-}" - printf 'HOST_BOOT alias=%s target=%s startup_unit=%s reachable=%s boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s\n' \ + printf 'HOST_BOOT alias=%s target=%s startup_unit=%s reachable=%s boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s' \ "$alias" "$target" "$unit" "$reachable" \ "$(escape_value "${boot_id:-unknown}")" \ "$(escape_value "${uptime_seconds:-unknown}")" \ "$(escape_value "${systemd_state:-unknown}")" \ "$(escape_value "${enabled:-unknown}")" \ "$(escape_value "${active:-unknown}")" + if [[ -n "$details" ]]; then + printf ' %s' "$details" + fi + printf '\n' } probe_local_host() { @@ -204,6 +209,39 @@ Write-Output ("boot_id=windows_{0} uptime_seconds={1} systemd_state=windows_ssh "${systemd_state:-windows_ssh}" "${enabled:-unknown}" "${active:-unknown}" } +probe_host112_guest() { + local alias="$1" + local target="$2" + local unit="$3" + local output boot_id uptime_seconds systemd_state enabled active + local key value details="" + + output="$( + run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" \ + ssh "${SSH_OPTS[@]}" "$target" ignored-by-forced-command 2>/dev/null + )" + [[ -n "$output" ]] || return 1 + + boot_id="$(sed -n 's/.*boot_id=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + uptime_seconds="$(sed -n 's/.*uptime_seconds=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + systemd_state="$(sed -n 's/.*systemd_state=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + enabled="$(sed -n 's/.*startup_enabled=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + active="$(sed -n 's/.*startup_active=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + [[ -n "$boot_id" && "$uptime_seconds" =~ ^[0-9]+$ ]] || return 1 + + for key in \ + default_target graphical_target display_manager lightdm vmware_tools xorg \ + wazuh_indexer wazuh_manager wazuh_dashboard filebeat console_ready \ + services_ready recovery_timer_ready guest_ready action_count actions; do + value="$(sed -n "s/.*${key}=\\([^ ]*\\).*/\\1/p" <<<"$output" | tail -n 1)" + value="$(escape_value "${value:-unknown}")" + details+="${details:+ }${key}=${value}" + done + + emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" \ + "${systemd_state:-unknown}" "${enabled:-unknown}" "${active:-unknown}" "$details" +} + probe_host() { local alias="$1" local target="$2" @@ -221,6 +259,10 @@ probe_host() { return 0 fi + if [[ "$alias" == "112" ]] && probe_host112_guest "$alias" "$target" "$unit"; then + return 0 + fi + if [[ "$target" != *@* ]]; then if probe_node_exporter "$alias" "$target" "$unit"; then return 0 diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index f3c87c0ef..eeca69d53 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -519,6 +519,7 @@ import json, sys payload = json.load(open(sys.argv[1], encoding="utf-8")) windows99 = payload.get("windows99_vmware_autostart") or {} collector = payload.get("windows99_vmware_verify_collector") or {} +host112 = (payload.get("host_boot_detection") or {}).get("host112_guest_readiness") or {} def label(value: object) -> str: @@ -534,6 +535,10 @@ def gauge(name: str, value: bool) -> None: print(f'{name}{{scope="windows99"}} {1 if value else 0}') +def host112_gauge(name: str, value: bool) -> None: + print(f'{name}{{scope="host112"}} {1 if value else 0}') + + def int_value(value: object, default: int = -1) -> int: try: return int(str(value), 0) @@ -613,6 +618,17 @@ if collector_present: f'{{scope="windows99",status="{label(collector_status)}",' f'safe_next_step="{label(collector_safe_next_step)}"}} 1' ) +host112_gauge( + "awoooi_host112_guest_readback_present", + host112.get("readback_present") is True, +) +host112_gauge("awoooi_host112_console_ready", host112.get("console_ready") is True) +host112_gauge("awoooi_host112_services_ready", host112.get("services_ready") is True) +host112_gauge( + "awoooi_host112_recovery_timer_ready", + host112.get("recovery_timer_ready") is True, +) +host112_gauge("awoooi_host112_guest_ready", host112.get("guest_ready") is True) PY )" active_blocker_metrics="$(python3 - "$scorecard_file" "$TARGET_MINUTES" <<'PY' @@ -701,6 +717,16 @@ awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_route # HELP awoooi_public_maintenance_edge_fallback_ready Whether live edge Nginx maintenance fallback config is installed. # TYPE awoooi_public_maintenance_edge_fallback_ready gauge awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} $public_edge_fallback_ready +# HELP awoooi_host112_guest_readback_present Whether host 112 returned its forced-command guest readiness row. +# TYPE awoooi_host112_guest_readback_present gauge +# HELP awoooi_host112_console_ready Whether host 112 graphical target, LightDM, Xorg and VMware Tools are ready. +# TYPE awoooi_host112_console_ready gauge +# HELP awoooi_host112_services_ready Whether the host 112 Wazuh stack and Filebeat are ready. +# TYPE awoooi_host112_services_ready gauge +# HELP awoooi_host112_recovery_timer_ready Whether host 112 bounded recovery timer is enabled and active. +# TYPE awoooi_host112_recovery_timer_ready gauge +# HELP awoooi_host112_guest_ready Whether host 112 guest, console, services and recovery timer are fully ready. +# TYPE awoooi_host112_guest_ready gauge # HELP awoooi_windows99_vmware_readback_present Whether Windows99 VMware verifier stdout was collected. # TYPE awoooi_windows99_vmware_readback_present gauge # HELP awoooi_windows99_vmware_verify_ready Whether Windows99 VMware autostart verifier is fully green. diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index 20f1f936f..1ec0582c1 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -155,6 +155,7 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: kv = parse_kv(text) vmx_present: dict[str, bool] = {} vm_power: dict[str, dict[str, Any]] = {} + vm_processes: dict[str, list[dict[str, Any]]] = {} services: dict[str, dict[str, Any]] = {} policies: dict[str, dict[str, Any]] = {} task: dict[str, Any] = {} @@ -177,6 +178,24 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: "source": match.group(4) or "vmrun", } continue + match = re.match( + r"^VM_PROCESS alias=(\S+) present=([01]) process_id=(\S+) " + r"session_id=(\S+) owner=(\S+) interactive_session=([01]) " + r"ui_truth=(\S+)$", + line, + ) + if match: + vm_processes.setdefault(match.group(1), []).append( + { + "present": match.group(2) == "1", + "process_id": match.group(3), + "session_id": match.group(4), + "owner": match.group(5), + "interactive_session": match.group(6) == "1", + "ui_truth": match.group(7), + } + ) + continue match = re.match( r"^VMWARE_SERVICE name=(\S+) present=([01]) status=(\S+) " r"startup_type=(\S+) ok=([01])$", @@ -258,6 +277,15 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: for alias in required_aliases if not vm_power.get(alias, {}).get("running") ) + ui_session_isolated_aliases = sorted( + alias + for alias, rows in vm_processes.items() + if any( + row.get("present") is True + and row.get("interactive_session") is not True + for row in rows + ) + ) service_blockers = sorted( name for name in ("VMAuthdService", "VMnetDHCP") @@ -328,6 +356,13 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: "vmware_services": services, "windows_update_policy": policies, "vm_power": vm_power, + "vm_processes": vm_processes, + "ui_session_isolated_aliases": ui_session_isolated_aliases, + "operator_power_truth": ( + "trust_vmx_process_and_guest_readiness_not_interactive_vmware_ui" + if ui_session_isolated_aliases + else "vmware_ui_and_runtime_share_interactive_session" + ), "blockers": blockers, } @@ -745,6 +780,35 @@ def source_controls() -> dict[str, bool]: "host_boot_probe_source_present": source_file( "scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh" ).exists(), + "host_112_guest_readiness_source_present": file_contains( + source_file("scripts/reboot-recovery/host112-guest-readiness.sh"), + "console_ready=", + "services_ready=", + "recovery_timer_ready=", + "guest_ready=", + ), + "host_112_guest_recovery_service_source_present": file_contains( + source_file( + "scripts/reboot-recovery/awoooi-host112-guest-recovery.service" + ), + "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply", + "TimeoutStartSec=300", + ), + "host_112_guest_recovery_timer_source_present": file_contains( + source_file( + "scripts/reboot-recovery/awoooi-host112-guest-recovery.timer" + ), + "OnBootSec=20s", + "OnUnitInactiveSec=60s", + ), + "host_112_guest_recovery_installer_source_present": file_contains( + source_file( + "scripts/reboot-recovery/install-host112-guest-recovery.sh" + ), + "--rollback", + "restrict,command=", + "visudo -cf", + ), "reboot_event_detector_source_present": source_file( "scripts/reboot-recovery/reboot-event-detector.py" ).exists(), @@ -1419,6 +1483,11 @@ def choose_safe_next_step( "restore_docker_stats_textfile_exporter_then_collect_sanitized_host_" "pressure_no_restart_no_secret_read" ) + if any(blocker.startswith("host112_") for blocker in blockers): + return ( + "run_host112_guest_recovery_controlled_apply_then_verify_forced_command_" + "readback_and_rerun_reboot_scorecard_no_host_reboot" + ) if "windows99_remote_execution_channel_unavailable" in blockers: return ( "restore_windows99_no_secret_management_channel_or_collect_local_console_" @@ -1494,6 +1563,9 @@ def build_required_checks(payload: dict[str, Any]) -> dict[str, bool]: host_boot_detection = payload.get("host_boot_detection") if not isinstance(host_boot_detection, dict): host_boot_detection = {} + host112_guest = host_boot_detection.get("host112_guest_readiness") + if not isinstance(host112_guest, dict): + host112_guest = {} post_reboot_readiness = payload.get("post_reboot_readiness") if not isinstance(post_reboot_readiness, dict): post_reboot_readiness = {} @@ -1522,6 +1594,34 @@ def build_required_checks(payload: dict[str, Any]) -> dict[str, bool]: "required_hosts_reachable": not strings( host_boot_detection.get("unreachable_hosts") ), + "host112_guest_readback_present": ( + isinstance(host_boot_detection.get("host112_guest_readiness"), dict) + and host_boot_detection.get("host112_guest_readiness", {}).get( + "readback_present" + ) + is True + ), + "host112_console_ready": ( + isinstance(host_boot_detection.get("host112_guest_readiness"), dict) + and host_boot_detection.get("host112_guest_readiness", {}).get( + "console_ready" + ) + is True + ), + "host112_services_ready": ( + isinstance(host_boot_detection.get("host112_guest_readiness"), dict) + and host_boot_detection.get("host112_guest_readiness", {}).get( + "services_ready" + ) + is True + ), + "host112_guest_ready": ( + isinstance(host_boot_detection.get("host112_guest_readiness"), dict) + and host_boot_detection.get("host112_guest_readiness", {}).get( + "guest_ready" + ) + is True + ), "windows99_vmware_autostart_ready": ( windows99.get("verify_ready") is True ), @@ -1583,6 +1683,11 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str "windows99_console_clipboard_unreliable", "windows99_console_focus_unreliable", "windows99_console_verify_output_truncated", + "host112_guest_readback_missing", + "host112_console_not_ready", + "host112_services_not_ready", + "host112_recovery_timer_not_ready", + "host112_system_state_not_running", } if any(blocker in host_boot_blockers for blocker in active_blockers): return "host_boot_detection_blocked" @@ -1627,6 +1732,11 @@ def reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "all_required_hosts_not_in_10_minute_reboot_window", "host_boot_observation_older_than_target_window", "host_uptime_unknown", + "host112_guest_readback_missing", + "host112_console_not_ready", + "host112_services_not_ready", + "host112_recovery_timer_not_ready", + "host112_system_state_not_running", "windows99_remote_execution_channel_unavailable", "windows99_vmware_autostart_readback_missing", "windows99_vmrun_missing", @@ -1774,7 +1884,34 @@ def active_blocker_action_row( "force_push_or_ref_delete", ] - if blocker.startswith("windows99_"): + if blocker.startswith("host112_"): + category = "host112_guest_recovery" + owner_lane = "agent99_windows99_and_host112_guest" + severity = "critical" + next_safe_action = ( + "run_sudo_n_usr_local_sbin_awoooi_host112_guest_readiness_apply_" + "then_verify_forced_command_readback_no_host_reboot" + ) + post_verifier = ( + "bash scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh " + "then_rerun_reboot_auto_recovery_slo_scorecard" + ) + evidence_inputs = [ + "host_boot_detection.host112_guest_readiness", + "host112_guest_recovery_journal", + "host112_last_readback", + ] + controlled_apply_mode = "host112_allowlisted_guest_recovery_with_cooldown" + forbidden_actions = [ + "host_reboot", + "vm_power_change", + "vm_reset", + "database_write_or_restore", + "secret_value_read", + "firewall_change", + "force_push_or_ref_delete", + ] + elif blocker.startswith("windows99_"): category = "windows99_vmware_autostart" owner_lane = "windows99_console_or_no_secret_management_channel" next_safe_action = ( @@ -1985,6 +2122,9 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: host_boot_detection = payload.get("host_boot_detection") if not isinstance(host_boot_detection, dict): host_boot_detection = {} + host112_guest = host_boot_detection.get("host112_guest_readiness") + if not isinstance(host112_guest, dict): + host112_guest = {} post_reboot_readiness = payload.get("post_reboot_readiness") if not isinstance(post_reboot_readiness, dict): post_reboot_readiness = {} @@ -2082,6 +2222,13 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "unknown_uptime_host_count": len( strings(host_boot_detection.get("unknown_uptime_hosts")) ), + "host112_guest_readback_present": host112_guest.get("readback_present") is True, + "host112_console_ready": host112_guest.get("console_ready") is True, + "host112_services_ready": host112_guest.get("services_ready") is True, + "host112_recovery_timer_ready": ( + host112_guest.get("recovery_timer_ready") is True + ), + "host112_guest_ready": host112_guest.get("guest_ready") is True, "post_start_blocked": int_value(post_reboot_readiness.get("post_start_blocked")), "service_green": post_reboot_readiness.get("service_green") is True, "product_data_green": post_reboot_readiness.get("product_data_green") is True, @@ -2259,6 +2406,11 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only, "required_checks": required_checks, "source_controls_present": source_controls_present, + "host112_guest_readback_present": rollups["host112_guest_readback_present"], + "host112_console_ready": rollups["host112_console_ready"], + "host112_services_ready": rollups["host112_services_ready"], + "host112_recovery_timer_ready": rollups["host112_recovery_timer_ready"], + "host112_guest_ready": rollups["host112_guest_ready"], "windows99_vmware_verify_ready": rollups["windows99_vmware_verify_ready"], "windows99_update_no_auto_reboot_ready": rollups[ "windows99_update_no_auto_reboot_ready" @@ -2366,6 +2518,19 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "reboot_auto_recovery_safe_next_step": readback["safe_next_step"], "reboot_auto_recovery_next_safe_action": readback["next_safe_action"], "reboot_auto_recovery_source_controls_present": source_controls_present, + "reboot_auto_recovery_host112_guest_readback_present": rollups[ + "host112_guest_readback_present" + ], + "reboot_auto_recovery_host112_console_ready": rollups[ + "host112_console_ready" + ], + "reboot_auto_recovery_host112_services_ready": rollups[ + "host112_services_ready" + ], + "reboot_auto_recovery_host112_recovery_timer_ready": rollups[ + "host112_recovery_timer_ready" + ], + "reboot_auto_recovery_host112_guest_ready": rollups["host112_guest_ready"], "reboot_auto_recovery_windows99_vmware_verify_ready": rollups[ "windows99_vmware_verify_ready" ], @@ -2450,6 +2615,30 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: for row in host_rows if row.get("reachable") and int_value(row.get("uptime_seconds"), -1) < 0 ) + host112_row = next( + (row for row in host_rows if str(row.get("alias")) == "112"), {} + ) + host112_guest_readiness = { + "readback_present": "guest_ready" in host112_row, + "reachable": host112_row.get("reachable") is True, + "systemd_state": str(host112_row.get("systemd_state") or "unknown"), + "default_target": str(host112_row.get("default_target") or "unknown"), + "graphical_target": str(host112_row.get("graphical_target") or "unknown"), + "display_manager": str(host112_row.get("display_manager") or "unknown"), + "lightdm": str(host112_row.get("lightdm") or "unknown"), + "vmware_tools": str(host112_row.get("vmware_tools") or "unknown"), + "xorg": str(host112_row.get("xorg") or "unknown"), + "wazuh_indexer": str(host112_row.get("wazuh_indexer") or "unknown"), + "wazuh_manager": str(host112_row.get("wazuh_manager") or "unknown"), + "wazuh_dashboard": str(host112_row.get("wazuh_dashboard") or "unknown"), + "filebeat": str(host112_row.get("filebeat") or "unknown"), + "console_ready": host112_row.get("console_ready") == "1", + "services_ready": host112_row.get("services_ready") == "1", + "recovery_timer_ready": host112_row.get("recovery_timer_ready") == "1", + "guest_ready": host112_row.get("guest_ready") == "1", + "last_action_count": int_value(host112_row.get("action_count"), 0), + "last_actions": str(host112_row.get("actions") or "unknown"), + } if not host_rows: blockers.append("all_host_reboot_detection_missing") if missing_hosts: @@ -2460,6 +2649,17 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: blockers.append("host_boot_observation_older_than_target_window") if unknown_uptime_hosts: blockers.append("host_uptime_unknown") + if not host112_guest_readiness["readback_present"]: + blockers.append("host112_guest_readback_missing") + else: + if not host112_guest_readiness["console_ready"]: + blockers.append("host112_console_not_ready") + if not host112_guest_readiness["services_ready"]: + blockers.append("host112_services_not_ready") + if not host112_guest_readiness["recovery_timer_ready"]: + blockers.append("host112_recovery_timer_not_ready") + if host112_guest_readiness["systemd_state"] != "running": + blockers.append("host112_system_state_not_running") reboot_event_missing_hosts = strings(reboot_event.get("missing_hosts")) reboot_event_unreachable_hosts = strings(reboot_event.get("unreachable_hosts")) @@ -2549,6 +2749,7 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]: "stale_hosts": stale_hosts, "unknown_uptime_hosts": unknown_uptime_hosts, "max_observed_uptime_seconds": max_uptime, + "host112_guest_readiness": host112_guest_readiness, "host_rows": host_rows, }, "reboot_event_detection": { diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py index 9510b6ff6..7e2acabc3 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py @@ -30,7 +30,7 @@ TARGET_HOSTS=99,110,111,112,120,121,188 HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=aa uptime_seconds=100 systemd_state=windows_exporter startup_enabled=enabled startup_active=active HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=a uptime_seconds=120 systemd_state=running startup_enabled=enabled startup_active=active HOST_BOOT alias=111 target=192.168.0.111 startup_unit=vm-host-boot reachable=1 boot_id=bb uptime_seconds=125 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown -HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=cc uptime_seconds=128 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown +HOST_BOOT alias=112 target=kali@192.168.0.112 startup_unit=awoooi-host112-guest-recovery.timer reachable=1 boot_id=cc uptime_seconds=128 systemd_state=running startup_enabled=enabled startup_active=active default_target=graphical.target graphical_target=active display_manager=active lightdm=active vmware_tools=active xorg=active wazuh_indexer=active wazuh_manager=active wazuh_dashboard=active filebeat=active console_ready=1 services_ready=1 recovery_timer_ready=1 guest_ready=1 action_count=0 actions=none HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=b uptime_seconds=130 systemd_state=running startup_enabled=enabled startup_active=active HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=c uptime_seconds=140 systemd_state=running startup_enabled=enabled startup_active=active HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=d uptime_seconds=150 systemd_state=running startup_enabled=enabled startup_active=active @@ -86,6 +86,7 @@ VM_POWER alias=188 vmx_present=1 running=1 VM_POWER alias=120 vmx_present=1 running=1 VM_POWER alias=121 vmx_present=1 running=1 VM_POWER alias=112 vmx_present=1 running=1 +VM_PROCESS alias=112 present=1 process_id=22288 session_id=0 owner=NT_AUTHORITY\\SYSTEM interactive_session=0 ui_truth=runtime_running_gui_session_isolated VMWARE_AUTOSTART_CONFIG_READY=1 VMWARE_AUTOSTART_POWER_READY=1 WINDOWS_UPDATE_NO_AUTO_REBOOT_READY=1 @@ -492,6 +493,10 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - is False ) assert payload["required_checks"]["windows99_management_channel_ready"] is True + assert payload["required_checks"]["host112_guest_readback_present"] is True + assert payload["required_checks"]["host112_console_ready"] is True + assert payload["required_checks"]["host112_services_ready"] is True + assert payload["required_checks"]["host112_guest_ready"] is True assert payload["required_checks"]["public_maintenance_fallback_runtime_ready"] is True assert payload["rollups"]["source_controls_present"] is True assert payload["rollups"]["public_maintenance_runtime_readback_present"] is True @@ -522,6 +527,10 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - "host_110_startup_controlled_drain_guarded_autostart_source_present" ] is True assert payload["host_boot_detection"]["max_observed_uptime_seconds"] == 150 + assert payload["host_boot_detection"]["host112_guest_readiness"]["guest_ready"] is True + assert payload["rollups"]["host112_console_ready"] is True + assert payload["rollups"]["host112_services_ready"] is True + assert payload["readback"]["host112_guest_ready"] is True assert payload["windows99_vmware_autostart"]["required_vm_aliases"] == [ "111", "112", @@ -530,6 +539,12 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - "188", ] assert payload["windows99_vmware_autostart"]["verify_ready"] is True + assert payload["windows99_vmware_autostart"]["ui_session_isolated_aliases"] == [ + "112" + ] + assert payload["windows99_vmware_autostart"]["operator_power_truth"] == ( + "trust_vmx_process_and_guest_readiness_not_interactive_vmware_ui" + ) task = payload["windows99_vmware_autostart"]["scheduled_task"] assert task["principal_user"] == "SYSTEM" assert task["run_level"] == "Highest" @@ -549,6 +564,36 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - ] is True +def test_host112_console_gap_blocks_reboot_completion_with_controlled_action( + tmp_path: Path, +) -> None: + probe = HOST_PROBE_GREEN.replace("xorg=active", "xorg=inactive").replace( + "console_ready=1", "console_ready=0" + ).replace("guest_ready=1", "guest_ready=0") + + payload = run_scorecard(tmp_path, GREEN_SUMMARY, probe=probe) + + assert "host112_console_not_ready" in payload["active_blockers"] + assert payload["host_boot_detection"]["host112_guest_readiness"][ + "console_ready" + ] is False + assert payload["current_phase"] == "host_boot_detection_blocked" + assert payload["safe_next_step"] == ( + "run_host112_guest_recovery_controlled_apply_then_verify_forced_command_" + "readback_and_rerun_reboot_scorecard_no_host_reboot" + ) + action = next( + item + for item in payload["active_blocker_action_matrix"]["items"] + if item["blocker"] == "host112_console_not_ready" + ) + assert action["category"] == "host112_guest_recovery" + assert action["controlled_apply_mode"] == ( + "host112_allowlisted_guest_recovery_with_cooldown" + ) + assert "host_reboot" in action["forbidden_actions"] + + def test_vm_power_provenance_suffix_is_preserved(tmp_path: Path) -> None: windows99 = "\n".join( f"{line} source=vmware_vmx_process" diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 002d7cf93..a7db9299b 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -83,6 +83,29 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "systemd_state=windows_ssh" in host_probe assert '[[ "$boot_id" == windows_* && "$uptime_seconds" =~ ^[0-9]+$ ]]' in host_probe assert '"111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy"' in host_probe + assert '"112=kali@192.168.0.112:awoooi-host112-guest-recovery.timer"' in host_probe + assert "probe_host112_guest()" in host_probe + for field in ["console_ready", "services_ready", "recovery_timer_ready", "guest_ready"]: + assert field in host_probe + + host112_readiness = read("scripts/reboot-recovery/host112-guest-readiness.sh") + host112_installer = read("scripts/reboot-recovery/install-host112-guest-recovery.sh") + host112_service = read( + "scripts/reboot-recovery/awoooi-host112-guest-recovery.service" + ) + host112_timer = read("scripts/reboot-recovery/awoooi-host112-guest-recovery.timer") + assert "systemctl set-default graphical.target" in host112_readiness + assert "start_if_inactive lightdm.service" in host112_readiness + assert "start_if_inactive open-vm-tools.service" in host112_readiness + assert "INDEXER_RETRY_COOLDOWN_SECONDS" in host112_readiness + assert "console_ready=$console_ready" in host112_readiness + assert "guest_ready=$guest_ready" in host112_readiness + assert "restrict,command=" in host112_installer + assert "visudo -cf" in host112_installer + assert "--rollback" in host112_installer + assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply" in host112_service + assert "OnBootSec=20s" in host112_timer + assert "OnUnitInactiveSec=60s" in host112_timer macos_readback = read("scripts/reboot-recovery/macos-host-boot-readback.sh") macos_installer = read("scripts/reboot-recovery/install-macos111-host-boot-readback.sh") @@ -135,6 +158,9 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "last_task_result" in windows99 assert "start_script_present" in windows99 assert "VMWARE_AUTOSTART_TASK_ACTION" in windows99 + assert "Get-VmwareVmxProcessReadback" in windows99 + assert "VM_PROCESS alias=$alias" in windows99 + assert "runtime_running_gui_session_isolated" in windows99 verify_exit = windows99.index('if ($Mode -eq "Verify")') apply_start = windows99.index("Write-StartupScript -VmMap $vmMap") @@ -387,8 +413,17 @@ def test_reboot_slo_scorecard_fails_closed_on_raw_public_502(tmp_path: Path) -> host_probe = tmp_path / "host-probe.txt" host_probe.write_text( "\n".join( - f"HOST_BOOT alias={alias} host=192.168.0.{alias} reachable=1 uptime_seconds=120" - for alias in ["99", "110", "111", "112", "120", "121", "188"] + [ + f"HOST_BOOT alias={alias} host=192.168.0.{alias} reachable=1 uptime_seconds=120" + for alias in ["99", "110", "111", "120", "121", "188"] + ] + + [ + "HOST_BOOT alias=112 host=192.168.0.112 reachable=1 uptime_seconds=120 " + "systemd_state=running default_target=graphical.target graphical_target=active " + "display_manager=active lightdm=active vmware_tools=active xorg=active " + "wazuh_indexer=active wazuh_manager=active wazuh_dashboard=active filebeat=active " + "console_ready=1 services_ready=1 recovery_timer_ready=1 guest_ready=1" + ] ), encoding="utf-8", ) diff --git a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 index b132b497d..a876d6cdf 100644 --- a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 +++ b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 @@ -187,6 +187,51 @@ function Get-VmwareVmxProcessCommandLines { } } +function Get-VmwareVmxProcessReadback { + param([hashtable]$VmMap) + + $processes = @() + try { + $processes = @(Get-CimInstance -ClassName Win32_Process -Filter "Name='vmware-vmx.exe'" -ErrorAction SilentlyContinue) + } catch { + $processes = @() + } + foreach ($alias in $VmOrder) { + $path = "$($VmMap[$alias])" + $matches = @($processes | Where-Object { + $commandLine = [string]$_.CommandLine + $path -and $commandLine -and $commandLine.IndexOf($path, [StringComparison]::OrdinalIgnoreCase) -ge 0 + }) + if ($matches.Count -eq 0) { + Write-Output "VM_PROCESS alias=$alias present=0 process_id=unknown session_id=unknown owner=unknown interactive_session=0 ui_truth=not_running" + continue + } + foreach ($process in $matches) { + $owner = "unknown" + try { + $ownerResult = Invoke-CimMethod -InputObject $process -MethodName GetOwner -ErrorAction SilentlyContinue + if ($ownerResult -and $ownerResult.ReturnValue -eq 0 -and $ownerResult.User) { + $owner = if ($ownerResult.Domain) { + "$($ownerResult.Domain)\$($ownerResult.User)" + } else { + "$($ownerResult.User)" + } + } + } catch { + $owner = "unknown" + } + $sessionId = if ($null -ne $process.SessionId) { [int]$process.SessionId } else { -1 } + $interactiveSession = [int]($sessionId -gt 0) + $uiTruth = if ($interactiveSession -eq 1) { + "runtime_running_interactive_session" + } else { + "runtime_running_gui_session_isolated" + } + Write-Output "VM_PROCESS alias=$alias present=1 process_id=$($process.ProcessId) session_id=$sessionId owner=$(Format-ReadbackValue $owner) interactive_session=$interactiveSession ui_truth=$uiTruth" + } + } +} + function Get-VmxRunningSource { param( [string]$VmxPath, @@ -350,6 +395,7 @@ Get-VmwareServiceReadback Get-AutostartTaskReadback Get-WindowsUpdatePolicyReadback Get-VmPowerReadback -VmMap $vmMap +Get-VmwareVmxProcessReadback -VmMap $vmMap $serviceReady = @("VMAuthdService", "VMnetDHCP") | ForEach-Object { $svc = Get-Service -Name $_ -ErrorAction SilentlyContinue