diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index d7486a2b6..ef4a2cd8c 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -152,7 +152,7 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: kv = parse_kv(text) vmx_present: dict[str, bool] = {} - vm_power: dict[str, dict[str, bool]] = {} + vm_power: dict[str, dict[str, Any]] = {} services: dict[str, dict[str, Any]] = {} policies: dict[str, dict[str, Any]] = {} task: dict[str, Any] = {} @@ -164,13 +164,15 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]: vmx_present[match.group(1)] = match.group(2) == "1" continue match = re.match( - r"^VM_POWER alias=(\S+) vmx_present=([01]) running=([01])$", + r"^VM_POWER alias=(\S+) vmx_present=([01]) running=([01])" + r"(?: source=(\S+))?$", line, ) if match: vm_power[match.group(1)] = { "vmx_present": match.group(2) == "1", "running": match.group(3) == "1", + "source": match.group(4) or "vmrun", } continue match = re.match( diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py index 2f064d741..9510b6ff6 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py @@ -549,6 +549,26 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - ] is True +def test_vm_power_provenance_suffix_is_preserved(tmp_path: Path) -> None: + windows99 = "\n".join( + f"{line} source=vmware_vmx_process" + if line.startswith("VM_POWER ") + else line + for line in WINDOWS99_VMWARE_GREEN.splitlines() + ) + + payload = run_scorecard(tmp_path, GREEN_SUMMARY, windows99=windows99) + vmware = payload["windows99_vmware_autostart"] + + assert vmware["missing_vmx_aliases"] == [] + assert vmware["powered_off_aliases"] == [] + assert vmware["power_ready"] is True + assert all( + row["source"] == "vmware_vmx_process" + for row in vmware["vm_power"].values() + ) + + def test_windows99_autostart_task_never_ran_blocks_powered_off_guests( tmp_path: Path, ) -> None: diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 727244100..b2f5d93d1 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -93,12 +93,24 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "VMWARE_AUTOSTART_VERIFY_READY" in windows99 assert "WINDOWS_UPDATE_POLICY" in windows99 assert "VM_POWER alias=$alias" in windows99 + assert "Get-VmwareVmxProcessCommandLines" in windows99 + assert "Get-VmxRunningSource" in windows99 + assert "vmware_vmx_process" in windows99 + assert "source=$runningSource" in windows99 + assert "Win32_Process" in windows99 assert "VM_ALREADY_RUNNING" in windows99 assert "VMWARE_AUTOSTART_TASK_DETAIL" in windows99 assert "last_task_result" in windows99 assert "start_script_present" in windows99 assert "VMWARE_AUTOSTART_TASK_ACTION" in windows99 + verify_exit = windows99.index('if ($Mode -eq "Verify")') + apply_start = windows99.index("Write-StartupScript -VmMap $vmMap") + assert verify_exit < apply_start + assert 'if ($Mode -eq "Verify") {\n exit 0\n}' in windows99 + assert "Restart-Computer" not in windows99 + assert "Stop-Computer" not in windows99 + def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: alerts = read("ops/monitoring/alerts-unified.yml") diff --git a/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py b/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py index 8360afccf..8e59ba504 100644 --- a/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py +++ b/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py @@ -101,6 +101,28 @@ def test_valid_console_verify_red_artifact_is_accepted_and_normalized( ) +def test_principal_aware_power_source_is_accepted_and_normalized( + tmp_path: Path, +) -> None: + raw = tmp_path / "console-process-source.txt" + normalized = tmp_path / "windows99-vmware-verify.txt" + raw.write_text( + VALID_RED_VERIFY.replace( + " running=0", + " running=1 source=vmware_vmx_process", + ), + encoding="utf-8", + ) + + result = _run(raw, normalized) + + assert result.returncode == 0 + values = _kv(result.stdout) + assert values["missing_required_field_count"] == "0" + assert values["powered_off_aliases"] == "" + assert "source=vmware_vmx_process" in normalized.read_text(encoding="utf-8") + + def test_collector_envelope_is_extracted(tmp_path: Path) -> None: raw = tmp_path / "collector.txt" normalized = tmp_path / "normalized.txt" diff --git a/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py b/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py index 16ee21437..2c379dede 100755 --- a/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py +++ b/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py @@ -5,7 +5,6 @@ from __future__ import annotations import argparse import re -import sys from pathlib import Path @@ -76,16 +75,19 @@ def validate(lines: list[str], required_aliases: tuple[str, ...]) -> tuple[int, powered_off_aliases: list[str] = [] for alias in required_aliases: vmx_pattern = rf"^VMX alias={re.escape(alias)}\s+.*\spresent=([01])$" - power_pattern = rf"^VM_POWER alias={re.escape(alias)}\s+.*\srunning=([01])$" + power_pattern = ( + rf"^VM_POWER alias={re.escape(alias)}\s+.*\srunning=([01])" + r"(?:\s+source=(\S+))?$" + ) vmx_line = next((line for line in lines if re.search(vmx_pattern, line)), "") power_line = next((line for line in lines if re.search(power_pattern, line)), "") if not vmx_line: missing_fields.append(f"VMX alias={alias}") - elif not _line_matches([vmx_line], rf"\spresent=1$"): + elif not _line_matches([vmx_line], r"\spresent=1$"): missing_vmx_aliases.append(alias) if not power_line: missing_fields.append(f"VM_POWER alias={alias}") - elif not _line_matches([power_line], rf"\srunning=1$"): + elif not _line_matches([power_line], r"\srunning=1(?:\s+source=\S+)?$"): powered_off_aliases.append(alias) status = "ready_console_verify_artifact" diff --git a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 index 6c5e22257..d842551d8 100644 --- a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 +++ b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 @@ -111,13 +111,16 @@ Start-Service -Name "VMAuthdService" -ErrorAction SilentlyContinue Start-Service -Name "VMnetDHCP" -ErrorAction SilentlyContinue Start-Sleep -Seconds 30 `$running = (& `$VmrunPath list 2>`$null) -join [Environment]::NewLine +`$runningProcesses = @(Get-CimInstance -ClassName Win32_Process -Filter "Name='vmware-vmx.exe'" -ErrorAction SilentlyContinue | ForEach-Object { [string]`$_.CommandLine }) foreach (`$vm in `$VMs) { if (-not (Test-Path -LiteralPath `$vm.Path)) { Write-Output "VMX_MISSING alias=`$(`$vm.Alias) path=`$(`$vm.Path)" continue } - if (`$running -like "*`$(`$vm.Path)*") { + `$runningByVmrun = `$running -like "*`$(`$vm.Path)*" + `$runningByProcess = @(`$runningProcesses | Where-Object { `$_ -and `$_.IndexOf(`$vm.Path, [StringComparison]::OrdinalIgnoreCase) -ge 0 }).Count -gt 0 + if (`$runningByVmrun -or `$runningByProcess) { Write-Output "VM_ALREADY_RUNNING alias=`$(`$vm.Alias) path=`$(`$vm.Path)" continue } @@ -173,6 +176,34 @@ function Get-VmrunList { } } +function Get-VmwareVmxProcessCommandLines { + try { + return @(Get-CimInstance -ClassName Win32_Process -Filter "Name='vmware-vmx.exe'" -ErrorAction SilentlyContinue | + ForEach-Object { [string]$_.CommandLine } | + Where-Object { -not [string]::IsNullOrWhiteSpace($_) }) + } catch { + return @() + } +} + +function Get-VmxRunningSource { + param( + [string]$VmxPath, + [string[]]$VmrunList, + [string[]]$ProcessCommandLines + ) + + if (-not $VmxPath) { return "none" } + $byVmrun = [bool]($VmrunList -contains $VmxPath) + $byProcess = [bool](@($ProcessCommandLines | Where-Object { + $_ -and $_.IndexOf($VmxPath, [StringComparison]::OrdinalIgnoreCase) -ge 0 + }).Count -gt 0) + if ($byVmrun -and $byProcess) { return "vmrun_and_process" } + if ($byVmrun) { return "vmrun" } + if ($byProcess) { return "vmware_vmx_process" } + return "none" +} + function Get-VmwareServiceReadback { $serviceNames = @("VMAuthdService", "VMnetDHCP") foreach ($name in $serviceNames) { @@ -289,14 +320,13 @@ function Get-VmPowerReadback { param([hashtable]$VmMap) $running = Get-VmrunList + $processCommands = Get-VmwareVmxProcessCommandLines foreach ($alias in $VmOrder) { $path = "$($VmMap[$alias])" $present = [int][bool]$path - $runningMatch = 0 - if ($path) { - $runningMatch = [int]($running -contains $path) - } - Write-Output "VM_POWER alias=$alias vmx_present=$present running=$runningMatch" + $runningSource = Get-VmxRunningSource -VmxPath $path -VmrunList $running -ProcessCommandLines $processCommands + $runningMatch = [int]($runningSource -ne "none") + Write-Output "VM_POWER alias=$alias vmx_present=$present running=$runningMatch source=$runningSource" } } @@ -353,10 +383,12 @@ $windowsUpdateReady = ( "$($policyValues["AUPowerManagement"])" -eq "0" ) $runningVmList = Get-VmrunList +$runningVmxProcessCommandLines = Get-VmwareVmxProcessCommandLines $vmPowerReady = $true foreach ($alias in $VmOrder) { $path = "$($vmMap[$alias])" - if (-not $path -or -not ($runningVmList -contains $path)) { + $runningSource = Get-VmxRunningSource -VmxPath $path -VmrunList $runningVmList -ProcessCommandLines $runningVmxProcessCommandLines + if (-not $path -or $runningSource -eq "none") { $vmPowerReady = $false } }