fix(recovery): detect cross-session vmware guests

scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py

@@ -152,7 +152,7 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]:
 
     kv = parse_kv(text)
     vmx_present: dict[str, bool] = {}
-    vm_power: dict[str, dict[str, bool]] = {}
+    vm_power: dict[str, dict[str, Any]] = {}
     services: dict[str, dict[str, Any]] = {}
     policies: dict[str, dict[str, Any]] = {}
     task: dict[str, Any] = {}
@@ -164,13 +164,15 @@ def parse_windows99_vmware_readback(text: str) -> dict[str, Any]:
             vmx_present[match.group(1)] = match.group(2) == "1"
             continue
         match = re.match(
-            r"^VM_POWER alias=(\S+) vmx_present=([01]) running=([01])$",
+            r"^VM_POWER alias=(\S+) vmx_present=([01]) running=([01])"
+            r"(?: source=(\S+))?$",
             line,
         )
         if match:
             vm_power[match.group(1)] = {
                 "vmx_present": match.group(2) == "1",
                 "running": match.group(3) == "1",
+                "source": match.group(4) or "vmrun",
             }
             continue
         match = re.match(

scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py

@@ -549,6 +549,26 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) -
     ] is True
 
 
+def test_vm_power_provenance_suffix_is_preserved(tmp_path: Path) -> None:
+    windows99 = "\n".join(
+        f"{line} source=vmware_vmx_process"
+        if line.startswith("VM_POWER ")
+        else line
+        for line in WINDOWS99_VMWARE_GREEN.splitlines()
+    )
+
+    payload = run_scorecard(tmp_path, GREEN_SUMMARY, windows99=windows99)
+    vmware = payload["windows99_vmware_autostart"]
+
+    assert vmware["missing_vmx_aliases"] == []
+    assert vmware["powered_off_aliases"] == []
+    assert vmware["power_ready"] is True
+    assert all(
+        row["source"] == "vmware_vmx_process"
+        for row in vmware["vm_power"].values()
+    )
+
+
 def test_windows99_autostart_task_never_ran_blocks_powered_off_guests(
     tmp_path: Path,
 ) -> None:

scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py

@@ -93,12 +93,24 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
     assert "VMWARE_AUTOSTART_VERIFY_READY" in windows99
     assert "WINDOWS_UPDATE_POLICY" in windows99
     assert "VM_POWER alias=$alias" in windows99
+    assert "Get-VmwareVmxProcessCommandLines" in windows99
+    assert "Get-VmxRunningSource" in windows99
+    assert "vmware_vmx_process" in windows99
+    assert "source=$runningSource" in windows99
+    assert "Win32_Process" in windows99
     assert "VM_ALREADY_RUNNING" in windows99
     assert "VMWARE_AUTOSTART_TASK_DETAIL" in windows99
     assert "last_task_result" in windows99
     assert "start_script_present" in windows99
     assert "VMWARE_AUTOSTART_TASK_ACTION" in windows99
 
+    verify_exit = windows99.index('if ($Mode -eq "Verify")')
+    apply_start = windows99.index("Write-StartupScript -VmMap $vmMap")
+    assert verify_exit < apply_start
+    assert 'if ($Mode -eq "Verify") {\n    exit 0\n}' in windows99
+    assert "Restart-Computer" not in windows99
+    assert "Stop-Computer" not in windows99
+
 
 def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None:
     alerts = read("ops/monitoring/alerts-unified.yml")

scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py

@@ -101,6 +101,28 @@ def test_valid_console_verify_red_artifact_is_accepted_and_normalized(
     )
 
 
+def test_principal_aware_power_source_is_accepted_and_normalized(
+    tmp_path: Path,
+) -> None:
+    raw = tmp_path / "console-process-source.txt"
+    normalized = tmp_path / "windows99-vmware-verify.txt"
+    raw.write_text(
+        VALID_RED_VERIFY.replace(
+            " running=0",
+            " running=1 source=vmware_vmx_process",
+        ),
+        encoding="utf-8",
+    )
+
+    result = _run(raw, normalized)
+
+    assert result.returncode == 0
+    values = _kv(result.stdout)
+    assert values["missing_required_field_count"] == "0"
+    assert values["powered_off_aliases"] == ""
+    assert "source=vmware_vmx_process" in normalized.read_text(encoding="utf-8")
+
+
 def test_collector_envelope_is_extracted(tmp_path: Path) -> None:
     raw = tmp_path / "collector.txt"
     normalized = tmp_path / "normalized.txt"

scripts/reboot-recovery/validate-windows99-console-verify-artifact.py

@@ -5,7 +5,6 @@ from __future__ import annotations
 
 import argparse
 import re
-import sys
 from pathlib import Path
 
 
@@ -76,16 +75,19 @@ def validate(lines: list[str], required_aliases: tuple[str, ...]) -> tuple[int,
     powered_off_aliases: list[str] = []
     for alias in required_aliases:
         vmx_pattern = rf"^VMX alias={re.escape(alias)}\s+.*\spresent=([01])$"
-        power_pattern = rf"^VM_POWER alias={re.escape(alias)}\s+.*\srunning=([01])$"
+        power_pattern = (
+            rf"^VM_POWER alias={re.escape(alias)}\s+.*\srunning=([01])"
+            r"(?:\s+source=(\S+))?$"
+        )
         vmx_line = next((line for line in lines if re.search(vmx_pattern, line)), "")
         power_line = next((line for line in lines if re.search(power_pattern, line)), "")
         if not vmx_line:
             missing_fields.append(f"VMX alias={alias}")
-        elif not _line_matches([vmx_line], rf"\spresent=1$"):
+        elif not _line_matches([vmx_line], r"\spresent=1$"):
             missing_vmx_aliases.append(alias)
         if not power_line:
             missing_fields.append(f"VM_POWER alias={alias}")
-        elif not _line_matches([power_line], rf"\srunning=1$"):
+        elif not _line_matches([power_line], r"\srunning=1(?:\s+source=\S+)?$"):
             powered_off_aliases.append(alias)
 
     status = "ready_console_verify_artifact"

scripts/reboot-recovery/windows99-vmware-autostart.ps1

@@ -111,13 +111,16 @@ Start-Service -Name "VMAuthdService" -ErrorAction SilentlyContinue
 Start-Service -Name "VMnetDHCP" -ErrorAction SilentlyContinue
 Start-Sleep -Seconds 30
 `$running = (& `$VmrunPath list 2>`$null) -join [Environment]::NewLine
+`$runningProcesses = @(Get-CimInstance -ClassName Win32_Process -Filter "Name='vmware-vmx.exe'" -ErrorAction SilentlyContinue | ForEach-Object { [string]`$_.CommandLine })
 
 foreach (`$vm in `$VMs) {
     if (-not (Test-Path -LiteralPath `$vm.Path)) {
         Write-Output "VMX_MISSING alias=`$(`$vm.Alias) path=`$(`$vm.Path)"
         continue
     }
-    if (`$running -like "*`$(`$vm.Path)*") {
+    `$runningByVmrun = `$running -like "*`$(`$vm.Path)*"
+    `$runningByProcess = @(`$runningProcesses | Where-Object { `$_ -and `$_.IndexOf(`$vm.Path, [StringComparison]::OrdinalIgnoreCase) -ge 0 }).Count -gt 0
+    if (`$runningByVmrun -or `$runningByProcess) {
         Write-Output "VM_ALREADY_RUNNING alias=`$(`$vm.Alias) path=`$(`$vm.Path)"
         continue
     }
@@ -173,6 +176,34 @@ function Get-VmrunList {
     }
 }
 
+function Get-VmwareVmxProcessCommandLines {
+    try {
+        return @(Get-CimInstance -ClassName Win32_Process -Filter "Name='vmware-vmx.exe'" -ErrorAction SilentlyContinue |
+            ForEach-Object { [string]$_.CommandLine } |
+            Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
+    } catch {
+        return @()
+    }
+}
+
+function Get-VmxRunningSource {
+    param(
+        [string]$VmxPath,
+        [string[]]$VmrunList,
+        [string[]]$ProcessCommandLines
+    )
+
+    if (-not $VmxPath) { return "none" }
+    $byVmrun = [bool]($VmrunList -contains $VmxPath)
+    $byProcess = [bool](@($ProcessCommandLines | Where-Object {
+        $_ -and $_.IndexOf($VmxPath, [StringComparison]::OrdinalIgnoreCase) -ge 0
+    }).Count -gt 0)
+    if ($byVmrun -and $byProcess) { return "vmrun_and_process" }
+    if ($byVmrun) { return "vmrun" }
+    if ($byProcess) { return "vmware_vmx_process" }
+    return "none"
+}
+
 function Get-VmwareServiceReadback {
     $serviceNames = @("VMAuthdService", "VMnetDHCP")
     foreach ($name in $serviceNames) {
@@ -289,14 +320,13 @@ function Get-VmPowerReadback {
     param([hashtable]$VmMap)
 
     $running = Get-VmrunList
+    $processCommands = Get-VmwareVmxProcessCommandLines
     foreach ($alias in $VmOrder) {
         $path = "$($VmMap[$alias])"
         $present = [int][bool]$path
-        $runningMatch = 0
-        if ($path) {
-            $runningMatch = [int]($running -contains $path)
-        }
-        Write-Output "VM_POWER alias=$alias vmx_present=$present running=$runningMatch"
+        $runningSource = Get-VmxRunningSource -VmxPath $path -VmrunList $running -ProcessCommandLines $processCommands
+        $runningMatch = [int]($runningSource -ne "none")
+        Write-Output "VM_POWER alias=$alias vmx_present=$present running=$runningMatch source=$runningSource"
     }
 }
 
@@ -353,10 +383,12 @@ $windowsUpdateReady = (
     "$($policyValues["AUPowerManagement"])" -eq "0"
 )
 $runningVmList = Get-VmrunList
+$runningVmxProcessCommandLines = Get-VmwareVmxProcessCommandLines
 $vmPowerReady = $true
 foreach ($alias in $VmOrder) {
     $path = "$($vmMap[$alias])"
-    if (-not $path -or -not ($runningVmList -contains $path)) {
+    $runningSource = Get-VmxRunningSource -VmxPath $path -VmrunList $runningVmList -ProcessCommandLines $runningVmxProcessCommandLines
+    if (-not $path -or $runningSource -eq "none") {
         $vmPowerReady = $false
     }
 }