feat(recovery): add controlled VMware autostart apply
This commit is contained in:
ogt
@@ -0,0 +1,36 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = (
|
||||
ROOT
|
||||
/ "scripts"
|
||||
/ "reboot-recovery"
|
||||
/ "windows99-vmware-autostart-controlled-apply.ps1"
|
||||
)
|
||||
|
||||
|
||||
def test_controlled_apply_has_check_apply_verify_and_rollback_contract() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
for mode in ("DryRun", "Apply", "Verify"):
|
||||
assert f'Invoke-AutostartMode -Mode {mode}' in text
|
||||
assert "Export-ScheduledTask" in text
|
||||
assert "Register-ScheduledTask -TaskName $TaskName -Xml $taskXml -Force" in text
|
||||
assert "Restore-PolicySnapshot" in text
|
||||
assert "Restore-ServiceStartModes" in text
|
||||
assert 'status = "failed_rolled_back"' in text
|
||||
assert 'status = "completed"' in text
|
||||
|
||||
|
||||
def test_controlled_apply_proves_no_reboot_or_vm_power_change() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "preVmxProcessCount" in text
|
||||
assert "postVmxProcessCount" in text
|
||||
assert "processCountUnchanged" in text
|
||||
assert 'hostRebootPerformed = $false' in text
|
||||
assert 'vmPowerChangePerformed = $false' in text
|
||||
assert 'secretReadPerformed = $false' in text
|
||||
assert "runningVmRows" in text
|
||||
assert "VMWARE_AUTOSTART_VERIFY_READY=1" in text
|
||||
@@ -0,0 +1,244 @@
|
||||
param(
|
||||
[string]$SourceScript = (Join-Path $PSScriptRoot "windows99-vmware-autostart.ps1"),
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99",
|
||||
[string]$SourceRevision = "unknown",
|
||||
[string]$TaskName = "AWOOOI-Start-VMware-VMs",
|
||||
[string]$StartScript = "C:\ProgramData\AWOOOI\Start-AWOOOI-VMs.ps1",
|
||||
[int]$ExpectedVmCount = 5
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$PolicyPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate\AU"
|
||||
$PolicyNames = @(
|
||||
"NoAutoRebootWithLoggedOnUsers",
|
||||
"AlwaysAutoRebootAtScheduledTime",
|
||||
"AUOptions",
|
||||
"ScheduledInstallDay",
|
||||
"ScheduledInstallTime",
|
||||
"AUPowerManagement"
|
||||
)
|
||||
$ServiceNames = @("VMAuthdService", "VMnetDHCP")
|
||||
$PowerShellExe = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$Stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
||||
$TraceId = "windows99-vmware-autostart-$Stamp"
|
||||
$BackupDir = Join-Path $AgentRoot "deploy\vmware-autostart-backup-$Stamp"
|
||||
$EvidencePath = Join-Path $AgentRoot "evidence\windows99-vmware-autostart-apply-$Stamp.json"
|
||||
|
||||
function Invoke-AutostartMode {
|
||||
param([ValidateSet("DryRun", "Apply", "Verify")][string]$Mode)
|
||||
|
||||
$output = @(& $PowerShellExe `
|
||||
-NoProfile `
|
||||
-NonInteractive `
|
||||
-ExecutionPolicy Bypass `
|
||||
-File $SourceScript `
|
||||
-Mode $Mode 2>&1 | ForEach-Object { "$_" })
|
||||
[pscustomobject]@{
|
||||
mode = $Mode
|
||||
exitCode = $LASTEXITCODE
|
||||
output = $output
|
||||
}
|
||||
}
|
||||
|
||||
function Get-PolicySnapshot {
|
||||
$pathExisted = Test-Path -LiteralPath $PolicyPath
|
||||
$properties = if ($pathExisted) {
|
||||
Get-ItemProperty -LiteralPath $PolicyPath -ErrorAction Stop
|
||||
} else {
|
||||
$null
|
||||
}
|
||||
$values = foreach ($name in $PolicyNames) {
|
||||
$property = if ($properties) { $properties.PSObject.Properties[$name] } else { $null }
|
||||
[pscustomobject]@{
|
||||
name = $name
|
||||
existed = [bool]$property
|
||||
value = if ($property) { [int]$property.Value } else { $null }
|
||||
}
|
||||
}
|
||||
[pscustomobject]@{
|
||||
pathExisted = $pathExisted
|
||||
values = @($values)
|
||||
}
|
||||
}
|
||||
|
||||
function Restore-PolicySnapshot {
|
||||
param([object]$Snapshot)
|
||||
|
||||
if (-not $Snapshot.pathExisted) {
|
||||
if (Test-Path -LiteralPath $PolicyPath) {
|
||||
Remove-Item -LiteralPath $PolicyPath -Recurse -Force
|
||||
}
|
||||
return
|
||||
}
|
||||
New-Item -Path $PolicyPath -Force | Out-Null
|
||||
foreach ($row in @($Snapshot.values)) {
|
||||
if ($row.existed) {
|
||||
New-ItemProperty `
|
||||
-Path $PolicyPath `
|
||||
-Name $row.name `
|
||||
-Value ([int]$row.value) `
|
||||
-PropertyType DWord `
|
||||
-Force | Out-Null
|
||||
} else {
|
||||
Remove-ItemProperty `
|
||||
-LiteralPath $PolicyPath `
|
||||
-Name $row.name `
|
||||
-ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Restore-ServiceStartModes {
|
||||
param([object[]]$Rows)
|
||||
|
||||
foreach ($row in $Rows) {
|
||||
$startupType = switch ($row.startMode) {
|
||||
"Auto" { "Automatic" }
|
||||
"Disabled" { "Disabled" }
|
||||
default { "Manual" }
|
||||
}
|
||||
Set-Service -Name $row.name -StartupType $startupType -ErrorAction Stop
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $SourceScript)) {
|
||||
throw "VMware autostart source script not found: $SourceScript"
|
||||
}
|
||||
$task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop
|
||||
if ($task.State -eq "Running") {
|
||||
throw "VMware autostart task is running; controlled apply refused."
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $BackupDir | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $EvidencePath) | Out-Null
|
||||
$hadStartScript = Test-Path -LiteralPath $StartScript
|
||||
if ($hadStartScript) {
|
||||
Copy-Item -LiteralPath $StartScript -Destination (Join-Path $BackupDir "Start-AWOOOI-VMs.ps1")
|
||||
}
|
||||
$taskXml = Export-ScheduledTask -TaskName $TaskName
|
||||
$taskXml | Set-Content -LiteralPath (Join-Path $BackupDir "$TaskName.xml") -Encoding Unicode
|
||||
$policySnapshot = Get-PolicySnapshot
|
||||
$policySnapshot | ConvertTo-Json -Depth 6 | Set-Content `
|
||||
-LiteralPath (Join-Path $BackupDir "windows-update-policy.json") `
|
||||
-Encoding UTF8
|
||||
$serviceSnapshot = @($ServiceNames | ForEach-Object {
|
||||
$service = Get-CimInstance -ClassName Win32_Service -Filter "Name='$_'" -ErrorAction Stop
|
||||
[pscustomobject]@{ name = $_; startMode = [string]$service.StartMode }
|
||||
})
|
||||
$serviceSnapshot | ConvertTo-Json -Depth 4 | Set-Content `
|
||||
-LiteralPath (Join-Path $BackupDir "vmware-service-start-modes.json") `
|
||||
-Encoding UTF8
|
||||
|
||||
$preVmxProcessCount = @(Get-CimInstance `
|
||||
-ClassName Win32_Process `
|
||||
-Filter "Name='vmware-vmx.exe'" `
|
||||
-ErrorAction Stop).Count
|
||||
$receipt = [ordered]@{
|
||||
schemaVersion = "windows99_vmware_autostart_controlled_apply_v1"
|
||||
traceId = $TraceId
|
||||
sourceRevision = $SourceRevision
|
||||
startedAt = (Get-Date -Format o)
|
||||
completedAt = $null
|
||||
ok = $false
|
||||
status = "started"
|
||||
sourceScript = $SourceScript
|
||||
backupDir = $BackupDir
|
||||
evidence = $EvidencePath
|
||||
expectedVmCount = $ExpectedVmCount
|
||||
preVmxProcessCount = $preVmxProcessCount
|
||||
postVmxProcessCount = $null
|
||||
dryRun = $null
|
||||
apply = $null
|
||||
verifier = $null
|
||||
rollbackAttempted = $false
|
||||
rollbackSucceeded = $null
|
||||
error = $null
|
||||
boundaries = [ordered]@{
|
||||
hostRebootPerformed = $false
|
||||
vmPowerChangePerformed = $false
|
||||
secretReadPerformed = $false
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
$receipt.dryRun = Invoke-AutostartMode -Mode DryRun
|
||||
if ($receipt.dryRun.exitCode -ne 0 -or -not ($receipt.dryRun.output -match "^DRY_RUN ")) {
|
||||
throw "VMware autostart dry-run gate failed."
|
||||
}
|
||||
|
||||
$receipt.apply = Invoke-AutostartMode -Mode Apply
|
||||
if ($receipt.apply.exitCode -ne 0) {
|
||||
throw "VMware autostart apply failed with exit code $($receipt.apply.exitCode)."
|
||||
}
|
||||
|
||||
$receipt.verifier = Invoke-AutostartMode -Mode Verify
|
||||
$postVmxProcessCount = @(Get-CimInstance `
|
||||
-ClassName Win32_Process `
|
||||
-Filter "Name='vmware-vmx.exe'" `
|
||||
-ErrorAction Stop).Count
|
||||
$receipt.postVmxProcessCount = $postVmxProcessCount
|
||||
$runningRows = @($receipt.verifier.output | Where-Object {
|
||||
$_ -match "^VM_POWER alias=.+ vmx_present=1 running=1 source=\S+$"
|
||||
})
|
||||
$verifyReady = $receipt.verifier.output -contains "VMWARE_AUTOSTART_VERIFY_READY=1"
|
||||
$processAwareGuard = [bool](Select-String `
|
||||
-LiteralPath $StartScript `
|
||||
-Pattern "runningProcesses" `
|
||||
-SimpleMatch `
|
||||
-Quiet)
|
||||
$taskAfter = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop
|
||||
$verified = (
|
||||
$receipt.verifier.exitCode -eq 0 -and
|
||||
$runningRows.Count -eq $ExpectedVmCount -and
|
||||
$verifyReady -and
|
||||
$processAwareGuard -and
|
||||
$taskAfter.State -ne "Disabled" -and
|
||||
$postVmxProcessCount -eq $preVmxProcessCount
|
||||
)
|
||||
$receipt.verifier = [ordered]@{
|
||||
exitCode = $receipt.verifier.exitCode
|
||||
output = $receipt.verifier.output
|
||||
runningVmRows = $runningRows.Count
|
||||
verifyReady = $verifyReady
|
||||
processAwareGuard = $processAwareGuard
|
||||
taskState = [string]$taskAfter.State
|
||||
processCountUnchanged = ($postVmxProcessCount -eq $preVmxProcessCount)
|
||||
}
|
||||
if (-not $verified) {
|
||||
throw "VMware autostart independent post-verifier failed."
|
||||
}
|
||||
$receipt.ok = $true
|
||||
$receipt.status = "completed"
|
||||
} catch {
|
||||
$receipt.status = "failed_rollback_required"
|
||||
$receipt.error = $_.Exception.Message
|
||||
$receipt.rollbackAttempted = $true
|
||||
try {
|
||||
if ($hadStartScript) {
|
||||
Copy-Item `
|
||||
-Force `
|
||||
-LiteralPath (Join-Path $BackupDir "Start-AWOOOI-VMs.ps1") `
|
||||
-Destination $StartScript
|
||||
} elseif (Test-Path -LiteralPath $StartScript) {
|
||||
Remove-Item -LiteralPath $StartScript -Force
|
||||
}
|
||||
Register-ScheduledTask -TaskName $TaskName -Xml $taskXml -Force | Out-Null
|
||||
Restore-PolicySnapshot -Snapshot $policySnapshot
|
||||
Restore-ServiceStartModes -Rows $serviceSnapshot
|
||||
$receipt.rollbackSucceeded = $true
|
||||
$receipt.status = "failed_rolled_back"
|
||||
} catch {
|
||||
$receipt.rollbackSucceeded = $false
|
||||
$receipt.status = "failed_rollback_failed"
|
||||
$receipt.error = "$($receipt.error); rollback=$($_.Exception.Message)"
|
||||
}
|
||||
} finally {
|
||||
$receipt.completedAt = Get-Date -Format o
|
||||
$receipt | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $EvidencePath -Encoding UTF8
|
||||
}
|
||||
|
||||
$receipt | ConvertTo-Json -Depth 12
|
||||
if (-not $receipt.ok) {
|
||||
exit 2
|
||||
}
|
||||
exit 0
|
||||
Reference in New Issue
Block a user