fix(agent99): harden control loop recovery
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 2m2s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 6m29s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 29s
CD Pipeline / build-and-deploy (push) Failing after 42s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 2m2s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 6m29s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 29s
CD Pipeline / build-and-deploy (push) Failing after 42s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
Your Name
283
scripts/reboot-recovery/agent99-control-loop-maintenance.ps1
Normal file
283
scripts/reboot-recovery/agent99-control-loop-maintenance.ps1
Normal file
@@ -0,0 +1,283 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("Check", "Apply")]
|
||||
[string]$Mode = "Check",
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateRange(1, 2147483647)]
|
||||
[int]$ExpectedControlPid,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[datetime]$ExpectedControlCreated,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateRange(1, 2147483647)]
|
||||
[int]$ExpectedRecoverPid,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[datetime]$ExpectedRecoverCreated,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateRange(1, 2147483647)]
|
||||
[int]$ExpectedRecoverParentPid,
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$taskName = "Wooo-Agent99-Control-Loop"
|
||||
$runtimeWritePerformed = $false
|
||||
|
||||
function Get-ExactProcess {
|
||||
param([int]$ProcessId)
|
||||
Get-CimInstance Win32_Process -ErrorAction Stop |
|
||||
Where-Object { [int]$_.ProcessId -eq $ProcessId } |
|
||||
Select-Object -First 1
|
||||
}
|
||||
|
||||
function Test-ExactCreationTime {
|
||||
param([object]$Process, [datetime]$ExpectedCreated)
|
||||
if (-not $Process -or -not $Process.CreationDate) { return $false }
|
||||
$actual = ([datetime]$Process.CreationDate).ToUniversalTime()
|
||||
$expected = $ExpectedCreated.ToUniversalTime()
|
||||
[bool]($actual.Ticks -eq $expected.Ticks)
|
||||
}
|
||||
|
||||
function Test-ControlIdentity {
|
||||
param([object]$Process)
|
||||
[bool](
|
||||
$Process -and
|
||||
[string]$Process.Name -eq "powershell.exe" -and
|
||||
(Test-ExactCreationTime $Process $ExpectedControlCreated) -and
|
||||
([string]$Process.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)'
|
||||
)
|
||||
}
|
||||
|
||||
function Test-RecoverIdentity {
|
||||
param([object]$Process)
|
||||
[bool](
|
||||
$Process -and
|
||||
[string]$Process.Name -eq "powershell.exe" -and
|
||||
(Test-ExactCreationTime $Process $ExpectedRecoverCreated) -and
|
||||
[int]$Process.ParentProcessId -eq $ExpectedRecoverParentPid -and
|
||||
([string]$Process.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply'
|
||||
)
|
||||
}
|
||||
|
||||
function Wait-ProcessAbsent {
|
||||
param([int]$ProcessId, [int]$TimeoutSeconds)
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
||||
if (-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue)) { return $true }
|
||||
Start-Sleep -Milliseconds 200
|
||||
}
|
||||
[bool](-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue))
|
||||
}
|
||||
|
||||
function Test-RecoveryMutexActive {
|
||||
$mutex = $null
|
||||
$acquired = $false
|
||||
try {
|
||||
$mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1")
|
||||
try { $acquired = [bool]$mutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $acquired = $true }
|
||||
[bool](-not $acquired)
|
||||
} finally {
|
||||
if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
|
||||
if ($mutex) { $mutex.Dispose() }
|
||||
}
|
||||
}
|
||||
|
||||
function Write-DurableReceipt {
|
||||
param([object]$Value, [string]$Path)
|
||||
$temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
|
||||
$stream = $null
|
||||
$writer = $null
|
||||
try {
|
||||
$encoding = New-Object Text.UTF8Encoding($false)
|
||||
$stream = [IO.File]::Open($temp, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
$writer = New-Object IO.StreamWriter($stream, $encoding)
|
||||
$writer.Write(($Value | ConvertTo-Json -Depth 8))
|
||||
$writer.Flush()
|
||||
$stream.Flush($true)
|
||||
$writer.Dispose(); $writer = $null
|
||||
$stream.Dispose(); $stream = $null
|
||||
$beforeHash = (Get-FileHash -LiteralPath $temp -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
Move-Item -LiteralPath $temp -Destination $Path -Force
|
||||
$afterHash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
if ($beforeHash -ne $afterHash) { throw "receipt_readback_mismatch" }
|
||||
} finally {
|
||||
if ($writer) { $writer.Dispose() }
|
||||
if ($stream) { $stream.Dispose() }
|
||||
Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
$maintenanceMutex = [Threading.Mutex]::new($false, "Global\WoooAgent99ControlLoopMaintenanceV1")
|
||||
$maintenanceMutexAcquired = $false
|
||||
try {
|
||||
try {
|
||||
$maintenanceMutexAcquired = [bool]$maintenanceMutex.WaitOne(0)
|
||||
} catch [Threading.AbandonedMutexException] {
|
||||
$maintenanceMutexAcquired = $true
|
||||
}
|
||||
if (-not $maintenanceMutexAcquired) {
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_control_loop_maintenance_v2"
|
||||
timestamp = (Get-Date).ToString("o")
|
||||
mode = $Mode
|
||||
precheckPassed = $false
|
||||
blockers = @("maintenance_writer_active")
|
||||
terminal = "blocked_single_writer_active"
|
||||
runtimeWritePerformed = $false
|
||||
secretValueRead = $false
|
||||
} | ConvertTo-Json -Depth 5 -Compress
|
||||
exit 75
|
||||
}
|
||||
|
||||
$blockers = New-Object System.Collections.Generic.List[string]
|
||||
if ($env:COMPUTERNAME -ne "WOOO-SUPER") { $blockers.Add("computer_identity_mismatch") }
|
||||
$task = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue
|
||||
if (-not $task) {
|
||||
$blockers.Add("control_task_missing")
|
||||
} else {
|
||||
$actions = @($task.Actions)
|
||||
if ($actions.Count -ne 1) { $blockers.Add("control_task_action_count_mismatch") }
|
||||
if ($actions.Count -eq 1) {
|
||||
if ([string]$actions[0].Execute -notmatch '(?i)\\WindowsPowerShell\\v1\.0\\powershell\.exe$') { $blockers.Add("control_task_executable_mismatch") }
|
||||
if ([string]$actions[0].Arguments -notmatch '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)') { $blockers.Add("control_task_arguments_mismatch") }
|
||||
}
|
||||
}
|
||||
|
||||
$control = Get-ExactProcess $ExpectedControlPid
|
||||
$recover = Get-ExactProcess $ExpectedRecoverPid
|
||||
if (-not $control) {
|
||||
$blockers.Add("control_process_missing")
|
||||
} else {
|
||||
if (-not (Test-ControlIdentity $control)) { $blockers.Add("control_process_identity_mismatch") }
|
||||
}
|
||||
if (-not $recover) {
|
||||
$blockers.Add("recover_process_missing")
|
||||
} else {
|
||||
if (-not (Test-RecoverIdentity $recover)) { $blockers.Add("recover_process_identity_mismatch") }
|
||||
}
|
||||
|
||||
$precheckPassed = [bool]($blockers.Count -eq 0)
|
||||
$controlStopped = $false
|
||||
$recoverStopped = $false
|
||||
$leaseMetadataRemoved = $false
|
||||
$mutexReleased = $false
|
||||
$taskAfter = $task
|
||||
$operationIntentRef = ""
|
||||
$deviceMutationAttempted = $false
|
||||
$operationFailureType = ""
|
||||
$terminal = if ($precheckPassed) { "maintenance_freeze_check_verified" } else { "blocked_identity_mismatch" }
|
||||
|
||||
if ($Mode -eq "Apply" -and $precheckPassed) {
|
||||
$intentStamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
|
||||
$operationIntentRef = "agent99-control-loop-maintenance-intent-$intentStamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json"
|
||||
$operationIntentPath = Join-Path $AgentRoot "evidence\$operationIntentRef"
|
||||
$operationIntent = [pscustomobject]@{
|
||||
schemaVersion = "agent99_control_loop_maintenance_intent_v1"
|
||||
timestamp = (Get-Date).ToString("o")
|
||||
taskName = $taskName
|
||||
taskEnabledBefore = [bool]$task.Settings.Enabled
|
||||
expectedControlPid = $ExpectedControlPid
|
||||
expectedControlCreated = $ExpectedControlCreated.ToUniversalTime().ToString("o")
|
||||
expectedRecoverPid = $ExpectedRecoverPid
|
||||
expectedRecoverCreated = $ExpectedRecoverCreated.ToUniversalTime().ToString("o")
|
||||
expectedRecoverParentPid = $ExpectedRecoverParentPid
|
||||
plannedActions = @("disable_control_task", "stop_exact_control_tree", "stop_exact_recover_tree", "remove_exact_recovery_lease")
|
||||
rollback = "keep task frozen on partial mutation; restore only after patched runtime verifier"
|
||||
vmPowerChangePerformed = $false
|
||||
vmwareVmxProcessTerminated = $false
|
||||
secretValueRead = $false
|
||||
}
|
||||
Write-DurableReceipt $operationIntent $operationIntentPath
|
||||
$runtimeWritePerformed = $true
|
||||
|
||||
try {
|
||||
$deviceMutationAttempted = $true
|
||||
Disable-ScheduledTask -TaskName $taskName -TaskPath "\" | Out-Null
|
||||
Stop-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue
|
||||
$controlStopped = Wait-ProcessAbsent $ExpectedControlPid 15
|
||||
if (-not $controlStopped) {
|
||||
$controlRecheck = Get-ExactProcess $ExpectedControlPid
|
||||
if (-not (Test-ControlIdentity $controlRecheck)) { throw "control_recheck_identity_mismatch" }
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID $ExpectedControlPid /T /F 2>&1 | Out-Null
|
||||
$controlStopped = Wait-ProcessAbsent $ExpectedControlPid 15
|
||||
}
|
||||
if (-not (Wait-ProcessAbsent $ExpectedRecoverPid 5)) {
|
||||
$recoverRecheck = Get-ExactProcess $ExpectedRecoverPid
|
||||
if (-not (Test-RecoverIdentity $recoverRecheck)) { throw "recover_recheck_identity_mismatch" }
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID $ExpectedRecoverPid /T /F 2>&1 | Out-Null
|
||||
}
|
||||
$recoverStopped = Wait-ProcessAbsent $ExpectedRecoverPid 15
|
||||
$controlStopped = [bool]($controlStopped -or (Wait-ProcessAbsent $ExpectedControlPid 5))
|
||||
|
||||
$leasePath = Join-Path $AgentRoot "state\recovery-transport-lease.json"
|
||||
$mutexReleased = [bool](-not (Test-RecoveryMutexActive))
|
||||
if ($recoverStopped -and $mutexReleased -and (Test-Path -LiteralPath $leasePath)) {
|
||||
$lease = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
[int]$lease.processId -eq $ExpectedRecoverPid -and
|
||||
[string]$lease.runId -match "^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|agent99-recovery-[0-9]{8}-[0-9]{6}-[0-9a-f]{8})$"
|
||||
) {
|
||||
Remove-Item -LiteralPath $leasePath -Force
|
||||
}
|
||||
}
|
||||
$leaseMetadataRemoved = [bool](-not (Test-Path -LiteralPath $leasePath))
|
||||
$taskAfter = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop
|
||||
$verified = [bool](-not $taskAfter.Settings.Enabled -and $controlStopped -and $recoverStopped -and $mutexReleased -and $leaseMetadataRemoved)
|
||||
$terminal = if ($verified) { "control_loop_maintenance_frozen" } else { "maintenance_freeze_unverified" }
|
||||
} catch {
|
||||
$operationFailureType = $_.Exception.GetType().Name
|
||||
$taskAfter = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue
|
||||
$controlStopped = Wait-ProcessAbsent $ExpectedControlPid 1
|
||||
$recoverStopped = Wait-ProcessAbsent $ExpectedRecoverPid 1
|
||||
$mutexReleased = [bool](-not (Test-RecoveryMutexActive))
|
||||
$leasePath = Join-Path $AgentRoot "state\recovery-transport-lease.json"
|
||||
$leaseMetadataRemoved = [bool](-not (Test-Path -LiteralPath $leasePath))
|
||||
$terminal = if ($deviceMutationAttempted) { "maintenance_partial_frozen_manual_recovery" } else { "maintenance_apply_failed_no_mutation" }
|
||||
}
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_control_loop_maintenance_v2"
|
||||
timestamp = (Get-Date).ToString("o")
|
||||
mode = $Mode
|
||||
taskName = $taskName
|
||||
expectedControlPid = $ExpectedControlPid
|
||||
expectedControlCreated = $ExpectedControlCreated.ToUniversalTime().ToString("o")
|
||||
expectedRecoverPid = $ExpectedRecoverPid
|
||||
expectedRecoverCreated = $ExpectedRecoverCreated.ToUniversalTime().ToString("o")
|
||||
expectedRecoverParentPid = $ExpectedRecoverParentPid
|
||||
precheckPassed = $precheckPassed
|
||||
blockers = @($blockers)
|
||||
taskEnabledBefore = if ($task) { [bool]$task.Settings.Enabled } else { $null }
|
||||
taskEnabledAfter = if ($taskAfter) { [bool]$taskAfter.Settings.Enabled } else { $null }
|
||||
operationIntentRef = $operationIntentRef
|
||||
deviceMutationAttempted = $deviceMutationAttempted
|
||||
operationFailureType = $operationFailureType
|
||||
rollbackStatus = if ($deviceMutationAttempted -and $terminal -ne "control_loop_maintenance_frozen") { "task_kept_frozen_manual_recovery_required" } else { "not_required" }
|
||||
controlStopped = $controlStopped
|
||||
recoverStopped = $recoverStopped
|
||||
leaseMetadataRemoved = $leaseMetadataRemoved
|
||||
terminal = $terminal
|
||||
runtimeWritePerformed = $runtimeWritePerformed
|
||||
vmPowerChangePerformed = $false
|
||||
vmwareVmxProcessTerminated = $false
|
||||
vmxLockDeleted = $false
|
||||
hostRebootPerformed = $false
|
||||
secretValueRead = $false
|
||||
}
|
||||
|
||||
if ($Mode -eq "Apply") {
|
||||
$path = Join-Path $AgentRoot ("evidence\agent99-control-loop-maintenance-" + (Get-Date -Format "yyyyMMdd-HHmmss-fff") + "-" + ([guid]::NewGuid().ToString('N').Substring(0, 8)) + ".json")
|
||||
Write-DurableReceipt $result $path
|
||||
$result | Add-Member -NotePropertyName receiptRef -NotePropertyValue (Split-Path -Leaf $path) -Force
|
||||
}
|
||||
$result | ConvertTo-Json -Depth 8 -Compress
|
||||
if (-not $precheckPassed) { exit 64 }
|
||||
if ($Mode -eq "Check") { exit 0 }
|
||||
if ($terminal -eq "control_loop_maintenance_frozen") { exit 0 }
|
||||
exit 70
|
||||
} finally {
|
||||
if ($maintenanceMutexAcquired -and $maintenanceMutex) {
|
||||
try { $maintenanceMutex.ReleaseMutex() } catch {}
|
||||
}
|
||||
if ($maintenanceMutex) { $maintenanceMutex.Dispose() }
|
||||
}
|
||||
516
scripts/reboot-recovery/agent99-stale-recovery-breakglass.ps1
Normal file
516
scripts/reboot-recovery/agent99-stale-recovery-breakglass.ps1
Normal file
@@ -0,0 +1,516 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("Check", "Apply")]
|
||||
[string]$Mode = "Check",
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|agent99-recovery-[0-9]{8}-[0-9]{6}-[0-9a-f]{8})$")]
|
||||
[string]$ExpectedRunId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateRange(1, 2147483647)]
|
||||
[int]$ExpectedParentPid,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[datetime]$ExpectedParentCreated,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateRange(1, 2147483647)]
|
||||
[int]$ExpectedChildPid,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[datetime]$ExpectedChildCreated,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ExpectedVmxPath,
|
||||
[ValidateRange(20, 1440)]
|
||||
[int]$MinimumAgeMinutes = 20,
|
||||
[ValidatePattern("^$|^[A-Za-z0-9._-]{1,180}\.json$")]
|
||||
[string]$PriorReceiptRef = "",
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$script:RuntimeWritePerformed = $false
|
||||
$script:ReceiptPath = ""
|
||||
$postFailureReadbackErrors = @()
|
||||
|
||||
function Get-Sha256Text {
|
||||
param([string]$Text)
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
([BitConverter]::ToString($sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($Text)))).Replace("-", "").ToLowerInvariant()
|
||||
} finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Write-AtomicJson {
|
||||
param([object]$Value, [string]$Path)
|
||||
$directory = Split-Path -Parent $Path
|
||||
if (-not (Test-Path -LiteralPath $directory)) {
|
||||
New-Item -ItemType Directory -Path $directory -Force | Out-Null
|
||||
}
|
||||
$temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
|
||||
$stream = $null
|
||||
$writer = $null
|
||||
try {
|
||||
$json = $Value | ConvertTo-Json -Depth 8
|
||||
$encoding = New-Object Text.UTF8Encoding($false)
|
||||
$stream = [IO.File]::Open($temp, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
$writer = New-Object IO.StreamWriter($stream, $encoding)
|
||||
$writer.Write($json)
|
||||
$writer.Flush()
|
||||
$stream.Flush($true)
|
||||
$writer.Dispose()
|
||||
$writer = $null
|
||||
$stream.Dispose()
|
||||
$stream = $null
|
||||
$tempHash = (Get-FileHash -LiteralPath $temp -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
Move-Item -LiteralPath $temp -Destination $Path -Force
|
||||
if (-not (Test-Path -LiteralPath $Path)) { throw "receipt_publish_failed" }
|
||||
$readbackHash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
if ($readbackHash -ne $tempHash) { throw "receipt_readback_mismatch" }
|
||||
} finally {
|
||||
if ($writer) { $writer.Dispose() }
|
||||
if ($stream) { $stream.Dispose() }
|
||||
Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ExactProcess {
|
||||
param([int]$ProcessId)
|
||||
Get-CimInstance Win32_Process -ErrorAction Stop | Where-Object { [int]$_.ProcessId -eq $ProcessId } | Select-Object -First 1
|
||||
}
|
||||
|
||||
function Test-ExactProcessIdentity {
|
||||
param(
|
||||
[object]$Process,
|
||||
[int]$ExpectedPid,
|
||||
[datetime]$ExpectedCreated,
|
||||
[string]$ExpectedName,
|
||||
[int]$ExpectedParentPid = -1
|
||||
)
|
||||
if (-not $Process -or -not $Process.CreationDate) { return $false }
|
||||
[bool](
|
||||
[int]$Process.ProcessId -eq $ExpectedPid -and
|
||||
[string]$Process.Name -eq $ExpectedName -and
|
||||
($ExpectedParentPid -lt 0 -or [int]$Process.ParentProcessId -eq $ExpectedParentPid) -and
|
||||
([datetime]$Process.CreationDate).ToUniversalTime().Ticks -eq $ExpectedCreated.ToUniversalTime().Ticks
|
||||
)
|
||||
}
|
||||
|
||||
function Wait-ExactProcessIdentityAbsent {
|
||||
param(
|
||||
[int]$ProcessId,
|
||||
[datetime]$ExpectedCreated,
|
||||
[int]$TimeoutSeconds = 10
|
||||
)
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
||||
$current = Get-ExactProcess $ProcessId
|
||||
if (-not $current -or ([datetime]$current.CreationDate).ToUniversalTime().Ticks -ne $ExpectedCreated.ToUniversalTime().Ticks) { return $true }
|
||||
Start-Sleep -Milliseconds 200
|
||||
}
|
||||
$current = Get-ExactProcess $ProcessId
|
||||
[bool](-not $current -or ([datetime]$current.CreationDate).ToUniversalTime().Ticks -ne $ExpectedCreated.ToUniversalTime().Ticks)
|
||||
}
|
||||
|
||||
function Get-FailedMutationReadback {
|
||||
param(
|
||||
[int]$ParentProcessId,
|
||||
[datetime]$ParentCreated,
|
||||
[int]$ChildProcessId,
|
||||
[datetime]$ChildCreated,
|
||||
[string]$LeasePath
|
||||
)
|
||||
$errors = @()
|
||||
try {
|
||||
$parentAbsent = Wait-ExactProcessIdentityAbsent $ParentProcessId $ParentCreated 1
|
||||
} catch {
|
||||
$parentAbsent = $false
|
||||
$errors += "parent_identity_unknown"
|
||||
}
|
||||
try {
|
||||
$childAbsent = Wait-ExactProcessIdentityAbsent $ChildProcessId $ChildCreated 1
|
||||
} catch {
|
||||
$childAbsent = $false
|
||||
$errors += "child_identity_unknown"
|
||||
}
|
||||
try {
|
||||
$recoveryMutexReleased = [bool](-not (Test-RecoveryMutexActive))
|
||||
} catch {
|
||||
$recoveryMutexReleased = $false
|
||||
$errors += "recovery_mutex_unknown"
|
||||
}
|
||||
try {
|
||||
$leaseRemoved = [bool](-not (Test-Path -LiteralPath $LeasePath))
|
||||
} catch {
|
||||
$leaseRemoved = $false
|
||||
$errors += "lease_metadata_unknown"
|
||||
}
|
||||
[pscustomobject]@{
|
||||
parentAbsent = [bool]$parentAbsent
|
||||
childAbsent = [bool]$childAbsent
|
||||
recoveryMutexReleased = [bool]$recoveryMutexReleased
|
||||
leaseRemoved = [bool]$leaseRemoved
|
||||
complete = [bool]($errors.Count -eq 0)
|
||||
errors = @($errors)
|
||||
}
|
||||
}
|
||||
|
||||
function Wait-ExactProcessAbsent {
|
||||
param(
|
||||
[int]$ProcessId,
|
||||
[int]$TimeoutSeconds = 10
|
||||
)
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) {
|
||||
if (-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue)) { return $true }
|
||||
Start-Sleep -Milliseconds 200
|
||||
}
|
||||
[bool](-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue))
|
||||
}
|
||||
|
||||
function Get-DescendantProcesses {
|
||||
param([int]$ParentProcessId)
|
||||
$all = @(Get-CimInstance Win32_Process -ErrorAction Stop)
|
||||
$known = New-Object 'System.Collections.Generic.HashSet[int]'
|
||||
[void]$known.Add($ParentProcessId)
|
||||
$rows = @()
|
||||
do {
|
||||
$added = $false
|
||||
foreach ($processInfo in $all) {
|
||||
if (
|
||||
$known.Contains([int]$processInfo.ParentProcessId) -and
|
||||
-not $known.Contains([int]$processInfo.ProcessId)
|
||||
) {
|
||||
[void]$known.Add([int]$processInfo.ProcessId)
|
||||
$rows += $processInfo
|
||||
$added = $true
|
||||
}
|
||||
}
|
||||
} while ($added)
|
||||
@($rows)
|
||||
}
|
||||
|
||||
function Invoke-ExactTaskkill {
|
||||
param([int]$ProcessId)
|
||||
$output = @(& "$env:SystemRoot\System32\taskkill.exe" /PID $ProcessId /T /F 2>&1)
|
||||
[pscustomobject]@{
|
||||
exitCode = [int]$LASTEXITCODE
|
||||
outputPresent = [bool]($output.Count -gt 0)
|
||||
rawOutputStored = $false
|
||||
}
|
||||
}
|
||||
|
||||
function Test-RecoveryMutexActive {
|
||||
$mutex = $null
|
||||
$acquired = $false
|
||||
try {
|
||||
$mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1")
|
||||
try { $acquired = [bool]$mutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $acquired = $true }
|
||||
[bool](-not $acquired)
|
||||
} finally {
|
||||
if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
|
||||
if ($mutex) { $mutex.Dispose() }
|
||||
}
|
||||
}
|
||||
|
||||
$reconcilerMutex = [Threading.Mutex]::new($false, "Global\WoooAgent99StaleRecoveryReconcilerV1")
|
||||
$reconcilerMutexAcquired = $false
|
||||
try { $reconcilerMutexAcquired = [bool]$reconcilerMutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $reconcilerMutexAcquired = $true }
|
||||
if (-not $reconcilerMutexAcquired) {
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_stale_recovery_breakglass_v3"
|
||||
mode = $Mode
|
||||
precheckPassed = $false
|
||||
blockers = @("stale_recovery_reconciler_active")
|
||||
terminal = "blocked_single_writer_active"
|
||||
runtimeWritePerformed = $false
|
||||
secretValueRead = $false
|
||||
} | ConvertTo-Json -Compress
|
||||
$reconcilerMutex.Dispose()
|
||||
exit 75
|
||||
}
|
||||
|
||||
try {
|
||||
$leasePath = Join-Path $AgentRoot "state\recovery-transport-lease.json"
|
||||
$preconditions = New-Object System.Collections.Generic.List[string]
|
||||
$lease = $null
|
||||
if (-not (Test-Path -LiteralPath $leasePath)) {
|
||||
$preconditions.Add("lease_missing")
|
||||
} else {
|
||||
try { $lease = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json } catch { $preconditions.Add("lease_parse_failed") }
|
||||
}
|
||||
|
||||
$parent = Get-ExactProcess $ExpectedParentPid
|
||||
$child = Get-ExactProcess $ExpectedChildPid
|
||||
$canonicalVmx = [IO.Path]::GetFullPath($ExpectedVmxPath).TrimEnd("\")
|
||||
$vmxIdentityDigest = Get-Sha256Text $canonicalVmx.ToLowerInvariant()
|
||||
$priorReceipt = $null
|
||||
$priorReceiptVerified = $false
|
||||
$now = Get-Date
|
||||
$parentAgeMinutes = -1
|
||||
if ($parent) {
|
||||
try { $parentAgeMinutes = [math]::Round(($now - (Get-Process -Id $ExpectedParentPid -ErrorAction Stop).StartTime).TotalMinutes, 2) } catch {}
|
||||
}
|
||||
|
||||
if (-not $lease) {
|
||||
$preconditions.Add("lease_unavailable")
|
||||
} else {
|
||||
if ([string]$lease.schemaVersion -ne "agent99_recovery_transport_lease_v1") { $preconditions.Add("lease_schema_mismatch") }
|
||||
if ([string]$lease.runId -ne $ExpectedRunId) { $preconditions.Add("lease_run_id_mismatch") }
|
||||
if ([int]$lease.processId -ne $ExpectedParentPid) { $preconditions.Add("lease_process_id_mismatch") }
|
||||
if ([bool]$lease.storesSecret) { $preconditions.Add("lease_secret_contract_invalid") }
|
||||
}
|
||||
if (-not $parent) {
|
||||
$preconditions.Add("parent_missing")
|
||||
} else {
|
||||
$parentCommand = [string]$parent.CommandLine
|
||||
if (-not (Test-ExactProcessIdentity $parent $ExpectedParentPid $ExpectedParentCreated "powershell.exe")) { $preconditions.Add("parent_identity_mismatch") }
|
||||
if ($parentCommand -notmatch '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply') { $preconditions.Add("parent_command_mismatch") }
|
||||
$runIdBoundToCommand = if ($ExpectedRunId -match '^agent99-recovery-') {
|
||||
$parentCommand -notmatch "(?i)-AutomationRunId\s+"
|
||||
} else {
|
||||
$parentCommand.IndexOf($ExpectedRunId, [StringComparison]::OrdinalIgnoreCase) -ge 0
|
||||
}
|
||||
if (-not $runIdBoundToCommand) { $preconditions.Add("parent_run_id_mismatch") }
|
||||
if ($lease -and $lease.acquiredAt) {
|
||||
$leaseAcquiredUtc = ([datetime]$lease.acquiredAt).ToUniversalTime()
|
||||
$parentCreatedUtc = $ExpectedParentCreated.ToUniversalTime()
|
||||
if ($leaseAcquiredUtc -lt $parentCreatedUtc -or $leaseAcquiredUtc -gt $parentCreatedUtc.AddMinutes(5)) { $preconditions.Add("lease_parent_creation_mismatch") }
|
||||
} else {
|
||||
$preconditions.Add("lease_acquired_at_missing")
|
||||
}
|
||||
if ($parentAgeMinutes -lt $MinimumAgeMinutes) { $preconditions.Add("parent_not_stale") }
|
||||
}
|
||||
if (-not $child) {
|
||||
if (-not $PriorReceiptRef) {
|
||||
$preconditions.Add("child_missing_without_prior_receipt")
|
||||
} else {
|
||||
$priorPath = Join-Path (Join-Path $AgentRoot "evidence") $PriorReceiptRef
|
||||
try {
|
||||
$priorReceipt = Get-Content -LiteralPath $priorPath -Raw | ConvertFrom-Json
|
||||
$priorSchema = [string]$priorReceipt.schemaVersion
|
||||
$priorCreationIdentityValid = if ($priorSchema -eq "agent99_stale_recovery_breakglass_v3") {
|
||||
[bool](
|
||||
$priorReceipt.PSObject.Properties["expectedParentCreated"] -and
|
||||
$priorReceipt.PSObject.Properties["expectedChildCreated"] -and
|
||||
([datetime]$priorReceipt.expectedParentCreated).ToUniversalTime().Ticks -eq $ExpectedParentCreated.ToUniversalTime().Ticks -and
|
||||
([datetime]$priorReceipt.expectedChildCreated).ToUniversalTime().Ticks -eq $ExpectedChildCreated.ToUniversalTime().Ticks
|
||||
)
|
||||
} else {
|
||||
[bool]($priorSchema -eq "agent99_stale_recovery_breakglass_v2" -and $priorReceipt.childTerminated -is [bool] -and $priorReceipt.childTerminated)
|
||||
}
|
||||
$priorReceiptVerified = [bool](
|
||||
$priorSchema -in @("agent99_stale_recovery_breakglass_v2", "agent99_stale_recovery_breakglass_v3") -and
|
||||
$priorCreationIdentityValid -and
|
||||
[string]$priorReceipt.mode -eq "Apply" -and
|
||||
[string]$priorReceipt.expectedRunId -eq $ExpectedRunId -and
|
||||
[int]$priorReceipt.expectedParentPid -eq $ExpectedParentPid -and
|
||||
[int]$priorReceipt.expectedChildPid -eq $ExpectedChildPid -and
|
||||
[string]$priorReceipt.vmxIdentityDigest -eq $vmxIdentityDigest -and
|
||||
$priorReceipt.precheckPassed -is [bool] -and
|
||||
$priorReceipt.precheckPassed -eq $true -and
|
||||
$priorReceipt.runtimeWritePerformed -is [bool] -and
|
||||
$priorReceipt.runtimeWritePerformed -eq $true -and
|
||||
[string]$priorReceipt.terminal -in @("child_termination_failed", "child_terminated_parent_draining")
|
||||
)
|
||||
if (-not $priorReceiptVerified) { $preconditions.Add("prior_receipt_identity_mismatch") }
|
||||
} catch {
|
||||
$preconditions.Add("prior_receipt_readback_failed")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$childCommand = [string]$child.CommandLine
|
||||
if (-not (Test-ExactProcessIdentity $child $ExpectedChildPid $ExpectedChildCreated "vmrun.exe" $ExpectedParentPid)) { $preconditions.Add("child_identity_mismatch") }
|
||||
if ($childCommand -notmatch "(?i)vmrun\.exe.*\sstart\s") { $preconditions.Add("child_command_mismatch") }
|
||||
if ($childCommand.IndexOf($canonicalVmx, [StringComparison]::OrdinalIgnoreCase) -lt 0) { $preconditions.Add("child_vmx_mismatch") }
|
||||
if ($childCommand -notmatch "(?i)\snogui(\s|$)") { $preconditions.Add("child_nogui_mismatch") }
|
||||
}
|
||||
if (-not (Test-RecoveryMutexActive)) { $preconditions.Add("recovery_mutex_not_active") }
|
||||
|
||||
$precheckPassed = [bool]($preconditions.Count -eq 0)
|
||||
$terminal = if ($precheckPassed) { "check_verified" } else { "blocked_identity_mismatch" }
|
||||
$childTerminated = $false
|
||||
$childAlreadyAbsent = [bool](-not $child -and $priorReceiptVerified)
|
||||
$parentDrained = $false
|
||||
$parentTerminated = $false
|
||||
$mutexReleased = $false
|
||||
$leaseMetadataRemoved = $false
|
||||
$childTaskkill = $null
|
||||
$parentTaskkill = $null
|
||||
$operationIntentRef = ""
|
||||
$parentCleanupIntentRef = ""
|
||||
$mutationAttempted = $false
|
||||
$operationFailureType = ""
|
||||
|
||||
if ($Mode -eq "Apply" -and $precheckPassed) {
|
||||
try {
|
||||
$intentStamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
|
||||
$operationIntentRef = "agent99-stale-recovery-breakglass-intent-$intentStamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json"
|
||||
$operationIntentPath = Join-Path (Join-Path $AgentRoot "evidence") $operationIntentRef
|
||||
$operationIntent = [pscustomobject]@{
|
||||
schemaVersion = "agent99_stale_recovery_breakglass_intent_v1"
|
||||
timestamp = (Get-Date).ToString("o")
|
||||
expectedRunId = $ExpectedRunId
|
||||
expectedParentPid = $ExpectedParentPid
|
||||
expectedParentCreated = $ExpectedParentCreated.ToUniversalTime().ToString("o")
|
||||
expectedChildPid = $ExpectedChildPid
|
||||
expectedChildCreated = $ExpectedChildCreated.ToUniversalTime().ToString("o")
|
||||
vmxIdentityDigest = $vmxIdentityDigest
|
||||
childAlreadyAbsent = $childAlreadyAbsent
|
||||
precheckPassed = $true
|
||||
plannedActions = @("terminate_exact_vmrun_child_if_present", "wait_or_terminate_exact_recover_parent", "remove_exact_lease_metadata")
|
||||
rollback = "process_termination_not_reversible; preserve intent and terminal receipt"
|
||||
vmPowerChangePerformed = $false
|
||||
vmwareVmxProcessTerminated = $false
|
||||
secretValueRead = $false
|
||||
}
|
||||
Write-AtomicJson $operationIntent $operationIntentPath
|
||||
$script:RuntimeWritePerformed = $true
|
||||
|
||||
if ($childAlreadyAbsent) {
|
||||
$childTerminated = $true
|
||||
} else {
|
||||
$childRecheck = Get-ExactProcess $ExpectedChildPid
|
||||
if (
|
||||
-not (Test-ExactProcessIdentity $childRecheck $ExpectedChildPid $ExpectedChildCreated "vmrun.exe" $ExpectedParentPid) -or
|
||||
([string]$childRecheck.CommandLine) -ne $childCommand
|
||||
) {
|
||||
$terminal = "child_recheck_identity_changed"
|
||||
} else {
|
||||
$mutationAttempted = $true
|
||||
$childTaskkill = Invoke-ExactTaskkill $ExpectedChildPid
|
||||
$script:RuntimeWritePerformed = $true
|
||||
$childTerminated = Wait-ExactProcessIdentityAbsent $ExpectedChildPid $ExpectedChildCreated 15
|
||||
}
|
||||
}
|
||||
if (-not $childTerminated) {
|
||||
if ($terminal -ne "child_recheck_identity_changed") { $terminal = "child_termination_failed" }
|
||||
} else {
|
||||
$terminal = "child_terminated_parent_draining"
|
||||
$parentDrained = Wait-ExactProcessIdentityAbsent $ExpectedParentPid $ExpectedParentCreated 30
|
||||
if (-not $parentDrained) {
|
||||
$parentRecheck = Get-ExactProcess $ExpectedParentPid
|
||||
$descendants = @(Get-DescendantProcesses $ExpectedParentPid)
|
||||
$parentRecheckValid = [bool](
|
||||
(Test-ExactProcessIdentity $parentRecheck $ExpectedParentPid $ExpectedParentCreated "powershell.exe") -and
|
||||
([string]$parentRecheck.CommandLine) -eq $parentCommand -and
|
||||
$descendants.Count -eq 0
|
||||
)
|
||||
if ($parentRecheckValid) {
|
||||
$parentIntentStamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
|
||||
$parentCleanupIntentRef = "agent99-stale-recovery-parent-cleanup-intent-$parentIntentStamp.json"
|
||||
$intentPath = Join-Path (Join-Path $AgentRoot "evidence") $parentCleanupIntentRef
|
||||
$intent = [pscustomobject]@{
|
||||
schemaVersion = "agent99_stale_recovery_parent_cleanup_intent_v1"
|
||||
timestamp = (Get-Date).ToString("o")
|
||||
expectedRunId = $ExpectedRunId
|
||||
expectedParentPid = $ExpectedParentPid
|
||||
expectedParentCreated = $ExpectedParentCreated.ToUniversalTime().ToString("o")
|
||||
expectedChildPid = $ExpectedChildPid
|
||||
expectedChildCreated = $ExpectedChildCreated.ToUniversalTime().ToString("o")
|
||||
vmxIdentityDigest = $vmxIdentityDigest
|
||||
childAbsent = $true
|
||||
descendantCount = 0
|
||||
leaseIdentityVerified = $true
|
||||
mutexActive = $true
|
||||
nextAction = "terminate_exact_stale_recover_parent"
|
||||
secretValueRead = $false
|
||||
}
|
||||
Write-AtomicJson $intent $intentPath
|
||||
$script:RuntimeWritePerformed = $true
|
||||
$mutationAttempted = $true
|
||||
$parentTaskkill = Invoke-ExactTaskkill $ExpectedParentPid
|
||||
$parentTerminated = Wait-ExactProcessIdentityAbsent $ExpectedParentPid $ExpectedParentCreated 15
|
||||
$parentDrained = $parentTerminated
|
||||
} else {
|
||||
$terminal = "parent_cleanup_identity_changed"
|
||||
}
|
||||
}
|
||||
$mutexReleased = [bool](-not (Test-RecoveryMutexActive))
|
||||
if ($parentDrained -and $mutexReleased) {
|
||||
$leaseRecheck = $null
|
||||
try { $leaseRecheck = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json } catch {}
|
||||
if (
|
||||
$leaseRecheck -and
|
||||
[string]$leaseRecheck.runId -eq $ExpectedRunId -and
|
||||
[int]$leaseRecheck.processId -eq $ExpectedParentPid -and
|
||||
-not (Get-ExactProcess $ExpectedParentPid)
|
||||
) {
|
||||
$mutationAttempted = $true
|
||||
Remove-Item -LiteralPath $leasePath -Force -ErrorAction Stop
|
||||
$script:RuntimeWritePerformed = $true
|
||||
}
|
||||
$leaseMetadataRemoved = [bool](-not (Test-Path -LiteralPath $leasePath))
|
||||
if ($leaseMetadataRemoved) { $terminal = "stale_recovery_released" } else { $terminal = "lease_metadata_cleanup_failed" }
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
$operationFailureType = $_.Exception.GetType().Name
|
||||
$terminal = if ($mutationAttempted) { "partial_write_failed_closed" } else { "apply_intent_or_precheck_failed_no_mutation" }
|
||||
$failureReadback = Get-FailedMutationReadback $ExpectedParentPid $ExpectedParentCreated $ExpectedChildPid $ExpectedChildCreated $leasePath
|
||||
$parentDrained = [bool]$failureReadback.parentAbsent
|
||||
$childTerminated = [bool]$failureReadback.childAbsent
|
||||
$mutexReleased = [bool]$failureReadback.recoveryMutexReleased
|
||||
$leaseMetadataRemoved = [bool]$failureReadback.leaseRemoved
|
||||
$postFailureReadbackErrors = @($failureReadback.errors)
|
||||
}
|
||||
}
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_stale_recovery_breakglass_v3"
|
||||
timestamp = (Get-Date).ToString("o")
|
||||
mode = $Mode
|
||||
host = $env:COMPUTERNAME
|
||||
expectedRunId = $ExpectedRunId
|
||||
expectedParentPid = $ExpectedParentPid
|
||||
expectedParentCreated = $ExpectedParentCreated.ToUniversalTime().ToString("o")
|
||||
expectedChildPid = $ExpectedChildPid
|
||||
expectedChildCreated = $ExpectedChildCreated.ToUniversalTime().ToString("o")
|
||||
vmxIdentityDigest = $vmxIdentityDigest
|
||||
minimumAgeMinutes = $MinimumAgeMinutes
|
||||
observedParentAgeMinutes = $parentAgeMinutes
|
||||
precheckPassed = $precheckPassed
|
||||
blockers = @($preconditions)
|
||||
priorReceiptRef = $PriorReceiptRef
|
||||
priorReceiptVerified = $priorReceiptVerified
|
||||
childAlreadyAbsent = $childAlreadyAbsent
|
||||
childTerminated = $childTerminated
|
||||
childTaskkillExitCode = if ($childTaskkill) { [int]$childTaskkill.exitCode } else { $null }
|
||||
parentDrained = $parentDrained
|
||||
parentTerminated = $parentTerminated
|
||||
parentTaskkillExitCode = if ($parentTaskkill) { [int]$parentTaskkill.exitCode } else { $null }
|
||||
operationIntentRef = $operationIntentRef
|
||||
parentCleanupIntentRef = $parentCleanupIntentRef
|
||||
mutationAttempted = $mutationAttempted
|
||||
operationFailureType = $operationFailureType
|
||||
postFailureReadbackComplete = [bool]($postFailureReadbackErrors.Count -eq 0)
|
||||
postFailureReadbackErrors = @($postFailureReadbackErrors)
|
||||
rollbackStatus = if ($mutationAttempted) { "not_reversible_process_termination_manual_verification_required" } else { "not_required_no_mutation" }
|
||||
mutexReleased = $mutexReleased
|
||||
leaseMetadataRemoved = $leaseMetadataRemoved
|
||||
terminal = $terminal
|
||||
runtimeWritePerformed = $script:RuntimeWritePerformed
|
||||
vmPowerChangePerformed = $false
|
||||
vmwareVmxProcessTerminated = $false
|
||||
vmxLockDeleted = $false
|
||||
hostRebootPerformed = $false
|
||||
secretValueRead = $false
|
||||
}
|
||||
|
||||
if ($Mode -eq "Apply") {
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
|
||||
$script:ReceiptPath = Join-Path $AgentRoot "evidence\agent99-stale-recovery-breakglass-$stamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json"
|
||||
Write-AtomicJson $result $script:ReceiptPath
|
||||
$result | Add-Member -NotePropertyName receiptRef -NotePropertyValue (Split-Path -Leaf $script:ReceiptPath) -Force
|
||||
}
|
||||
} finally {
|
||||
if ($reconcilerMutexAcquired -and $reconcilerMutex) {
|
||||
try { $reconcilerMutex.ReleaseMutex() } catch {}
|
||||
}
|
||||
if ($reconcilerMutex) { $reconcilerMutex.Dispose() }
|
||||
}
|
||||
|
||||
$result | ConvertTo-Json -Depth 8 -Compress
|
||||
if (-not $precheckPassed) { exit 64 }
|
||||
if ($Mode -eq "Check") { exit 0 }
|
||||
if ($terminal -eq "stale_recovery_released") { exit 0 }
|
||||
if ($terminal -eq "child_terminated_parent_draining") { exit 2 }
|
||||
exit 70
|
||||
345
scripts/reboot-recovery/tests/agent99-bounded-process-replay.ps1
Normal file
345
scripts/reboot-recovery/tests/agent99-bounded-process-replay.ps1
Normal file
@@ -0,0 +1,345 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ControlPlaneSource
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Get-ExactFunctionSource {
|
||||
param([object]$Ast, [string]$Name)
|
||||
$matches = @($Ast.FindAll({
|
||||
param($node)
|
||||
$node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $Name
|
||||
}, $true))
|
||||
if ($matches.Count -ne 1) { throw "function_identity_mismatch:$Name" }
|
||||
[string]$matches[0].Extent.Text
|
||||
}
|
||||
|
||||
function Assert-Replay {
|
||||
param([bool]$Condition, [string]$Reason)
|
||||
if (-not $Condition) { throw $Reason }
|
||||
}
|
||||
|
||||
$tokens = $null
|
||||
$errors = $null
|
||||
$ast = [Management.Automation.Language.Parser]::ParseInput(
|
||||
$ControlPlaneSource,
|
||||
[ref]$tokens,
|
||||
[ref]$errors
|
||||
)
|
||||
if ($errors.Count -ne 0) { throw "control_plane_parse_failed" }
|
||||
foreach ($name in @(
|
||||
"Get-AgentProcessDescendants",
|
||||
"ConvertTo-AgentNativeProcessArgument",
|
||||
"ConvertTo-AgentProcessIdentitySnapshot",
|
||||
"Get-AgentCreationBoundedDescendants",
|
||||
"Test-AgentProcessSnapshotMatch",
|
||||
"Select-AgentProcessTreeIdentitySnapshot",
|
||||
"Get-AgentProcessTreeIdentitySnapshot",
|
||||
"Get-AgentProcessIdentityReadback",
|
||||
"Test-AgentProcessIdentityActive",
|
||||
"Wait-AgentProcessTreeIdentityAbsent",
|
||||
"Stop-AgentBoundedProcessSnapshot",
|
||||
"Read-AgentBoundedTextFile",
|
||||
"Invoke-AgentBoundedProcess"
|
||||
)) {
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast $name)))
|
||||
}
|
||||
|
||||
$root = Join-Path $env:TEMP ("agent99-bounded-process-replay-" + [guid]::NewGuid().ToString("N"))
|
||||
$normal = $null
|
||||
$timeout = $null
|
||||
$flood = $null
|
||||
$rootOnly = $null
|
||||
$rootOnlyChildIdentity = $null
|
||||
$orphanRace = $null
|
||||
$orphanRaceChildIdentity = $null
|
||||
$pidReuseSelection = $null
|
||||
$creationBoundSelection = $null
|
||||
$stopReadbackFailure = $null
|
||||
$stopReadbackFailureChildIdentity = $null
|
||||
$cimFailure = $null
|
||||
$cimFailureChildIdentity = $null
|
||||
$cleanupVerified = $false
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $root -Force | Out-Null
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$cmd = Join-Path $env:SystemRoot "System32\cmd.exe"
|
||||
|
||||
$normal = Invoke-AgentBoundedProcess `
|
||||
-FilePath $cmd `
|
||||
-Arguments @("/d", "/c", "echo", "AG99_BOUNDED_OK") `
|
||||
-TimeoutSeconds 5 `
|
||||
-MaximumOutputBytes 4096
|
||||
Assert-Replay ($normal.ok -and $normal.completed -and $normal.stdout -match "AG99_BOUNDED_OK") "normal_process_failed"
|
||||
|
||||
$childPath = Join-Path $root "child.ps1"
|
||||
$parentPath = Join-Path $root "parent.ps1"
|
||||
[IO.File]::WriteAllText($childPath, 'Start-Sleep -Seconds 30', (New-Object Text.UTF8Encoding($false)))
|
||||
[IO.File]::WriteAllText($parentPath, @'
|
||||
param([string]$ChildPath)
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`""
|
||||
Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null
|
||||
Start-Sleep -Seconds 30
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
$timeout = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $parentPath, "-ChildPath", $childPath) `
|
||||
-TimeoutSeconds 2 `
|
||||
-MaximumOutputBytes 4096
|
||||
Assert-Replay ($timeout.timedOut -and -not $timeout.completed) "timeout_not_detected"
|
||||
Assert-Replay ($timeout.processTreeStopped -and -not $timeout.treeReadbackFailed) "timeout_tree_not_stopped"
|
||||
Assert-Replay ([int]$timeout.killedIdentityCount -ge 2) "timeout_descendant_not_observed"
|
||||
|
||||
$floodPath = Join-Path $root "flood.ps1"
|
||||
[IO.File]::WriteAllText($floodPath, @'
|
||||
for ($index = 0; $index -lt 1000; $index++) {
|
||||
[Console]::Out.Write(('X' * 1024))
|
||||
}
|
||||
Start-Sleep -Seconds 5
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
$flood = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $floodPath) `
|
||||
-TimeoutSeconds 10 `
|
||||
-MaximumOutputBytes 4096
|
||||
Assert-Replay ($flood.outputLimitExceeded -and -not $flood.completed) "streaming_output_limit_not_detected"
|
||||
Assert-Replay ([long]$flood.capturedBytes -le [long]$flood.maximumOutputBytes) "captured_output_exceeded_contract"
|
||||
Assert-Replay ($flood.processTreeStopped -and -not $flood.treeReadbackFailed) "flood_tree_not_stopped"
|
||||
|
||||
$rootOnlyChildPidPath = Join-Path $root "root-only-child.pid"
|
||||
$rootOnlyChildPath = Join-Path $root "root-only-child.ps1"
|
||||
$rootOnlyParentPath = Join-Path $root "root-only-parent.ps1"
|
||||
[IO.File]::WriteAllText($rootOnlyChildPath, @'
|
||||
param([string]$PidPath)
|
||||
[IO.File]::WriteAllText($PidPath, [string]$PID, (New-Object Text.UTF8Encoding($false)))
|
||||
Start-Sleep -Seconds 30
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
[IO.File]::WriteAllText($rootOnlyParentPath, @'
|
||||
param([string]$ChildPath, [string]$ChildPidPath)
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`" -PidPath `"$ChildPidPath`""
|
||||
Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null
|
||||
Start-Sleep -Seconds 30
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
$rootOnly = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $rootOnlyChildPidPath) `
|
||||
-TimeoutSeconds 2 `
|
||||
-MaximumOutputBytes 4096 `
|
||||
-TerminationScope Root
|
||||
Assert-Replay ($rootOnly.timedOut -and $rootOnly.forcedTerminationPerformed) "root_only_timeout_not_detected"
|
||||
Assert-Replay ($rootOnly.processScopeStopped -and -not $rootOnly.processTreeStopped) "root_only_scope_not_preserved"
|
||||
Assert-Replay ([int]$rootOnly.preservedDescendantCount -ge 1) "root_only_descendant_not_preserved"
|
||||
Assert-Replay (Test-Path -LiteralPath $rootOnlyChildPidPath -PathType Leaf) "root_only_child_pid_missing"
|
||||
$rootOnlyChildPid = [int]([IO.File]::ReadAllText($rootOnlyChildPidPath))
|
||||
$rootOnlyChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $rootOnlyChildPid" -ErrorAction Stop
|
||||
$rootOnlyChildIdentity = [pscustomobject]@{
|
||||
processId = [int]$rootOnlyChildProcess.ProcessId
|
||||
parentProcessId = [int]$rootOnlyChildProcess.ParentProcessId
|
||||
creationUtc = ([datetime]$rootOnlyChildProcess.CreationDate).ToUniversalTime().ToString("o")
|
||||
name = [string]$rootOnlyChildProcess.Name
|
||||
}
|
||||
Assert-Replay (Test-AgentProcessIdentityActive $rootOnlyChildIdentity) "root_only_child_not_active_after_parent_stop"
|
||||
|
||||
$orphanRaceChildPidPath = Join-Path $root "orphan-race-child.pid"
|
||||
$orphanRaceParentPath = Join-Path $root "orphan-race-parent.ps1"
|
||||
[IO.File]::WriteAllText($orphanRaceParentPath, @'
|
||||
param([string]$ChildPath, [string]$ChildPidPath)
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`" -PidPath `"$ChildPidPath`""
|
||||
Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
while (-not (Test-Path -LiteralPath $ChildPidPath) -and $stopwatch.Elapsed.TotalSeconds -lt 5) {
|
||||
Start-Sleep -Milliseconds 50
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
$orphanStartInfo = New-Object Diagnostics.ProcessStartInfo
|
||||
$orphanStartInfo.FileName = $powershell
|
||||
$orphanStartInfo.Arguments = "-NoProfile -NonInteractive -File `"$orphanRaceParentPath`" -ChildPath `"$rootOnlyChildPath`" -ChildPidPath `"$orphanRaceChildPidPath`""
|
||||
$orphanStartInfo.UseShellExecute = $false
|
||||
$orphanStartInfo.CreateNoWindow = $true
|
||||
$orphanRoot = New-Object Diagnostics.Process
|
||||
$orphanRoot.StartInfo = $orphanStartInfo
|
||||
Assert-Replay $orphanRoot.Start() "orphan_race_parent_start_failed"
|
||||
$orphanRootInfo = Get-CimInstance Win32_Process -Filter "ProcessId = $([int]$orphanRoot.Id)" -ErrorAction Stop | Select-Object -First 1
|
||||
$orphanRootIdentity = ConvertTo-AgentProcessIdentitySnapshot $orphanRootInfo
|
||||
Assert-Replay ([bool]$orphanRootIdentity) "orphan_race_root_identity_capture_failed"
|
||||
Assert-Replay $orphanRoot.WaitForExit(8000) "orphan_race_parent_did_not_exit"
|
||||
Assert-Replay (Test-Path -LiteralPath $orphanRaceChildPidPath -PathType Leaf) "orphan_race_child_pid_missing"
|
||||
$orphanIdentities = @(Get-AgentProcessTreeIdentitySnapshot $orphanRootIdentity $orphanRoot.ExitTime.ToUniversalTime())
|
||||
Assert-Replay ($orphanIdentities.Count -ge 1) "orphan_race_descendant_snapshot_missing"
|
||||
Assert-Replay (@($orphanIdentities | Where-Object { [int]$_.processId -eq [int]$orphanRoot.Id }).Count -eq 0) "orphan_race_root_unexpectedly_present"
|
||||
$orphanRaceChildIdentity = @($orphanIdentities | Select-Object -First 1)[0]
|
||||
$orphanRace = Stop-AgentBoundedProcessSnapshot $orphanRoot $orphanIdentities "Tree"
|
||||
Assert-Replay ($orphanRace.processTreeStopped -and $orphanRace.processScopeStopped) "orphan_race_tree_false_green"
|
||||
Assert-Replay (-not (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) "orphan_race_child_still_active"
|
||||
|
||||
$pidReuseRootCreated = [datetime]::UtcNow.AddMinutes(-5)
|
||||
$pidReuseRootExited = $pidReuseRootCreated.AddSeconds(2)
|
||||
$pidReuseExpectedRoot = [pscustomobject]@{
|
||||
processId = 9001
|
||||
parentProcessId = 7000
|
||||
creationUtc = $pidReuseRootCreated.ToString("o")
|
||||
name = "powershell.exe"
|
||||
}
|
||||
$pidReuseProcesses = @(
|
||||
[pscustomobject]@{ ProcessId = 9001; ParentProcessId = 8000; CreationDate = $pidReuseRootExited.AddSeconds(1); Name = "powershell.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9002; ParentProcessId = 9001; CreationDate = $pidReuseRootCreated.AddSeconds(1); Name = "cmd.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9003; ParentProcessId = 9002; CreationDate = $pidReuseRootExited.AddSeconds(1); Name = "conhost.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9004; ParentProcessId = 9001; CreationDate = $pidReuseRootExited.AddSeconds(2); Name = "not-agent99.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9005; ParentProcessId = 9002; CreationDate = $pidReuseRootCreated.AddMilliseconds(500); Name = "older-than-parent.exe" }
|
||||
)
|
||||
$pidReuseSelection = @(Select-AgentProcessTreeIdentitySnapshot $pidReuseProcesses $pidReuseExpectedRoot $pidReuseRootExited)
|
||||
$pidReuseSelectedIds = @($pidReuseSelection | ForEach-Object { [int]$_.processId } | Sort-Object)
|
||||
Assert-Replay (($pidReuseSelectedIds -join ",") -eq "9002,9003") "pid_reuse_selection_not_identity_bounded"
|
||||
|
||||
$liveRootCreated = [datetime]::UtcNow.AddMinutes(-1)
|
||||
$liveRootIdentity = [pscustomobject]@{ processId = 9101; parentProcessId = 7000; creationUtc = $liveRootCreated.ToString("o"); name = "powershell.exe" }
|
||||
$liveRootProcesses = @(
|
||||
[pscustomobject]@{ ProcessId = 9101; ParentProcessId = 7000; CreationDate = $liveRootCreated; Name = "powershell.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9102; ParentProcessId = 9101; CreationDate = $liveRootCreated.AddSeconds(1); Name = "valid-child.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9103; ParentProcessId = 9101; CreationDate = $liveRootCreated.AddSeconds(-1); Name = "older-unrelated.exe" }
|
||||
)
|
||||
$creationBoundSelection = @(Select-AgentProcessTreeIdentitySnapshot $liveRootProcesses $liveRootIdentity ([datetime]::MinValue))
|
||||
$creationBoundIds = @($creationBoundSelection | ForEach-Object { [int]$_.processId } | Sort-Object)
|
||||
Assert-Replay (($creationBoundIds -join ",") -eq "9101,9102") "live_root_child_creation_bound_failed"
|
||||
|
||||
$stopReadbackChildPidPath = Join-Path $root "stop-readback-failure-child.pid"
|
||||
function Get-AgentProcessIdentityReadback { [pscustomobject]@{ known = $false; active = $false; reason = "synthetic_stop_readback_failure" } }
|
||||
$stopReadbackFailure = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $stopReadbackChildPidPath) `
|
||||
-TimeoutSeconds 2 `
|
||||
-MaximumOutputBytes 4096
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessIdentityReadback")))
|
||||
Assert-Replay ($stopReadbackFailure.timedOut -and $stopReadbackFailure.identityReadbackFailed) "stop_readback_failure_not_detected"
|
||||
Assert-Replay ($stopReadbackFailure.treeReadbackFailed -and $stopReadbackFailure.rootFallbackTerminationUsed) "stop_readback_failure_not_fail_closed"
|
||||
Assert-Replay ($stopReadbackFailure.processScopeStopped -and -not $stopReadbackFailure.processTreeStopped) "stop_readback_failure_tree_false_green"
|
||||
Assert-Replay (Test-Path -LiteralPath $stopReadbackChildPidPath -PathType Leaf) "stop_readback_failure_child_pid_missing"
|
||||
$stopReadbackChildPid = [int]([IO.File]::ReadAllText($stopReadbackChildPidPath))
|
||||
$stopReadbackChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $stopReadbackChildPid" -ErrorAction Stop
|
||||
$stopReadbackFailureChildIdentity = ConvertTo-AgentProcessIdentitySnapshot $stopReadbackChildProcess
|
||||
Assert-Replay (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity) "stop_readback_failure_child_unexpectedly_stopped"
|
||||
|
||||
$cimFailureChildPidPath = Join-Path $root "cim-failure-child.pid"
|
||||
function Get-AgentProcessTreeIdentitySnapshot { throw "synthetic_cim_failure" }
|
||||
$cimFailure = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $cimFailureChildPidPath) `
|
||||
-TimeoutSeconds 2 `
|
||||
-MaximumOutputBytes 4096
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessTreeIdentitySnapshot")))
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessIdentityReadback")))
|
||||
Assert-Replay ($cimFailure.timedOut -and $cimFailure.treeReadbackFailed) "cim_failure_not_detected"
|
||||
Assert-Replay ($cimFailure.rootFallbackTerminationUsed -and $cimFailure.processScopeStopped) "cim_failure_root_not_stopped"
|
||||
Assert-Replay (-not $cimFailure.processTreeStopped) "cim_failure_tree_false_green"
|
||||
Assert-Replay (Test-Path -LiteralPath $cimFailureChildPidPath -PathType Leaf) "cim_failure_child_pid_missing"
|
||||
$cimFailureChildPid = [int]([IO.File]::ReadAllText($cimFailureChildPidPath))
|
||||
$cimFailureChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $cimFailureChildPid" -ErrorAction Stop
|
||||
$cimFailureChildIdentity = [pscustomobject]@{
|
||||
processId = [int]$cimFailureChildProcess.ProcessId
|
||||
parentProcessId = [int]$cimFailureChildProcess.ParentProcessId
|
||||
creationUtc = ([datetime]$cimFailureChildProcess.CreationDate).ToUniversalTime().ToString("o")
|
||||
name = [string]$cimFailureChildProcess.Name
|
||||
}
|
||||
Assert-Replay (Test-AgentProcessIdentityActive $cimFailureChildIdentity) "cim_failure_child_unexpectedly_stopped"
|
||||
} finally {
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessTreeIdentitySnapshot")))
|
||||
if ($rootOnlyChildIdentity -and (Test-AgentProcessIdentityActive $rootOnlyChildIdentity)) {
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$rootOnlyChildIdentity.processId) /T /F 2>&1 | Out-Null
|
||||
[void](Wait-AgentProcessTreeIdentityAbsent @($rootOnlyChildIdentity) 8)
|
||||
}
|
||||
if ($cimFailureChildIdentity -and (Test-AgentProcessIdentityActive $cimFailureChildIdentity)) {
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$cimFailureChildIdentity.processId) /T /F 2>&1 | Out-Null
|
||||
[void](Wait-AgentProcessTreeIdentityAbsent @($cimFailureChildIdentity) 8)
|
||||
}
|
||||
if ($orphanRaceChildIdentity -and (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) {
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$orphanRaceChildIdentity.processId) /T /F 2>&1 | Out-Null
|
||||
[void](Wait-AgentProcessTreeIdentityAbsent @($orphanRaceChildIdentity) 8)
|
||||
}
|
||||
if ($stopReadbackFailureChildIdentity -and (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity)) {
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$stopReadbackFailureChildIdentity.processId) /T /F 2>&1 | Out-Null
|
||||
[void](Wait-AgentProcessTreeIdentityAbsent @($stopReadbackFailureChildIdentity) 8)
|
||||
}
|
||||
Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue
|
||||
$cleanupVerified = [bool](
|
||||
-not (Test-Path -LiteralPath $root) -and
|
||||
(-not $rootOnlyChildIdentity -or -not (Test-AgentProcessIdentityActive $rootOnlyChildIdentity)) -and
|
||||
(-not $orphanRaceChildIdentity -or -not (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) -and
|
||||
(-not $stopReadbackFailureChildIdentity -or -not (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity)) -and
|
||||
(-not $cimFailureChildIdentity -or -not (Test-AgentProcessIdentityActive $cimFailureChildIdentity))
|
||||
)
|
||||
}
|
||||
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_bounded_process_replay_v2"
|
||||
ok = [bool]($normal.ok -and $timeout.timedOut -and $timeout.processTreeStopped -and $flood.outputLimitExceeded -and $flood.processTreeStopped -and $rootOnly.timedOut -and $rootOnly.processScopeStopped -and -not $rootOnly.processTreeStopped -and $orphanRace.processTreeStopped -and $pidReuseSelection.Count -eq 2 -and $creationBoundSelection.Count -eq 2 -and $stopReadbackFailure.identityReadbackFailed -and -not $stopReadbackFailure.processTreeStopped -and $cimFailure.timedOut -and $cimFailure.treeReadbackFailed -and $cimFailure.rootFallbackTerminationUsed -and $cimFailure.processScopeStopped -and -not $cimFailure.processTreeStopped -and $cleanupVerified)
|
||||
normal = [pscustomobject]@{
|
||||
exitCode = [int]$normal.exitCode
|
||||
completed = [bool]$normal.completed
|
||||
}
|
||||
timeout = [pscustomobject]@{
|
||||
exitCode = [int]$timeout.exitCode
|
||||
timedOut = [bool]$timeout.timedOut
|
||||
processTreeStopped = [bool]$timeout.processTreeStopped
|
||||
killedIdentityCount = [int]$timeout.killedIdentityCount
|
||||
}
|
||||
flood = [pscustomobject]@{
|
||||
exitCode = [int]$flood.exitCode
|
||||
outputLimitExceeded = [bool]$flood.outputLimitExceeded
|
||||
capturedBytes = [long]$flood.capturedBytes
|
||||
maximumOutputBytes = [long]$flood.maximumOutputBytes
|
||||
processTreeStopped = [bool]$flood.processTreeStopped
|
||||
}
|
||||
rootOnly = [pscustomobject]@{
|
||||
exitCode = [int]$rootOnly.exitCode
|
||||
timedOut = [bool]$rootOnly.timedOut
|
||||
terminationScope = [string]$rootOnly.terminationScope
|
||||
processScopeStopped = [bool]$rootOnly.processScopeStopped
|
||||
processTreeStopped = [bool]$rootOnly.processTreeStopped
|
||||
preservedDescendantCount = [int]$rootOnly.preservedDescendantCount
|
||||
}
|
||||
orphanRace = [pscustomobject]@{
|
||||
rootExitedBeforeSnapshot = $true
|
||||
descendantSnapshotCount = [int]$orphanIdentities.Count
|
||||
processScopeStopped = [bool]$orphanRace.processScopeStopped
|
||||
processTreeStopped = [bool]$orphanRace.processTreeStopped
|
||||
}
|
||||
pidReuseSelection = [pscustomobject]@{
|
||||
reusedRootExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9001 }).Count -eq 0)
|
||||
originalChildIncluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9002 }).Count -eq 1)
|
||||
originalGrandchildIncluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9003 }).Count -eq 1)
|
||||
reusedRootChildExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9004 }).Count -eq 0)
|
||||
intermediateOlderProcessExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9005 }).Count -eq 0)
|
||||
}
|
||||
creationBounds = [pscustomobject]@{
|
||||
liveRootIncluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9101 }).Count -eq 1)
|
||||
validChildIncluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9102 }).Count -eq 1)
|
||||
olderUnrelatedChildExcluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9103 }).Count -eq 0)
|
||||
}
|
||||
stopReadbackFailure = [pscustomobject]@{
|
||||
identityReadbackFailed = [bool]$stopReadbackFailure.identityReadbackFailed
|
||||
treeReadbackFailed = [bool]$stopReadbackFailure.treeReadbackFailed
|
||||
rootFallbackTerminationUsed = [bool]$stopReadbackFailure.rootFallbackTerminationUsed
|
||||
processScopeStopped = [bool]$stopReadbackFailure.processScopeStopped
|
||||
processTreeStopped = [bool]$stopReadbackFailure.processTreeStopped
|
||||
}
|
||||
cimFailure = [pscustomobject]@{
|
||||
exitCode = [int]$cimFailure.exitCode
|
||||
timedOut = [bool]$cimFailure.timedOut
|
||||
treeReadbackFailed = [bool]$cimFailure.treeReadbackFailed
|
||||
rootFallbackTerminationUsed = [bool]$cimFailure.rootFallbackTerminationUsed
|
||||
processScopeStopped = [bool]$cimFailure.processScopeStopped
|
||||
processTreeStopped = [bool]$cimFailure.processTreeStopped
|
||||
}
|
||||
cleanupVerified = $cleanupVerified
|
||||
productionPathWritePerformed = $false
|
||||
vmPowerChangePerformed = $false
|
||||
hostRebootPerformed = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress -Depth 5
|
||||
@@ -0,0 +1,55 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BreakglassSource
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$tokens = $null
|
||||
$errors = $null
|
||||
$ast = [Management.Automation.Language.Parser]::ParseInput(
|
||||
$BreakglassSource,
|
||||
[ref]$tokens,
|
||||
[ref]$errors
|
||||
)
|
||||
if ($errors.Count -ne 0) { throw "breakglass_parse_failed" }
|
||||
$matches = @($ast.FindAll({
|
||||
param($node)
|
||||
$node -is [Management.Automation.Language.FunctionDefinitionAst] -and
|
||||
$node.Name -eq "Get-FailedMutationReadback"
|
||||
}, $true))
|
||||
if ($matches.Count -ne 1) { throw "failure_readback_function_identity_mismatch" }
|
||||
. ([ScriptBlock]::Create([string]$matches[0].Extent.Text))
|
||||
|
||||
function Wait-ExactProcessIdentityAbsent { throw "synthetic_cim_failure" }
|
||||
function Test-RecoveryMutexActive { throw "synthetic_mutex_readback_failure" }
|
||||
function Test-Path { param([string]$LiteralPath); throw "synthetic_filesystem_readback_failure" }
|
||||
|
||||
$readback = Get-FailedMutationReadback `
|
||||
1001 ([datetime]::UtcNow.AddHours(-1)) `
|
||||
1002 ([datetime]::UtcNow.AddHours(-1)) `
|
||||
"C:\synthetic\recovery-transport-lease.json"
|
||||
$expected = @(
|
||||
"parent_identity_unknown",
|
||||
"child_identity_unknown",
|
||||
"recovery_mutex_unknown",
|
||||
"lease_metadata_unknown"
|
||||
)
|
||||
if ($readback.complete) { throw "failure_readback_false_green" }
|
||||
if ($readback.parentAbsent -or $readback.childAbsent -or $readback.recoveryMutexReleased -or $readback.leaseRemoved) {
|
||||
throw "failure_readback_unknown_projected_true"
|
||||
}
|
||||
if ((Compare-Object $expected @($readback.errors)).Count -ne 0) { throw "failure_readback_errors_incomplete" }
|
||||
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_breakglass_failure_readback_replay_v1"
|
||||
ok = $true
|
||||
complete = [bool]$readback.complete
|
||||
errors = @($readback.errors)
|
||||
terminalReceiptConstructionCanContinue = $true
|
||||
runtimeWritePerformed = $false
|
||||
processTerminationPerformed = $false
|
||||
vmPowerChangePerformed = $false
|
||||
hostRebootPerformed = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress -Depth 4
|
||||
@@ -0,0 +1,160 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$TelegramInboxSource,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$SubmitRequestSource
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-Replay {
|
||||
param([bool]$Condition, [string]$Reason)
|
||||
if (-not $Condition) { throw $Reason }
|
||||
}
|
||||
|
||||
function Invoke-SyntheticCase {
|
||||
param(
|
||||
[string]$Name,
|
||||
[string]$Text
|
||||
)
|
||||
|
||||
$root = Join-Path $env:TEMP ("agent99-telegram-ingress-replay-" + $Name + "-" + [guid]::NewGuid().ToString("N"))
|
||||
$bin = Join-Path $root "bin"
|
||||
$config = Join-Path $root "config"
|
||||
$telegramPath = Join-Path $bin "agent99-telegram-inbox.ps1"
|
||||
$submitPath = Join-Path $bin "agent99-submit-request.ps1"
|
||||
$stdoutPath = Join-Path $root "telegram.out"
|
||||
$stderrPath = Join-Path $root "telegram.err"
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $bin, $config -Force | Out-Null
|
||||
$encoding = New-Object Text.UTF8Encoding($true)
|
||||
[IO.File]::WriteAllText($telegramPath, $TelegramInboxSource, $encoding)
|
||||
[IO.File]::WriteAllText($submitPath, $SubmitRequestSource, $encoding)
|
||||
[IO.File]::WriteAllText(
|
||||
(Join-Path $config "agent99.config.json"),
|
||||
'{"telegram":{"autoIngestMonitoringAlerts":true}}',
|
||||
(New-Object Text.UTF8Encoding($false))
|
||||
)
|
||||
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$output = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $Text -SyntheticChatId "synthetic-chat" 2>$stderrPath)
|
||||
$exitCode = [int]$LASTEXITCODE
|
||||
[IO.File]::WriteAllLines($stdoutPath, @($output | ForEach-Object { [string]$_ }), (New-Object Text.UTF8Encoding($false)))
|
||||
$payload = (($output | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json
|
||||
$queueFiles = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
$evidenceFiles = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-TelegramInbox-*.json" -ErrorAction SilentlyContinue)
|
||||
$submitOutFile = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-telegram-submit-*.out" -ErrorAction SilentlyContinue | Select-Object -First 1)
|
||||
$submitErrFile = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-telegram-submit-*.err" -ErrorAction SilentlyContinue | Select-Object -First 1)
|
||||
[pscustomobject]@{
|
||||
name = $Name
|
||||
exitCode = $exitCode
|
||||
ok = [bool]$payload.ok
|
||||
reason = [string]$payload.reason
|
||||
processedCount = [int]$payload.processedCount
|
||||
failedUpdateCount = [int]$payload.failedUpdateCount
|
||||
durableQueued = [bool](@($payload.processed | Where-Object { $_.durableQueued }).Count -gt 0)
|
||||
submitExitCode = if (@($payload.processed).Count -gt 0) { [int]$payload.processed[0].submitExitCode } else { $null }
|
||||
failureReason = if (@($payload.processed).Count -gt 0) { [string]$payload.processed[0].failureReason } else { "" }
|
||||
submitStdoutBytes = if ($submitOutFile.Count -eq 1) { [long]$submitOutFile[0].Length } else { 0 }
|
||||
submitStderrBytes = if ($submitErrFile.Count -eq 1) { [long]$submitErrFile[0].Length } else { 0 }
|
||||
offsetBefore = [int64]$payload.offsetBefore
|
||||
offsetAfter = [int64]$payload.offsetAfter
|
||||
queueCount = $queueFiles.Count
|
||||
evidenceCount = $evidenceFiles.Count
|
||||
offsetFilePresent = Test-Path -LiteralPath (Join-Path $root "state\telegram-inbox-offset.json")
|
||||
stderrPresent = [bool]((Test-Path -LiteralPath $stderrPath) -and (Get-Item -LiteralPath $stderrPath).Length -gt 0)
|
||||
}
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path -LiteralPath $root) { throw "synthetic_case_cleanup_failed:$Name" }
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-QueueCommitReplayCase {
|
||||
$root = Join-Path $env:TEMP ("agent99-telegram-ingress-replay-reentry-" + [guid]::NewGuid().ToString("N"))
|
||||
$bin = Join-Path $root "bin"
|
||||
$config = Join-Path $root "config"
|
||||
$telegramPath = Join-Path $bin "agent99-telegram-inbox.ps1"
|
||||
$submitPath = Join-Path $bin "agent99-submit-request.ps1"
|
||||
$fixedUpdateId = [int64]1784464999
|
||||
$instruction = "/agent99 status readback"
|
||||
$needle = '$submitReceipt = Write-AgentDurableJson $result $submitEvidence $id $ExternalId'
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $bin, $config -Force | Out-Null
|
||||
$encoding = New-Object Text.UTF8Encoding($true)
|
||||
[IO.File]::WriteAllText($telegramPath, $TelegramInboxSource, $encoding)
|
||||
[IO.File]::WriteAllText(
|
||||
(Join-Path $config "agent99.config.json"),
|
||||
'{"telegram":{"autoIngestMonitoringAlerts":true}}',
|
||||
(New-Object Text.UTF8Encoding($false))
|
||||
)
|
||||
Assert-Replay $SubmitRequestSource.Contains($needle) "queue_commit_fault_marker_missing"
|
||||
$faultedSubmit = $SubmitRequestSource.Replace($needle, ('throw "synthetic_after_queue_commit"' + "`r`n" + $needle))
|
||||
[IO.File]::WriteAllText($submitPath, $faultedSubmit, $encoding)
|
||||
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$firstOutput = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $instruction -SyntheticUpdateId $fixedUpdateId -SyntheticChatId "synthetic-chat" 2>$null)
|
||||
$firstExit = [int]$LASTEXITCODE
|
||||
$firstPayload = (($firstOutput | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json
|
||||
$firstQueue = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
$firstRequests = @(Get-ChildItem -LiteralPath (Join-Path $root "requests") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
Assert-Replay ($firstExit -ne 0 -and -not [bool]$firstPayload.ok) "queue_commit_fault_false_green"
|
||||
Assert-Replay ($firstQueue.Count -eq 1 -and $firstRequests.Count -eq 1) "queue_commit_fault_artifact_count_invalid"
|
||||
$firstQueuePayload = Get-Content -LiteralPath $firstQueue[0].FullName -Raw | ConvertFrom-Json
|
||||
|
||||
[IO.File]::WriteAllText($submitPath, $SubmitRequestSource, $encoding)
|
||||
$secondOutput = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $instruction -SyntheticUpdateId $fixedUpdateId -SyntheticChatId "synthetic-chat" 2>$null)
|
||||
$secondExit = [int]$LASTEXITCODE
|
||||
$secondPayload = (($secondOutput | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json
|
||||
$secondQueue = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
$secondRequests = @(Get-ChildItem -LiteralPath (Join-Path $root "requests") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
$processed = @($secondPayload.processed)
|
||||
Assert-Replay ($secondExit -eq 0 -and [bool]$secondPayload.ok) "queue_commit_retry_failed"
|
||||
Assert-Replay ($secondQueue.Count -eq 1 -and $secondRequests.Count -eq 1) "queue_commit_retry_duplicated_artifact"
|
||||
Assert-Replay ($processed.Count -eq 1 -and [bool]$processed[0].durableQueued -and [bool]$processed[0].idempotentReplay) "queue_commit_retry_not_idempotent"
|
||||
Assert-Replay ([string]$processed[0].queueId -eq [string]$firstQueuePayload.id) "queue_commit_retry_identity_changed"
|
||||
[pscustomobject]@{
|
||||
firstExitCode = $firstExit
|
||||
firstReason = [string]$firstPayload.reason
|
||||
secondExitCode = $secondExit
|
||||
secondReason = [string]$secondPayload.reason
|
||||
requestId = [string]$firstQueuePayload.id
|
||||
queueCountAfterRetry = $secondQueue.Count
|
||||
requestCountAfterRetry = $secondRequests.Count
|
||||
idempotentReplay = [bool]$processed[0].idempotentReplay
|
||||
offsetBefore = [int64]$secondPayload.offsetBefore
|
||||
offsetAfter = [int64]$secondPayload.offsetAfter
|
||||
}
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path -LiteralPath $root) { throw "queue_commit_replay_cleanup_failed" }
|
||||
}
|
||||
}
|
||||
|
||||
$success = Invoke-SyntheticCase "success" "/agent99 status readback"
|
||||
$failure = Invoke-SyntheticCase "enqueue-failure" ("/agent99 " + ("A" * 9000))
|
||||
$queueCommitReplay = Invoke-QueueCommitReplayCase
|
||||
|
||||
Assert-Replay ($success.exitCode -eq 0 -and $success.ok) ("success_case_failed:" + ($success | ConvertTo-Json -Compress))
|
||||
Assert-Replay ($success.processedCount -eq 1 -and $success.durableQueued -and $success.queueCount -eq 1) "success_queue_not_durable"
|
||||
Assert-Replay ($success.evidenceCount -eq 1 -and -not $success.offsetFilePresent) "success_evidence_or_synthetic_offset_invalid"
|
||||
Assert-Replay ($failure.exitCode -ne 0 -and -not $failure.ok) ("failure_case_false_green:" + ($failure | ConvertTo-Json -Compress))
|
||||
Assert-Replay ($failure.reason -eq "update_processing_failed_no_checkpoint_past_failure") "failure_reason_mismatch"
|
||||
Assert-Replay ($failure.failedUpdateCount -eq 1 -and $failure.queueCount -eq 0) "failure_queue_or_count_invalid"
|
||||
Assert-Replay ($failure.offsetBefore -eq $failure.offsetAfter -and -not $failure.offsetFilePresent) "failure_offset_advanced"
|
||||
Assert-Replay ($failure.evidenceCount -eq 1) "failure_evidence_missing"
|
||||
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_telegram_durable_ingress_replay_v1"
|
||||
ok = $true
|
||||
success = $success
|
||||
enqueueFailure = $failure
|
||||
queueCommitReplay = $queueCommitReplay
|
||||
cleanupVerified = $true
|
||||
productionPathWritePerformed = $false
|
||||
scheduledTaskStarted = $false
|
||||
telegramApiCalled = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress -Depth 6
|
||||
@@ -0,0 +1,276 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
CONTROL = ROOT / "agent99-control-plane.ps1"
|
||||
BREAKGLASS = ROOT / "scripts" / "reboot-recovery" / "agent99-stale-recovery-breakglass.ps1"
|
||||
MAINTENANCE = ROOT / "scripts" / "reboot-recovery" / "agent99-control-loop-maintenance.ps1"
|
||||
REGISTER_TASKS = ROOT / "agent99-register-tasks.ps1"
|
||||
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
|
||||
TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1"
|
||||
SUBMIT_REQUEST = ROOT / "agent99-submit-request.ps1"
|
||||
BOUNDED_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-bounded-process-replay.ps1"
|
||||
TELEGRAM_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-telegram-durable-ingress-replay.ps1"
|
||||
BREAKGLASS_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-breakglass-failure-readback-replay.ps1"
|
||||
|
||||
|
||||
def test_vmrun_start_is_process_aware_and_bounded() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
|
||||
assert "function Get-AgentVmwareVmxProcessReadback" in source
|
||||
assert "vmx_process_already_running" in source
|
||||
assert "vmx_process_readback_failed" in source
|
||||
assert "function Invoke-AgentBoundedVmrunStart" in source
|
||||
assert "Get-AgentVmwareVmxProcessReadback ([string]$vm.vmx)" in source
|
||||
assert "Invoke-AgentBoundedVmrunStart" in source
|
||||
assert '& $vmrun start $vm.vmx nogui' not in source
|
||||
|
||||
|
||||
def test_bounded_runner_kills_trees_but_vmrun_preserves_started_vm_descendants() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
function = source.split("function Invoke-AgentBoundedProcess", 1)[1].split(
|
||||
"function Test-PublicRoutes", 1
|
||||
)[0]
|
||||
|
||||
assert "$stopwatch.Elapsed.TotalSeconds -ge $TimeoutSeconds" in function
|
||||
assert "StandardOutput.ReadAsync" in function
|
||||
assert "StandardError.ReadAsync" in function
|
||||
assert "($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes" in function
|
||||
assert function.index("($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes") < function.index("$process.WaitForExit()")
|
||||
assert '[ValidateSet("Tree", "Root")]' in function
|
||||
assert "processScopeStopped" in function
|
||||
assert "preservedDescendantCount" in function
|
||||
assert "Get-AgentProcessTreeIdentitySnapshot $rootProcessIdentity $rootExitedUtc" in function
|
||||
assert "ConvertTo-AgentProcessIdentitySnapshot $rootProcessInfo" in function
|
||||
assert "rootIdentityCaptured" in function
|
||||
assert "Stop-AgentBoundedProcessSnapshot $process $treeIdentities $TerminationScope" in function
|
||||
assert "processTreeStopped" in function
|
||||
assert "Read-AgentBoundedTextFile $stdout" in function
|
||||
assert "Remove-Item -LiteralPath $stdout, $stderr" in function
|
||||
stopper = source.split("function Stop-AgentBoundedProcessSnapshot", 1)[1].split(
|
||||
"function Test-AgentProcessIdentityActive", 1
|
||||
)[0]
|
||||
assert 'taskkill.exe" /PID ([string]$rootId) /T /F' in stopper
|
||||
assert 'taskkill.exe" /PID ([string]$rootId) /F' in stopper
|
||||
assert 'taskkill.exe" /PID ([string]([int]$identity.processId)) /T /F' in stopper
|
||||
snapshot = source.split("function Select-AgentProcessTreeIdentitySnapshot", 1)[1].split(
|
||||
"function Get-AgentProcessTreeIdentitySnapshot", 1
|
||||
)[0]
|
||||
assert "Test-AgentProcessSnapshotMatch $RootIdentity $currentRoot" in snapshot
|
||||
assert 'throw "bounded_process_root_identity_changed_before_exit_readback"' in snapshot
|
||||
assert "-le $RootExitedUtc.ToUniversalTime()" in snapshot
|
||||
assert "Get-AgentCreationBoundedDescendants $Processes @($currentRoot)" in snapshot
|
||||
descendants = source.split("function Get-AgentCreationBoundedDescendants", 1)[1].split(
|
||||
"function Test-AgentProcessSnapshotMatch", 1
|
||||
)[0]
|
||||
assert "-ge $parentCreated" in descendants
|
||||
assert "System.Collections.Generic.HashSet[int]" in descendants
|
||||
assert "function Get-AgentProcessIdentityReadback" in source
|
||||
assert 'reason = "identity_readback_failed"' in source
|
||||
assert 'throw "process_tree_identity_readback_failed"' in source
|
||||
assert "identityReadbackFailed" in stopper
|
||||
vmrun = source.split("function Invoke-AgentBoundedVmrunStart", 1)[1].split(
|
||||
"function Start-ConfiguredVMs", 1
|
||||
)[0]
|
||||
assert "Invoke-AgentBoundedProcess" in vmrun
|
||||
assert "-TerminationScope Root" in vmrun
|
||||
assert "processScopeStopped" in vmrun
|
||||
|
||||
|
||||
def test_perf_guard_can_release_an_exact_stale_recovery_before_deferral() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
|
||||
assert "function Invoke-AgentStaleRecoveryTransportGuard" in source
|
||||
assert 'if ($Mode -in @("Perf", "LoadShed") -and $ControlledApply)' in source
|
||||
invocation = source.index("$staleRecoveryTransportGuard = Invoke-AgentStaleRecoveryTransportGuard")
|
||||
lease_probe = source.index("$recoveryLeaseObservation = Test-AgentRecoveryTransportLease", invocation)
|
||||
assert invocation < lease_probe
|
||||
assert "agent99_stale_recovery_guard_intent_v1" in source
|
||||
assert "agent99_stale_recovery_guard_receipt_v1" in source
|
||||
assert "$stream.Flush($true)" in source
|
||||
assert "stale_recovery_receipt_readback_mismatch" in source
|
||||
assert "stale_recovery_descendant_identity_unapproved" in source
|
||||
assert "Get-AgentProcessDescendants $recheckAll ([int]$lease.processId)" in source
|
||||
assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source
|
||||
assert "Wait-AgentCimProcessIdentityAbsent $parent 15" in source
|
||||
assert "Test-AgentCimProcessIdentityMatch $child $childBeforeKill" in source
|
||||
assert "$leaseAcquiredUtc.Ticks" in source
|
||||
assert "vmPowerChangePerformed = $false" in source
|
||||
assert "vmwareVmxProcessTerminated = $false" in source
|
||||
|
||||
|
||||
def test_breakglass_targets_only_exact_stale_vmrun_processes() -> None:
|
||||
source = BREAKGLASS.read_text(encoding="utf-8")
|
||||
|
||||
assert "lease_run_id_mismatch" in source
|
||||
assert "lease_process_id_mismatch" in source
|
||||
assert "parent_run_id_mismatch" in source
|
||||
assert "child_identity_mismatch" in source
|
||||
assert "child_vmx_mismatch" in source
|
||||
assert "recovery_mutex_not_active" in source
|
||||
assert 'taskkill.exe" /PID $ProcessId /T /F' in source
|
||||
assert "Invoke-ExactTaskkill $ExpectedChildPid" in source
|
||||
assert "prior_receipt_identity_mismatch" in source
|
||||
assert '"agent99_stale_recovery_breakglass_v2", "agent99_stale_recovery_breakglass_v3"' in source
|
||||
assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source
|
||||
assert "Wait-ExactProcessIdentityAbsent" in source
|
||||
assert "parent_cleanup_identity_changed" in source
|
||||
assert "Get-DescendantProcesses $ExpectedParentPid" in source
|
||||
assert "descendantCount = 0" in source
|
||||
assert "agent99_stale_recovery_parent_cleanup_intent_v1" in source
|
||||
assert "Write-AtomicJson $intent $intentPath" in source
|
||||
assert "Invoke-ExactTaskkill $ExpectedParentPid" in source
|
||||
assert "vmwareVmxProcessTerminated = $false" in source
|
||||
assert "vmxLockDeleted = $false" in source
|
||||
assert "hostRebootPerformed = $false" in source
|
||||
assert "Remove-Item" in source
|
||||
assert ".lck" not in source
|
||||
|
||||
|
||||
def test_breakglass_apply_is_receipted_and_check_is_no_write() -> None:
|
||||
source = BREAKGLASS.read_text(encoding="utf-8")
|
||||
|
||||
check_pos = source.index('if ($Mode -eq "Apply" -and $precheckPassed)')
|
||||
operation_intent_pos = source.index("Write-AtomicJson $operationIntent $operationIntentPath")
|
||||
child_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedChildPid")
|
||||
intent_write_pos = source.index("Write-AtomicJson $intent $intentPath")
|
||||
parent_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedParentPid")
|
||||
receipt_pos = source.index('if ($Mode -eq "Apply") {', parent_kill_pos)
|
||||
assert check_pos < operation_intent_pos < child_kill_pos < intent_write_pos < parent_kill_pos < receipt_pos
|
||||
assert "agent99_stale_recovery_breakglass_intent_v1" in source
|
||||
assert '"partial_write_failed_closed"' in source
|
||||
assert "operationFailureType" in source
|
||||
assert "Write-AtomicJson $result $script:ReceiptPath" in source
|
||||
assert "function Get-FailedMutationReadback" in source
|
||||
assert '"parent_identity_unknown"' in source
|
||||
assert '"child_identity_unknown"' in source
|
||||
assert '"recovery_mutex_unknown"' in source
|
||||
assert '"lease_metadata_unknown"' in source
|
||||
assert "postFailureReadbackComplete" in source
|
||||
assert "$stream.Flush($true)" in source
|
||||
assert "receipt_readback_mismatch" in source
|
||||
assert 'runtimeWritePerformed = $script:RuntimeWritePerformed' in source
|
||||
assert 'if ($Mode -eq "Check") { exit 0 }' in source
|
||||
|
||||
|
||||
def test_control_loop_maintenance_freeze_is_exact_and_receipted() -> None:
|
||||
source = MAINTENANCE.read_text(encoding="utf-8")
|
||||
|
||||
assert 'if ($env:COMPUTERNAME -ne "WOOO-SUPER")' in source
|
||||
assert 'agent99-run\\.ps1"?\\s+-Mode\\s+ControlTick' in source
|
||||
assert 'agent99-run\\.ps1"?\\s+-Mode\\s+Recover\\s+-ControlledApply' in source
|
||||
assert "Global\\WoooAgent99ControlLoopMaintenanceV1" in source
|
||||
assert "blocked_single_writer_active" in source
|
||||
assert "Test-ExactCreationTime $Process $ExpectedControlCreated" in source
|
||||
assert "Test-ExactCreationTime $Process $ExpectedRecoverCreated" in source
|
||||
assert "[int]$Process.ParentProcessId -eq $ExpectedRecoverParentPid" in source
|
||||
assert 'Disable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source
|
||||
assert 'Stop-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source
|
||||
assert "control_recheck_identity_mismatch" in source
|
||||
assert 'taskkill.exe" /PID $ExpectedRecoverPid /T /F' in source
|
||||
assert "$stream.Flush($true)" in source
|
||||
assert "control_loop_maintenance_frozen" in source
|
||||
assert "agent99_control_loop_maintenance_intent_v1" in source
|
||||
assert source.index("Write-DurableReceipt $operationIntent $operationIntentPath") < source.index(
|
||||
'Disable-ScheduledTask -TaskName $taskName'
|
||||
)
|
||||
assert "maintenance_partial_frozen_manual_recovery" in source
|
||||
assert "taskEnabledBefore" in source
|
||||
assert "taskEnabledAfter" in source
|
||||
assert "operationFailureType" in source
|
||||
assert "vmwareVmxProcessTerminated = $false" in source
|
||||
assert "hostRebootPerformed = $false" in source
|
||||
|
||||
|
||||
def test_control_loop_deadline_and_queue_batch_are_compatible() -> None:
|
||||
control = CONTROL.read_text(encoding="utf-8")
|
||||
tasks = REGISTER_TASKS.read_text(encoding="utf-8")
|
||||
|
||||
assert "$script:Agent99QueueOuterDeadlineSeconds = 1680" in control
|
||||
assert "-ExecutionTimeLimit (New-TimeSpan -Minutes 35)" in tasks
|
||||
control_registration = tasks.split('$controlArgs = ', 1)[1].split('$telegramInboxScript = ', 1)[0]
|
||||
assert "-Settings $controlSettings" in control_registration
|
||||
queue_selection = control.split("function Invoke-AgentQueuedCommands", 1)[1].split("$results = @()", 1)[0]
|
||||
assert "Select-Object -First 1" in queue_selection
|
||||
|
||||
|
||||
def test_ingress_signals_tasks_without_waiting_for_control_execution() -> None:
|
||||
sre = SRE_INBOX.read_text(encoding="utf-8")
|
||||
telegram = TELEGRAM_INBOX.read_text(encoding="utf-8")
|
||||
submit = SUBMIT_REQUEST.read_text(encoding="utf-8")
|
||||
|
||||
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in sre
|
||||
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in submit
|
||||
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-SRE-Alert-Inbox"' in telegram
|
||||
assert '"-InstructionBase64", (Convert-AgentBase64 $instruction)' in telegram
|
||||
assert '"-RequestedByBase64", (Convert-AgentBase64 ([string]$update.from))' in telegram
|
||||
assert '"-AgentRoot", $AgentRoot' in telegram
|
||||
assert "$null = $submit.Handle" in telegram
|
||||
assert "instruction_transport_ambiguous" in submit
|
||||
assert "scheduled_task_signaled" in sre
|
||||
assert "scheduled_task_signaled" in submit
|
||||
assert "function Write-AgentDurableJson" in submit
|
||||
assert '"request-" + $ingressDigest.Substring(0, 32)' in submit
|
||||
assert "Global\\Wooo-Agent99-Submit-" in submit
|
||||
assert "function Get-AgentIngressLifecycleReceipt" in submit
|
||||
assert 'throw "ingress_lifecycle_ambiguous"' in submit
|
||||
assert "idempotentReplay = [bool]$queueReceipt.existing" in submit
|
||||
assert "durableQueue = [bool]$queueReceipt.ok" in submit
|
||||
assert "submitEvidenceSha256" in submit
|
||||
assert "function Get-AgentDurableSubmitReadback" in telegram
|
||||
assert "Sort-Object update_id" in telegram
|
||||
assert "lastDurablyHandledUpdateId" in telegram
|
||||
assert "update_processing_failed_no_checkpoint_past_failure" in telegram
|
||||
assert "offset_checkpoint_failed_safe_replay_required" in telegram
|
||||
assert "Write-AgentDurableJson $result $evidencePath" in telegram
|
||||
assert "Write-AgentDurableJson $offsetState $offsetPath" in telegram
|
||||
assert "$maxUpdateId + 1" not in telegram
|
||||
|
||||
|
||||
def test_windows_native_bounded_process_replay_covers_timeout_tree_and_stream_cap() -> None:
|
||||
source = BOUNDED_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "timeout_descendant_not_observed" in source
|
||||
assert "streaming_output_limit_not_detected" in source
|
||||
assert "root_only_descendant_not_preserved" in source
|
||||
assert "-TerminationScope Root" in source
|
||||
assert "cim_failure_root_not_stopped" in source
|
||||
assert "cim_failure_tree_false_green" in source
|
||||
assert "orphan_race_descendant_snapshot_missing" in source
|
||||
assert "orphan_race_tree_false_green" in source
|
||||
assert "pid_reuse_selection_not_identity_bounded" in source
|
||||
assert "reusedRootExcluded" in source
|
||||
assert "reusedRootChildExcluded" in source
|
||||
assert "live_root_child_creation_bound_failed" in source
|
||||
assert "intermediateOlderProcessExcluded" in source
|
||||
assert "stop_readback_failure_not_detected" in source
|
||||
assert "stop_readback_failure_not_fail_closed" in source
|
||||
assert "stop_readback_failure_tree_false_green" in source
|
||||
assert "captured_output_exceeded_contract" in source
|
||||
assert "cleanupVerified" in source
|
||||
assert "productionPathWritePerformed = $false" in source
|
||||
|
||||
|
||||
def test_windows_native_telegram_replay_covers_durable_success_and_enqueue_failure() -> None:
|
||||
source = TELEGRAM_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "success_queue_not_durable" in source
|
||||
assert "failure_case_false_green" in source
|
||||
assert "failure_offset_advanced" in source
|
||||
assert "synthetic_after_queue_commit" in source
|
||||
assert "queue_commit_retry_duplicated_artifact" in source
|
||||
assert "queue_commit_retry_not_idempotent" in source
|
||||
assert '"A" * 9000' in source
|
||||
assert "telegramApiCalled = $false" in source
|
||||
assert "productionPathWritePerformed = $false" in source
|
||||
|
||||
|
||||
def test_windows_native_breakglass_failure_readback_replay_is_fail_closed() -> None:
|
||||
source = BREAKGLASS_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "synthetic_cim_failure" in source
|
||||
assert "synthetic_mutex_readback_failure" in source
|
||||
assert "synthetic_filesystem_readback_failure" in source
|
||||
assert "failure_readback_false_green" in source
|
||||
assert "terminalReceiptConstructionCanContinue = $true" in source
|
||||
assert "runtimeWritePerformed = $false" in source
|
||||
@@ -248,7 +248,12 @@ def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
|
||||
assert "executorReserveSeconds -ge 60" in control
|
||||
assert "queueReserveSeconds -ge 300" in control
|
||||
assert "clientReserveSeconds -ge 120" in control
|
||||
assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' in control
|
||||
queue_runner = control[
|
||||
control.index("$execution = Invoke-AgentBoundedProcess") :
|
||||
]
|
||||
assert "-TimeoutSeconds $script:Agent99QueueOuterDeadlineSeconds" in queue_runner
|
||||
assert "-MaximumOutputBytes 65536" in queue_runner
|
||||
assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' not in control
|
||||
|
||||
|
||||
def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> None:
|
||||
@@ -516,7 +521,9 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
|
||||
assert f"[string]${parameter} = \"\"" in control
|
||||
assert f'[string]`${parameter} = ""' in bootstrap
|
||||
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
|
||||
assert '[string]$dispatchIdentityValidation.reason -eq "identity_control_value_unsafe"' in control
|
||||
assert '"controlled_dispatch_identity_unsafe"' in control
|
||||
assert "reason = $identityRejectionReason" in control
|
||||
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
|
||||
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
|
||||
assert "-not $dryRun.artifactTransactionBlocked" in function
|
||||
|
||||
Reference in New Issue
Block a user