Merge remote-tracking branch 'gitea/main' into codex/host188-checkmode-diagnostics-v3-20260718
# Conflicts: # scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py
This commit is contained in:
Your Name
345
scripts/reboot-recovery/tests/agent99-bounded-process-replay.ps1
Normal file
345
scripts/reboot-recovery/tests/agent99-bounded-process-replay.ps1
Normal file
@@ -0,0 +1,345 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ControlPlaneSource
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Get-ExactFunctionSource {
|
||||
param([object]$Ast, [string]$Name)
|
||||
$matches = @($Ast.FindAll({
|
||||
param($node)
|
||||
$node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $Name
|
||||
}, $true))
|
||||
if ($matches.Count -ne 1) { throw "function_identity_mismatch:$Name" }
|
||||
[string]$matches[0].Extent.Text
|
||||
}
|
||||
|
||||
function Assert-Replay {
|
||||
param([bool]$Condition, [string]$Reason)
|
||||
if (-not $Condition) { throw $Reason }
|
||||
}
|
||||
|
||||
$tokens = $null
|
||||
$errors = $null
|
||||
$ast = [Management.Automation.Language.Parser]::ParseInput(
|
||||
$ControlPlaneSource,
|
||||
[ref]$tokens,
|
||||
[ref]$errors
|
||||
)
|
||||
if ($errors.Count -ne 0) { throw "control_plane_parse_failed" }
|
||||
foreach ($name in @(
|
||||
"Get-AgentProcessDescendants",
|
||||
"ConvertTo-AgentNativeProcessArgument",
|
||||
"ConvertTo-AgentProcessIdentitySnapshot",
|
||||
"Get-AgentCreationBoundedDescendants",
|
||||
"Test-AgentProcessSnapshotMatch",
|
||||
"Select-AgentProcessTreeIdentitySnapshot",
|
||||
"Get-AgentProcessTreeIdentitySnapshot",
|
||||
"Get-AgentProcessIdentityReadback",
|
||||
"Test-AgentProcessIdentityActive",
|
||||
"Wait-AgentProcessTreeIdentityAbsent",
|
||||
"Stop-AgentBoundedProcessSnapshot",
|
||||
"Read-AgentBoundedTextFile",
|
||||
"Invoke-AgentBoundedProcess"
|
||||
)) {
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast $name)))
|
||||
}
|
||||
|
||||
$root = Join-Path $env:TEMP ("agent99-bounded-process-replay-" + [guid]::NewGuid().ToString("N"))
|
||||
$normal = $null
|
||||
$timeout = $null
|
||||
$flood = $null
|
||||
$rootOnly = $null
|
||||
$rootOnlyChildIdentity = $null
|
||||
$orphanRace = $null
|
||||
$orphanRaceChildIdentity = $null
|
||||
$pidReuseSelection = $null
|
||||
$creationBoundSelection = $null
|
||||
$stopReadbackFailure = $null
|
||||
$stopReadbackFailureChildIdentity = $null
|
||||
$cimFailure = $null
|
||||
$cimFailureChildIdentity = $null
|
||||
$cleanupVerified = $false
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $root -Force | Out-Null
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$cmd = Join-Path $env:SystemRoot "System32\cmd.exe"
|
||||
|
||||
$normal = Invoke-AgentBoundedProcess `
|
||||
-FilePath $cmd `
|
||||
-Arguments @("/d", "/c", "echo", "AG99_BOUNDED_OK") `
|
||||
-TimeoutSeconds 5 `
|
||||
-MaximumOutputBytes 4096
|
||||
Assert-Replay ($normal.ok -and $normal.completed -and $normal.stdout -match "AG99_BOUNDED_OK") "normal_process_failed"
|
||||
|
||||
$childPath = Join-Path $root "child.ps1"
|
||||
$parentPath = Join-Path $root "parent.ps1"
|
||||
[IO.File]::WriteAllText($childPath, 'Start-Sleep -Seconds 30', (New-Object Text.UTF8Encoding($false)))
|
||||
[IO.File]::WriteAllText($parentPath, @'
|
||||
param([string]$ChildPath)
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`""
|
||||
Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null
|
||||
Start-Sleep -Seconds 30
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
$timeout = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $parentPath, "-ChildPath", $childPath) `
|
||||
-TimeoutSeconds 2 `
|
||||
-MaximumOutputBytes 4096
|
||||
Assert-Replay ($timeout.timedOut -and -not $timeout.completed) "timeout_not_detected"
|
||||
Assert-Replay ($timeout.processTreeStopped -and -not $timeout.treeReadbackFailed) "timeout_tree_not_stopped"
|
||||
Assert-Replay ([int]$timeout.killedIdentityCount -ge 2) "timeout_descendant_not_observed"
|
||||
|
||||
$floodPath = Join-Path $root "flood.ps1"
|
||||
[IO.File]::WriteAllText($floodPath, @'
|
||||
for ($index = 0; $index -lt 1000; $index++) {
|
||||
[Console]::Out.Write(('X' * 1024))
|
||||
}
|
||||
Start-Sleep -Seconds 5
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
$flood = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $floodPath) `
|
||||
-TimeoutSeconds 10 `
|
||||
-MaximumOutputBytes 4096
|
||||
Assert-Replay ($flood.outputLimitExceeded -and -not $flood.completed) "streaming_output_limit_not_detected"
|
||||
Assert-Replay ([long]$flood.capturedBytes -le [long]$flood.maximumOutputBytes) "captured_output_exceeded_contract"
|
||||
Assert-Replay ($flood.processTreeStopped -and -not $flood.treeReadbackFailed) "flood_tree_not_stopped"
|
||||
|
||||
$rootOnlyChildPidPath = Join-Path $root "root-only-child.pid"
|
||||
$rootOnlyChildPath = Join-Path $root "root-only-child.ps1"
|
||||
$rootOnlyParentPath = Join-Path $root "root-only-parent.ps1"
|
||||
[IO.File]::WriteAllText($rootOnlyChildPath, @'
|
||||
param([string]$PidPath)
|
||||
[IO.File]::WriteAllText($PidPath, [string]$PID, (New-Object Text.UTF8Encoding($false)))
|
||||
Start-Sleep -Seconds 30
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
[IO.File]::WriteAllText($rootOnlyParentPath, @'
|
||||
param([string]$ChildPath, [string]$ChildPidPath)
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`" -PidPath `"$ChildPidPath`""
|
||||
Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null
|
||||
Start-Sleep -Seconds 30
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
$rootOnly = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $rootOnlyChildPidPath) `
|
||||
-TimeoutSeconds 2 `
|
||||
-MaximumOutputBytes 4096 `
|
||||
-TerminationScope Root
|
||||
Assert-Replay ($rootOnly.timedOut -and $rootOnly.forcedTerminationPerformed) "root_only_timeout_not_detected"
|
||||
Assert-Replay ($rootOnly.processScopeStopped -and -not $rootOnly.processTreeStopped) "root_only_scope_not_preserved"
|
||||
Assert-Replay ([int]$rootOnly.preservedDescendantCount -ge 1) "root_only_descendant_not_preserved"
|
||||
Assert-Replay (Test-Path -LiteralPath $rootOnlyChildPidPath -PathType Leaf) "root_only_child_pid_missing"
|
||||
$rootOnlyChildPid = [int]([IO.File]::ReadAllText($rootOnlyChildPidPath))
|
||||
$rootOnlyChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $rootOnlyChildPid" -ErrorAction Stop
|
||||
$rootOnlyChildIdentity = [pscustomobject]@{
|
||||
processId = [int]$rootOnlyChildProcess.ProcessId
|
||||
parentProcessId = [int]$rootOnlyChildProcess.ParentProcessId
|
||||
creationUtc = ([datetime]$rootOnlyChildProcess.CreationDate).ToUniversalTime().ToString("o")
|
||||
name = [string]$rootOnlyChildProcess.Name
|
||||
}
|
||||
Assert-Replay (Test-AgentProcessIdentityActive $rootOnlyChildIdentity) "root_only_child_not_active_after_parent_stop"
|
||||
|
||||
$orphanRaceChildPidPath = Join-Path $root "orphan-race-child.pid"
|
||||
$orphanRaceParentPath = Join-Path $root "orphan-race-parent.ps1"
|
||||
[IO.File]::WriteAllText($orphanRaceParentPath, @'
|
||||
param([string]$ChildPath, [string]$ChildPidPath)
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`" -PidPath `"$ChildPidPath`""
|
||||
Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
while (-not (Test-Path -LiteralPath $ChildPidPath) -and $stopwatch.Elapsed.TotalSeconds -lt 5) {
|
||||
Start-Sleep -Milliseconds 50
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
'@, (New-Object Text.UTF8Encoding($false)))
|
||||
$orphanStartInfo = New-Object Diagnostics.ProcessStartInfo
|
||||
$orphanStartInfo.FileName = $powershell
|
||||
$orphanStartInfo.Arguments = "-NoProfile -NonInteractive -File `"$orphanRaceParentPath`" -ChildPath `"$rootOnlyChildPath`" -ChildPidPath `"$orphanRaceChildPidPath`""
|
||||
$orphanStartInfo.UseShellExecute = $false
|
||||
$orphanStartInfo.CreateNoWindow = $true
|
||||
$orphanRoot = New-Object Diagnostics.Process
|
||||
$orphanRoot.StartInfo = $orphanStartInfo
|
||||
Assert-Replay $orphanRoot.Start() "orphan_race_parent_start_failed"
|
||||
$orphanRootInfo = Get-CimInstance Win32_Process -Filter "ProcessId = $([int]$orphanRoot.Id)" -ErrorAction Stop | Select-Object -First 1
|
||||
$orphanRootIdentity = ConvertTo-AgentProcessIdentitySnapshot $orphanRootInfo
|
||||
Assert-Replay ([bool]$orphanRootIdentity) "orphan_race_root_identity_capture_failed"
|
||||
Assert-Replay $orphanRoot.WaitForExit(8000) "orphan_race_parent_did_not_exit"
|
||||
Assert-Replay (Test-Path -LiteralPath $orphanRaceChildPidPath -PathType Leaf) "orphan_race_child_pid_missing"
|
||||
$orphanIdentities = @(Get-AgentProcessTreeIdentitySnapshot $orphanRootIdentity $orphanRoot.ExitTime.ToUniversalTime())
|
||||
Assert-Replay ($orphanIdentities.Count -ge 1) "orphan_race_descendant_snapshot_missing"
|
||||
Assert-Replay (@($orphanIdentities | Where-Object { [int]$_.processId -eq [int]$orphanRoot.Id }).Count -eq 0) "orphan_race_root_unexpectedly_present"
|
||||
$orphanRaceChildIdentity = @($orphanIdentities | Select-Object -First 1)[0]
|
||||
$orphanRace = Stop-AgentBoundedProcessSnapshot $orphanRoot $orphanIdentities "Tree"
|
||||
Assert-Replay ($orphanRace.processTreeStopped -and $orphanRace.processScopeStopped) "orphan_race_tree_false_green"
|
||||
Assert-Replay (-not (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) "orphan_race_child_still_active"
|
||||
|
||||
$pidReuseRootCreated = [datetime]::UtcNow.AddMinutes(-5)
|
||||
$pidReuseRootExited = $pidReuseRootCreated.AddSeconds(2)
|
||||
$pidReuseExpectedRoot = [pscustomobject]@{
|
||||
processId = 9001
|
||||
parentProcessId = 7000
|
||||
creationUtc = $pidReuseRootCreated.ToString("o")
|
||||
name = "powershell.exe"
|
||||
}
|
||||
$pidReuseProcesses = @(
|
||||
[pscustomobject]@{ ProcessId = 9001; ParentProcessId = 8000; CreationDate = $pidReuseRootExited.AddSeconds(1); Name = "powershell.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9002; ParentProcessId = 9001; CreationDate = $pidReuseRootCreated.AddSeconds(1); Name = "cmd.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9003; ParentProcessId = 9002; CreationDate = $pidReuseRootExited.AddSeconds(1); Name = "conhost.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9004; ParentProcessId = 9001; CreationDate = $pidReuseRootExited.AddSeconds(2); Name = "not-agent99.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9005; ParentProcessId = 9002; CreationDate = $pidReuseRootCreated.AddMilliseconds(500); Name = "older-than-parent.exe" }
|
||||
)
|
||||
$pidReuseSelection = @(Select-AgentProcessTreeIdentitySnapshot $pidReuseProcesses $pidReuseExpectedRoot $pidReuseRootExited)
|
||||
$pidReuseSelectedIds = @($pidReuseSelection | ForEach-Object { [int]$_.processId } | Sort-Object)
|
||||
Assert-Replay (($pidReuseSelectedIds -join ",") -eq "9002,9003") "pid_reuse_selection_not_identity_bounded"
|
||||
|
||||
$liveRootCreated = [datetime]::UtcNow.AddMinutes(-1)
|
||||
$liveRootIdentity = [pscustomobject]@{ processId = 9101; parentProcessId = 7000; creationUtc = $liveRootCreated.ToString("o"); name = "powershell.exe" }
|
||||
$liveRootProcesses = @(
|
||||
[pscustomobject]@{ ProcessId = 9101; ParentProcessId = 7000; CreationDate = $liveRootCreated; Name = "powershell.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9102; ParentProcessId = 9101; CreationDate = $liveRootCreated.AddSeconds(1); Name = "valid-child.exe" },
|
||||
[pscustomobject]@{ ProcessId = 9103; ParentProcessId = 9101; CreationDate = $liveRootCreated.AddSeconds(-1); Name = "older-unrelated.exe" }
|
||||
)
|
||||
$creationBoundSelection = @(Select-AgentProcessTreeIdentitySnapshot $liveRootProcesses $liveRootIdentity ([datetime]::MinValue))
|
||||
$creationBoundIds = @($creationBoundSelection | ForEach-Object { [int]$_.processId } | Sort-Object)
|
||||
Assert-Replay (($creationBoundIds -join ",") -eq "9101,9102") "live_root_child_creation_bound_failed"
|
||||
|
||||
$stopReadbackChildPidPath = Join-Path $root "stop-readback-failure-child.pid"
|
||||
function Get-AgentProcessIdentityReadback { [pscustomobject]@{ known = $false; active = $false; reason = "synthetic_stop_readback_failure" } }
|
||||
$stopReadbackFailure = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $stopReadbackChildPidPath) `
|
||||
-TimeoutSeconds 2 `
|
||||
-MaximumOutputBytes 4096
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessIdentityReadback")))
|
||||
Assert-Replay ($stopReadbackFailure.timedOut -and $stopReadbackFailure.identityReadbackFailed) "stop_readback_failure_not_detected"
|
||||
Assert-Replay ($stopReadbackFailure.treeReadbackFailed -and $stopReadbackFailure.rootFallbackTerminationUsed) "stop_readback_failure_not_fail_closed"
|
||||
Assert-Replay ($stopReadbackFailure.processScopeStopped -and -not $stopReadbackFailure.processTreeStopped) "stop_readback_failure_tree_false_green"
|
||||
Assert-Replay (Test-Path -LiteralPath $stopReadbackChildPidPath -PathType Leaf) "stop_readback_failure_child_pid_missing"
|
||||
$stopReadbackChildPid = [int]([IO.File]::ReadAllText($stopReadbackChildPidPath))
|
||||
$stopReadbackChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $stopReadbackChildPid" -ErrorAction Stop
|
||||
$stopReadbackFailureChildIdentity = ConvertTo-AgentProcessIdentitySnapshot $stopReadbackChildProcess
|
||||
Assert-Replay (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity) "stop_readback_failure_child_unexpectedly_stopped"
|
||||
|
||||
$cimFailureChildPidPath = Join-Path $root "cim-failure-child.pid"
|
||||
function Get-AgentProcessTreeIdentitySnapshot { throw "synthetic_cim_failure" }
|
||||
$cimFailure = Invoke-AgentBoundedProcess `
|
||||
-FilePath $powershell `
|
||||
-Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $cimFailureChildPidPath) `
|
||||
-TimeoutSeconds 2 `
|
||||
-MaximumOutputBytes 4096
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessTreeIdentitySnapshot")))
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessIdentityReadback")))
|
||||
Assert-Replay ($cimFailure.timedOut -and $cimFailure.treeReadbackFailed) "cim_failure_not_detected"
|
||||
Assert-Replay ($cimFailure.rootFallbackTerminationUsed -and $cimFailure.processScopeStopped) "cim_failure_root_not_stopped"
|
||||
Assert-Replay (-not $cimFailure.processTreeStopped) "cim_failure_tree_false_green"
|
||||
Assert-Replay (Test-Path -LiteralPath $cimFailureChildPidPath -PathType Leaf) "cim_failure_child_pid_missing"
|
||||
$cimFailureChildPid = [int]([IO.File]::ReadAllText($cimFailureChildPidPath))
|
||||
$cimFailureChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $cimFailureChildPid" -ErrorAction Stop
|
||||
$cimFailureChildIdentity = [pscustomobject]@{
|
||||
processId = [int]$cimFailureChildProcess.ProcessId
|
||||
parentProcessId = [int]$cimFailureChildProcess.ParentProcessId
|
||||
creationUtc = ([datetime]$cimFailureChildProcess.CreationDate).ToUniversalTime().ToString("o")
|
||||
name = [string]$cimFailureChildProcess.Name
|
||||
}
|
||||
Assert-Replay (Test-AgentProcessIdentityActive $cimFailureChildIdentity) "cim_failure_child_unexpectedly_stopped"
|
||||
} finally {
|
||||
. ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessTreeIdentitySnapshot")))
|
||||
if ($rootOnlyChildIdentity -and (Test-AgentProcessIdentityActive $rootOnlyChildIdentity)) {
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$rootOnlyChildIdentity.processId) /T /F 2>&1 | Out-Null
|
||||
[void](Wait-AgentProcessTreeIdentityAbsent @($rootOnlyChildIdentity) 8)
|
||||
}
|
||||
if ($cimFailureChildIdentity -and (Test-AgentProcessIdentityActive $cimFailureChildIdentity)) {
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$cimFailureChildIdentity.processId) /T /F 2>&1 | Out-Null
|
||||
[void](Wait-AgentProcessTreeIdentityAbsent @($cimFailureChildIdentity) 8)
|
||||
}
|
||||
if ($orphanRaceChildIdentity -and (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) {
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$orphanRaceChildIdentity.processId) /T /F 2>&1 | Out-Null
|
||||
[void](Wait-AgentProcessTreeIdentityAbsent @($orphanRaceChildIdentity) 8)
|
||||
}
|
||||
if ($stopReadbackFailureChildIdentity -and (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity)) {
|
||||
& "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$stopReadbackFailureChildIdentity.processId) /T /F 2>&1 | Out-Null
|
||||
[void](Wait-AgentProcessTreeIdentityAbsent @($stopReadbackFailureChildIdentity) 8)
|
||||
}
|
||||
Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue
|
||||
$cleanupVerified = [bool](
|
||||
-not (Test-Path -LiteralPath $root) -and
|
||||
(-not $rootOnlyChildIdentity -or -not (Test-AgentProcessIdentityActive $rootOnlyChildIdentity)) -and
|
||||
(-not $orphanRaceChildIdentity -or -not (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) -and
|
||||
(-not $stopReadbackFailureChildIdentity -or -not (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity)) -and
|
||||
(-not $cimFailureChildIdentity -or -not (Test-AgentProcessIdentityActive $cimFailureChildIdentity))
|
||||
)
|
||||
}
|
||||
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_bounded_process_replay_v2"
|
||||
ok = [bool]($normal.ok -and $timeout.timedOut -and $timeout.processTreeStopped -and $flood.outputLimitExceeded -and $flood.processTreeStopped -and $rootOnly.timedOut -and $rootOnly.processScopeStopped -and -not $rootOnly.processTreeStopped -and $orphanRace.processTreeStopped -and $pidReuseSelection.Count -eq 2 -and $creationBoundSelection.Count -eq 2 -and $stopReadbackFailure.identityReadbackFailed -and -not $stopReadbackFailure.processTreeStopped -and $cimFailure.timedOut -and $cimFailure.treeReadbackFailed -and $cimFailure.rootFallbackTerminationUsed -and $cimFailure.processScopeStopped -and -not $cimFailure.processTreeStopped -and $cleanupVerified)
|
||||
normal = [pscustomobject]@{
|
||||
exitCode = [int]$normal.exitCode
|
||||
completed = [bool]$normal.completed
|
||||
}
|
||||
timeout = [pscustomobject]@{
|
||||
exitCode = [int]$timeout.exitCode
|
||||
timedOut = [bool]$timeout.timedOut
|
||||
processTreeStopped = [bool]$timeout.processTreeStopped
|
||||
killedIdentityCount = [int]$timeout.killedIdentityCount
|
||||
}
|
||||
flood = [pscustomobject]@{
|
||||
exitCode = [int]$flood.exitCode
|
||||
outputLimitExceeded = [bool]$flood.outputLimitExceeded
|
||||
capturedBytes = [long]$flood.capturedBytes
|
||||
maximumOutputBytes = [long]$flood.maximumOutputBytes
|
||||
processTreeStopped = [bool]$flood.processTreeStopped
|
||||
}
|
||||
rootOnly = [pscustomobject]@{
|
||||
exitCode = [int]$rootOnly.exitCode
|
||||
timedOut = [bool]$rootOnly.timedOut
|
||||
terminationScope = [string]$rootOnly.terminationScope
|
||||
processScopeStopped = [bool]$rootOnly.processScopeStopped
|
||||
processTreeStopped = [bool]$rootOnly.processTreeStopped
|
||||
preservedDescendantCount = [int]$rootOnly.preservedDescendantCount
|
||||
}
|
||||
orphanRace = [pscustomobject]@{
|
||||
rootExitedBeforeSnapshot = $true
|
||||
descendantSnapshotCount = [int]$orphanIdentities.Count
|
||||
processScopeStopped = [bool]$orphanRace.processScopeStopped
|
||||
processTreeStopped = [bool]$orphanRace.processTreeStopped
|
||||
}
|
||||
pidReuseSelection = [pscustomobject]@{
|
||||
reusedRootExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9001 }).Count -eq 0)
|
||||
originalChildIncluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9002 }).Count -eq 1)
|
||||
originalGrandchildIncluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9003 }).Count -eq 1)
|
||||
reusedRootChildExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9004 }).Count -eq 0)
|
||||
intermediateOlderProcessExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9005 }).Count -eq 0)
|
||||
}
|
||||
creationBounds = [pscustomobject]@{
|
||||
liveRootIncluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9101 }).Count -eq 1)
|
||||
validChildIncluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9102 }).Count -eq 1)
|
||||
olderUnrelatedChildExcluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9103 }).Count -eq 0)
|
||||
}
|
||||
stopReadbackFailure = [pscustomobject]@{
|
||||
identityReadbackFailed = [bool]$stopReadbackFailure.identityReadbackFailed
|
||||
treeReadbackFailed = [bool]$stopReadbackFailure.treeReadbackFailed
|
||||
rootFallbackTerminationUsed = [bool]$stopReadbackFailure.rootFallbackTerminationUsed
|
||||
processScopeStopped = [bool]$stopReadbackFailure.processScopeStopped
|
||||
processTreeStopped = [bool]$stopReadbackFailure.processTreeStopped
|
||||
}
|
||||
cimFailure = [pscustomobject]@{
|
||||
exitCode = [int]$cimFailure.exitCode
|
||||
timedOut = [bool]$cimFailure.timedOut
|
||||
treeReadbackFailed = [bool]$cimFailure.treeReadbackFailed
|
||||
rootFallbackTerminationUsed = [bool]$cimFailure.rootFallbackTerminationUsed
|
||||
processScopeStopped = [bool]$cimFailure.processScopeStopped
|
||||
processTreeStopped = [bool]$cimFailure.processTreeStopped
|
||||
}
|
||||
cleanupVerified = $cleanupVerified
|
||||
productionPathWritePerformed = $false
|
||||
vmPowerChangePerformed = $false
|
||||
hostRebootPerformed = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress -Depth 5
|
||||
@@ -0,0 +1,55 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BreakglassSource
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$tokens = $null
|
||||
$errors = $null
|
||||
$ast = [Management.Automation.Language.Parser]::ParseInput(
|
||||
$BreakglassSource,
|
||||
[ref]$tokens,
|
||||
[ref]$errors
|
||||
)
|
||||
if ($errors.Count -ne 0) { throw "breakglass_parse_failed" }
|
||||
$matches = @($ast.FindAll({
|
||||
param($node)
|
||||
$node -is [Management.Automation.Language.FunctionDefinitionAst] -and
|
||||
$node.Name -eq "Get-FailedMutationReadback"
|
||||
}, $true))
|
||||
if ($matches.Count -ne 1) { throw "failure_readback_function_identity_mismatch" }
|
||||
. ([ScriptBlock]::Create([string]$matches[0].Extent.Text))
|
||||
|
||||
function Wait-ExactProcessIdentityAbsent { throw "synthetic_cim_failure" }
|
||||
function Test-RecoveryMutexActive { throw "synthetic_mutex_readback_failure" }
|
||||
function Test-Path { param([string]$LiteralPath); throw "synthetic_filesystem_readback_failure" }
|
||||
|
||||
$readback = Get-FailedMutationReadback `
|
||||
1001 ([datetime]::UtcNow.AddHours(-1)) `
|
||||
1002 ([datetime]::UtcNow.AddHours(-1)) `
|
||||
"C:\synthetic\recovery-transport-lease.json"
|
||||
$expected = @(
|
||||
"parent_identity_unknown",
|
||||
"child_identity_unknown",
|
||||
"recovery_mutex_unknown",
|
||||
"lease_metadata_unknown"
|
||||
)
|
||||
if ($readback.complete) { throw "failure_readback_false_green" }
|
||||
if ($readback.parentAbsent -or $readback.childAbsent -or $readback.recoveryMutexReleased -or $readback.leaseRemoved) {
|
||||
throw "failure_readback_unknown_projected_true"
|
||||
}
|
||||
if ((Compare-Object $expected @($readback.errors)).Count -ne 0) { throw "failure_readback_errors_incomplete" }
|
||||
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_breakglass_failure_readback_replay_v1"
|
||||
ok = $true
|
||||
complete = [bool]$readback.complete
|
||||
errors = @($readback.errors)
|
||||
terminalReceiptConstructionCanContinue = $true
|
||||
runtimeWritePerformed = $false
|
||||
processTerminationPerformed = $false
|
||||
vmPowerChangePerformed = $false
|
||||
hostRebootPerformed = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress -Depth 4
|
||||
@@ -0,0 +1,160 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$TelegramInboxSource,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$SubmitRequestSource
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
function Assert-Replay {
|
||||
param([bool]$Condition, [string]$Reason)
|
||||
if (-not $Condition) { throw $Reason }
|
||||
}
|
||||
|
||||
function Invoke-SyntheticCase {
|
||||
param(
|
||||
[string]$Name,
|
||||
[string]$Text
|
||||
)
|
||||
|
||||
$root = Join-Path $env:TEMP ("agent99-telegram-ingress-replay-" + $Name + "-" + [guid]::NewGuid().ToString("N"))
|
||||
$bin = Join-Path $root "bin"
|
||||
$config = Join-Path $root "config"
|
||||
$telegramPath = Join-Path $bin "agent99-telegram-inbox.ps1"
|
||||
$submitPath = Join-Path $bin "agent99-submit-request.ps1"
|
||||
$stdoutPath = Join-Path $root "telegram.out"
|
||||
$stderrPath = Join-Path $root "telegram.err"
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $bin, $config -Force | Out-Null
|
||||
$encoding = New-Object Text.UTF8Encoding($true)
|
||||
[IO.File]::WriteAllText($telegramPath, $TelegramInboxSource, $encoding)
|
||||
[IO.File]::WriteAllText($submitPath, $SubmitRequestSource, $encoding)
|
||||
[IO.File]::WriteAllText(
|
||||
(Join-Path $config "agent99.config.json"),
|
||||
'{"telegram":{"autoIngestMonitoringAlerts":true}}',
|
||||
(New-Object Text.UTF8Encoding($false))
|
||||
)
|
||||
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$output = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $Text -SyntheticChatId "synthetic-chat" 2>$stderrPath)
|
||||
$exitCode = [int]$LASTEXITCODE
|
||||
[IO.File]::WriteAllLines($stdoutPath, @($output | ForEach-Object { [string]$_ }), (New-Object Text.UTF8Encoding($false)))
|
||||
$payload = (($output | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json
|
||||
$queueFiles = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
$evidenceFiles = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-TelegramInbox-*.json" -ErrorAction SilentlyContinue)
|
||||
$submitOutFile = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-telegram-submit-*.out" -ErrorAction SilentlyContinue | Select-Object -First 1)
|
||||
$submitErrFile = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-telegram-submit-*.err" -ErrorAction SilentlyContinue | Select-Object -First 1)
|
||||
[pscustomobject]@{
|
||||
name = $Name
|
||||
exitCode = $exitCode
|
||||
ok = [bool]$payload.ok
|
||||
reason = [string]$payload.reason
|
||||
processedCount = [int]$payload.processedCount
|
||||
failedUpdateCount = [int]$payload.failedUpdateCount
|
||||
durableQueued = [bool](@($payload.processed | Where-Object { $_.durableQueued }).Count -gt 0)
|
||||
submitExitCode = if (@($payload.processed).Count -gt 0) { [int]$payload.processed[0].submitExitCode } else { $null }
|
||||
failureReason = if (@($payload.processed).Count -gt 0) { [string]$payload.processed[0].failureReason } else { "" }
|
||||
submitStdoutBytes = if ($submitOutFile.Count -eq 1) { [long]$submitOutFile[0].Length } else { 0 }
|
||||
submitStderrBytes = if ($submitErrFile.Count -eq 1) { [long]$submitErrFile[0].Length } else { 0 }
|
||||
offsetBefore = [int64]$payload.offsetBefore
|
||||
offsetAfter = [int64]$payload.offsetAfter
|
||||
queueCount = $queueFiles.Count
|
||||
evidenceCount = $evidenceFiles.Count
|
||||
offsetFilePresent = Test-Path -LiteralPath (Join-Path $root "state\telegram-inbox-offset.json")
|
||||
stderrPresent = [bool]((Test-Path -LiteralPath $stderrPath) -and (Get-Item -LiteralPath $stderrPath).Length -gt 0)
|
||||
}
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path -LiteralPath $root) { throw "synthetic_case_cleanup_failed:$Name" }
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-QueueCommitReplayCase {
|
||||
$root = Join-Path $env:TEMP ("agent99-telegram-ingress-replay-reentry-" + [guid]::NewGuid().ToString("N"))
|
||||
$bin = Join-Path $root "bin"
|
||||
$config = Join-Path $root "config"
|
||||
$telegramPath = Join-Path $bin "agent99-telegram-inbox.ps1"
|
||||
$submitPath = Join-Path $bin "agent99-submit-request.ps1"
|
||||
$fixedUpdateId = [int64]1784464999
|
||||
$instruction = "/agent99 status readback"
|
||||
$needle = '$submitReceipt = Write-AgentDurableJson $result $submitEvidence $id $ExternalId'
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $bin, $config -Force | Out-Null
|
||||
$encoding = New-Object Text.UTF8Encoding($true)
|
||||
[IO.File]::WriteAllText($telegramPath, $TelegramInboxSource, $encoding)
|
||||
[IO.File]::WriteAllText(
|
||||
(Join-Path $config "agent99.config.json"),
|
||||
'{"telegram":{"autoIngestMonitoringAlerts":true}}',
|
||||
(New-Object Text.UTF8Encoding($false))
|
||||
)
|
||||
Assert-Replay $SubmitRequestSource.Contains($needle) "queue_commit_fault_marker_missing"
|
||||
$faultedSubmit = $SubmitRequestSource.Replace($needle, ('throw "synthetic_after_queue_commit"' + "`r`n" + $needle))
|
||||
[IO.File]::WriteAllText($submitPath, $faultedSubmit, $encoding)
|
||||
|
||||
$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$firstOutput = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $instruction -SyntheticUpdateId $fixedUpdateId -SyntheticChatId "synthetic-chat" 2>$null)
|
||||
$firstExit = [int]$LASTEXITCODE
|
||||
$firstPayload = (($firstOutput | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json
|
||||
$firstQueue = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
$firstRequests = @(Get-ChildItem -LiteralPath (Join-Path $root "requests") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
Assert-Replay ($firstExit -ne 0 -and -not [bool]$firstPayload.ok) "queue_commit_fault_false_green"
|
||||
Assert-Replay ($firstQueue.Count -eq 1 -and $firstRequests.Count -eq 1) "queue_commit_fault_artifact_count_invalid"
|
||||
$firstQueuePayload = Get-Content -LiteralPath $firstQueue[0].FullName -Raw | ConvertFrom-Json
|
||||
|
||||
[IO.File]::WriteAllText($submitPath, $SubmitRequestSource, $encoding)
|
||||
$secondOutput = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $instruction -SyntheticUpdateId $fixedUpdateId -SyntheticChatId "synthetic-chat" 2>$null)
|
||||
$secondExit = [int]$LASTEXITCODE
|
||||
$secondPayload = (($secondOutput | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json
|
||||
$secondQueue = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
$secondRequests = @(Get-ChildItem -LiteralPath (Join-Path $root "requests") -File -Filter "*.json" -ErrorAction SilentlyContinue)
|
||||
$processed = @($secondPayload.processed)
|
||||
Assert-Replay ($secondExit -eq 0 -and [bool]$secondPayload.ok) "queue_commit_retry_failed"
|
||||
Assert-Replay ($secondQueue.Count -eq 1 -and $secondRequests.Count -eq 1) "queue_commit_retry_duplicated_artifact"
|
||||
Assert-Replay ($processed.Count -eq 1 -and [bool]$processed[0].durableQueued -and [bool]$processed[0].idempotentReplay) "queue_commit_retry_not_idempotent"
|
||||
Assert-Replay ([string]$processed[0].queueId -eq [string]$firstQueuePayload.id) "queue_commit_retry_identity_changed"
|
||||
[pscustomobject]@{
|
||||
firstExitCode = $firstExit
|
||||
firstReason = [string]$firstPayload.reason
|
||||
secondExitCode = $secondExit
|
||||
secondReason = [string]$secondPayload.reason
|
||||
requestId = [string]$firstQueuePayload.id
|
||||
queueCountAfterRetry = $secondQueue.Count
|
||||
requestCountAfterRetry = $secondRequests.Count
|
||||
idempotentReplay = [bool]$processed[0].idempotentReplay
|
||||
offsetBefore = [int64]$secondPayload.offsetBefore
|
||||
offsetAfter = [int64]$secondPayload.offsetAfter
|
||||
}
|
||||
} finally {
|
||||
Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue
|
||||
if (Test-Path -LiteralPath $root) { throw "queue_commit_replay_cleanup_failed" }
|
||||
}
|
||||
}
|
||||
|
||||
$success = Invoke-SyntheticCase "success" "/agent99 status readback"
|
||||
$failure = Invoke-SyntheticCase "enqueue-failure" ("/agent99 " + ("A" * 9000))
|
||||
$queueCommitReplay = Invoke-QueueCommitReplayCase
|
||||
|
||||
Assert-Replay ($success.exitCode -eq 0 -and $success.ok) ("success_case_failed:" + ($success | ConvertTo-Json -Compress))
|
||||
Assert-Replay ($success.processedCount -eq 1 -and $success.durableQueued -and $success.queueCount -eq 1) "success_queue_not_durable"
|
||||
Assert-Replay ($success.evidenceCount -eq 1 -and -not $success.offsetFilePresent) "success_evidence_or_synthetic_offset_invalid"
|
||||
Assert-Replay ($failure.exitCode -ne 0 -and -not $failure.ok) ("failure_case_false_green:" + ($failure | ConvertTo-Json -Compress))
|
||||
Assert-Replay ($failure.reason -eq "update_processing_failed_no_checkpoint_past_failure") "failure_reason_mismatch"
|
||||
Assert-Replay ($failure.failedUpdateCount -eq 1 -and $failure.queueCount -eq 0) "failure_queue_or_count_invalid"
|
||||
Assert-Replay ($failure.offsetBefore -eq $failure.offsetAfter -and -not $failure.offsetFilePresent) "failure_offset_advanced"
|
||||
Assert-Replay ($failure.evidenceCount -eq 1) "failure_evidence_missing"
|
||||
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_telegram_durable_ingress_replay_v1"
|
||||
ok = $true
|
||||
success = $success
|
||||
enqueueFailure = $failure
|
||||
queueCommitReplay = $queueCommitReplay
|
||||
cleanupVerified = $true
|
||||
productionPathWritePerformed = $false
|
||||
scheduledTaskStarted = $false
|
||||
telegramApiCalled = $false
|
||||
secretValuesRead = $false
|
||||
} | ConvertTo-Json -Compress -Depth 6
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -24,7 +25,7 @@ def _function_source() -> str:
|
||||
|
||||
def _evidence_function_source() -> str:
|
||||
source = PREFLIGHT.read_text(encoding="utf-8")
|
||||
start = source.index("function Get-EvidenceReadback")
|
||||
start = source.index("function Get-AgentBoundedNewestFiles")
|
||||
end = source.index("$requiredTasks = @(", start)
|
||||
return source[start:end]
|
||||
|
||||
@@ -52,7 +53,8 @@ def _base_case() -> dict[str, Any]:
|
||||
"staleAlertExecutionCount": 0,
|
||||
"alertIncomingCount": 0,
|
||||
"staleAlertIncomingCount": 0,
|
||||
"pendingCommandCount": 0,
|
||||
"pendingCommandBlockingCount": 0,
|
||||
"promotionDeferredCommandCount": 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -96,7 +98,8 @@ $parameters = @{{
|
||||
StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
|
||||
AlertIncomingCount = [int]$case.alertIncomingCount
|
||||
StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount
|
||||
PendingCommandCount = [int]$case.pendingCommandCount
|
||||
PendingCommandBlockingCount = [int]$case.pendingCommandBlockingCount
|
||||
PromotionDeferredCommandCount = [int]$case.promotionDeferredCommandCount
|
||||
}}
|
||||
Get-AgentLivePreflightDecision @parameters |
|
||||
ConvertTo-Json -Depth 6 -Compress
|
||||
@@ -121,6 +124,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
assert '$warningReasons += "self_health_warning"' in decision
|
||||
assert '$warningReasons += "performance_warning"' in decision
|
||||
assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision
|
||||
assert (
|
||||
'$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"'
|
||||
in decision
|
||||
)
|
||||
assert '"critical" { $blockingReasons += "self_health_not_ok" }' in decision
|
||||
assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
|
||||
assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
|
||||
@@ -147,9 +154,45 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
assert '"agent99_background_mode_deferred_v1"' in source
|
||||
assert '"agent99_recovery_transport_deferred_v1"' in source
|
||||
assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source
|
||||
assert "[IO.Directory]::EnumerateFiles(" in source
|
||||
assert "[Collections.Generic.SortedSet[string]]::new(" in source
|
||||
assert "$null = $selected.Remove($selected.Min)" in source
|
||||
assert "candidateWindowExhausted = $candidateWindowExhausted" in source
|
||||
assert "function Get-AgentControlLoopMaintenanceReadback" in source
|
||||
assert '"agent99-control-loop-maintenance-2*.json"' in source
|
||||
assert '$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 })' in source
|
||||
assert '$legacyPropertySetMatches' in source
|
||||
assert '$legacyIdentityFieldsValid' in source
|
||||
assert '$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled' in source
|
||||
assert '"legacy_v2_exact_upgrade_bridge"' in source
|
||||
assert '$receipt.precheckPassed -is [bool]' in source
|
||||
assert '$receipt.taskEnabledAfter -is [bool]' in source
|
||||
assert '$receipt.runtimeWritePerformed -is [bool]' in source
|
||||
assert '$intent.secretValueRead -is [bool]' in source
|
||||
assert '$activeControlProcesses.Count -eq 0' in source
|
||||
assert '-not $leasePresent' in source
|
||||
assert '-not $recoveryMutexActive' in source
|
||||
assert '$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")' in source
|
||||
assert '$task | Add-Member -NotePropertyName runtimeHealthy' in source
|
||||
assert '$task | Add-Member -NotePropertyName promotionReady' in source
|
||||
assert '$controlLoopMaintenance.valid' in source
|
||||
assert "function Get-AgentPendingCommandPromotionReadback" in source
|
||||
assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source
|
||||
assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
|
||||
assert '$payload.controlledApply -is [bool] -and $payload.controlledApply' in source
|
||||
assert '$payload.observation -is [pscustomobject]' in source
|
||||
assert 'instructionContentReadback = $false' in source
|
||||
assert '$rootTaskPath = "\\"' in source
|
||||
assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source
|
||||
assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source
|
||||
assert "[string[]]$RequiredArgumentMarkers" in source
|
||||
assert "$actions.Count -eq 1" in source
|
||||
assert "actionDefinitionReady = $actionDefinitionReady" in source
|
||||
assert "principalReady = $principalReady" in source
|
||||
assert "definitionReady = $definitionReady" in source
|
||||
assert '$task.Principal.LogonType -eq "S4U"' in source
|
||||
assert '$task.Principal.RunLevel -eq "Highest"' in source
|
||||
assert "$task.definitionReady" in source
|
||||
|
||||
|
||||
def test_evidence_selection_skips_deferred_but_not_malformed_candidates(
|
||||
@@ -274,6 +317,165 @@ Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 |
|
||||
assert payload["healthy"] is False
|
||||
|
||||
|
||||
def test_evidence_candidate_window_fails_closed_when_recent_files_are_all_deferred(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
||||
|
||||
evidence = tmp_path / "evidence"
|
||||
evidence.mkdir()
|
||||
canonical = evidence / "agent99-SelfCheck-20260715-190000.json"
|
||||
canonical.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"mode": "SelfCheck",
|
||||
"selfHealth": {"severity": "ok"},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
base_time = canonical.stat().st_mtime
|
||||
os.utime(canonical, (base_time - 1000, base_time - 1000))
|
||||
for index in range(40):
|
||||
deferred = evidence / f"agent99-SelfCheck-20260715-20{index:04d}.json"
|
||||
deferred.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schemaVersion": "agent99_background_mode_deferred_v1",
|
||||
"deferred": True,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
candidate_time = base_time + index
|
||||
os.utime(deferred, (candidate_time, candidate_time))
|
||||
|
||||
agent_root = str(tmp_path).replace("'", "''")
|
||||
command = f"""
|
||||
{_evidence_function_source()}
|
||||
$AgentRoot = '{agent_root}'
|
||||
Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true |
|
||||
ConvertTo-Json -Depth 8 -Compress
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
payload = json.loads(
|
||||
next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{"))
|
||||
)
|
||||
assert payload["enumeratedCandidateCount"] == 41
|
||||
assert payload["candidateWindowLimit"] == 32
|
||||
assert payload["candidateWindowTruncated"] is True
|
||||
assert payload["candidateWindowExhausted"] is True
|
||||
assert payload["deferredCandidateCount"] == 32
|
||||
assert payload["parseError"] == "EvidenceCandidateWindowExhausted"
|
||||
assert payload["healthy"] is False
|
||||
|
||||
|
||||
def test_promotion_reserve_defers_only_exact_maintenance_auto_recover(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
||||
|
||||
queue = tmp_path / "queue"
|
||||
queue.mkdir()
|
||||
now = datetime.now(timezone.utc)
|
||||
incident_id = "agent99-auto-" + ("a" * 20)
|
||||
command_id = "auto-recover-20260719-152746-39ca4a72"
|
||||
command_path = queue / f"{command_id}.json"
|
||||
payload = {
|
||||
"approvalId": "",
|
||||
"automationRunId": "096c8173-96b9-5659-94d3-5679c3fb770f",
|
||||
"canonicalDigest": "b" * 64,
|
||||
"controlledApply": True,
|
||||
"correlationKey": "agent99-controlled:" + ("c" * 64),
|
||||
"createdAt": now.isoformat(),
|
||||
"dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1",
|
||||
"dispatchRouteId": "agent99_local_observer",
|
||||
"executionGeneration": "1",
|
||||
"externalId": "host_unreachable_detected",
|
||||
"id": command_id,
|
||||
"idempotencyKey": "agent99-controlled:" + ("c" * 64),
|
||||
"incidentId": incident_id,
|
||||
"instruction": "recover exact observed unreachable host through controlled playbook",
|
||||
"lockOwner": "096c8173-96b9-5659-94d3-5679c3fb770f",
|
||||
"mode": "Recover",
|
||||
"observation": {},
|
||||
"projectId": "awoooi",
|
||||
"reason": "host_unreachable_detected",
|
||||
"requestedBy": "Agent99",
|
||||
"singleFlightKey": "agent99-controlled-flight:" + ("d" * 64),
|
||||
"source": "agent99-recovery-observer",
|
||||
"sourceFingerprint": "e" * 64,
|
||||
"traceId": "00-" + ("f" * 32) + "-" + ("1" * 16) + "-01",
|
||||
"workItemId": f"agent99-auto:{incident_id}:host_unreachable_detected",
|
||||
}
|
||||
|
||||
function_source = _evidence_function_source()
|
||||
agent_root = str(tmp_path).replace("'", "''")
|
||||
maintenance_timestamp = (now - timedelta(minutes=10)).isoformat()
|
||||
|
||||
def run(scope: str) -> dict[str, Any]:
|
||||
command = f"""
|
||||
{function_source}
|
||||
$AgentRoot = '{agent_root}'
|
||||
$maintenance = [pscustomobject]@{{
|
||||
valid = $true
|
||||
receiptFieldsValid = $true
|
||||
intentFieldsValid = $true
|
||||
receiptTimestamp = '{maintenance_timestamp}'
|
||||
}}
|
||||
Get-AgentPendingCommandPromotionReadback -MaintenanceReadback $maintenance -Scope {scope} |
|
||||
ConvertTo-Json -Depth 8 -Compress
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
return json.loads(
|
||||
next(
|
||||
line
|
||||
for line in reversed(result.stdout.splitlines())
|
||||
if line.startswith("{")
|
||||
)
|
||||
)
|
||||
|
||||
command_path.write_bytes(b"\xef\xbb\xbf" + json.dumps(payload).encode("utf-8"))
|
||||
reserve = run("PromotionReserve")
|
||||
assert reserve["observedCount"] == 1
|
||||
assert reserve["deferredCount"] == 1
|
||||
assert reserve["blockingCount"] == 0
|
||||
assert reserve["contractValid"] is True
|
||||
assert reserve["items"][0]["instructionContentReadback"] is False
|
||||
assert reserve["items"][0]["identity"].startswith(f"{command_id}|")
|
||||
|
||||
full_runtime = run("FullRuntime")
|
||||
assert full_runtime["deferredCount"] == 0
|
||||
assert full_runtime["blockingCount"] == 1
|
||||
assert full_runtime["eligibleMaintenanceAutoRecoverCount"] == 1
|
||||
assert full_runtime["reason"] == (
|
||||
"full_runtime_requires_maintenance_auto_recover_to_drain"
|
||||
)
|
||||
|
||||
payload["mode"] = "Status"
|
||||
command_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
invalid = run("PromotionReserve")
|
||||
assert invalid["deferredCount"] == 0
|
||||
assert invalid["blockingCount"] == 1
|
||||
assert invalid["contractValid"] is False
|
||||
|
||||
|
||||
def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
@@ -348,11 +550,17 @@ def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(pendingCommandCount=1),
|
||||
_with_overrides(pendingCommandBlockingCount=1),
|
||||
False,
|
||||
("pending_work_before_reboot",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(promotionDeferredCommandCount=1),
|
||||
True,
|
||||
(),
|
||||
("maintenance_pending_commands_deferred_until_post_promotion",),
|
||||
),
|
||||
)
|
||||
|
||||
for case, expected_eligible, expected_blockers, expected_warnings in cases:
|
||||
|
||||
@@ -391,11 +391,39 @@ def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() ->
|
||||
assert "failedTasks" in source
|
||||
assert "failedEvidenceNames" in source
|
||||
assert "failedEvidence" in source
|
||||
assert "actionDefinitionReady = [bool]$_.actionDefinitionReady" in source
|
||||
assert "principalReady = [bool]$_.principalReady" in source
|
||||
assert "definitionReady = [bool]$_.definitionReady" in source
|
||||
assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source
|
||||
assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source
|
||||
assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source
|
||||
|
||||
|
||||
def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
assert "$LivePreflightTimeoutSeconds = 60" in source
|
||||
assert "$PostVerifyMaxAttempts = 4" in source
|
||||
assert "$PostVerifyRetryWaitSeconds = 45" in source
|
||||
assert "function Test-AgentPostPromotionReadinessRetryable" in source
|
||||
assert '[string]$Readiness.readinessScope -ne "FullRuntime"' in source
|
||||
assert '"required_evidence_stale"' in source
|
||||
assert '"required_evidence_content_unhealthy"' in source
|
||||
assert '$_.name -notin @("telegram_inbox", "sre_alert_inbox")' in source
|
||||
assert "-not $_.definitionReady" in source
|
||||
assert "[int]$Readiness.alertRunningCount -ne 0" in source
|
||||
assert "$pendingCommandsRetryable" in source
|
||||
assert "$ExpectedMaintenancePendingCommandIdentities" in source
|
||||
assert '"pending_work_before_reboot"' in source
|
||||
assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
|
||||
assert "maintenancePendingCommandIdentities" in source
|
||||
assert "$postVerifyAttempt -le $PostVerifyMaxAttempts" in source
|
||||
assert "Start-Sleep -Seconds $PostVerifyRetryWaitSeconds" in source
|
||||
assert "automaticTaskTriggerAllowed = $false" in source
|
||||
assert "postVerifyAttempts = $postVerifyAttempts" in source
|
||||
assert "postVerifyRetryCount = $postVerifyRetryCount" in source
|
||||
|
||||
|
||||
def test_check_and_apply_receipts_project_the_same_run_identity() -> None:
|
||||
sender = SENDER.read_text(encoding="utf-8")
|
||||
projector = sender[
|
||||
@@ -607,7 +635,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
assert "$RelayPollIntervalSeconds = 1" in source
|
||||
assert "$ValidateOnlyTimeoutSeconds = 300" in source
|
||||
assert "$LiveDeployTimeoutSeconds = 300" in source
|
||||
assert "$LivePreflightTimeoutSeconds = 120" in source
|
||||
assert "$LivePreflightTimeoutSeconds = 60" in source
|
||||
assert "$PromotionReadinessMaxAttempts = 2" in source
|
||||
assert "$PromotionReadinessRetryWaitSeconds = 30" in source
|
||||
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
@@ -648,18 +676,23 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
promotion_readiness_attempts = 2
|
||||
promotion_readiness_retry_wait_seconds = 30
|
||||
deploy_seconds = 300
|
||||
live_preflight_seconds = 120
|
||||
live_preflight_seconds = 60
|
||||
post_verify_attempts = 4
|
||||
post_verify_retry_wait_seconds = 45
|
||||
windows_control_baseline_seconds = 300
|
||||
worst_case_seconds = (
|
||||
validate_seconds
|
||||
+ (promotion_readiness_attempts * live_preflight_seconds)
|
||||
+ promotion_readiness_retry_wait_seconds
|
||||
+ deploy_seconds
|
||||
+ live_preflight_seconds
|
||||
+ (post_verify_attempts * live_preflight_seconds)
|
||||
+ ((post_verify_attempts - 1) * post_verify_retry_wait_seconds)
|
||||
+ (2 * (60 + 360))
|
||||
+ windows_control_baseline_seconds
|
||||
)
|
||||
assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source
|
||||
assert worst_case_seconds == 1830
|
||||
assert wrapper_budget_seconds - worst_case_seconds == 570
|
||||
assert worst_case_seconds == 2265
|
||||
assert wrapper_budget_seconds - worst_case_seconds == 135
|
||||
|
||||
deploy = source.index(
|
||||
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
|
||||
@@ -781,6 +814,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() -
|
||||
assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight
|
||||
assert '[int]$summary.alertIncomingCount -eq 0' not in preflight
|
||||
assert '[int]$summary.alertRunningCount -eq 0' in preflight
|
||||
assert '[int]$summary.pendingCommandBlockingCount -eq 0' in preflight
|
||||
assert '$ReadinessScope -eq "PromotionReserve"' in preflight
|
||||
assert '[int]$summary.pendingCommandCount -eq 0' in preflight
|
||||
assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight
|
||||
assert '[string[]]$blockingReasons = @()' in preflight
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1"
|
||||
|
||||
|
||||
def test_stale_queue_quarantine_is_exact_recoverable_and_receipted() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert '[ValidateSet("Check", "Apply")]' in source
|
||||
assert "ExpectedQueueSha256" in source
|
||||
assert "ExpectedMaintenanceSha256" in source
|
||||
assert '"Global\\WoooAgent99StaleQueueMaintenanceV1"' in source
|
||||
assert '"Global\\WoooAgent99RecoveryTransportV1"' in source
|
||||
assert 'Get-ScheduledTask `' in source
|
||||
assert '"Wooo-Agent99-Control-Loop"' in source
|
||||
assert 'throw "control_loop_not_frozen"' in source
|
||||
assert 'throw "active_control_or_recover_process"' in source
|
||||
assert 'throw "recovery_lease_present"' in source
|
||||
assert 'throw "recovery_mutex_active"' in source
|
||||
assert '[string]$command.mode -ne "Recover"' in source
|
||||
assert '[string]$command.source -ne "agent99-recovery-observer"' in source
|
||||
assert "$command.controlledApply -isnot [bool]" in source
|
||||
assert '"atomic_move_exact_queue_item_to_failed"' in source
|
||||
assert "[IO.File]::Move($sourcePath, $destinationPath)" in source
|
||||
assert "[IO.File]::Move($destinationPath, $sourcePath)" in source
|
||||
assert '"verified_quarantined"' in source
|
||||
assert '"failed_rolled_back"' in source
|
||||
assert '"rollback_unverified"' in source
|
||||
|
||||
|
||||
def test_stale_queue_quarantine_does_not_execute_or_delete_command() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "commandExecuted = $false" in source
|
||||
assert "deletePerformed = $false" in source
|
||||
assert "otherQueueMutationPerformed = $false" in source
|
||||
assert "rawInstructionStored = $false" in source
|
||||
assert "secretValueRead = $false" in source
|
||||
assert "Remove-Item -LiteralPath $sourcePath" not in source
|
||||
assert "Start-ScheduledTask" not in source
|
||||
assert "Invoke-Expression" not in source
|
||||
assert "& $command" not in source
|
||||
@@ -0,0 +1,276 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
CONTROL = ROOT / "agent99-control-plane.ps1"
|
||||
BREAKGLASS = ROOT / "scripts" / "reboot-recovery" / "agent99-stale-recovery-breakglass.ps1"
|
||||
MAINTENANCE = ROOT / "scripts" / "reboot-recovery" / "agent99-control-loop-maintenance.ps1"
|
||||
REGISTER_TASKS = ROOT / "agent99-register-tasks.ps1"
|
||||
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
|
||||
TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1"
|
||||
SUBMIT_REQUEST = ROOT / "agent99-submit-request.ps1"
|
||||
BOUNDED_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-bounded-process-replay.ps1"
|
||||
TELEGRAM_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-telegram-durable-ingress-replay.ps1"
|
||||
BREAKGLASS_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-breakglass-failure-readback-replay.ps1"
|
||||
|
||||
|
||||
def test_vmrun_start_is_process_aware_and_bounded() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
|
||||
assert "function Get-AgentVmwareVmxProcessReadback" in source
|
||||
assert "vmx_process_already_running" in source
|
||||
assert "vmx_process_readback_failed" in source
|
||||
assert "function Invoke-AgentBoundedVmrunStart" in source
|
||||
assert "Get-AgentVmwareVmxProcessReadback ([string]$vm.vmx)" in source
|
||||
assert "Invoke-AgentBoundedVmrunStart" in source
|
||||
assert '& $vmrun start $vm.vmx nogui' not in source
|
||||
|
||||
|
||||
def test_bounded_runner_kills_trees_but_vmrun_preserves_started_vm_descendants() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
function = source.split("function Invoke-AgentBoundedProcess", 1)[1].split(
|
||||
"function Test-PublicRoutes", 1
|
||||
)[0]
|
||||
|
||||
assert "$stopwatch.Elapsed.TotalSeconds -ge $TimeoutSeconds" in function
|
||||
assert "StandardOutput.ReadAsync" in function
|
||||
assert "StandardError.ReadAsync" in function
|
||||
assert "($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes" in function
|
||||
assert function.index("($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes") < function.index("$process.WaitForExit()")
|
||||
assert '[ValidateSet("Tree", "Root")]' in function
|
||||
assert "processScopeStopped" in function
|
||||
assert "preservedDescendantCount" in function
|
||||
assert "Get-AgentProcessTreeIdentitySnapshot $rootProcessIdentity $rootExitedUtc" in function
|
||||
assert "ConvertTo-AgentProcessIdentitySnapshot $rootProcessInfo" in function
|
||||
assert "rootIdentityCaptured" in function
|
||||
assert "Stop-AgentBoundedProcessSnapshot $process $treeIdentities $TerminationScope" in function
|
||||
assert "processTreeStopped" in function
|
||||
assert "Read-AgentBoundedTextFile $stdout" in function
|
||||
assert "Remove-Item -LiteralPath $stdout, $stderr" in function
|
||||
stopper = source.split("function Stop-AgentBoundedProcessSnapshot", 1)[1].split(
|
||||
"function Test-AgentProcessIdentityActive", 1
|
||||
)[0]
|
||||
assert 'taskkill.exe" /PID ([string]$rootId) /T /F' in stopper
|
||||
assert 'taskkill.exe" /PID ([string]$rootId) /F' in stopper
|
||||
assert 'taskkill.exe" /PID ([string]([int]$identity.processId)) /T /F' in stopper
|
||||
snapshot = source.split("function Select-AgentProcessTreeIdentitySnapshot", 1)[1].split(
|
||||
"function Get-AgentProcessTreeIdentitySnapshot", 1
|
||||
)[0]
|
||||
assert "Test-AgentProcessSnapshotMatch $RootIdentity $currentRoot" in snapshot
|
||||
assert 'throw "bounded_process_root_identity_changed_before_exit_readback"' in snapshot
|
||||
assert "-le $RootExitedUtc.ToUniversalTime()" in snapshot
|
||||
assert "Get-AgentCreationBoundedDescendants $Processes @($currentRoot)" in snapshot
|
||||
descendants = source.split("function Get-AgentCreationBoundedDescendants", 1)[1].split(
|
||||
"function Test-AgentProcessSnapshotMatch", 1
|
||||
)[0]
|
||||
assert "-ge $parentCreated" in descendants
|
||||
assert "System.Collections.Generic.HashSet[int]" in descendants
|
||||
assert "function Get-AgentProcessIdentityReadback" in source
|
||||
assert 'reason = "identity_readback_failed"' in source
|
||||
assert 'throw "process_tree_identity_readback_failed"' in source
|
||||
assert "identityReadbackFailed" in stopper
|
||||
vmrun = source.split("function Invoke-AgentBoundedVmrunStart", 1)[1].split(
|
||||
"function Start-ConfiguredVMs", 1
|
||||
)[0]
|
||||
assert "Invoke-AgentBoundedProcess" in vmrun
|
||||
assert "-TerminationScope Root" in vmrun
|
||||
assert "processScopeStopped" in vmrun
|
||||
|
||||
|
||||
def test_perf_guard_can_release_an_exact_stale_recovery_before_deferral() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
|
||||
assert "function Invoke-AgentStaleRecoveryTransportGuard" in source
|
||||
assert 'if ($Mode -in @("Perf", "LoadShed") -and $ControlledApply)' in source
|
||||
invocation = source.index("$staleRecoveryTransportGuard = Invoke-AgentStaleRecoveryTransportGuard")
|
||||
lease_probe = source.index("$recoveryLeaseObservation = Test-AgentRecoveryTransportLease", invocation)
|
||||
assert invocation < lease_probe
|
||||
assert "agent99_stale_recovery_guard_intent_v1" in source
|
||||
assert "agent99_stale_recovery_guard_receipt_v1" in source
|
||||
assert "$stream.Flush($true)" in source
|
||||
assert "stale_recovery_receipt_readback_mismatch" in source
|
||||
assert "stale_recovery_descendant_identity_unapproved" in source
|
||||
assert "Get-AgentProcessDescendants $recheckAll ([int]$lease.processId)" in source
|
||||
assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source
|
||||
assert "Wait-AgentCimProcessIdentityAbsent $parent 15" in source
|
||||
assert "Test-AgentCimProcessIdentityMatch $child $childBeforeKill" in source
|
||||
assert "$leaseAcquiredUtc.Ticks" in source
|
||||
assert "vmPowerChangePerformed = $false" in source
|
||||
assert "vmwareVmxProcessTerminated = $false" in source
|
||||
|
||||
|
||||
def test_breakglass_targets_only_exact_stale_vmrun_processes() -> None:
|
||||
source = BREAKGLASS.read_text(encoding="utf-8")
|
||||
|
||||
assert "lease_run_id_mismatch" in source
|
||||
assert "lease_process_id_mismatch" in source
|
||||
assert "parent_run_id_mismatch" in source
|
||||
assert "child_identity_mismatch" in source
|
||||
assert "child_vmx_mismatch" in source
|
||||
assert "recovery_mutex_not_active" in source
|
||||
assert 'taskkill.exe" /PID $ProcessId /T /F' in source
|
||||
assert "Invoke-ExactTaskkill $ExpectedChildPid" in source
|
||||
assert "prior_receipt_identity_mismatch" in source
|
||||
assert '"agent99_stale_recovery_breakglass_v2", "agent99_stale_recovery_breakglass_v3"' in source
|
||||
assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source
|
||||
assert "Wait-ExactProcessIdentityAbsent" in source
|
||||
assert "parent_cleanup_identity_changed" in source
|
||||
assert "Get-DescendantProcesses $ExpectedParentPid" in source
|
||||
assert "descendantCount = 0" in source
|
||||
assert "agent99_stale_recovery_parent_cleanup_intent_v1" in source
|
||||
assert "Write-AtomicJson $intent $intentPath" in source
|
||||
assert "Invoke-ExactTaskkill $ExpectedParentPid" in source
|
||||
assert "vmwareVmxProcessTerminated = $false" in source
|
||||
assert "vmxLockDeleted = $false" in source
|
||||
assert "hostRebootPerformed = $false" in source
|
||||
assert "Remove-Item" in source
|
||||
assert ".lck" not in source
|
||||
|
||||
|
||||
def test_breakglass_apply_is_receipted_and_check_is_no_write() -> None:
|
||||
source = BREAKGLASS.read_text(encoding="utf-8")
|
||||
|
||||
check_pos = source.index('if ($Mode -eq "Apply" -and $precheckPassed)')
|
||||
operation_intent_pos = source.index("Write-AtomicJson $operationIntent $operationIntentPath")
|
||||
child_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedChildPid")
|
||||
intent_write_pos = source.index("Write-AtomicJson $intent $intentPath")
|
||||
parent_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedParentPid")
|
||||
receipt_pos = source.index('if ($Mode -eq "Apply") {', parent_kill_pos)
|
||||
assert check_pos < operation_intent_pos < child_kill_pos < intent_write_pos < parent_kill_pos < receipt_pos
|
||||
assert "agent99_stale_recovery_breakglass_intent_v1" in source
|
||||
assert '"partial_write_failed_closed"' in source
|
||||
assert "operationFailureType" in source
|
||||
assert "Write-AtomicJson $result $script:ReceiptPath" in source
|
||||
assert "function Get-FailedMutationReadback" in source
|
||||
assert '"parent_identity_unknown"' in source
|
||||
assert '"child_identity_unknown"' in source
|
||||
assert '"recovery_mutex_unknown"' in source
|
||||
assert '"lease_metadata_unknown"' in source
|
||||
assert "postFailureReadbackComplete" in source
|
||||
assert "$stream.Flush($true)" in source
|
||||
assert "receipt_readback_mismatch" in source
|
||||
assert 'runtimeWritePerformed = $script:RuntimeWritePerformed' in source
|
||||
assert 'if ($Mode -eq "Check") { exit 0 }' in source
|
||||
|
||||
|
||||
def test_control_loop_maintenance_freeze_is_exact_and_receipted() -> None:
|
||||
source = MAINTENANCE.read_text(encoding="utf-8")
|
||||
|
||||
assert 'if ($env:COMPUTERNAME -ne "WOOO-SUPER")' in source
|
||||
assert 'agent99-run\\.ps1"?\\s+-Mode\\s+ControlTick' in source
|
||||
assert 'agent99-run\\.ps1"?\\s+-Mode\\s+Recover\\s+-ControlledApply' in source
|
||||
assert "Global\\WoooAgent99ControlLoopMaintenanceV1" in source
|
||||
assert "blocked_single_writer_active" in source
|
||||
assert "Test-ExactCreationTime $Process $ExpectedControlCreated" in source
|
||||
assert "Test-ExactCreationTime $Process $ExpectedRecoverCreated" in source
|
||||
assert "[int]$Process.ParentProcessId -eq $ExpectedRecoverParentPid" in source
|
||||
assert 'Disable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source
|
||||
assert 'Stop-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source
|
||||
assert "control_recheck_identity_mismatch" in source
|
||||
assert 'taskkill.exe" /PID $ExpectedRecoverPid /T /F' in source
|
||||
assert "$stream.Flush($true)" in source
|
||||
assert "control_loop_maintenance_frozen" in source
|
||||
assert "agent99_control_loop_maintenance_intent_v1" in source
|
||||
assert source.index("Write-DurableReceipt $operationIntent $operationIntentPath") < source.index(
|
||||
'Disable-ScheduledTask -TaskName $taskName'
|
||||
)
|
||||
assert "maintenance_partial_frozen_manual_recovery" in source
|
||||
assert "taskEnabledBefore" in source
|
||||
assert "taskEnabledAfter" in source
|
||||
assert "operationFailureType" in source
|
||||
assert "vmwareVmxProcessTerminated = $false" in source
|
||||
assert "hostRebootPerformed = $false" in source
|
||||
|
||||
|
||||
def test_control_loop_deadline_and_queue_batch_are_compatible() -> None:
|
||||
control = CONTROL.read_text(encoding="utf-8")
|
||||
tasks = REGISTER_TASKS.read_text(encoding="utf-8")
|
||||
|
||||
assert "$script:Agent99QueueOuterDeadlineSeconds = 1680" in control
|
||||
assert "-ExecutionTimeLimit (New-TimeSpan -Minutes 35)" in tasks
|
||||
control_registration = tasks.split('$controlArgs = ', 1)[1].split('$telegramInboxScript = ', 1)[0]
|
||||
assert "-Settings $controlSettings" in control_registration
|
||||
queue_selection = control.split("function Invoke-AgentQueuedCommands", 1)[1].split("$results = @()", 1)[0]
|
||||
assert "Select-Object -First 1" in queue_selection
|
||||
|
||||
|
||||
def test_ingress_signals_tasks_without_waiting_for_control_execution() -> None:
|
||||
sre = SRE_INBOX.read_text(encoding="utf-8")
|
||||
telegram = TELEGRAM_INBOX.read_text(encoding="utf-8")
|
||||
submit = SUBMIT_REQUEST.read_text(encoding="utf-8")
|
||||
|
||||
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in sre
|
||||
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in submit
|
||||
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-SRE-Alert-Inbox"' in telegram
|
||||
assert '"-InstructionBase64", (Convert-AgentBase64 $instruction)' in telegram
|
||||
assert '"-RequestedByBase64", (Convert-AgentBase64 ([string]$update.from))' in telegram
|
||||
assert '"-AgentRoot", $AgentRoot' in telegram
|
||||
assert "$null = $submit.Handle" in telegram
|
||||
assert "instruction_transport_ambiguous" in submit
|
||||
assert "scheduled_task_signaled" in sre
|
||||
assert "scheduled_task_signaled" in submit
|
||||
assert "function Write-AgentDurableJson" in submit
|
||||
assert '"request-" + $ingressDigest.Substring(0, 32)' in submit
|
||||
assert "Global\\Wooo-Agent99-Submit-" in submit
|
||||
assert "function Get-AgentIngressLifecycleReceipt" in submit
|
||||
assert 'throw "ingress_lifecycle_ambiguous"' in submit
|
||||
assert "idempotentReplay = [bool]$queueReceipt.existing" in submit
|
||||
assert "durableQueue = [bool]$queueReceipt.ok" in submit
|
||||
assert "submitEvidenceSha256" in submit
|
||||
assert "function Get-AgentDurableSubmitReadback" in telegram
|
||||
assert "Sort-Object update_id" in telegram
|
||||
assert "lastDurablyHandledUpdateId" in telegram
|
||||
assert "update_processing_failed_no_checkpoint_past_failure" in telegram
|
||||
assert "offset_checkpoint_failed_safe_replay_required" in telegram
|
||||
assert "Write-AgentDurableJson $result $evidencePath" in telegram
|
||||
assert "Write-AgentDurableJson $offsetState $offsetPath" in telegram
|
||||
assert "$maxUpdateId + 1" not in telegram
|
||||
|
||||
|
||||
def test_windows_native_bounded_process_replay_covers_timeout_tree_and_stream_cap() -> None:
|
||||
source = BOUNDED_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "timeout_descendant_not_observed" in source
|
||||
assert "streaming_output_limit_not_detected" in source
|
||||
assert "root_only_descendant_not_preserved" in source
|
||||
assert "-TerminationScope Root" in source
|
||||
assert "cim_failure_root_not_stopped" in source
|
||||
assert "cim_failure_tree_false_green" in source
|
||||
assert "orphan_race_descendant_snapshot_missing" in source
|
||||
assert "orphan_race_tree_false_green" in source
|
||||
assert "pid_reuse_selection_not_identity_bounded" in source
|
||||
assert "reusedRootExcluded" in source
|
||||
assert "reusedRootChildExcluded" in source
|
||||
assert "live_root_child_creation_bound_failed" in source
|
||||
assert "intermediateOlderProcessExcluded" in source
|
||||
assert "stop_readback_failure_not_detected" in source
|
||||
assert "stop_readback_failure_not_fail_closed" in source
|
||||
assert "stop_readback_failure_tree_false_green" in source
|
||||
assert "captured_output_exceeded_contract" in source
|
||||
assert "cleanupVerified" in source
|
||||
assert "productionPathWritePerformed = $false" in source
|
||||
|
||||
|
||||
def test_windows_native_telegram_replay_covers_durable_success_and_enqueue_failure() -> None:
|
||||
source = TELEGRAM_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "success_queue_not_durable" in source
|
||||
assert "failure_case_false_green" in source
|
||||
assert "failure_offset_advanced" in source
|
||||
assert "synthetic_after_queue_commit" in source
|
||||
assert "queue_commit_retry_duplicated_artifact" in source
|
||||
assert "queue_commit_retry_not_idempotent" in source
|
||||
assert '"A" * 9000' in source
|
||||
assert "telegramApiCalled = $false" in source
|
||||
assert "productionPathWritePerformed = $false" in source
|
||||
|
||||
|
||||
def test_windows_native_breakglass_failure_readback_replay_is_fail_closed() -> None:
|
||||
source = BREAKGLASS_REPLAY.read_text(encoding="utf-8")
|
||||
|
||||
assert "synthetic_cim_failure" in source
|
||||
assert "synthetic_mutex_readback_failure" in source
|
||||
assert "synthetic_filesystem_readback_failure" in source
|
||||
assert "failure_readback_false_green" in source
|
||||
assert "terminalReceiptConstructionCanContinue = $true" in source
|
||||
assert "runtimeWritePerformed = $false" in source
|
||||
@@ -248,7 +248,12 @@ def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
|
||||
assert "executorReserveSeconds -ge 60" in control
|
||||
assert "queueReserveSeconds -ge 300" in control
|
||||
assert "clientReserveSeconds -ge 120" in control
|
||||
assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' in control
|
||||
queue_runner = control[
|
||||
control.index("$execution = Invoke-AgentBoundedProcess") :
|
||||
]
|
||||
assert "-TimeoutSeconds $script:Agent99QueueOuterDeadlineSeconds" in queue_runner
|
||||
assert "-MaximumOutputBytes 65536" in queue_runner
|
||||
assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' not in control
|
||||
|
||||
|
||||
def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> None:
|
||||
@@ -516,7 +521,7 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
|
||||
assert f"[string]${parameter} = \"\"" in control
|
||||
assert f'[string]`${parameter} = ""' in bootstrap
|
||||
assert '$identityRejectionReason = if (' in control
|
||||
assert '[string]$dispatchIdentityValidation.reason -eq "identity_control_value_unsafe"' in control
|
||||
assert '"controlled_dispatch_identity_unsafe"' in control
|
||||
assert '"controlled_dispatch_identity_invalid"' in control
|
||||
assert "reason = $identityRejectionReason" in control
|
||||
|
||||
Reference in New Issue
Block a user