From 3a81d9bedc4d2699698a73907ebc0406a85092be Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 13:32:44 +0800 Subject: [PATCH 1/4] fix(agent99): harden control loop recovery --- agent99-control-plane.ps1 | 1118 ++++++++++++++++- agent99-register-tasks.ps1 | 8 +- agent99-sre-alert-inbox.ps1 | 15 +- agent99-submit-request.ps1 | 209 ++- agent99-telegram-inbox.ps1 | 246 +++- ...nt99_transport_recovery_deploy_contract.py | 3 +- .../agent99-control-loop-maintenance.ps1 | 283 +++++ .../agent99-stale-recovery-breakglass.ps1 | 516 ++++++++ .../tests/agent99-bounded-process-replay.ps1 | 345 +++++ ...t99-breakglass-failure-readback-replay.ps1 | 55 + ...gent99-telegram-durable-ingress-replay.ps1 | 160 +++ .../tests/test_agent99_vmrun_timeout_guard.py | 276 ++++ .../test_host112_manager_recovery_contract.py | 11 +- 13 files changed, 3170 insertions(+), 75 deletions(-) create mode 100644 scripts/reboot-recovery/agent99-control-loop-maintenance.ps1 create mode 100644 scripts/reboot-recovery/agent99-stale-recovery-breakglass.ps1 create mode 100644 scripts/reboot-recovery/tests/agent99-bounded-process-replay.ps1 create mode 100644 scripts/reboot-recovery/tests/agent99-breakglass-failure-readback-replay.ps1 create mode 100644 scripts/reboot-recovery/tests/agent99-telegram-durable-ingress-replay.ps1 create mode 100644 scripts/reboot-recovery/tests/test_agent99_vmrun_timeout_guard.py diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 656ceabac..74b6fd4ab 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -105,6 +105,8 @@ $script:Agent99Host112ApplyTransportTimeoutSeconds = 510 $script:Agent99Host112ReceiptTimeoutSeconds = 30 $script:Agent99Host112AfterTimeoutSeconds = 30 $script:Agent99Host112SshCallCount = 5 +$script:Agent99RecoveryRunIdPattern = '^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|agent99-recovery-[0-9]{8}-[0-9]{6}-[0-9a-f]{8})$' +$script:Agent99FallbackRecoveryRunIdPattern = '^agent99-recovery-[0-9]{8}-[0-9]{6}-[0-9a-f]{8}$' function Convert-AgentBool { param([object]$Value) @@ -396,6 +398,411 @@ function Exit-AgentRecoveryTransportLease { return $true } +function Get-AgentStaleRecoveryGuardConfig { + $configured = if ($Config.PSObject.Properties["staleRecoveryGuard"]) { $Config.staleRecoveryGuard } else { $null } + [pscustomobject]@{ + enabled = [bool](-not $configured -or -not $configured.PSObject.Properties["enabled"] -or (Convert-AgentBool $configured.enabled)) + staleMinutes = if ($configured -and $configured.PSObject.Properties["staleMinutes"]) { [math]::Min(120, [math]::Max(20, [int]$configured.staleMinutes)) } else { 20 } + hardStaleMinutes = if ($configured -and $configured.PSObject.Properties["hardStaleMinutes"]) { [math]::Min(240, [math]::Max(60, [int]$configured.hardStaleMinutes)) } else { 60 } + } +} + +function Write-AgentStaleRecoveryGuardReceipt { + param( + [object]$Value, + [string]$Kind + ) + $directory = Join-Path $EvidenceDir "stale-recovery-guard" + New-Item -ItemType Directory -Path $directory -Force | Out-Null + $stamp = Get-Date -Format "yyyyMMdd-HHmmss" + $path = Join-Path $directory "agent99-stale-recovery-$Kind-$stamp-$PID.json" + $temp = "$path.tmp-$([guid]::NewGuid().ToString('N'))" + $stream = $null + $writer = $null + try { + $json = $Value | ConvertTo-Json -Depth 8 + $encoding = New-Object Text.UTF8Encoding($false) + $stream = [IO.File]::Open($temp, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) + $writer = New-Object IO.StreamWriter($stream, $encoding) + $writer.Write($json) + $writer.Flush() + $stream.Flush($true) + $writer.Dispose() + $writer = $null + $stream.Dispose() + $stream = $null + $tempHash = (Get-FileHash -LiteralPath $temp -Algorithm SHA256).Hash.ToLowerInvariant() + Move-Item -LiteralPath $temp -Destination $path -Force + if (-not (Test-Path -LiteralPath $path)) { throw "stale_recovery_receipt_publish_failed" } + $readbackHash = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($readbackHash -ne $tempHash) { throw "stale_recovery_receipt_readback_mismatch" } + Split-Path -Leaf $path + } finally { + if ($writer) { $writer.Dispose() } + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue + } +} + +function Wait-AgentExactProcessAbsent { + param( + [int]$ProcessId, + [int]$TimeoutSeconds + ) + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + if (-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue)) { return $true } + Start-Sleep -Milliseconds 200 + } + [bool](-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue)) +} + +function Get-AgentProcessDescendants { + param( + [object[]]$Processes, + [int]$ParentProcessId + ) + $known = New-Object 'System.Collections.Generic.HashSet[int]' + [void]$known.Add($ParentProcessId) + $rows = @() + do { + $added = $false + foreach ($processInfo in $Processes) { + if ( + $known.Contains([int]$processInfo.ParentProcessId) -and + -not $known.Contains([int]$processInfo.ProcessId) + ) { + [void]$known.Add([int]$processInfo.ProcessId) + $rows += $processInfo + $added = $true + } + } + } while ($added) + @($rows) +} + +function Test-AgentCimProcessIdentityMatch { + param( + [object]$Expected, + [object]$Actual + ) + if (-not $Expected -or -not $Actual -or -not $Expected.CreationDate -or -not $Actual.CreationDate) { return $false } + [bool]( + [int]$Expected.ProcessId -eq [int]$Actual.ProcessId -and + [int]$Expected.ParentProcessId -eq [int]$Actual.ParentProcessId -and + [string]$Expected.Name -eq [string]$Actual.Name -and + ([datetime]$Expected.CreationDate).ToUniversalTime().Ticks -eq ([datetime]$Actual.CreationDate).ToUniversalTime().Ticks + ) +} + +function Wait-AgentCimProcessIdentityAbsent { + param( + [object]$Identity, + [int]$TimeoutSeconds + ) + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + $current = Get-CimInstance Win32_Process -Filter "ProcessId = $([int]$Identity.ProcessId)" -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not (Test-AgentCimProcessIdentityMatch $Identity $current)) { return $true } + Start-Sleep -Milliseconds 200 + } + $current = Get-CimInstance Win32_Process -Filter "ProcessId = $([int]$Identity.ProcessId)" -ErrorAction SilentlyContinue | Select-Object -First 1 + [bool](-not (Test-AgentCimProcessIdentityMatch $Identity $current)) +} + +function Invoke-AgentStaleRecoveryTransportGuardUnsafe { + $guard = Get-AgentStaleRecoveryGuardConfig + $guardWritePerformed = $false + $intentRef = "" + $childStops = @() + $parentStopped = $false + $mutexReleased = $false + $leaseRemoved = $false + $leasePath = Join-Path (Join-Path $Config.agentRoot "state") "recovery-transport-lease.json" + if (-not $guard.enabled -or -not $ControlledApply) { + return [pscustomobject]@{ ok = $true; skipped = $true; reason = "guard_disabled_or_observe_mode"; runtimeWritePerformed = $false } + } + if (-not (Test-Path -LiteralPath $leasePath)) { + return [pscustomobject]@{ ok = $true; skipped = $true; reason = "recovery_lease_metadata_absent"; runtimeWritePerformed = $false } + } + + try { + $lease = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json + if ( + [string]$lease.schemaVersion -ne "agent99_recovery_transport_lease_v1" -or + [string]$lease.runId -notmatch $script:Agent99RecoveryRunIdPattern -or + [int]$lease.processId -le 0 -or + -not $lease.acquiredAt -or + [bool]$lease.storesSecret + ) { + return [pscustomobject]@{ ok = $false; skipped = $false; reason = "recovery_lease_identity_invalid"; runtimeWritePerformed = $false } + } + + $all = @(Get-CimInstance Win32_Process -ErrorAction Stop) + $parent = $all | Where-Object { [int]$_.ProcessId -eq [int]$lease.processId } | Select-Object -First 1 + $leaseObservation = Test-AgentRecoveryTransportLease + if (-not $parent) { + if (-not $leaseObservation.active) { + $guardWritePerformed = $true + Remove-Item -LiteralPath $leasePath -Force -ErrorAction Stop + $removed = [bool](-not (Test-Path -LiteralPath $leasePath)) + return [pscustomobject]@{ + ok = $removed + skipped = $false + reason = if ($removed) { "orphan_lease_metadata_removed" } else { "orphan_lease_metadata_cleanup_failed" } + runtimeWritePerformed = $removed + parentPid = [int]$lease.processId + runId = [string]$lease.runId + } + } + return [pscustomobject]@{ ok = $false; skipped = $false; reason = "recovery_mutex_active_parent_missing"; runtimeWritePerformed = $false } + } + + $parentProcess = Get-Process -Id ([int]$lease.processId) -ErrorAction Stop + $parentAgeMinutes = [math]::Round(((Get-Date) - $parentProcess.StartTime).TotalMinutes, 2) + $parentCommand = [string]$parent.CommandLine + $parentCreationUtc = ([datetime]$parent.CreationDate).ToUniversalTime() + $leaseAcquiredUtc = ([datetime]$lease.acquiredAt).ToUniversalTime() + $leaseBoundToParentCreation = [bool]( + $leaseAcquiredUtc -ge $parentCreationUtc -and + $leaseAcquiredUtc -le $parentCreationUtc.AddMinutes(5) + ) + $runIdBoundToCommand = if ([string]$lease.runId -match $script:Agent99FallbackRecoveryRunIdPattern) { + $parentCommand -notmatch "(?i)-AutomationRunId\s+" + } else { + $parentCommand.IndexOf([string]$lease.runId, [StringComparison]::OrdinalIgnoreCase) -ge 0 + } + $parentIdentityValid = [bool]( + [string]$parent.Name -eq "powershell.exe" -and + $parentCommand -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply' -and + $leaseBoundToParentCreation -and + $runIdBoundToCommand + ) + if (-not $parentIdentityValid) { + return [pscustomobject]@{ ok = $false; skipped = $false; reason = "recovery_parent_identity_mismatch"; runtimeWritePerformed = $false; parentPid = [int]$lease.processId } + } + if ($parentAgeMinutes -lt $guard.staleMinutes) { + return [pscustomobject]@{ ok = $true; skipped = $true; reason = "recovery_within_time_budget"; runtimeWritePerformed = $false; parentAgeMinutes = $parentAgeMinutes } + } + + $descendants = @(Get-AgentProcessDescendants $all ([int]$lease.processId)) + $allowedVmx = @($Config.vms | Where-Object { $_.vmx } | ForEach-Object { [IO.Path]::GetFullPath([string]$_.vmx).TrimEnd("\") }) + $vmrunChildren = @() + $unexpectedChildren = @() + foreach ($child in $descendants) { + $childCommand = [string]$child.CommandLine + $matchedVmx = $allowedVmx | Where-Object { $childCommand.IndexOf($_, [StringComparison]::OrdinalIgnoreCase) -ge 0 } | Select-Object -First 1 + if ( + [string]$child.Name -eq "vmrun.exe" -and + [int]$child.ParentProcessId -eq [int]$lease.processId -and + $childCommand -match "(?i)vmrun\.exe.*\sstart\s" -and + $childCommand -match "(?i)\snogui(\s|$)" -and + $matchedVmx + ) { + $vmrunChildren += $child + } else { + $unexpectedChildren += $child + } + } + $eligible = [bool]( + $unexpectedChildren.Count -eq 0 -and + ( + ($vmrunChildren.Count -eq 1) -or + ($descendants.Count -eq 0 -and $parentAgeMinutes -ge $guard.hardStaleMinutes) + ) + ) + if (-not $eligible) { + return [pscustomobject]@{ + ok = $false + skipped = $false + reason = "stale_recovery_descendant_identity_unapproved" + runtimeWritePerformed = $false + parentPid = [int]$lease.processId + parentAgeMinutes = $parentAgeMinutes + descendantCount = $descendants.Count + vmrunChildCount = $vmrunChildren.Count + unexpectedChildCount = $unexpectedChildren.Count + } + } + + $intent = [pscustomobject]@{ + schemaVersion = "agent99_stale_recovery_guard_intent_v1" + timestamp = (Get-Date).ToString("o") + runId = [string]$lease.runId + parentPid = [int]$lease.processId + parentCreationUtc = $parentCreationUtc.ToString("o") + leaseAcquiredAt = $leaseAcquiredUtc.ToString("o") + parentAgeMinutes = $parentAgeMinutes + vmrunChildPids = @($vmrunChildren | ForEach-Object { [int]$_.ProcessId }) + vmrunChildIdentities = @($vmrunChildren | ForEach-Object { + [pscustomobject]@{ + processId = [int]$_.ProcessId + parentProcessId = [int]$_.ParentProcessId + creationUtc = ([datetime]$_.CreationDate).ToUniversalTime().ToString("o") + name = [string]$_.Name + } + }) + descendantCount = $descendants.Count + action = "terminate_exact_stale_recovery_tree" + vmPowerChangePerformed = $false + vmwareVmxProcessTerminated = $false + secretValueRead = $false + } + $intentRef = Write-AgentStaleRecoveryGuardReceipt $intent "intent" + $guardWritePerformed = $true + $childStops = @() + foreach ($child in $vmrunChildren) { + $leaseBeforeKill = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json + $childBeforeKill = Get-CimInstance Win32_Process -Filter "ProcessId = $([int]$child.ProcessId)" -ErrorAction SilentlyContinue | Select-Object -First 1 + if ( + [string]$leaseBeforeKill.runId -ne [string]$lease.runId -or + [int]$leaseBeforeKill.processId -ne [int]$lease.processId -or + ([datetime]$leaseBeforeKill.acquiredAt).ToUniversalTime().Ticks -ne $leaseAcquiredUtc.Ticks -or + -not (Test-AgentCimProcessIdentityMatch $child $childBeforeKill) -or + ([string]$childBeforeKill.CommandLine) -ne ([string]$child.CommandLine) + ) { + throw "stale_vmrun_child_recheck_identity_changed" + } + & "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$child.ProcessId) /T /F 2>&1 | Out-Null + $childStops += [pscustomobject]@{ + pid = [int]$child.ProcessId + exitCode = [int]$LASTEXITCODE + stopped = Wait-AgentCimProcessIdentityAbsent $child 15 + } + } + if (@($childStops | Where-Object { -not $_.stopped }).Count -gt 0) { + throw "stale_vmrun_child_stop_failed" + } + + $parentStopped = Wait-AgentCimProcessIdentityAbsent $parent 30 + if (-not $parentStopped) { + $recheckAll = @(Get-CimInstance Win32_Process -ErrorAction Stop) + $recheckParent = $recheckAll | Where-Object { [int]$_.ProcessId -eq [int]$lease.processId } | Select-Object -First 1 + $recheckDescendants = @(Get-AgentProcessDescendants $recheckAll ([int]$lease.processId)) + $recheckCommand = if ($recheckParent) { [string]$recheckParent.CommandLine } else { "" } + $leaseBeforeParentKill = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json + $recheckValid = [bool]( + $recheckParent -and + (Test-AgentCimProcessIdentityMatch $parent $recheckParent) -and + $recheckCommand -eq $parentCommand -and + [string]$leaseBeforeParentKill.runId -eq [string]$lease.runId -and + [int]$leaseBeforeParentKill.processId -eq [int]$lease.processId -and + ([datetime]$leaseBeforeParentKill.acquiredAt).ToUniversalTime().Ticks -eq $leaseAcquiredUtc.Ticks -and + $recheckDescendants.Count -eq 0 + ) + if (-not $recheckValid) { + throw "stale_recovery_parent_recheck_failed" + } + & "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$lease.processId) /T /F 2>&1 | Out-Null + $parentStopped = Wait-AgentCimProcessIdentityAbsent $parent 15 + } + + $mutexReleased = [bool](-not (Test-AgentRecoveryTransportLease).active) + if ($parentStopped -and $mutexReleased) { + if (Test-Path -LiteralPath $leasePath) { + $leaseRecheck = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json + if ( + [string]$leaseRecheck.runId -eq [string]$lease.runId -and + [int]$leaseRecheck.processId -eq [int]$lease.processId -and + ([datetime]$leaseRecheck.acquiredAt).ToUniversalTime().Ticks -eq $leaseAcquiredUtc.Ticks + ) { + $guardWritePerformed = $true + Remove-Item -LiteralPath $leasePath -Force -ErrorAction Stop + } + } + } + $leaseRemoved = [bool](-not (Test-Path -LiteralPath $leasePath)) + $verified = [bool]($parentStopped -and $mutexReleased -and $leaseRemoved) + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_stale_recovery_guard_receipt_v1" + timestamp = (Get-Date).ToString("o") + runId = [string]$lease.runId + parentPid = [int]$lease.processId + parentStopped = $parentStopped + mutexReleased = $mutexReleased + leaseMetadataRemoved = $leaseRemoved + childStops = $childStops + intentRef = $intentRef + terminal = if ($verified) { "stale_recovery_released" } else { "stale_recovery_cleanup_unverified" } + vmPowerChangePerformed = $false + vmwareVmxProcessTerminated = $false + secretValueRead = $false + } + $receiptRef = Write-AgentStaleRecoveryGuardReceipt $receipt "receipt" + [pscustomobject]@{ + ok = $verified + skipped = $false + reason = $receipt.terminal + runtimeWritePerformed = $true + intentRef = $intentRef + receiptRef = $receiptRef + parentPid = [int]$lease.processId + parentAgeMinutes = $parentAgeMinutes + childStops = $childStops + mutexReleased = $mutexReleased + leaseMetadataRemoved = $leaseRemoved + } + } catch { + $caughtFailure = $_ + $failureReason = if ($caughtFailure.Exception.Message -in @( + "stale_vmrun_child_recheck_identity_changed", + "stale_vmrun_child_stop_failed", + "stale_recovery_parent_recheck_failed" + )) { $caughtFailure.Exception.Message } else { "stale_recovery_guard_failed" } + $failureReceiptRef = "" + if ($intentRef) { + try { + $failureReceipt = [pscustomobject]@{ + schemaVersion = "agent99_stale_recovery_guard_receipt_v1" + timestamp = (Get-Date).ToString("o") + runId = if ($lease) { [string]$lease.runId } else { "" } + parentPid = if ($lease) { [int]$lease.processId } else { 0 } + parentStopped = $parentStopped + mutexReleased = $mutexReleased + leaseMetadataRemoved = $leaseRemoved + childStops = $childStops + intentRef = $intentRef + terminal = $failureReason + vmPowerChangePerformed = $false + vmwareVmxProcessTerminated = $false + secretValueRead = $false + } + $failureReceiptRef = Write-AgentStaleRecoveryGuardReceipt $failureReceipt "receipt" + } catch {} + } + [pscustomobject]@{ + ok = $false + skipped = $false + reason = $failureReason + errorType = $caughtFailure.Exception.GetType().Name + runtimeWritePerformed = $guardWritePerformed + intentRef = $intentRef + receiptRef = $failureReceiptRef + } + } +} + +function Invoke-AgentStaleRecoveryTransportGuard { + $mutex = $null + $acquired = $false + try { + $mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99StaleRecoveryReconcilerV1") + try { $acquired = [bool]$mutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $acquired = $true } + if (-not $acquired) { + return [pscustomobject]@{ + ok = $true + skipped = $true + reason = "stale_recovery_reconciler_active" + runtimeWritePerformed = $false + } + } + Invoke-AgentStaleRecoveryTransportGuardUnsafe + } finally { + if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { $mutex.Dispose() } + } +} + function Enter-AgentSshTransportLock { $transport = Get-AgentSshTransportConfig if (-not $transport.serialized) { @@ -7277,7 +7684,7 @@ function Invoke-AgentQueuedCommands { @(Get-ChildItem -Path $queueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" } | Sort-Object LastWriteTime | - Select-Object -First 3) + Select-Object -First 1) } $results = @() foreach ($file in $commands) { @@ -7502,29 +7909,23 @@ function Invoke-AgentQueuedCommands { $args += @("-ReconcileOnly", "-ReconcileEvidenceId", $reconcileEvidenceId) } if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") } - $process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru - $completed = $process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000) - if (-not $completed) { - $process.Kill() - $process.WaitForExit(2000) | Out-Null - } - $process.WaitForExit() | Out-Null - $process.Refresh() + $powershellPath = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $execution = Invoke-AgentBoundedProcess ` + -FilePath $powershellPath ` + -Arguments $args ` + -TimeoutSeconds $script:Agent99QueueOuterDeadlineSeconds ` + -MaximumOutputBytes 65536 ` + -StdoutPath $stdoutPath ` + -StderrPath $stderrPath ` + -PreserveOutputFiles + $completed = [bool]$execution.completed $latest = if ($reconcileOnly) { $exactEvidencePath = Join-Path $EvidenceDir "agent99-Status-$reconcileEvidenceId.json" Get-AgentEvidenceAtPath $exactEvidencePath } else { Get-AgentLatestEvidence "agent99-$modeName-*.json" } - $exitCode = if (-not $completed) { - -2 - } elseif ($null -ne $process.ExitCode) { - $process.ExitCode - } elseif ($latest.exists -and -not $latest.parseError) { - 0 - } else { - -3 - } + $exitCode = if ($null -ne $execution.exitCode) { [int]$execution.exitCode } elseif ($latest.exists -and -not $latest.parseError) { 0 } else { -5 } $outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence -SourceEventResolutionPolicy $sourceEventResolutionPolicy -RequireReconcileOnly $reconcileOnly -ExpectedReconcileEvidenceId $reconcileEvidenceId $outcome | Add-Member -MemberType NoteProperty -Name automationRunId -Value $automationRunId -Force $outcome | Add-Member -MemberType NoteProperty -Name traceId -Value $traceId -Force @@ -7556,6 +7957,7 @@ function Invoke-AgentQueuedCommands { mode = $modeName controlledApply = $controlled source = $commandSource + externalId = if ($command.PSObject.Properties["externalId"]) { [string]$command.externalId } else { "" } reason = $commandReason instruction = $commandInstruction correlationKey = $correlationKey @@ -7592,6 +7994,11 @@ function Invoke-AgentQueuedCommands { identity = $dispatchIdentity outcome = $outcome exitCode = $exitCode + transportTimedOut = [bool]$execution.timedOut + transportOutputLimitExceeded = [bool]$execution.outputLimitExceeded + transportProcessTreeStopped = [bool]$execution.processTreeStopped + transportTreeReadbackFailed = [bool]$execution.treeReadbackFailed + transportTaskkillExitCode = $execution.taskkillExitCode durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1) evidence = $latest.path stdout = $stdoutPath @@ -9080,12 +9487,523 @@ function Test-HostReachability { $results } +function ConvertTo-AgentNativeProcessArgument { + param([string]$Value) + if ($Value.Contains('"') -or $Value.Contains("`r") -or $Value.Contains("`n") -or $Value.Contains([char]0)) { + throw "native_process_argument_invalid" + } + if ($Value -match "\s") { + if ($Value.EndsWith("\")) { throw "native_process_argument_trailing_backslash_invalid" } + return '"' + $Value + '"' + } + $Value +} + +function ConvertTo-AgentProcessIdentitySnapshot { + param([object]$ProcessInfo) + if (-not $ProcessInfo -or -not $ProcessInfo.CreationDate) { return $null } + [pscustomobject]@{ + processId = [int]$ProcessInfo.ProcessId + parentProcessId = [int]$ProcessInfo.ParentProcessId + creationUtc = ([datetime]$ProcessInfo.CreationDate).ToUniversalTime().ToString("o") + name = [string]$ProcessInfo.Name + } +} + +function Get-AgentCreationBoundedDescendants { + param( + [object[]]$Processes, + [object[]]$Parents + ) + $selected = @() + $frontier = @($Parents) + $seen = New-Object 'System.Collections.Generic.HashSet[int]' + foreach ($parent in $frontier) { [void]$seen.Add([int]$parent.ProcessId) } + while ($frontier.Count -gt 0) { + $next = @() + foreach ($parent in $frontier) { + $parentCreated = ([datetime]$parent.CreationDate).ToUniversalTime() + foreach ($child in @($Processes | Where-Object { + [int]$_.ParentProcessId -eq [int]$parent.ProcessId -and + $_.CreationDate -and + ([datetime]$_.CreationDate).ToUniversalTime() -ge $parentCreated + })) { + if ($seen.Add([int]$child.ProcessId)) { + $selected += $child + $next += $child + } + } + } + $frontier = @($next) + } + @($selected) +} + +function Test-AgentProcessSnapshotMatch { + param([object]$Expected, [object]$Actual) + if (-not $Expected -or -not $Actual -or -not $Actual.CreationDate) { return $false } + [bool]( + [int]$Actual.ProcessId -eq [int]$Expected.processId -and + [int]$Actual.ParentProcessId -eq [int]$Expected.parentProcessId -and + [string]$Actual.Name -eq [string]$Expected.name -and + ([datetime]$Actual.CreationDate).ToUniversalTime().Ticks -eq ([datetime]$Expected.creationUtc).ToUniversalTime().Ticks + ) +} + +function Select-AgentProcessTreeIdentitySnapshot { + param( + [object[]]$Processes, + [object]$RootIdentity, + [datetime]$RootExitedUtc = [datetime]::MinValue + ) + if (-not $RootIdentity) { throw "bounded_process_root_identity_not_captured" } + $rootId = [int]$RootIdentity.processId + $currentRoot = $Processes | Where-Object { [int]$_.ProcessId -eq $rootId } | Select-Object -First 1 + $rootMatches = Test-AgentProcessSnapshotMatch $RootIdentity $currentRoot + if ($rootMatches) { + $tree = @($currentRoot) + @(Get-AgentCreationBoundedDescendants $Processes @($currentRoot)) + } else { + if ($RootExitedUtc -eq [datetime]::MinValue) { + throw "bounded_process_root_identity_changed_before_exit_readback" + } + $rootCreatedUtc = ([datetime]$RootIdentity.creationUtc).ToUniversalTime() + $directChildren = @($Processes | Where-Object { + [int]$_.ParentProcessId -eq $rootId -and + $_.CreationDate -and + ([datetime]$_.CreationDate).ToUniversalTime() -ge $rootCreatedUtc -and + ([datetime]$_.CreationDate).ToUniversalTime() -le $RootExitedUtc.ToUniversalTime() + }) + $tree = @($directChildren) + $tree += @(Get-AgentCreationBoundedDescendants $Processes $directChildren) + } + @($tree | Sort-Object ProcessId -Unique | ForEach-Object { + ConvertTo-AgentProcessIdentitySnapshot $_ + }) +} + +function Get-AgentProcessTreeIdentitySnapshot { + param( + [object]$RootIdentity, + [datetime]$RootExitedUtc = [datetime]::MinValue + ) + $all = @(Get-CimInstance Win32_Process -ErrorAction Stop) + @(Select-AgentProcessTreeIdentitySnapshot $all $RootIdentity $RootExitedUtc) +} + +function Stop-AgentBoundedProcessSnapshot { + param( + [Diagnostics.Process]$RootProcess, + [object[]]$Identities, + [ValidateSet("Tree", "Root")] + [string]$TerminationScope + ) + $rootId = [int]$RootProcess.Id + $rootIdentity = @($Identities | Where-Object { [int]$_.processId -eq $rootId } | Select-Object -First 1) + $taskkillExitCode = $null + $forcedTerminationPerformed = $false + $identityReadbackFailed = $false + + if ($TerminationScope -eq "Root") { + if (-not $RootProcess.HasExited) { + & "$env:SystemRoot\System32\taskkill.exe" /PID ([string]$rootId) /F 2>&1 | Out-Null + $taskkillExitCode = [int]$LASTEXITCODE + $forcedTerminationPerformed = $true + } + try { [void]$RootProcess.WaitForExit(8000); $RootProcess.Refresh() } catch {} + $preserved = 0 + foreach ($identity in @($Identities | Where-Object { [int]$_.processId -ne $rootId })) { + $state = Get-AgentProcessIdentityReadback $identity + if (-not $state.known) { + $identityReadbackFailed = $true + } elseif ($state.active) { + $preserved += 1 + } + } + return [pscustomobject]@{ + forcedTerminationPerformed = $forcedTerminationPerformed + processScopeStopped = [bool]$RootProcess.HasExited + processTreeStopped = $false + preservedDescendantCount = [int]$preserved + taskkillExitCode = $taskkillExitCode + identityReadbackFailed = $identityReadbackFailed + } + } + + if ($rootIdentity.Count -eq 1) { + $rootState = Get-AgentProcessIdentityReadback $rootIdentity[0] + if (-not $rootState.known) { + $identityReadbackFailed = $true + } elseif ($rootState.active) { + & "$env:SystemRoot\System32\taskkill.exe" /PID ([string]$rootId) /T /F 2>&1 | Out-Null + $taskkillExitCode = [int]$LASTEXITCODE + $forcedTerminationPerformed = $true + } + } + foreach ($identity in @($Identities | Where-Object { [int]$_.processId -ne $rootId })) { + $state = Get-AgentProcessIdentityReadback $identity + if (-not $state.known) { + $identityReadbackFailed = $true + } elseif ($state.active) { + & "$env:SystemRoot\System32\taskkill.exe" /PID ([string]([int]$identity.processId)) /T /F 2>&1 | Out-Null + $currentExit = [int]$LASTEXITCODE + if ($null -eq $taskkillExitCode -or $currentExit -ne 0) { $taskkillExitCode = $currentExit } + $forcedTerminationPerformed = $true + } + } + try { [void]$RootProcess.WaitForExit(8000); $RootProcess.Refresh() } catch {} + $treeStopped = $false + if (-not $identityReadbackFailed) { + try { + $treeStopped = [bool]( + $RootProcess.HasExited -and + (Wait-AgentProcessTreeIdentityAbsent @($Identities) 8) + ) + } catch { + $identityReadbackFailed = $true + $treeStopped = $false + } + } + [pscustomobject]@{ + forcedTerminationPerformed = $forcedTerminationPerformed + processScopeStopped = $treeStopped + processTreeStopped = $treeStopped + preservedDescendantCount = 0 + taskkillExitCode = $taskkillExitCode + identityReadbackFailed = $identityReadbackFailed + } +} + +function Get-AgentProcessIdentityReadback { + param([object]$Identity) + try { + $current = Get-CimInstance Win32_Process -Filter "ProcessId = $([int]$Identity.processId)" -ErrorAction Stop | Select-Object -First 1 + $active = [bool]( + $current -and + [int]$current.ParentProcessId -eq [int]$Identity.parentProcessId -and + [string]$current.Name -eq [string]$Identity.name -and + ([datetime]$current.CreationDate).ToUniversalTime().Ticks -eq ([datetime]$Identity.creationUtc).ToUniversalTime().Ticks + ) + [pscustomobject]@{ known = $true; active = $active; reason = if ($active) { "identity_active" } else { "identity_absent_or_changed" } } + } catch { + [pscustomobject]@{ known = $false; active = $false; reason = "identity_readback_failed" } + } +} + +function Test-AgentProcessIdentityActive { + param([object]$Identity) + $readback = Get-AgentProcessIdentityReadback $Identity + if (-not $readback.known) { throw "process_identity_readback_failed" } + [bool]$readback.active +} + +function Wait-AgentProcessTreeIdentityAbsent { + param( + [object[]]$Identities, + [int]$TimeoutSeconds = 5 + ) + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + $states = @($Identities | ForEach-Object { Get-AgentProcessIdentityReadback $_ }) + if (@($states | Where-Object { -not $_.known }).Count -gt 0) { throw "process_tree_identity_readback_failed" } + if (@($states | Where-Object { $_.active }).Count -eq 0) { return $true } + Start-Sleep -Milliseconds 100 + } + $states = @($Identities | ForEach-Object { Get-AgentProcessIdentityReadback $_ }) + if (@($states | Where-Object { -not $_.known }).Count -gt 0) { throw "process_tree_identity_readback_failed" } + [bool](@($states | Where-Object { $_.active }).Count -eq 0) +} + +function Invoke-AgentBoundedProcess { + param( + [Parameter(Mandatory = $true)] + [string]$FilePath, + [string[]]$Arguments = @(), + [ValidateRange(1, 3600)] + [int]$TimeoutSeconds = 30, + [ValidateRange(1024, 1048576)] + [int]$MaximumOutputBytes = 65536, + [ValidateSet("Tree", "Root")] + [string]$TerminationScope = "Tree", + [string]$StdoutPath = "", + [string]$StderrPath = "", + [switch]$PreserveOutputFiles + ) + + $id = [guid]::NewGuid().ToString("N") + $stdout = if ($StdoutPath) { $StdoutPath } else { Join-Path $env:TEMP "agent99-process-$id.out" } + $stderr = if ($StderrPath) { $StderrPath } else { Join-Path $env:TEMP "agent99-process-$id.err" } + $process = $null + $stdoutStream = $null + $stderrStream = $null + $stdoutWriter = $null + $stderrWriter = $null + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + $timedOut = $false + $outputLimitExceeded = $false + $treeReadbackFailed = $false + $identityReadbackFailed = $false + $processScopeStopped = $false + $processTreeStopped = $false + $forcedTerminationPerformed = $false + $rootFallbackTerminationUsed = $false + $preservedDescendantCount = 0 + $taskkillExitCode = $null + $treeIdentities = @() + $rootProcessIdentity = $null + $capturedBytes = [long]0 + try { + foreach ($path in @($stdout, $stderr)) { + $directory = Split-Path -Parent $path + if ($directory -and -not (Test-Path -LiteralPath $directory)) { + New-Item -ItemType Directory -Path $directory -Force | Out-Null + } + Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue + } + $argumentLine = (@($Arguments | ForEach-Object { ConvertTo-AgentNativeProcessArgument ([string]$_) }) -join " ") + $startInfo = New-Object Diagnostics.ProcessStartInfo + $startInfo.FileName = $FilePath + $startInfo.Arguments = $argumentLine + $startInfo.UseShellExecute = $false + $startInfo.CreateNoWindow = $true + $startInfo.RedirectStandardOutput = $true + $startInfo.RedirectStandardError = $true + $startInfo.StandardOutputEncoding = New-Object Text.UTF8Encoding($false) + $startInfo.StandardErrorEncoding = New-Object Text.UTF8Encoding($false) + $process = New-Object Diagnostics.Process + $process.StartInfo = $startInfo + if (-not $process.Start()) { throw "bounded_process_start_failed" } + try { + $rootProcessInfo = Get-CimInstance Win32_Process -Filter "ProcessId = $([int]$process.Id)" -ErrorAction Stop | Select-Object -First 1 + $rootProcessIdentity = ConvertTo-AgentProcessIdentitySnapshot $rootProcessInfo + } catch {} + + $encoding = New-Object Text.UTF8Encoding($false) + $stdoutStream = [IO.File]::Open($stdout, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::Read) + $stderrStream = [IO.File]::Open($stderr, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::Read) + $stdoutWriter = New-Object IO.StreamWriter($stdoutStream, $encoding, 4096, $true) + $stderrWriter = New-Object IO.StreamWriter($stderrStream, $encoding, 4096, $true) + $stdoutBuffer = New-Object char[] 2048 + $stderrBuffer = New-Object char[] 2048 + $stdoutRead = $process.StandardOutput.ReadAsync($stdoutBuffer, 0, $stdoutBuffer.Length) + $stderrRead = $process.StandardError.ReadAsync($stderrBuffer, 0, $stderrBuffer.Length) + $stdoutEof = $false + $stderrEof = $false + + while ($true) { + if (-not $stdoutEof -and $stdoutRead.IsCompleted) { + $count = [int]$stdoutRead.GetAwaiter().GetResult() + if ($count -le 0) { + $stdoutEof = $true + } else { + $chunkBytes = [long]$encoding.GetByteCount($stdoutBuffer, 0, $count) + if (($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes) { + $outputLimitExceeded = $true + } else { + $stdoutWriter.Write($stdoutBuffer, 0, $count) + $stdoutWriter.Flush() + $capturedBytes += $chunkBytes + $stdoutRead = $process.StandardOutput.ReadAsync($stdoutBuffer, 0, $stdoutBuffer.Length) + } + } + } + if (-not $stderrEof -and -not $outputLimitExceeded -and $stderrRead.IsCompleted) { + $count = [int]$stderrRead.GetAwaiter().GetResult() + if ($count -le 0) { + $stderrEof = $true + } else { + $chunkBytes = [long]$encoding.GetByteCount($stderrBuffer, 0, $count) + if (($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes) { + $outputLimitExceeded = $true + } else { + $stderrWriter.Write($stderrBuffer, 0, $count) + $stderrWriter.Flush() + $capturedBytes += $chunkBytes + $stderrRead = $process.StandardError.ReadAsync($stderrBuffer, 0, $stderrBuffer.Length) + } + } + } + if ($outputLimitExceeded) { break } + if ($stopwatch.Elapsed.TotalSeconds -ge $TimeoutSeconds) { $timedOut = $true; break } + if ($process.HasExited -and $stdoutEof -and $stderrEof) { break } + Start-Sleep -Milliseconds 10 + } + + if ($timedOut -or $outputLimitExceeded) { + $rootExitedUtc = [datetime]::MinValue + try { + $process.Refresh() + if ($process.HasExited) { $rootExitedUtc = $process.ExitTime.ToUniversalTime() } + } catch {} + try { $treeIdentities = @(Get-AgentProcessTreeIdentitySnapshot $rootProcessIdentity $rootExitedUtc) } catch { $treeReadbackFailed = $true } + if (-not $treeReadbackFailed -and $treeIdentities.Count -gt 0) { + $stopReceipt = Stop-AgentBoundedProcessSnapshot $process $treeIdentities $TerminationScope + $taskkillExitCode = $stopReceipt.taskkillExitCode + $forcedTerminationPerformed = [bool]$stopReceipt.forcedTerminationPerformed + $processScopeStopped = [bool]$stopReceipt.processScopeStopped + $processTreeStopped = [bool]$stopReceipt.processTreeStopped + $preservedDescendantCount = [int]$stopReceipt.preservedDescendantCount + $identityReadbackFailed = [bool]$stopReceipt.identityReadbackFailed + if ($stopReceipt.identityReadbackFailed) { + $treeReadbackFailed = $true + if ($TerminationScope -eq "Tree") { + $rootFallbackTerminationUsed = $true + try { + if (-not $process.HasExited) { + $process.Kill() + $forcedTerminationPerformed = $true + } + } catch {} + try { [void]$process.WaitForExit(8000); $process.Refresh() } catch {} + $processScopeStopped = [bool]$process.HasExited + } + $processTreeStopped = $false + } + } elseif (-not $treeReadbackFailed) { + $processScopeStopped = [bool]$process.HasExited + $processTreeStopped = $processScopeStopped + } else { + $rootFallbackTerminationUsed = $true + try { + if (-not $process.HasExited) { + $process.Kill() + $forcedTerminationPerformed = $true + } + } catch {} + try { [void]$process.WaitForExit(8000); $process.Refresh() } catch {} + $processScopeStopped = [bool]$process.HasExited + $processTreeStopped = $false + } + } else { + $process.WaitForExit() | Out-Null + $process.Refresh() + $processScopeStopped = [bool]$process.HasExited + $processTreeStopped = $processScopeStopped + } + + $stdoutWriter.Flush() + $stderrWriter.Flush() + $stdoutText = Read-AgentBoundedTextFile $stdout ([math]::Min($MaximumOutputBytes, 32768)) + $stderrText = Read-AgentBoundedTextFile $stderr ([math]::Min($MaximumOutputBytes, 32768)) + $exitCode = if ($timedOut) { -2 } elseif ($outputLimitExceeded) { -3 } elseif (-not $processScopeStopped) { -4 } elseif ($process.HasExited) { [int]$process.ExitCode } else { -5 } + [pscustomobject]@{ + ok = [bool]($exitCode -eq 0 -and -not $timedOut -and -not $outputLimitExceeded) + completed = [bool](-not $timedOut -and -not $outputLimitExceeded -and $process.HasExited) + exitCode = $exitCode + timedOut = $timedOut + outputLimitExceeded = $outputLimitExceeded + terminationScope = $TerminationScope + forcedTerminationPerformed = $forcedTerminationPerformed + rootFallbackTerminationUsed = $rootFallbackTerminationUsed + processScopeStopped = $processScopeStopped + processTreeStopped = $processTreeStopped + preservedDescendantCount = $preservedDescendantCount + treeReadbackFailed = $treeReadbackFailed + identityReadbackFailed = $identityReadbackFailed + rootIdentityCaptured = [bool]$rootProcessIdentity + taskkillExitCode = $taskkillExitCode + killedIdentityCount = [int]$treeIdentities.Count + capturedBytes = $capturedBytes + maximumOutputBytes = $MaximumOutputBytes + elapsedMs = [long]$stopwatch.ElapsedMilliseconds + stdout = $stdoutText + stderr = $stderrText + stdoutPath = $stdout + stderrPath = $stderr + runtimeWritePerformed = $true + } + } catch { + if ($process -and -not $process.HasExited) { + try { $treeIdentities = @(Get-AgentProcessTreeIdentitySnapshot $rootProcessIdentity ([datetime]::MinValue)) } catch { $treeReadbackFailed = $true } + if (-not $treeReadbackFailed -and $treeIdentities.Count -gt 0) { + try { + $stopReceipt = Stop-AgentBoundedProcessSnapshot $process $treeIdentities $TerminationScope + $taskkillExitCode = $stopReceipt.taskkillExitCode + $forcedTerminationPerformed = [bool]$stopReceipt.forcedTerminationPerformed + $processScopeStopped = [bool]$stopReceipt.processScopeStopped + $processTreeStopped = [bool]$stopReceipt.processTreeStopped + $preservedDescendantCount = [int]$stopReceipt.preservedDescendantCount + $identityReadbackFailed = [bool]$stopReceipt.identityReadbackFailed + if ($stopReceipt.identityReadbackFailed) { + $treeReadbackFailed = $true + if ($TerminationScope -eq "Tree") { + $rootFallbackTerminationUsed = $true + try { + if (-not $process.HasExited) { + $process.Kill() + $forcedTerminationPerformed = $true + } + } catch {} + try { [void]$process.WaitForExit(8000); $process.Refresh() } catch {} + $processScopeStopped = [bool]$process.HasExited + } + $processTreeStopped = $false + } + } catch {} + } elseif ($treeReadbackFailed) { + $rootFallbackTerminationUsed = $true + try { + if (-not $process.HasExited) { + $process.Kill() + $forcedTerminationPerformed = $true + } + } catch {} + try { [void]$process.WaitForExit(8000); $process.Refresh() } catch {} + $processScopeStopped = [bool]$process.HasExited + $processTreeStopped = $false + } + } + [pscustomobject]@{ + ok = $false + completed = $false + exitCode = -5 + timedOut = $timedOut + outputLimitExceeded = $outputLimitExceeded + terminationScope = $TerminationScope + forcedTerminationPerformed = $forcedTerminationPerformed + rootFallbackTerminationUsed = $rootFallbackTerminationUsed + processScopeStopped = $processScopeStopped + processTreeStopped = $processTreeStopped + preservedDescendantCount = $preservedDescendantCount + treeReadbackFailed = $treeReadbackFailed + identityReadbackFailed = $identityReadbackFailed + rootIdentityCaptured = [bool]$rootProcessIdentity + taskkillExitCode = $taskkillExitCode + killedIdentityCount = [int]$treeIdentities.Count + capturedBytes = $capturedBytes + maximumOutputBytes = $MaximumOutputBytes + elapsedMs = [long]$stopwatch.ElapsedMilliseconds + stdout = Read-AgentBoundedTextFile $stdout 32768 + stderr = Read-AgentBoundedTextFile $stderr 32768 + stdoutPath = $stdout + stderrPath = $stderr + runtimeWritePerformed = [bool]$process + reason = "bounded_process_failed" + } + } finally { + $stopwatch.Stop() + if ($stdoutWriter) { $stdoutWriter.Dispose() } + if ($stderrWriter) { $stderrWriter.Dispose() } + if ($stdoutStream) { $stdoutStream.Dispose() } + if ($stderrStream) { $stderrStream.Dispose() } + if ($process) { $process.Dispose() } + if (-not $PreserveOutputFiles) { + Remove-Item -LiteralPath $stdout, $stderr -Force -ErrorAction SilentlyContinue + } + } +} + function Test-PublicRoutes { $results = @() + $curlPath = Join-Path $env:SystemRoot "System32\curl.exe" foreach ($url in $Config.publicUrls) { try { - $output = & curl.exe --location --head --max-time 12 $url 2>&1 - $text = $output -join "`n" + $probe = Invoke-AgentBoundedProcess ` + -FilePath $curlPath ` + -Arguments @("--location", "--head", "--silent", "--show-error", "--max-time", "12", [string]$url) ` + -TimeoutSeconds 15 ` + -MaximumOutputBytes 32768 + if (-not $probe.completed -or [int]$probe.exitCode -ne 0) { throw "public_route_probe_failed" } + $text = (@($probe.stdout, $probe.stderr) | Where-Object { $_ }) -join "`n" $matches = [regex]::Matches($text, "HTTP/\S+\s+(\d+)") $status = if ($matches.Count -gt 0) { [int]$matches[$matches.Count - 1].Groups[1].Value } else { $null } $non502 = ($null -ne $status -and $status -ne 502) @@ -9112,12 +10030,18 @@ function Test-PublicRoutes { function Test-AiServices { $results = @() + $curlPath = Join-Path $env:SystemRoot "System32\curl.exe" foreach ($svc in @($Config.aiServices)) { if (-not $svc.url) { continue } $name = if ($svc.name) { [string]$svc.name } else { [string]$svc.url } try { - $output = & curl.exe --max-time 12 -sS $svc.url 2>&1 - $text = ($output -join "`n").Trim() + $probe = Invoke-AgentBoundedProcess ` + -FilePath $curlPath ` + -Arguments @("--max-time", "12", "--silent", "--show-error", [string]$svc.url) ` + -TimeoutSeconds 15 ` + -MaximumOutputBytes 32768 + if (-not $probe.completed -or [int]$probe.exitCode -ne 0) { throw "ai_service_probe_failed" } + $text = ([string]$probe.stdout).Trim() $statusValue = $null try { $json = $text | ConvertFrom-Json @@ -9292,6 +10216,110 @@ function Find-Vmrun { return $null } +function Get-AgentVmrunTimeoutSeconds { + $configured = if ($Config.PSObject.Properties["vmrunTimeoutSeconds"]) { + [int]$Config.vmrunTimeoutSeconds + } else { + 45 + } + [math]::Min(90, [math]::Max(15, $configured)) +} + +function Get-AgentVmwareVmxProcessReadback { + param([string]$VmxPath) + + try { + $canonicalVmx = [IO.Path]::GetFullPath($VmxPath).TrimEnd("\") + $matches = @() + foreach ($processInfo in @(Get-CimInstance Win32_Process -ErrorAction Stop | Where-Object { $_.Name -eq "vmware-vmx.exe" })) { + $commandLine = [string]$processInfo.CommandLine + if ( + $commandLine -and + $commandLine.IndexOf($canonicalVmx, [StringComparison]::OrdinalIgnoreCase) -ge 0 + ) { + $matches += [pscustomobject]@{ + pid = [int]$processInfo.ProcessId + parentPid = [int]$processInfo.ParentProcessId + } + } + } + [pscustomobject]@{ + ok = $true + canonicalVmx = $canonicalVmx + running = [bool]($matches.Count -gt 0) + matchCount = [int]$matches.Count + processes = $matches + commandLineCaptured = $false + runtimeWritePerformed = $false + } + } catch { + [pscustomobject]@{ + ok = $false + canonicalVmx = "" + running = $false + matchCount = 0 + processes = @() + commandLineCaptured = $false + runtimeWritePerformed = $false + reason = "vmx_process_readback_failed" + } + } +} + +function Read-AgentBoundedTextFile { + param( + [string]$Path, + [int]$MaximumBytes = 8192 + ) + + if (-not (Test-Path -LiteralPath $Path)) { return "" } + $stream = $null + try { + $stream = [IO.File]::Open($Path, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite) + $length = [math]::Min([long]$MaximumBytes, $stream.Length) + $buffer = New-Object byte[] $length + $read = $stream.Read($buffer, 0, [int]$length) + [Text.Encoding]::UTF8.GetString($buffer, 0, $read) + } catch { + "" + } finally { + if ($stream) { $stream.Dispose() } + } +} + +function Invoke-AgentBoundedVmrunStart { + param( + [string]$VmrunPath, + [string]$VmxPath, + [int]$TimeoutSeconds = 45, + [int]$MaximumOutputBytes = 16384 + ) + + $execution = Invoke-AgentBoundedProcess ` + -FilePath $VmrunPath ` + -Arguments @("start", $VmxPath, "nogui") ` + -TimeoutSeconds $TimeoutSeconds ` + -MaximumOutputBytes $MaximumOutputBytes ` + -TerminationScope Root + [pscustomobject]@{ + ok = [bool]$execution.ok + exitCode = [int]$execution.exitCode + timedOut = [bool]$execution.timedOut + outputLimitExceeded = [bool]$execution.outputLimitExceeded + terminationScope = [string]$execution.terminationScope + forcedTerminationPerformed = [bool]$execution.forcedTerminationPerformed + processScopeStopped = [bool]$execution.processScopeStopped + processTreeStopped = [bool]$execution.processTreeStopped + preservedDescendantCount = [int]$execution.preservedDescendantCount + treeReadbackFailed = [bool]$execution.treeReadbackFailed + taskkillExitCode = $execution.taskkillExitCode + killedIdentityCount = [int]$execution.killedIdentityCount + elapsedMs = [long]$execution.elapsedMs + output = Limit-AgentTextLine ((@($execution.stdout, $execution.stderr) | Where-Object { $_ }) -join "`n") 512 + runtimeWritePerformed = [bool]$execution.runtimeWritePerformed + } +} + function Start-ConfiguredVMs { $vmrun = Find-Vmrun $results = @() @@ -9317,18 +10345,46 @@ function Start-ConfiguredVMs { continue } + $vmxProcess = Get-AgentVmwareVmxProcessReadback ([string]$vm.vmx) + if (-not $vmxProcess.ok) { + $results += [pscustomobject]@{ name = $vm.name; ok = $false; skipped = $true; reason = "vmx_process_readback_failed" } + continue + } + if ($vmxProcess.running) { + $results += [pscustomobject]@{ + name = $vm.name + ok = $true + skipped = $true + reason = "vmx_process_already_running" + vmxProcessCount = $vmxProcess.matchCount + } + continue + } + if (-not $ControlledApply) { $results += [pscustomobject]@{ name = $vm.name; ok = $false; skipped = $true; reason = "controlled_apply_required" } continue } Write-AgentLog "CONTROLLED_APPLY vm_start name=$($vm.name) vmx=$($vm.vmx)" - $output = & $vmrun start $vm.vmx nogui 2>&1 + $start = Invoke-AgentBoundedVmrunStart ` + -VmrunPath $vmrun ` + -VmxPath ([string]$vm.vmx) ` + -TimeoutSeconds (Get-AgentVmrunTimeoutSeconds) $results += [pscustomobject]@{ name = $vm.name - ok = ($LASTEXITCODE -eq 0) + ok = $start.ok skipped = $false - output = ($output -join "`n") + reason = if ($start.timedOut) { "vmrun_timeout" } elseif ($start.outputLimitExceeded) { "vmrun_output_limit_exceeded" } elseif (-not $start.ok) { "vmrun_start_failed" } else { "vmrun_start_completed" } + exitCode = $start.exitCode + timedOut = $start.timedOut + outputLimitExceeded = $start.outputLimitExceeded + terminationScope = $start.terminationScope + processScopeStopped = $start.processScopeStopped + processTreeStopped = $start.processTreeStopped + preservedDescendantCount = $start.preservedDescendantCount + elapsedMs = $start.elapsedMs + output = $start.output } } @@ -9418,6 +10474,13 @@ $recoveryRunId = if ($Mode -eq "Recover") { } $recoveryTransportLease = $null $recoveryTransportLeaseReceipt = $null +$staleRecoveryTransportGuard = $null +if ($Mode -in @("Perf", "LoadShed") -and $ControlledApply) { + $staleRecoveryTransportGuard = Invoke-AgentStaleRecoveryTransportGuard + if (-not $staleRecoveryTransportGuard.skipped) { + Record-AgentEvent "stale_recovery_transport_guard" $(if ($staleRecoveryTransportGuard.ok) { "warning" } else { "critical" }) "result=$($staleRecoveryTransportGuard.reason) receipt=$($staleRecoveryTransportGuard.receiptRef)" $staleRecoveryTransportGuard -Alert + } +} if ($Mode -eq "Recover") { $recoveryTransportLease = Enter-AgentRecoveryTransportLease $recoveryRunId $recoveryTransportLeaseReceipt = [pscustomobject]@{ @@ -9457,7 +10520,8 @@ if ($Mode -eq "Recover") { deferred = $true reason = [string]$recoveryLeaseObservation.reason recoveryTransportLease = $recoveryLeaseObservation - runtimeWritePerformed = $false + staleRecoveryTransportGuard = $staleRecoveryTransportGuard + runtimeWritePerformed = [bool]($staleRecoveryTransportGuard -and $staleRecoveryTransportGuard.runtimeWritePerformed) evidenceLog = $logPath } Write-AgentLog "background_mode_deferred mode=$Mode reason=$($recoveryLeaseObservation.reason)" @@ -9742,11 +10806,13 @@ $result = [pscustomobject]@{ @($externalHostRecovery | Where-Object { $_.runtimeWritePerformed }).Count -gt 0 -or ($host112Recovery -and $host112Recovery.runtimeWritePerformed) -or ($host188EdgeRecovery -and $host188EdgeRecovery.runtimeWritePerformed) -or + ($staleRecoveryTransportGuard -and $staleRecoveryTransportGuard.runtimeWritePerformed) -or ($sshProcessGuard -and $sshProcessGuard.runtimeWritePerformed) -or ($windowsProbeProcessGuard -and $windowsProbeProcessGuard.runtimeWritePerformed) ) sshProcessGuard = $sshProcessGuard windowsProbeProcessGuard = $windowsProbeProcessGuard + staleRecoveryTransportGuard = $staleRecoveryTransportGuard bootState = $bootState recoveryTrigger = $recoveryTrigger recoverySlo = $recoverySlo diff --git a/agent99-register-tasks.ps1 b/agent99-register-tasks.ps1 index cc3a93bd5..b0608e396 100644 --- a/agent99-register-tasks.ps1 +++ b/agent99-register-tasks.ps1 @@ -28,6 +28,12 @@ $recoverySettings = New-ScheduledTaskSettingsSet ` -StartWhenAvailable ` -MultipleInstances IgnoreNew ` -ExecutionTimeLimit (New-TimeSpan -Minutes 20) +$controlSettings = New-ScheduledTaskSettingsSet ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -StartWhenAvailable ` + -MultipleInstances IgnoreNew ` + -ExecutionTimeLimit (New-TimeSpan -Minutes 35) $serviceSettings = New-ScheduledTaskSettingsSet ` -AllowStartIfOnBatteries ` -DontStopIfGoingOnBatteries ` @@ -93,7 +99,7 @@ Register-ScheduledTask ` -Action $controlAction ` -Trigger $controlTrigger ` -Principal $principal ` - -Settings $settings ` + -Settings $controlSettings ` -Description "Agent99 visible control center, desktop shortcuts, and allowlisted operator command queue processor." ` -Force | Out-Null diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index 60fe4a714..ad456a6b0 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -1370,16 +1370,20 @@ foreach ($file in $files) { } $controlExitCode = $null +$controlDispatchStatus = "not_requested" if ($RunNow -and @($queued).Count -gt 0) { - $launcher = Join-Path $AgentRoot "agent99-run.ps1" - $process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", "ControlTick") -Wait -PassThru -WindowStyle Hidden - $process.Refresh() - $controlExitCode = $process.ExitCode + try { + Start-ScheduledTask -TaskPath "\" -TaskName "Wooo-Agent99-Control-Loop" -ErrorAction Stop + $controlDispatchStatus = "scheduled_task_signaled" + } catch { + $controlDispatchStatus = "scheduled_task_signal_failed" + $controlExitCode = -1 + } } $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) - ok = [bool](@($failed).Count -eq 0) + ok = [bool](@($failed).Count -eq 0 -and $controlDispatchStatus -ne "scheduled_task_signal_failed") incomingPath = $IncomingDir queuedCount = @($queued).Count ignoredCount = @($ignored).Count @@ -1390,6 +1394,7 @@ $result = [pscustomobject]@{ ignored = $ignored failed = $failed runNow = [bool]$RunNow + controlDispatchStatus = $controlDispatchStatus controlExitCode = $controlExitCode } diff --git a/agent99-submit-request.ps1 b/agent99-submit-request.ps1 index 4dd1ddcc3..edcc1fa4b 100644 --- a/agent99-submit-request.ps1 +++ b/agent99-submit-request.ps1 @@ -1,10 +1,12 @@ param( [string]$Instruction = "", + [string]$InstructionBase64 = "", [string]$Mode = "", [switch]$ControlledApply, [switch]$RunNow, [string]$Source = "agent99-submit-request", [string]$RequestedBy = "", + [string]$RequestedByBase64 = "", [string]$ExternalId = "", [string]$ReplyChatId = "", [string]$AgentRoot = "C:\Wooo\Agent99" @@ -12,6 +14,25 @@ param( $ErrorActionPreference = "Stop" +function ConvertFrom-AgentBase64Argument { + param([string]$Value, [string]$FieldName, [int]$MaximumBytes) + try { + $bytes = [Convert]::FromBase64String($Value) + if ($bytes.Length -gt $MaximumBytes) { throw "${FieldName}_too_large" } + $encoding = New-Object Text.UTF8Encoding($false, $true) + $text = $encoding.GetString($bytes) + if ($text.IndexOf([char]0) -ge 0) { throw "${FieldName}_nul_invalid" } + $text + } catch { + throw "${FieldName}_base64_invalid" + } +} + +if ($Instruction -and $InstructionBase64) { throw "instruction_transport_ambiguous" } +if ($RequestedBy -and $RequestedByBase64) { throw "requested_by_transport_ambiguous" } +if ($InstructionBase64) { $Instruction = ConvertFrom-AgentBase64Argument $InstructionBase64 "instruction" 8192 } +if ($RequestedByBase64) { $RequestedBy = ConvertFrom-AgentBase64Argument $RequestedByBase64 "requested_by" 1024 } + if (-not $Instruction) { $Instruction = Read-Host "Agent99 request" } @@ -51,6 +72,128 @@ function Get-AgentSha256 { } } +function Write-AgentDurableJson { + param( + [object]$Value, + [string]$Path, + [string]$ExpectedId, + [string]$ExpectedExternalId + ) + $temporaryPath = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))" + $stream = $null + $writer = $null + try { + $json = $Value | ConvertTo-Json -Depth 10 + $encoding = New-Object Text.UTF8Encoding($false) + $stream = [IO.File]::Open($temporaryPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) + $writer = New-Object IO.StreamWriter($stream, $encoding) + $writer.Write($json) + $writer.Flush() + $stream.Flush($true) + $writer.Dispose(); $writer = $null + $stream.Dispose(); $stream = $null + $temporaryHash = (Get-FileHash -LiteralPath $temporaryPath -Algorithm SHA256).Hash.ToLowerInvariant() + Move-Item -LiteralPath $temporaryPath -Destination $Path -Force + $readbackHash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($readbackHash -ne $temporaryHash) { throw "durable_json_hash_mismatch" } + $readback = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json + if ([string]$readback.id -ne $ExpectedId) { throw "durable_json_id_mismatch" } + if ([string]$readback.externalId -ne $ExpectedExternalId) { throw "durable_json_external_id_mismatch" } + [pscustomobject]@{ + ok = $true + path = $Path + sha256 = $readbackHash + id = [string]$readback.id + externalId = [string]$readback.externalId + } + } finally { + if ($writer) { $writer.Dispose() } + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $temporaryPath -Force -ErrorAction SilentlyContinue + } +} + +function Read-AgentDurableIngressArtifact { + param( + [string]$Path, + [string]$ExpectedId, + [string]$ExpectedExternalId, + [string]$ExpectedSource, + [string]$ExpectedInstruction, + [string]$ExpectedMode, + [bool]$ExpectedControlledApply + ) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { throw "ingress_artifact_missing" } + if ((Get-Item -LiteralPath $Path).Length -gt 262144) { throw "ingress_artifact_oversized" } + $hash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + $readback = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json + $modeValue = if ($readback.PSObject.Properties["mode"]) { + [string]$readback.mode + } elseif ($readback.PSObject.Properties["resolvedMode"]) { + [string]$readback.resolvedMode + } else { + "" + } + if ( + [string]$readback.id -ne $ExpectedId -or + [string]$readback.externalId -ne $ExpectedExternalId -or + [string]$readback.source -ne $ExpectedSource -or + [string]$readback.instruction -ne $ExpectedInstruction -or + $modeValue -ne $ExpectedMode -or + -not $readback.PSObject.Properties["controlledApply"] -or + $readback.controlledApply -isnot [bool] -or + [bool]$readback.controlledApply -ne $ExpectedControlledApply + ) { + throw "ingress_artifact_identity_conflict" + } + [pscustomobject]@{ + ok = $true + existing = $true + path = $Path + sha256 = $hash + id = [string]$readback.id + externalId = [string]$readback.externalId + } +} + +function Get-AgentIngressLifecycleReceipt { + param( + [string]$QueueDirectory, + [string]$RequestId, + [string]$ExpectedExternalId, + [string]$ExpectedSource, + [string]$ExpectedInstruction, + [string]$ExpectedMode, + [bool]$ExpectedControlledApply + ) + $processedDirectory = Join-Path $QueueDirectory "processed" + $failedDirectory = Join-Path $QueueDirectory "failed" + $known = @( + [pscustomobject]@{ state = "pending"; path = (Join-Path $QueueDirectory "$RequestId.json") }, + [pscustomobject]@{ state = "running"; path = (Join-Path $QueueDirectory "running-$RequestId.json") }, + [pscustomobject]@{ state = "processed"; path = (Join-Path $processedDirectory "processed-$RequestId.json") }, + [pscustomobject]@{ state = "failed"; path = (Join-Path $failedDirectory "failed-$RequestId.json") } + ) + if (Test-Path -LiteralPath $failedDirectory -PathType Container) { + foreach ($stale in @(Get-ChildItem -LiteralPath $failedDirectory -File -Filter "stale-*-*$RequestId.json" -ErrorAction SilentlyContinue)) { + $known += [pscustomobject]@{ state = "stale_quarantined"; path = $stale.FullName } + } + } + $present = @($known | Where-Object { Test-Path -LiteralPath $_.path -PathType Leaf } | Sort-Object path -Unique) + if ($present.Count -gt 1) { throw "ingress_lifecycle_ambiguous" } + if ($present.Count -eq 0) { return $null } + $receipt = Read-AgentDurableIngressArtifact ` + -Path $present[0].path ` + -ExpectedId $RequestId ` + -ExpectedExternalId $ExpectedExternalId ` + -ExpectedSource $ExpectedSource ` + -ExpectedInstruction $ExpectedInstruction ` + -ExpectedMode $ExpectedMode ` + -ExpectedControlledApply $ExpectedControlledApply + $receipt | Add-Member -NotePropertyName lifecycleState -NotePropertyValue ([string]$present[0].state) -Force + $receipt +} + function Convert-AgentGuidToNetworkBytes { param([guid]$Guid) [byte[]]$bytes = $Guid.ToByteArray() @@ -215,7 +358,24 @@ $requestDir = Join-Path $AgentRoot "requests" $evidenceDir = Join-Path $AgentRoot "evidence" New-Item -ItemType Directory -Force -Path $queueDir, $requestDir, $evidenceDir | Out-Null -$id = "request-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) +$deterministicIngress = [bool]$ExternalId +$ingressDigest = Get-AgentSha256 (([string]$Source) + "`n" + ([string]$ExternalId)) +$id = if ($deterministicIngress) { + "request-" + $ingressDigest.Substring(0, 32) +} else { + "request-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) +} +$ingressMutex = $null +$ingressMutexAcquired = $false +try { + $ingressMutex = New-Object Threading.Mutex($false, ("Global\Wooo-Agent99-Submit-" + $ingressDigest.Substring(0, 32))) + try { + $ingressMutexAcquired = [bool]$ingressMutex.WaitOne([TimeSpan]::FromSeconds(30)) + } catch [Threading.AbandonedMutexException] { + $ingressMutexAcquired = $true + } + if (-not $ingressMutexAcquired) { throw "ingress_single_writer_timeout" } + $dispatchIdentity = [pscustomobject]@{ schemaVersion = "" projectId = "" @@ -262,8 +422,12 @@ $request = [pscustomobject]@{ $requestPath = Join-Path $requestDir "$id.json" $queuePath = Join-Path $queueDir "$id.json" -$request | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8 -[pscustomobject]@{ +$requestReceipt = if (Test-Path -LiteralPath $requestPath -PathType Leaf) { + Read-AgentDurableIngressArtifact $requestPath $id $ExternalId $Source $Instruction $modeName ([bool]$ControlledApply) +} else { + Write-AgentDurableJson $request $requestPath $id $ExternalId +} +$queueItem = [pscustomobject]@{ id = $id mode = $modeName controlledApply = [bool]$ControlledApply @@ -290,7 +454,14 @@ $request | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UT lockOwner = $dispatchIdentity.lockOwner canonicalDigest = $dispatchIdentity.canonicalDigest createdAt = (Get-Date -Format o) -} | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8 +} +$queueReceipt = Get-AgentIngressLifecycleReceipt $queueDir $id $ExternalId $Source $Instruction $modeName ([bool]$ControlledApply) +if (-not $queueReceipt) { + $queueReceipt = Write-AgentDurableJson $queueItem $queuePath $id $ExternalId + $queueReceipt | Add-Member -NotePropertyName existing -NotePropertyValue $false -Force + $queueReceipt | Add-Member -NotePropertyName lifecycleState -NotePropertyValue "pending" -Force +} +$submitEvidence = Join-Path $evidenceDir "agent99-submit-$id.json" $result = [pscustomobject]@{ ok = $true @@ -298,24 +469,42 @@ $result = [pscustomobject]@{ instruction = $Instruction resolvedMode = $modeName controlledApply = [bool]$ControlledApply + externalId = $ExternalId requestPath = $requestPath queuePath = $queuePath + queueLifecyclePath = [string]$queueReceipt.path + queueLifecycleState = [string]$queueReceipt.lifecycleState + idempotentReplay = [bool]$queueReceipt.existing + durableRequest = [bool]$requestReceipt.ok + requestSha256 = [string]$requestReceipt.sha256 + durableQueue = [bool]$queueReceipt.ok + queueSha256 = [string]$queueReceipt.sha256 + submitEvidencePath = $submitEvidence runNow = [bool]$RunNow automationRunId = $dispatchIdentity.automationRunId traceId = $dispatchIdentity.traceId workItemId = $dispatchIdentity.workItemId idempotencyKey = $dispatchIdentity.idempotencyKey canonicalDigest = $dispatchIdentity.canonicalDigest + controlDispatchStatus = "not_requested" controlExitCode = $null } if ($RunNow) { - $launcher = Join-Path $AgentRoot "agent99-run.ps1" - $process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", "ControlTick") -Wait -PassThru -WindowStyle Hidden - $process.Refresh() - $result.controlExitCode = $process.ExitCode + try { + Start-ScheduledTask -TaskPath "\" -TaskName "Wooo-Agent99-Control-Loop" -ErrorAction Stop + $result.controlDispatchStatus = "scheduled_task_signaled" + } catch { + $result.ok = $false + $result.controlDispatchStatus = "scheduled_task_signal_failed" + $result.controlExitCode = -1 + } } -$submitEvidence = Join-Path $evidenceDir "agent99-submit-$id.json" -$result | ConvertTo-Json -Depth 8 | Set-Content -Path $submitEvidence -Encoding UTF8 +$submitReceipt = Write-AgentDurableJson $result $submitEvidence $id $ExternalId +$result | Add-Member -NotePropertyName submitEvidenceSha256 -NotePropertyValue ([string]$submitReceipt.sha256) -Force $result | ConvertTo-Json -Depth 8 +} finally { + if ($ingressMutexAcquired -and $ingressMutex) { try { $ingressMutex.ReleaseMutex() } catch {} } + if ($ingressMutex) { try { $ingressMutex.Dispose() } catch {} } +} diff --git a/agent99-telegram-inbox.ps1 b/agent99-telegram-inbox.ps1 index 0439533d6..fad00c14c 100644 --- a/agent99-telegram-inbox.ps1 +++ b/agent99-telegram-inbox.ps1 @@ -3,6 +3,8 @@ param( [switch]$RunNow, [switch]$SelfTest, [string]$SyntheticUpdateText = "", + [ValidateRange(0, [int64]::MaxValue)] + [int64]$SyntheticUpdateId = 0, [string]$SyntheticChatTitle = "AwoooI SRE", [string]$SyntheticChatId = "" ) @@ -72,6 +74,95 @@ function Convert-AgentBase64 { [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($Text)) } +function Write-AgentDurableJson { + param( + [object]$Value, + [string]$Path, + [string]$ExpectedProperty = "", + [string]$ExpectedValue = "" + ) + $temporaryPath = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))" + $stream = $null + $writer = $null + try { + $directory = Split-Path -Parent $Path + if (-not (Test-Path -LiteralPath $directory -PathType Container)) { + New-Item -ItemType Directory -Path $directory -Force | Out-Null + } + $encoding = New-Object Text.UTF8Encoding($false) + $stream = [IO.File]::Open($temporaryPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) + $writer = New-Object IO.StreamWriter($stream, $encoding) + $writer.Write(($Value | ConvertTo-Json -Depth 12)) + $writer.Flush() + $stream.Flush($true) + $writer.Dispose(); $writer = $null + $stream.Dispose(); $stream = $null + $temporaryHash = (Get-FileHash -LiteralPath $temporaryPath -Algorithm SHA256).Hash.ToLowerInvariant() + Move-Item -LiteralPath $temporaryPath -Destination $Path -Force + $readbackHash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($readbackHash -ne $temporaryHash) { throw "durable_json_hash_mismatch" } + $readback = Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json + if ($ExpectedProperty) { + $property = $readback.PSObject.Properties[$ExpectedProperty] + if (-not $property -or [string]$property.Value -ne $ExpectedValue) { + throw "durable_json_identity_mismatch" + } + } + [pscustomobject]@{ ok = $true; path = $Path; sha256 = $readbackHash } + } finally { + if ($writer) { $writer.Dispose() } + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $temporaryPath -Force -ErrorAction SilentlyContinue + } +} + +function Get-AgentDurableSubmitReadback { + param( + [string]$OutputPath, + [int64]$UpdateId + ) + try { + if (-not (Test-Path -LiteralPath $OutputPath -PathType Leaf)) { throw "submit_output_missing" } + if ((Get-Item -LiteralPath $OutputPath).Length -gt 32768) { throw "submit_output_oversized" } + $submit = Get-Content -LiteralPath $OutputPath -Raw | ConvertFrom-Json + $expectedExternalId = "telegram-update-$UpdateId" + if ($submit.durableQueue -isnot [bool] -or -not $submit.durableQueue) { throw "submit_queue_not_durable" } + if ([string]$submit.externalId -ne $expectedExternalId) { throw "submit_external_id_mismatch" } + if ([string]$submit.id -notmatch "^request-[A-Za-z0-9-]{1,80}$") { throw "submit_id_invalid" } + if ($submit.idempotentReplay -isnot [bool]) { throw "submit_idempotent_replay_type_invalid" } + $evidencePath = [IO.Path]::GetFullPath([string]$submit.submitEvidencePath) + $expectedEvidenceRoot = [IO.Path]::GetFullPath($EvidenceDir).TrimEnd("\") + "\" + if (-not $evidencePath.StartsWith($expectedEvidenceRoot, [StringComparison]::OrdinalIgnoreCase)) { throw "submit_evidence_path_outside_root" } + if (-not (Test-Path -LiteralPath $evidencePath -PathType Leaf)) { throw "submit_evidence_missing" } + $evidenceHash = (Get-FileHash -LiteralPath $evidencePath -Algorithm SHA256).Hash.ToLowerInvariant() + if ($evidenceHash -ne [string]$submit.submitEvidenceSha256) { throw "submit_evidence_hash_mismatch" } + $evidence = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json + if ( + [string]$evidence.id -ne [string]$submit.id -or + [string]$evidence.externalId -ne $expectedExternalId -or + $evidence.durableQueue -isnot [bool] -or + -not $evidence.durableQueue -or + [string]$evidence.queueSha256 -ne [string]$submit.queueSha256 -or + $evidence.idempotentReplay -isnot [bool] -or + [bool]$evidence.idempotentReplay -ne [bool]$submit.idempotentReplay + ) { + throw "submit_evidence_identity_mismatch" + } + [pscustomobject]@{ + ok = $true + id = [string]$submit.id + queueSha256 = [string]$submit.queueSha256 + evidencePath = $evidencePath + evidenceSha256 = $evidenceHash + controlDispatchStatus = [string]$submit.controlDispatchStatus + idempotentReplay = [bool]$submit.idempotentReplay + queueLifecycleState = [string]$submit.queueLifecycleState + } + } catch { + [pscustomobject]@{ ok = $false; reason = $_.Exception.Message } + } +} + function Get-AgentTelegramUpdates { param( [object]$Config, @@ -487,17 +578,24 @@ function New-AgentMonitoringAlertFromTelegram { force = $true createdAt = (Get-Date -Format o) } - $alert | ConvertTo-Json -Depth 8 | Set-Content -Path $alertPath -Encoding UTF8 + $alertReceipt = Write-AgentDurableJson $alert $alertPath "id" $alertId $sreExitCode = $null + $sreInboxDispatchStatus = "not_requested" if ($RunNow -and (Test-Path $SreAlertInboxScript)) { - $process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $SreAlertInboxScript, "-AgentRoot", $AgentRoot, "-RunNow") -Wait -PassThru -WindowStyle Hidden - $process.Refresh() - $sreExitCode = $process.ExitCode + try { + Start-ScheduledTask -TaskPath "\" -TaskName "Wooo-Agent99-SRE-Alert-Inbox" -ErrorAction Stop + $sreInboxDispatchStatus = "scheduled_task_signaled" + } catch { + $sreInboxDispatchStatus = "scheduled_task_signal_failed" + $sreExitCode = -1 + } } [pscustomobject]@{ - ok = $true + ok = [bool]($sreInboxDispatchStatus -ne "scheduled_task_signal_failed") + durableAlert = [bool]$alertReceipt.ok + alertSha256 = [string]$alertReceipt.sha256 skipped = $false alertId = $alertId alertPath = $alertPath @@ -506,6 +604,7 @@ function New-AgentMonitoringAlertFromTelegram { service = $alert.service host = $alert.host suppressTelegram = $suppressTelegram + sreInboxDispatchStatus = $sreInboxDispatchStatus sreInboxExitCode = $sreExitCode } } @@ -716,8 +815,8 @@ function Invoke-AgentTelegramInboxSelfTest { exit 0 } -$stamp = Get-Date -Format "yyyyMMdd-HHmmss" -$evidencePath = Join-Path $EvidenceDir "agent99-TelegramInbox-$stamp.json" +$stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" +$evidencePath = Join-Path $EvidenceDir "agent99-TelegramInbox-$stamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json" $offsetPath = Join-Path $StateDir "telegram-inbox-offset.json" $offset = 0 if ((-not $SyntheticUpdateText) -and (Test-Path $offsetPath)) { @@ -743,7 +842,7 @@ if ($SyntheticUpdateText) { ok = $true error = $null updates = @([pscustomobject]@{ - update_id = [int64](Get-Date -UFormat %s) + update_id = if ($SyntheticUpdateId -gt 0) { $SyntheticUpdateId } else { [int64](Get-Date -UFormat %s) } message_id = 1 date = [int64](Get-Date -UFormat %s) text = $SyntheticUpdateText @@ -761,15 +860,16 @@ if ($SyntheticUpdateText) { $processed = @() $alerted = @() $ignored = @() -$maxUpdateId = if ($offset -gt 0) { $offset - 1 } else { 0 } +$failedUpdates = @() +$lastDurablyHandledUpdateId = if ($offset -gt 0) { $offset - 1 } else { -1 } +$processingFailed = $false -foreach ($update in @($updatesResult.updates)) { - if ($update.update_id -gt $maxUpdateId) { $maxUpdateId = [int64]$update.update_id } +foreach ($update in @($updatesResult.updates | Sort-Object update_id)) { $instruction = Resolve-AgentTelegramInstruction ([string]$update.text) if (-not $instruction) { if ($autoIngestMonitoringAlerts -and (Test-AgentMonitoringAlertText ([string]$update.text))) { $alertResult = New-AgentMonitoringAlertFromTelegram $update - if ($alertResult.ok) { + if ($alertResult.durableAlert) { $alerted += [pscustomobject]@{ updateId = $update.update_id messageId = $update.message_id @@ -784,13 +884,25 @@ foreach ($update in @($updatesResult.updates)) { host = $alertResult.host suppressTelegram = $alertResult.suppressTelegram alertPath = $alertResult.alertPath + alertSha256 = $alertResult.alertSha256 + durableAlert = $true + sreInboxDispatchStatus = $alertResult.sreInboxDispatchStatus sreInboxExitCode = $alertResult.sreInboxExitCode } - } else { + $lastDurablyHandledUpdateId = [int64]$update.update_id + } elseif ($alertResult.skipped -and [string]$alertResult.reason -eq "resolved_alert_no_remediation") { $ignored += [pscustomobject]@{ updateId = $update.update_id - reason = $alertResult.reason + reason = [string]$alertResult.reason } + $lastDurablyHandledUpdateId = [int64]$update.update_id + } else { + $failedUpdates += [pscustomobject]@{ + updateId = $update.update_id + reason = if ($alertResult.reason) { [string]$alertResult.reason } else { "monitoring_alert_enqueue_failed" } + } + $processingFailed = $true + break } continue } @@ -798,22 +910,52 @@ foreach ($update in @($updatesResult.updates)) { updateId = $update.update_id reason = "no_agent99_prefix" } + $lastDurablyHandledUpdateId = [int64]$update.update_id continue } $submitArgs = @( "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $SubmitScript, - "-Instruction", $instruction, + "-InstructionBase64", (Convert-AgentBase64 $instruction), "-ControlledApply", "-Source", "telegram-sre-war-room", - "-RequestedBy", ([string]$update.from), + "-RequestedByBase64", (Convert-AgentBase64 ([string]$update.from)), "-ExternalId", ("telegram-update-" + [string]$update.update_id), - "-ReplyChatId", ([string]$update.chat_id) + "-ReplyChatId", ([string]$update.chat_id), + "-AgentRoot", $AgentRoot ) if ($RunNow) { $submitArgs += "-RunNow" } - $submit = Start-Process -FilePath "powershell.exe" -ArgumentList $submitArgs -Wait -PassThru -NoNewWindow -RedirectStandardOutput (Join-Path $EvidenceDir "agent99-telegram-submit-$($update.update_id).out") -RedirectStandardError (Join-Path $EvidenceDir "agent99-telegram-submit-$($update.update_id).err") + $submitOut = Join-Path $EvidenceDir "agent99-telegram-submit-$($update.update_id).out" + $submitErr = Join-Path $EvidenceDir "agent99-telegram-submit-$($update.update_id).err" + $submit = Start-Process -FilePath "powershell.exe" -ArgumentList $submitArgs -PassThru -NoNewWindow -RedirectStandardOutput $submitOut -RedirectStandardError $submitErr + $null = $submit.Handle + $submitWatch = [Diagnostics.Stopwatch]::StartNew() + $submitTimedOut = $false + $submitOutputLimitExceeded = $false + while (-not $submit.HasExited) { + $submit.Refresh() + $submitBytes = 0 + foreach ($path in @($submitOut, $submitErr)) { + if (Test-Path -LiteralPath $path) { $submitBytes += (Get-Item -LiteralPath $path).Length } + } + if ($submitBytes -gt 32768) { $submitOutputLimitExceeded = $true; break } + if ($submitWatch.Elapsed.TotalSeconds -ge 30) { $submitTimedOut = $true; break } + Start-Sleep -Milliseconds 100 + } + if ($submitTimedOut -or $submitOutputLimitExceeded) { + & "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$submit.Id) /T /F 2>&1 | Out-Null + try { $submit.WaitForExit(5000) | Out-Null } catch {} + } else { + $submit.WaitForExit() | Out-Null + } $submit.Refresh() + $submitExitCode = if ($submitTimedOut) { -2 } elseif ($submitOutputLimitExceeded) { -3 } elseif ($submit.HasExited -and $null -ne $submit.ExitCode) { $submit.ExitCode } else { -4 } + $durableSubmit = if ($submitExitCode -eq 0) { + Get-AgentDurableSubmitReadback $submitOut ([int64]$update.update_id) + } else { + [pscustomobject]@{ ok = $false; reason = "submit_process_failed" } + } $processed += [pscustomobject]@{ updateId = $update.update_id messageId = $update.message_id @@ -822,34 +964,78 @@ foreach ($update in @($updatesResult.updates)) { chatType = $update.chat_type chatMatch = $update.chat_match instruction = $instruction - submitExitCode = if ($null -ne $submit.ExitCode) { $submit.ExitCode } else { 0 } + submitExitCode = $submitExitCode + submitTimedOut = $submitTimedOut + submitOutputLimitExceeded = $submitOutputLimitExceeded + submitProcessStopped = [bool]$submit.HasExited + durableQueued = [bool]$durableSubmit.ok + queueId = if ($durableSubmit.ok) { [string]$durableSubmit.id } else { "" } + queueSha256 = if ($durableSubmit.ok) { [string]$durableSubmit.queueSha256 } else { "" } + submitEvidencePath = if ($durableSubmit.ok) { [string]$durableSubmit.evidencePath } else { "" } + submitEvidenceSha256 = if ($durableSubmit.ok) { [string]$durableSubmit.evidenceSha256 } else { "" } + idempotentReplay = if ($durableSubmit.ok) { [bool]$durableSubmit.idempotentReplay } else { $false } + queueLifecycleState = if ($durableSubmit.ok) { [string]$durableSubmit.queueLifecycleState } else { "" } + failureReason = if ($durableSubmit.ok) { "" } else { [string]$durableSubmit.reason } + } + $submit.Dispose() + if ($durableSubmit.ok) { + $lastDurablyHandledUpdateId = [int64]$update.update_id + } else { + $failedUpdates += [pscustomobject]@{ + updateId = $update.update_id + reason = [string]$durableSubmit.reason + submitExitCode = $submitExitCode + } + $processingFailed = $true + break } } -if ((-not $SyntheticUpdateText) -and $updatesResult.ok -and $maxUpdateId -ge 0) { - [pscustomobject]@{ - nextOffset = $maxUpdateId + 1 - updatedAt = (Get-Date -Format o) - } | ConvertTo-Json -Depth 4 | Set-Content -Path $offsetPath -Encoding UTF8 -} - +$plannedOffset = if ($lastDurablyHandledUpdateId -ge $offset) { $lastDurablyHandledUpdateId + 1 } else { $offset } +$batchOk = [bool]($updatesResult.ok -and -not $processingFailed) $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) - ok = [bool]$updatesResult.ok - reason = if ($updatesResult.ok -and @($updatesResult.updates).Count -eq 0) { "no_updates" } elseif ($updatesResult.ok) { "updates_read" } else { "updates_read_failed" } + ok = $batchOk + reason = if (-not $updatesResult.ok) { "updates_read_failed" } elseif ($processingFailed) { "update_processing_failed_no_checkpoint_past_failure" } elseif (@($updatesResult.updates).Count -eq 0) { "no_updates" } else { "updates_durably_handled" } error = $updatesResult.error relay = $updatesResult.relay offsetBefore = $offset - offsetAfter = if ($SyntheticUpdateText) { $offset } elseif ($updatesResult.ok) { $maxUpdateId + 1 } else { $offset } + offsetAfter = if ($SyntheticUpdateText) { $offset } else { $plannedOffset } + lastDurablyHandledUpdateId = $lastDurablyHandledUpdateId synthetic = [bool]$SyntheticUpdateText updatesSeen = @($updatesResult.updates).Count processedCount = @($processed).Count alertedCount = @($alerted).Count ignoredCount = @($ignored).Count + failedUpdateCount = @($failedUpdates).Count processed = $processed alerted = $alerted ignored = $ignored + failedUpdates = $failedUpdates offsetPath = $offsetPath } -$result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8 +$evidenceReceipt = Write-AgentDurableJson $result $evidencePath +$offsetReceipt = $null +if ((-not $SyntheticUpdateText) -and $updatesResult.ok -and $plannedOffset -gt $offset) { + try { + $offsetState = [pscustomobject]@{ + schemaVersion = "agent99_telegram_offset_v2" + nextOffset = $plannedOffset + lastDurablyHandledUpdateId = $lastDurablyHandledUpdateId + evidencePath = $evidencePath + evidenceSha256 = [string]$evidenceReceipt.sha256 + updatedAt = (Get-Date -Format o) + } + $offsetReceipt = Write-AgentDurableJson $offsetState $offsetPath "nextOffset" ([string]$plannedOffset) + } catch { + $result.ok = $false + $result.reason = "offset_checkpoint_failed_safe_replay_required" + $result.offsetAfter = $offset + $result | Add-Member -NotePropertyName offsetFailureType -NotePropertyValue $_.Exception.GetType().Name -Force + $evidenceReceipt = Write-AgentDurableJson $result $evidencePath + } +} +$result | Add-Member -NotePropertyName evidenceSha256 -NotePropertyValue ([string]$evidenceReceipt.sha256) -Force +$result | Add-Member -NotePropertyName offsetCheckpointSha256 -NotePropertyValue $(if ($offsetReceipt) { [string]$offsetReceipt.sha256 } else { "" }) -Force $result | ConvertTo-Json -Depth 10 +if (-not $result.ok) { exit 70 } diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 1cb04b65f..fe86a3f44 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -113,8 +113,9 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None: assert "Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope" in consumer assert '"controlled_dispatch_identity_invalid"' in consumer assert consumer.index('"controlled_dispatch_identity_invalid"') < consumer.index( - '$process = Start-Process -FilePath "powershell.exe"' + "$execution = Invoke-AgentBoundedProcess" ) + assert '$process = Start-Process -FilePath "powershell.exe"' not in consumer assert 'Add-Check "recovery:auto_trigger_controlled_identity"' in contract assert 'Add-Check "recovery:queue_identity_fail_closed"' in contract assert 'correlationKey = $dispatchIdentity.idempotencyKey' in contract diff --git a/scripts/reboot-recovery/agent99-control-loop-maintenance.ps1 b/scripts/reboot-recovery/agent99-control-loop-maintenance.ps1 new file mode 100644 index 000000000..c3713c550 --- /dev/null +++ b/scripts/reboot-recovery/agent99-control-loop-maintenance.ps1 @@ -0,0 +1,283 @@ +[CmdletBinding()] +param( + [ValidateSet("Check", "Apply")] + [string]$Mode = "Check", + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2147483647)] + [int]$ExpectedControlPid, + [Parameter(Mandatory = $true)] + [datetime]$ExpectedControlCreated, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2147483647)] + [int]$ExpectedRecoverPid, + [Parameter(Mandatory = $true)] + [datetime]$ExpectedRecoverCreated, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2147483647)] + [int]$ExpectedRecoverParentPid, + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" +$taskName = "Wooo-Agent99-Control-Loop" +$runtimeWritePerformed = $false + +function Get-ExactProcess { + param([int]$ProcessId) + Get-CimInstance Win32_Process -ErrorAction Stop | + Where-Object { [int]$_.ProcessId -eq $ProcessId } | + Select-Object -First 1 +} + +function Test-ExactCreationTime { + param([object]$Process, [datetime]$ExpectedCreated) + if (-not $Process -or -not $Process.CreationDate) { return $false } + $actual = ([datetime]$Process.CreationDate).ToUniversalTime() + $expected = $ExpectedCreated.ToUniversalTime() + [bool]($actual.Ticks -eq $expected.Ticks) +} + +function Test-ControlIdentity { + param([object]$Process) + [bool]( + $Process -and + [string]$Process.Name -eq "powershell.exe" -and + (Test-ExactCreationTime $Process $ExpectedControlCreated) -and + ([string]$Process.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' + ) +} + +function Test-RecoverIdentity { + param([object]$Process) + [bool]( + $Process -and + [string]$Process.Name -eq "powershell.exe" -and + (Test-ExactCreationTime $Process $ExpectedRecoverCreated) -and + [int]$Process.ParentProcessId -eq $ExpectedRecoverParentPid -and + ([string]$Process.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply' + ) +} + +function Wait-ProcessAbsent { + param([int]$ProcessId, [int]$TimeoutSeconds) + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + if (-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue)) { return $true } + Start-Sleep -Milliseconds 200 + } + [bool](-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue)) +} + +function Test-RecoveryMutexActive { + $mutex = $null + $acquired = $false + try { + $mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1") + try { $acquired = [bool]$mutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $acquired = $true } + [bool](-not $acquired) + } finally { + if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { $mutex.Dispose() } + } +} + +function Write-DurableReceipt { + param([object]$Value, [string]$Path) + $temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))" + $stream = $null + $writer = $null + try { + $encoding = New-Object Text.UTF8Encoding($false) + $stream = [IO.File]::Open($temp, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) + $writer = New-Object IO.StreamWriter($stream, $encoding) + $writer.Write(($Value | ConvertTo-Json -Depth 8)) + $writer.Flush() + $stream.Flush($true) + $writer.Dispose(); $writer = $null + $stream.Dispose(); $stream = $null + $beforeHash = (Get-FileHash -LiteralPath $temp -Algorithm SHA256).Hash.ToLowerInvariant() + Move-Item -LiteralPath $temp -Destination $Path -Force + $afterHash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($beforeHash -ne $afterHash) { throw "receipt_readback_mismatch" } + } finally { + if ($writer) { $writer.Dispose() } + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue + } +} + +$maintenanceMutex = [Threading.Mutex]::new($false, "Global\WoooAgent99ControlLoopMaintenanceV1") +$maintenanceMutexAcquired = $false +try { + try { + $maintenanceMutexAcquired = [bool]$maintenanceMutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $maintenanceMutexAcquired = $true + } + if (-not $maintenanceMutexAcquired) { + [pscustomobject]@{ + schemaVersion = "agent99_control_loop_maintenance_v2" + timestamp = (Get-Date).ToString("o") + mode = $Mode + precheckPassed = $false + blockers = @("maintenance_writer_active") + terminal = "blocked_single_writer_active" + runtimeWritePerformed = $false + secretValueRead = $false + } | ConvertTo-Json -Depth 5 -Compress + exit 75 + } + +$blockers = New-Object System.Collections.Generic.List[string] +if ($env:COMPUTERNAME -ne "WOOO-SUPER") { $blockers.Add("computer_identity_mismatch") } +$task = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue +if (-not $task) { + $blockers.Add("control_task_missing") +} else { + $actions = @($task.Actions) + if ($actions.Count -ne 1) { $blockers.Add("control_task_action_count_mismatch") } + if ($actions.Count -eq 1) { + if ([string]$actions[0].Execute -notmatch '(?i)\\WindowsPowerShell\\v1\.0\\powershell\.exe$') { $blockers.Add("control_task_executable_mismatch") } + if ([string]$actions[0].Arguments -notmatch '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)') { $blockers.Add("control_task_arguments_mismatch") } + } +} + +$control = Get-ExactProcess $ExpectedControlPid +$recover = Get-ExactProcess $ExpectedRecoverPid +if (-not $control) { + $blockers.Add("control_process_missing") +} else { + if (-not (Test-ControlIdentity $control)) { $blockers.Add("control_process_identity_mismatch") } +} +if (-not $recover) { + $blockers.Add("recover_process_missing") +} else { + if (-not (Test-RecoverIdentity $recover)) { $blockers.Add("recover_process_identity_mismatch") } +} + +$precheckPassed = [bool]($blockers.Count -eq 0) +$controlStopped = $false +$recoverStopped = $false +$leaseMetadataRemoved = $false +$mutexReleased = $false +$taskAfter = $task +$operationIntentRef = "" +$deviceMutationAttempted = $false +$operationFailureType = "" +$terminal = if ($precheckPassed) { "maintenance_freeze_check_verified" } else { "blocked_identity_mismatch" } + +if ($Mode -eq "Apply" -and $precheckPassed) { + $intentStamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" + $operationIntentRef = "agent99-control-loop-maintenance-intent-$intentStamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json" + $operationIntentPath = Join-Path $AgentRoot "evidence\$operationIntentRef" + $operationIntent = [pscustomobject]@{ + schemaVersion = "agent99_control_loop_maintenance_intent_v1" + timestamp = (Get-Date).ToString("o") + taskName = $taskName + taskEnabledBefore = [bool]$task.Settings.Enabled + expectedControlPid = $ExpectedControlPid + expectedControlCreated = $ExpectedControlCreated.ToUniversalTime().ToString("o") + expectedRecoverPid = $ExpectedRecoverPid + expectedRecoverCreated = $ExpectedRecoverCreated.ToUniversalTime().ToString("o") + expectedRecoverParentPid = $ExpectedRecoverParentPid + plannedActions = @("disable_control_task", "stop_exact_control_tree", "stop_exact_recover_tree", "remove_exact_recovery_lease") + rollback = "keep task frozen on partial mutation; restore only after patched runtime verifier" + vmPowerChangePerformed = $false + vmwareVmxProcessTerminated = $false + secretValueRead = $false + } + Write-DurableReceipt $operationIntent $operationIntentPath + $runtimeWritePerformed = $true + + try { + $deviceMutationAttempted = $true + Disable-ScheduledTask -TaskName $taskName -TaskPath "\" | Out-Null + Stop-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue + $controlStopped = Wait-ProcessAbsent $ExpectedControlPid 15 + if (-not $controlStopped) { + $controlRecheck = Get-ExactProcess $ExpectedControlPid + if (-not (Test-ControlIdentity $controlRecheck)) { throw "control_recheck_identity_mismatch" } + & "$env:SystemRoot\System32\taskkill.exe" /PID $ExpectedControlPid /T /F 2>&1 | Out-Null + $controlStopped = Wait-ProcessAbsent $ExpectedControlPid 15 + } + if (-not (Wait-ProcessAbsent $ExpectedRecoverPid 5)) { + $recoverRecheck = Get-ExactProcess $ExpectedRecoverPid + if (-not (Test-RecoverIdentity $recoverRecheck)) { throw "recover_recheck_identity_mismatch" } + & "$env:SystemRoot\System32\taskkill.exe" /PID $ExpectedRecoverPid /T /F 2>&1 | Out-Null + } + $recoverStopped = Wait-ProcessAbsent $ExpectedRecoverPid 15 + $controlStopped = [bool]($controlStopped -or (Wait-ProcessAbsent $ExpectedControlPid 5)) + + $leasePath = Join-Path $AgentRoot "state\recovery-transport-lease.json" + $mutexReleased = [bool](-not (Test-RecoveryMutexActive)) + if ($recoverStopped -and $mutexReleased -and (Test-Path -LiteralPath $leasePath)) { + $lease = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json + if ( + [int]$lease.processId -eq $ExpectedRecoverPid -and + [string]$lease.runId -match "^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|agent99-recovery-[0-9]{8}-[0-9]{6}-[0-9a-f]{8})$" + ) { + Remove-Item -LiteralPath $leasePath -Force + } + } + $leaseMetadataRemoved = [bool](-not (Test-Path -LiteralPath $leasePath)) + $taskAfter = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop + $verified = [bool](-not $taskAfter.Settings.Enabled -and $controlStopped -and $recoverStopped -and $mutexReleased -and $leaseMetadataRemoved) + $terminal = if ($verified) { "control_loop_maintenance_frozen" } else { "maintenance_freeze_unverified" } + } catch { + $operationFailureType = $_.Exception.GetType().Name + $taskAfter = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue + $controlStopped = Wait-ProcessAbsent $ExpectedControlPid 1 + $recoverStopped = Wait-ProcessAbsent $ExpectedRecoverPid 1 + $mutexReleased = [bool](-not (Test-RecoveryMutexActive)) + $leasePath = Join-Path $AgentRoot "state\recovery-transport-lease.json" + $leaseMetadataRemoved = [bool](-not (Test-Path -LiteralPath $leasePath)) + $terminal = if ($deviceMutationAttempted) { "maintenance_partial_frozen_manual_recovery" } else { "maintenance_apply_failed_no_mutation" } + } +} + +$result = [pscustomobject]@{ + schemaVersion = "agent99_control_loop_maintenance_v2" + timestamp = (Get-Date).ToString("o") + mode = $Mode + taskName = $taskName + expectedControlPid = $ExpectedControlPid + expectedControlCreated = $ExpectedControlCreated.ToUniversalTime().ToString("o") + expectedRecoverPid = $ExpectedRecoverPid + expectedRecoverCreated = $ExpectedRecoverCreated.ToUniversalTime().ToString("o") + expectedRecoverParentPid = $ExpectedRecoverParentPid + precheckPassed = $precheckPassed + blockers = @($blockers) + taskEnabledBefore = if ($task) { [bool]$task.Settings.Enabled } else { $null } + taskEnabledAfter = if ($taskAfter) { [bool]$taskAfter.Settings.Enabled } else { $null } + operationIntentRef = $operationIntentRef + deviceMutationAttempted = $deviceMutationAttempted + operationFailureType = $operationFailureType + rollbackStatus = if ($deviceMutationAttempted -and $terminal -ne "control_loop_maintenance_frozen") { "task_kept_frozen_manual_recovery_required" } else { "not_required" } + controlStopped = $controlStopped + recoverStopped = $recoverStopped + leaseMetadataRemoved = $leaseMetadataRemoved + terminal = $terminal + runtimeWritePerformed = $runtimeWritePerformed + vmPowerChangePerformed = $false + vmwareVmxProcessTerminated = $false + vmxLockDeleted = $false + hostRebootPerformed = $false + secretValueRead = $false +} + +if ($Mode -eq "Apply") { + $path = Join-Path $AgentRoot ("evidence\agent99-control-loop-maintenance-" + (Get-Date -Format "yyyyMMdd-HHmmss-fff") + "-" + ([guid]::NewGuid().ToString('N').Substring(0, 8)) + ".json") + Write-DurableReceipt $result $path + $result | Add-Member -NotePropertyName receiptRef -NotePropertyValue (Split-Path -Leaf $path) -Force +} +$result | ConvertTo-Json -Depth 8 -Compress +if (-not $precheckPassed) { exit 64 } +if ($Mode -eq "Check") { exit 0 } +if ($terminal -eq "control_loop_maintenance_frozen") { exit 0 } +exit 70 +} finally { + if ($maintenanceMutexAcquired -and $maintenanceMutex) { + try { $maintenanceMutex.ReleaseMutex() } catch {} + } + if ($maintenanceMutex) { $maintenanceMutex.Dispose() } +} diff --git a/scripts/reboot-recovery/agent99-stale-recovery-breakglass.ps1 b/scripts/reboot-recovery/agent99-stale-recovery-breakglass.ps1 new file mode 100644 index 000000000..24ec04d85 --- /dev/null +++ b/scripts/reboot-recovery/agent99-stale-recovery-breakglass.ps1 @@ -0,0 +1,516 @@ +[CmdletBinding()] +param( + [ValidateSet("Check", "Apply")] + [string]$Mode = "Check", + [Parameter(Mandatory = $true)] + [ValidatePattern("^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|agent99-recovery-[0-9]{8}-[0-9]{6}-[0-9a-f]{8})$")] + [string]$ExpectedRunId, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2147483647)] + [int]$ExpectedParentPid, + [Parameter(Mandatory = $true)] + [datetime]$ExpectedParentCreated, + [Parameter(Mandatory = $true)] + [ValidateRange(1, 2147483647)] + [int]$ExpectedChildPid, + [Parameter(Mandatory = $true)] + [datetime]$ExpectedChildCreated, + [Parameter(Mandatory = $true)] + [string]$ExpectedVmxPath, + [ValidateRange(20, 1440)] + [int]$MinimumAgeMinutes = 20, + [ValidatePattern("^$|^[A-Za-z0-9._-]{1,180}\.json$")] + [string]$PriorReceiptRef = "", + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" +$script:RuntimeWritePerformed = $false +$script:ReceiptPath = "" +$postFailureReadbackErrors = @() + +function Get-Sha256Text { + param([string]$Text) + $sha = [Security.Cryptography.SHA256]::Create() + try { + ([BitConverter]::ToString($sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($Text)))).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +function Write-AtomicJson { + param([object]$Value, [string]$Path) + $directory = Split-Path -Parent $Path + if (-not (Test-Path -LiteralPath $directory)) { + New-Item -ItemType Directory -Path $directory -Force | Out-Null + } + $temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))" + $stream = $null + $writer = $null + try { + $json = $Value | ConvertTo-Json -Depth 8 + $encoding = New-Object Text.UTF8Encoding($false) + $stream = [IO.File]::Open($temp, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) + $writer = New-Object IO.StreamWriter($stream, $encoding) + $writer.Write($json) + $writer.Flush() + $stream.Flush($true) + $writer.Dispose() + $writer = $null + $stream.Dispose() + $stream = $null + $tempHash = (Get-FileHash -LiteralPath $temp -Algorithm SHA256).Hash.ToLowerInvariant() + Move-Item -LiteralPath $temp -Destination $Path -Force + if (-not (Test-Path -LiteralPath $Path)) { throw "receipt_publish_failed" } + $readbackHash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($readbackHash -ne $tempHash) { throw "receipt_readback_mismatch" } + } finally { + if ($writer) { $writer.Dispose() } + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue + } +} + +function Get-ExactProcess { + param([int]$ProcessId) + Get-CimInstance Win32_Process -ErrorAction Stop | Where-Object { [int]$_.ProcessId -eq $ProcessId } | Select-Object -First 1 +} + +function Test-ExactProcessIdentity { + param( + [object]$Process, + [int]$ExpectedPid, + [datetime]$ExpectedCreated, + [string]$ExpectedName, + [int]$ExpectedParentPid = -1 + ) + if (-not $Process -or -not $Process.CreationDate) { return $false } + [bool]( + [int]$Process.ProcessId -eq $ExpectedPid -and + [string]$Process.Name -eq $ExpectedName -and + ($ExpectedParentPid -lt 0 -or [int]$Process.ParentProcessId -eq $ExpectedParentPid) -and + ([datetime]$Process.CreationDate).ToUniversalTime().Ticks -eq $ExpectedCreated.ToUniversalTime().Ticks + ) +} + +function Wait-ExactProcessIdentityAbsent { + param( + [int]$ProcessId, + [datetime]$ExpectedCreated, + [int]$TimeoutSeconds = 10 + ) + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + $current = Get-ExactProcess $ProcessId + if (-not $current -or ([datetime]$current.CreationDate).ToUniversalTime().Ticks -ne $ExpectedCreated.ToUniversalTime().Ticks) { return $true } + Start-Sleep -Milliseconds 200 + } + $current = Get-ExactProcess $ProcessId + [bool](-not $current -or ([datetime]$current.CreationDate).ToUniversalTime().Ticks -ne $ExpectedCreated.ToUniversalTime().Ticks) +} + +function Get-FailedMutationReadback { + param( + [int]$ParentProcessId, + [datetime]$ParentCreated, + [int]$ChildProcessId, + [datetime]$ChildCreated, + [string]$LeasePath + ) + $errors = @() + try { + $parentAbsent = Wait-ExactProcessIdentityAbsent $ParentProcessId $ParentCreated 1 + } catch { + $parentAbsent = $false + $errors += "parent_identity_unknown" + } + try { + $childAbsent = Wait-ExactProcessIdentityAbsent $ChildProcessId $ChildCreated 1 + } catch { + $childAbsent = $false + $errors += "child_identity_unknown" + } + try { + $recoveryMutexReleased = [bool](-not (Test-RecoveryMutexActive)) + } catch { + $recoveryMutexReleased = $false + $errors += "recovery_mutex_unknown" + } + try { + $leaseRemoved = [bool](-not (Test-Path -LiteralPath $LeasePath)) + } catch { + $leaseRemoved = $false + $errors += "lease_metadata_unknown" + } + [pscustomobject]@{ + parentAbsent = [bool]$parentAbsent + childAbsent = [bool]$childAbsent + recoveryMutexReleased = [bool]$recoveryMutexReleased + leaseRemoved = [bool]$leaseRemoved + complete = [bool]($errors.Count -eq 0) + errors = @($errors) + } +} + +function Wait-ExactProcessAbsent { + param( + [int]$ProcessId, + [int]$TimeoutSeconds = 10 + ) + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + if (-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue)) { return $true } + Start-Sleep -Milliseconds 200 + } + [bool](-not (Get-Process -Id $ProcessId -ErrorAction SilentlyContinue)) +} + +function Get-DescendantProcesses { + param([int]$ParentProcessId) + $all = @(Get-CimInstance Win32_Process -ErrorAction Stop) + $known = New-Object 'System.Collections.Generic.HashSet[int]' + [void]$known.Add($ParentProcessId) + $rows = @() + do { + $added = $false + foreach ($processInfo in $all) { + if ( + $known.Contains([int]$processInfo.ParentProcessId) -and + -not $known.Contains([int]$processInfo.ProcessId) + ) { + [void]$known.Add([int]$processInfo.ProcessId) + $rows += $processInfo + $added = $true + } + } + } while ($added) + @($rows) +} + +function Invoke-ExactTaskkill { + param([int]$ProcessId) + $output = @(& "$env:SystemRoot\System32\taskkill.exe" /PID $ProcessId /T /F 2>&1) + [pscustomobject]@{ + exitCode = [int]$LASTEXITCODE + outputPresent = [bool]($output.Count -gt 0) + rawOutputStored = $false + } +} + +function Test-RecoveryMutexActive { + $mutex = $null + $acquired = $false + try { + $mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1") + try { $acquired = [bool]$mutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $acquired = $true } + [bool](-not $acquired) + } finally { + if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { $mutex.Dispose() } + } +} + +$reconcilerMutex = [Threading.Mutex]::new($false, "Global\WoooAgent99StaleRecoveryReconcilerV1") +$reconcilerMutexAcquired = $false +try { $reconcilerMutexAcquired = [bool]$reconcilerMutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $reconcilerMutexAcquired = $true } +if (-not $reconcilerMutexAcquired) { + [pscustomobject]@{ + schemaVersion = "agent99_stale_recovery_breakglass_v3" + mode = $Mode + precheckPassed = $false + blockers = @("stale_recovery_reconciler_active") + terminal = "blocked_single_writer_active" + runtimeWritePerformed = $false + secretValueRead = $false + } | ConvertTo-Json -Compress + $reconcilerMutex.Dispose() + exit 75 +} + +try { +$leasePath = Join-Path $AgentRoot "state\recovery-transport-lease.json" +$preconditions = New-Object System.Collections.Generic.List[string] +$lease = $null +if (-not (Test-Path -LiteralPath $leasePath)) { + $preconditions.Add("lease_missing") +} else { + try { $lease = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json } catch { $preconditions.Add("lease_parse_failed") } +} + +$parent = Get-ExactProcess $ExpectedParentPid +$child = Get-ExactProcess $ExpectedChildPid +$canonicalVmx = [IO.Path]::GetFullPath($ExpectedVmxPath).TrimEnd("\") +$vmxIdentityDigest = Get-Sha256Text $canonicalVmx.ToLowerInvariant() +$priorReceipt = $null +$priorReceiptVerified = $false +$now = Get-Date +$parentAgeMinutes = -1 +if ($parent) { + try { $parentAgeMinutes = [math]::Round(($now - (Get-Process -Id $ExpectedParentPid -ErrorAction Stop).StartTime).TotalMinutes, 2) } catch {} +} + +if (-not $lease) { + $preconditions.Add("lease_unavailable") +} else { + if ([string]$lease.schemaVersion -ne "agent99_recovery_transport_lease_v1") { $preconditions.Add("lease_schema_mismatch") } + if ([string]$lease.runId -ne $ExpectedRunId) { $preconditions.Add("lease_run_id_mismatch") } + if ([int]$lease.processId -ne $ExpectedParentPid) { $preconditions.Add("lease_process_id_mismatch") } + if ([bool]$lease.storesSecret) { $preconditions.Add("lease_secret_contract_invalid") } +} +if (-not $parent) { + $preconditions.Add("parent_missing") +} else { + $parentCommand = [string]$parent.CommandLine + if (-not (Test-ExactProcessIdentity $parent $ExpectedParentPid $ExpectedParentCreated "powershell.exe")) { $preconditions.Add("parent_identity_mismatch") } + if ($parentCommand -notmatch '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply') { $preconditions.Add("parent_command_mismatch") } + $runIdBoundToCommand = if ($ExpectedRunId -match '^agent99-recovery-') { + $parentCommand -notmatch "(?i)-AutomationRunId\s+" + } else { + $parentCommand.IndexOf($ExpectedRunId, [StringComparison]::OrdinalIgnoreCase) -ge 0 + } + if (-not $runIdBoundToCommand) { $preconditions.Add("parent_run_id_mismatch") } + if ($lease -and $lease.acquiredAt) { + $leaseAcquiredUtc = ([datetime]$lease.acquiredAt).ToUniversalTime() + $parentCreatedUtc = $ExpectedParentCreated.ToUniversalTime() + if ($leaseAcquiredUtc -lt $parentCreatedUtc -or $leaseAcquiredUtc -gt $parentCreatedUtc.AddMinutes(5)) { $preconditions.Add("lease_parent_creation_mismatch") } + } else { + $preconditions.Add("lease_acquired_at_missing") + } + if ($parentAgeMinutes -lt $MinimumAgeMinutes) { $preconditions.Add("parent_not_stale") } +} +if (-not $child) { + if (-not $PriorReceiptRef) { + $preconditions.Add("child_missing_without_prior_receipt") + } else { + $priorPath = Join-Path (Join-Path $AgentRoot "evidence") $PriorReceiptRef + try { + $priorReceipt = Get-Content -LiteralPath $priorPath -Raw | ConvertFrom-Json + $priorSchema = [string]$priorReceipt.schemaVersion + $priorCreationIdentityValid = if ($priorSchema -eq "agent99_stale_recovery_breakglass_v3") { + [bool]( + $priorReceipt.PSObject.Properties["expectedParentCreated"] -and + $priorReceipt.PSObject.Properties["expectedChildCreated"] -and + ([datetime]$priorReceipt.expectedParentCreated).ToUniversalTime().Ticks -eq $ExpectedParentCreated.ToUniversalTime().Ticks -and + ([datetime]$priorReceipt.expectedChildCreated).ToUniversalTime().Ticks -eq $ExpectedChildCreated.ToUniversalTime().Ticks + ) + } else { + [bool]($priorSchema -eq "agent99_stale_recovery_breakglass_v2" -and $priorReceipt.childTerminated -is [bool] -and $priorReceipt.childTerminated) + } + $priorReceiptVerified = [bool]( + $priorSchema -in @("agent99_stale_recovery_breakglass_v2", "agent99_stale_recovery_breakglass_v3") -and + $priorCreationIdentityValid -and + [string]$priorReceipt.mode -eq "Apply" -and + [string]$priorReceipt.expectedRunId -eq $ExpectedRunId -and + [int]$priorReceipt.expectedParentPid -eq $ExpectedParentPid -and + [int]$priorReceipt.expectedChildPid -eq $ExpectedChildPid -and + [string]$priorReceipt.vmxIdentityDigest -eq $vmxIdentityDigest -and + $priorReceipt.precheckPassed -is [bool] -and + $priorReceipt.precheckPassed -eq $true -and + $priorReceipt.runtimeWritePerformed -is [bool] -and + $priorReceipt.runtimeWritePerformed -eq $true -and + [string]$priorReceipt.terminal -in @("child_termination_failed", "child_terminated_parent_draining") + ) + if (-not $priorReceiptVerified) { $preconditions.Add("prior_receipt_identity_mismatch") } + } catch { + $preconditions.Add("prior_receipt_readback_failed") + } + } +} else { + $childCommand = [string]$child.CommandLine + if (-not (Test-ExactProcessIdentity $child $ExpectedChildPid $ExpectedChildCreated "vmrun.exe" $ExpectedParentPid)) { $preconditions.Add("child_identity_mismatch") } + if ($childCommand -notmatch "(?i)vmrun\.exe.*\sstart\s") { $preconditions.Add("child_command_mismatch") } + if ($childCommand.IndexOf($canonicalVmx, [StringComparison]::OrdinalIgnoreCase) -lt 0) { $preconditions.Add("child_vmx_mismatch") } + if ($childCommand -notmatch "(?i)\snogui(\s|$)") { $preconditions.Add("child_nogui_mismatch") } +} +if (-not (Test-RecoveryMutexActive)) { $preconditions.Add("recovery_mutex_not_active") } + +$precheckPassed = [bool]($preconditions.Count -eq 0) +$terminal = if ($precheckPassed) { "check_verified" } else { "blocked_identity_mismatch" } +$childTerminated = $false +$childAlreadyAbsent = [bool](-not $child -and $priorReceiptVerified) +$parentDrained = $false +$parentTerminated = $false +$mutexReleased = $false +$leaseMetadataRemoved = $false +$childTaskkill = $null +$parentTaskkill = $null +$operationIntentRef = "" +$parentCleanupIntentRef = "" +$mutationAttempted = $false +$operationFailureType = "" + +if ($Mode -eq "Apply" -and $precheckPassed) { + try { + $intentStamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" + $operationIntentRef = "agent99-stale-recovery-breakglass-intent-$intentStamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json" + $operationIntentPath = Join-Path (Join-Path $AgentRoot "evidence") $operationIntentRef + $operationIntent = [pscustomobject]@{ + schemaVersion = "agent99_stale_recovery_breakglass_intent_v1" + timestamp = (Get-Date).ToString("o") + expectedRunId = $ExpectedRunId + expectedParentPid = $ExpectedParentPid + expectedParentCreated = $ExpectedParentCreated.ToUniversalTime().ToString("o") + expectedChildPid = $ExpectedChildPid + expectedChildCreated = $ExpectedChildCreated.ToUniversalTime().ToString("o") + vmxIdentityDigest = $vmxIdentityDigest + childAlreadyAbsent = $childAlreadyAbsent + precheckPassed = $true + plannedActions = @("terminate_exact_vmrun_child_if_present", "wait_or_terminate_exact_recover_parent", "remove_exact_lease_metadata") + rollback = "process_termination_not_reversible; preserve intent and terminal receipt" + vmPowerChangePerformed = $false + vmwareVmxProcessTerminated = $false + secretValueRead = $false + } + Write-AtomicJson $operationIntent $operationIntentPath + $script:RuntimeWritePerformed = $true + + if ($childAlreadyAbsent) { + $childTerminated = $true + } else { + $childRecheck = Get-ExactProcess $ExpectedChildPid + if ( + -not (Test-ExactProcessIdentity $childRecheck $ExpectedChildPid $ExpectedChildCreated "vmrun.exe" $ExpectedParentPid) -or + ([string]$childRecheck.CommandLine) -ne $childCommand + ) { + $terminal = "child_recheck_identity_changed" + } else { + $mutationAttempted = $true + $childTaskkill = Invoke-ExactTaskkill $ExpectedChildPid + $script:RuntimeWritePerformed = $true + $childTerminated = Wait-ExactProcessIdentityAbsent $ExpectedChildPid $ExpectedChildCreated 15 + } + } + if (-not $childTerminated) { + if ($terminal -ne "child_recheck_identity_changed") { $terminal = "child_termination_failed" } + } else { + $terminal = "child_terminated_parent_draining" + $parentDrained = Wait-ExactProcessIdentityAbsent $ExpectedParentPid $ExpectedParentCreated 30 + if (-not $parentDrained) { + $parentRecheck = Get-ExactProcess $ExpectedParentPid + $descendants = @(Get-DescendantProcesses $ExpectedParentPid) + $parentRecheckValid = [bool]( + (Test-ExactProcessIdentity $parentRecheck $ExpectedParentPid $ExpectedParentCreated "powershell.exe") -and + ([string]$parentRecheck.CommandLine) -eq $parentCommand -and + $descendants.Count -eq 0 + ) + if ($parentRecheckValid) { + $parentIntentStamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" + $parentCleanupIntentRef = "agent99-stale-recovery-parent-cleanup-intent-$parentIntentStamp.json" + $intentPath = Join-Path (Join-Path $AgentRoot "evidence") $parentCleanupIntentRef + $intent = [pscustomobject]@{ + schemaVersion = "agent99_stale_recovery_parent_cleanup_intent_v1" + timestamp = (Get-Date).ToString("o") + expectedRunId = $ExpectedRunId + expectedParentPid = $ExpectedParentPid + expectedParentCreated = $ExpectedParentCreated.ToUniversalTime().ToString("o") + expectedChildPid = $ExpectedChildPid + expectedChildCreated = $ExpectedChildCreated.ToUniversalTime().ToString("o") + vmxIdentityDigest = $vmxIdentityDigest + childAbsent = $true + descendantCount = 0 + leaseIdentityVerified = $true + mutexActive = $true + nextAction = "terminate_exact_stale_recover_parent" + secretValueRead = $false + } + Write-AtomicJson $intent $intentPath + $script:RuntimeWritePerformed = $true + $mutationAttempted = $true + $parentTaskkill = Invoke-ExactTaskkill $ExpectedParentPid + $parentTerminated = Wait-ExactProcessIdentityAbsent $ExpectedParentPid $ExpectedParentCreated 15 + $parentDrained = $parentTerminated + } else { + $terminal = "parent_cleanup_identity_changed" + } + } + $mutexReleased = [bool](-not (Test-RecoveryMutexActive)) + if ($parentDrained -and $mutexReleased) { + $leaseRecheck = $null + try { $leaseRecheck = Get-Content -LiteralPath $leasePath -Raw | ConvertFrom-Json } catch {} + if ( + $leaseRecheck -and + [string]$leaseRecheck.runId -eq $ExpectedRunId -and + [int]$leaseRecheck.processId -eq $ExpectedParentPid -and + -not (Get-ExactProcess $ExpectedParentPid) + ) { + $mutationAttempted = $true + Remove-Item -LiteralPath $leasePath -Force -ErrorAction Stop + $script:RuntimeWritePerformed = $true + } + $leaseMetadataRemoved = [bool](-not (Test-Path -LiteralPath $leasePath)) + if ($leaseMetadataRemoved) { $terminal = "stale_recovery_released" } else { $terminal = "lease_metadata_cleanup_failed" } + } + } + } catch { + $operationFailureType = $_.Exception.GetType().Name + $terminal = if ($mutationAttempted) { "partial_write_failed_closed" } else { "apply_intent_or_precheck_failed_no_mutation" } + $failureReadback = Get-FailedMutationReadback $ExpectedParentPid $ExpectedParentCreated $ExpectedChildPid $ExpectedChildCreated $leasePath + $parentDrained = [bool]$failureReadback.parentAbsent + $childTerminated = [bool]$failureReadback.childAbsent + $mutexReleased = [bool]$failureReadback.recoveryMutexReleased + $leaseMetadataRemoved = [bool]$failureReadback.leaseRemoved + $postFailureReadbackErrors = @($failureReadback.errors) + } +} + +$result = [pscustomobject]@{ + schemaVersion = "agent99_stale_recovery_breakglass_v3" + timestamp = (Get-Date).ToString("o") + mode = $Mode + host = $env:COMPUTERNAME + expectedRunId = $ExpectedRunId + expectedParentPid = $ExpectedParentPid + expectedParentCreated = $ExpectedParentCreated.ToUniversalTime().ToString("o") + expectedChildPid = $ExpectedChildPid + expectedChildCreated = $ExpectedChildCreated.ToUniversalTime().ToString("o") + vmxIdentityDigest = $vmxIdentityDigest + minimumAgeMinutes = $MinimumAgeMinutes + observedParentAgeMinutes = $parentAgeMinutes + precheckPassed = $precheckPassed + blockers = @($preconditions) + priorReceiptRef = $PriorReceiptRef + priorReceiptVerified = $priorReceiptVerified + childAlreadyAbsent = $childAlreadyAbsent + childTerminated = $childTerminated + childTaskkillExitCode = if ($childTaskkill) { [int]$childTaskkill.exitCode } else { $null } + parentDrained = $parentDrained + parentTerminated = $parentTerminated + parentTaskkillExitCode = if ($parentTaskkill) { [int]$parentTaskkill.exitCode } else { $null } + operationIntentRef = $operationIntentRef + parentCleanupIntentRef = $parentCleanupIntentRef + mutationAttempted = $mutationAttempted + operationFailureType = $operationFailureType + postFailureReadbackComplete = [bool]($postFailureReadbackErrors.Count -eq 0) + postFailureReadbackErrors = @($postFailureReadbackErrors) + rollbackStatus = if ($mutationAttempted) { "not_reversible_process_termination_manual_verification_required" } else { "not_required_no_mutation" } + mutexReleased = $mutexReleased + leaseMetadataRemoved = $leaseMetadataRemoved + terminal = $terminal + runtimeWritePerformed = $script:RuntimeWritePerformed + vmPowerChangePerformed = $false + vmwareVmxProcessTerminated = $false + vmxLockDeleted = $false + hostRebootPerformed = $false + secretValueRead = $false +} + +if ($Mode -eq "Apply") { + $stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" + $script:ReceiptPath = Join-Path $AgentRoot "evidence\agent99-stale-recovery-breakglass-$stamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json" + Write-AtomicJson $result $script:ReceiptPath + $result | Add-Member -NotePropertyName receiptRef -NotePropertyValue (Split-Path -Leaf $script:ReceiptPath) -Force +} +} finally { + if ($reconcilerMutexAcquired -and $reconcilerMutex) { + try { $reconcilerMutex.ReleaseMutex() } catch {} + } + if ($reconcilerMutex) { $reconcilerMutex.Dispose() } +} + +$result | ConvertTo-Json -Depth 8 -Compress +if (-not $precheckPassed) { exit 64 } +if ($Mode -eq "Check") { exit 0 } +if ($terminal -eq "stale_recovery_released") { exit 0 } +if ($terminal -eq "child_terminated_parent_draining") { exit 2 } +exit 70 diff --git a/scripts/reboot-recovery/tests/agent99-bounded-process-replay.ps1 b/scripts/reboot-recovery/tests/agent99-bounded-process-replay.ps1 new file mode 100644 index 000000000..81dc5bfe3 --- /dev/null +++ b/scripts/reboot-recovery/tests/agent99-bounded-process-replay.ps1 @@ -0,0 +1,345 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$ControlPlaneSource +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Get-ExactFunctionSource { + param([object]$Ast, [string]$Name) + $matches = @($Ast.FindAll({ + param($node) + $node -is [Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -eq $Name + }, $true)) + if ($matches.Count -ne 1) { throw "function_identity_mismatch:$Name" } + [string]$matches[0].Extent.Text +} + +function Assert-Replay { + param([bool]$Condition, [string]$Reason) + if (-not $Condition) { throw $Reason } +} + +$tokens = $null +$errors = $null +$ast = [Management.Automation.Language.Parser]::ParseInput( + $ControlPlaneSource, + [ref]$tokens, + [ref]$errors +) +if ($errors.Count -ne 0) { throw "control_plane_parse_failed" } +foreach ($name in @( + "Get-AgentProcessDescendants", + "ConvertTo-AgentNativeProcessArgument", + "ConvertTo-AgentProcessIdentitySnapshot", + "Get-AgentCreationBoundedDescendants", + "Test-AgentProcessSnapshotMatch", + "Select-AgentProcessTreeIdentitySnapshot", + "Get-AgentProcessTreeIdentitySnapshot", + "Get-AgentProcessIdentityReadback", + "Test-AgentProcessIdentityActive", + "Wait-AgentProcessTreeIdentityAbsent", + "Stop-AgentBoundedProcessSnapshot", + "Read-AgentBoundedTextFile", + "Invoke-AgentBoundedProcess" +)) { + . ([ScriptBlock]::Create((Get-ExactFunctionSource $ast $name))) +} + +$root = Join-Path $env:TEMP ("agent99-bounded-process-replay-" + [guid]::NewGuid().ToString("N")) +$normal = $null +$timeout = $null +$flood = $null +$rootOnly = $null +$rootOnlyChildIdentity = $null +$orphanRace = $null +$orphanRaceChildIdentity = $null +$pidReuseSelection = $null +$creationBoundSelection = $null +$stopReadbackFailure = $null +$stopReadbackFailureChildIdentity = $null +$cimFailure = $null +$cimFailureChildIdentity = $null +$cleanupVerified = $false +try { + New-Item -ItemType Directory -Path $root -Force | Out-Null + $powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $cmd = Join-Path $env:SystemRoot "System32\cmd.exe" + + $normal = Invoke-AgentBoundedProcess ` + -FilePath $cmd ` + -Arguments @("/d", "/c", "echo", "AG99_BOUNDED_OK") ` + -TimeoutSeconds 5 ` + -MaximumOutputBytes 4096 + Assert-Replay ($normal.ok -and $normal.completed -and $normal.stdout -match "AG99_BOUNDED_OK") "normal_process_failed" + + $childPath = Join-Path $root "child.ps1" + $parentPath = Join-Path $root "parent.ps1" + [IO.File]::WriteAllText($childPath, 'Start-Sleep -Seconds 30', (New-Object Text.UTF8Encoding($false))) + [IO.File]::WriteAllText($parentPath, @' +param([string]$ChildPath) +$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" +$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`"" +Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null +Start-Sleep -Seconds 30 +'@, (New-Object Text.UTF8Encoding($false))) + $timeout = Invoke-AgentBoundedProcess ` + -FilePath $powershell ` + -Arguments @("-NoProfile", "-NonInteractive", "-File", $parentPath, "-ChildPath", $childPath) ` + -TimeoutSeconds 2 ` + -MaximumOutputBytes 4096 + Assert-Replay ($timeout.timedOut -and -not $timeout.completed) "timeout_not_detected" + Assert-Replay ($timeout.processTreeStopped -and -not $timeout.treeReadbackFailed) "timeout_tree_not_stopped" + Assert-Replay ([int]$timeout.killedIdentityCount -ge 2) "timeout_descendant_not_observed" + + $floodPath = Join-Path $root "flood.ps1" + [IO.File]::WriteAllText($floodPath, @' +for ($index = 0; $index -lt 1000; $index++) { + [Console]::Out.Write(('X' * 1024)) +} +Start-Sleep -Seconds 5 +'@, (New-Object Text.UTF8Encoding($false))) + $flood = Invoke-AgentBoundedProcess ` + -FilePath $powershell ` + -Arguments @("-NoProfile", "-NonInteractive", "-File", $floodPath) ` + -TimeoutSeconds 10 ` + -MaximumOutputBytes 4096 + Assert-Replay ($flood.outputLimitExceeded -and -not $flood.completed) "streaming_output_limit_not_detected" + Assert-Replay ([long]$flood.capturedBytes -le [long]$flood.maximumOutputBytes) "captured_output_exceeded_contract" + Assert-Replay ($flood.processTreeStopped -and -not $flood.treeReadbackFailed) "flood_tree_not_stopped" + + $rootOnlyChildPidPath = Join-Path $root "root-only-child.pid" + $rootOnlyChildPath = Join-Path $root "root-only-child.ps1" + $rootOnlyParentPath = Join-Path $root "root-only-parent.ps1" + [IO.File]::WriteAllText($rootOnlyChildPath, @' +param([string]$PidPath) +[IO.File]::WriteAllText($PidPath, [string]$PID, (New-Object Text.UTF8Encoding($false))) +Start-Sleep -Seconds 30 +'@, (New-Object Text.UTF8Encoding($false))) + [IO.File]::WriteAllText($rootOnlyParentPath, @' +param([string]$ChildPath, [string]$ChildPidPath) +$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" +$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`" -PidPath `"$ChildPidPath`"" +Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null +Start-Sleep -Seconds 30 +'@, (New-Object Text.UTF8Encoding($false))) + $rootOnly = Invoke-AgentBoundedProcess ` + -FilePath $powershell ` + -Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $rootOnlyChildPidPath) ` + -TimeoutSeconds 2 ` + -MaximumOutputBytes 4096 ` + -TerminationScope Root + Assert-Replay ($rootOnly.timedOut -and $rootOnly.forcedTerminationPerformed) "root_only_timeout_not_detected" + Assert-Replay ($rootOnly.processScopeStopped -and -not $rootOnly.processTreeStopped) "root_only_scope_not_preserved" + Assert-Replay ([int]$rootOnly.preservedDescendantCount -ge 1) "root_only_descendant_not_preserved" + Assert-Replay (Test-Path -LiteralPath $rootOnlyChildPidPath -PathType Leaf) "root_only_child_pid_missing" + $rootOnlyChildPid = [int]([IO.File]::ReadAllText($rootOnlyChildPidPath)) + $rootOnlyChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $rootOnlyChildPid" -ErrorAction Stop + $rootOnlyChildIdentity = [pscustomobject]@{ + processId = [int]$rootOnlyChildProcess.ProcessId + parentProcessId = [int]$rootOnlyChildProcess.ParentProcessId + creationUtc = ([datetime]$rootOnlyChildProcess.CreationDate).ToUniversalTime().ToString("o") + name = [string]$rootOnlyChildProcess.Name + } + Assert-Replay (Test-AgentProcessIdentityActive $rootOnlyChildIdentity) "root_only_child_not_active_after_parent_stop" + + $orphanRaceChildPidPath = Join-Path $root "orphan-race-child.pid" + $orphanRaceParentPath = Join-Path $root "orphan-race-parent.ps1" + [IO.File]::WriteAllText($orphanRaceParentPath, @' +param([string]$ChildPath, [string]$ChildPidPath) +$powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" +$arguments = "-NoProfile -NonInteractive -File `"$ChildPath`" -PidPath `"$ChildPidPath`"" +Start-Process -FilePath $powershell -ArgumentList $arguments -WindowStyle Hidden | Out-Null +$stopwatch = [Diagnostics.Stopwatch]::StartNew() +while (-not (Test-Path -LiteralPath $ChildPidPath) -and $stopwatch.Elapsed.TotalSeconds -lt 5) { + Start-Sleep -Milliseconds 50 +} +Start-Sleep -Seconds 1 +'@, (New-Object Text.UTF8Encoding($false))) + $orphanStartInfo = New-Object Diagnostics.ProcessStartInfo + $orphanStartInfo.FileName = $powershell + $orphanStartInfo.Arguments = "-NoProfile -NonInteractive -File `"$orphanRaceParentPath`" -ChildPath `"$rootOnlyChildPath`" -ChildPidPath `"$orphanRaceChildPidPath`"" + $orphanStartInfo.UseShellExecute = $false + $orphanStartInfo.CreateNoWindow = $true + $orphanRoot = New-Object Diagnostics.Process + $orphanRoot.StartInfo = $orphanStartInfo + Assert-Replay $orphanRoot.Start() "orphan_race_parent_start_failed" + $orphanRootInfo = Get-CimInstance Win32_Process -Filter "ProcessId = $([int]$orphanRoot.Id)" -ErrorAction Stop | Select-Object -First 1 + $orphanRootIdentity = ConvertTo-AgentProcessIdentitySnapshot $orphanRootInfo + Assert-Replay ([bool]$orphanRootIdentity) "orphan_race_root_identity_capture_failed" + Assert-Replay $orphanRoot.WaitForExit(8000) "orphan_race_parent_did_not_exit" + Assert-Replay (Test-Path -LiteralPath $orphanRaceChildPidPath -PathType Leaf) "orphan_race_child_pid_missing" + $orphanIdentities = @(Get-AgentProcessTreeIdentitySnapshot $orphanRootIdentity $orphanRoot.ExitTime.ToUniversalTime()) + Assert-Replay ($orphanIdentities.Count -ge 1) "orphan_race_descendant_snapshot_missing" + Assert-Replay (@($orphanIdentities | Where-Object { [int]$_.processId -eq [int]$orphanRoot.Id }).Count -eq 0) "orphan_race_root_unexpectedly_present" + $orphanRaceChildIdentity = @($orphanIdentities | Select-Object -First 1)[0] + $orphanRace = Stop-AgentBoundedProcessSnapshot $orphanRoot $orphanIdentities "Tree" + Assert-Replay ($orphanRace.processTreeStopped -and $orphanRace.processScopeStopped) "orphan_race_tree_false_green" + Assert-Replay (-not (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) "orphan_race_child_still_active" + + $pidReuseRootCreated = [datetime]::UtcNow.AddMinutes(-5) + $pidReuseRootExited = $pidReuseRootCreated.AddSeconds(2) + $pidReuseExpectedRoot = [pscustomobject]@{ + processId = 9001 + parentProcessId = 7000 + creationUtc = $pidReuseRootCreated.ToString("o") + name = "powershell.exe" + } + $pidReuseProcesses = @( + [pscustomobject]@{ ProcessId = 9001; ParentProcessId = 8000; CreationDate = $pidReuseRootExited.AddSeconds(1); Name = "powershell.exe" }, + [pscustomobject]@{ ProcessId = 9002; ParentProcessId = 9001; CreationDate = $pidReuseRootCreated.AddSeconds(1); Name = "cmd.exe" }, + [pscustomobject]@{ ProcessId = 9003; ParentProcessId = 9002; CreationDate = $pidReuseRootExited.AddSeconds(1); Name = "conhost.exe" }, + [pscustomobject]@{ ProcessId = 9004; ParentProcessId = 9001; CreationDate = $pidReuseRootExited.AddSeconds(2); Name = "not-agent99.exe" }, + [pscustomobject]@{ ProcessId = 9005; ParentProcessId = 9002; CreationDate = $pidReuseRootCreated.AddMilliseconds(500); Name = "older-than-parent.exe" } + ) + $pidReuseSelection = @(Select-AgentProcessTreeIdentitySnapshot $pidReuseProcesses $pidReuseExpectedRoot $pidReuseRootExited) + $pidReuseSelectedIds = @($pidReuseSelection | ForEach-Object { [int]$_.processId } | Sort-Object) + Assert-Replay (($pidReuseSelectedIds -join ",") -eq "9002,9003") "pid_reuse_selection_not_identity_bounded" + + $liveRootCreated = [datetime]::UtcNow.AddMinutes(-1) + $liveRootIdentity = [pscustomobject]@{ processId = 9101; parentProcessId = 7000; creationUtc = $liveRootCreated.ToString("o"); name = "powershell.exe" } + $liveRootProcesses = @( + [pscustomobject]@{ ProcessId = 9101; ParentProcessId = 7000; CreationDate = $liveRootCreated; Name = "powershell.exe" }, + [pscustomobject]@{ ProcessId = 9102; ParentProcessId = 9101; CreationDate = $liveRootCreated.AddSeconds(1); Name = "valid-child.exe" }, + [pscustomobject]@{ ProcessId = 9103; ParentProcessId = 9101; CreationDate = $liveRootCreated.AddSeconds(-1); Name = "older-unrelated.exe" } + ) + $creationBoundSelection = @(Select-AgentProcessTreeIdentitySnapshot $liveRootProcesses $liveRootIdentity ([datetime]::MinValue)) + $creationBoundIds = @($creationBoundSelection | ForEach-Object { [int]$_.processId } | Sort-Object) + Assert-Replay (($creationBoundIds -join ",") -eq "9101,9102") "live_root_child_creation_bound_failed" + + $stopReadbackChildPidPath = Join-Path $root "stop-readback-failure-child.pid" + function Get-AgentProcessIdentityReadback { [pscustomobject]@{ known = $false; active = $false; reason = "synthetic_stop_readback_failure" } } + $stopReadbackFailure = Invoke-AgentBoundedProcess ` + -FilePath $powershell ` + -Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $stopReadbackChildPidPath) ` + -TimeoutSeconds 2 ` + -MaximumOutputBytes 4096 + . ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessIdentityReadback"))) + Assert-Replay ($stopReadbackFailure.timedOut -and $stopReadbackFailure.identityReadbackFailed) "stop_readback_failure_not_detected" + Assert-Replay ($stopReadbackFailure.treeReadbackFailed -and $stopReadbackFailure.rootFallbackTerminationUsed) "stop_readback_failure_not_fail_closed" + Assert-Replay ($stopReadbackFailure.processScopeStopped -and -not $stopReadbackFailure.processTreeStopped) "stop_readback_failure_tree_false_green" + Assert-Replay (Test-Path -LiteralPath $stopReadbackChildPidPath -PathType Leaf) "stop_readback_failure_child_pid_missing" + $stopReadbackChildPid = [int]([IO.File]::ReadAllText($stopReadbackChildPidPath)) + $stopReadbackChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $stopReadbackChildPid" -ErrorAction Stop + $stopReadbackFailureChildIdentity = ConvertTo-AgentProcessIdentitySnapshot $stopReadbackChildProcess + Assert-Replay (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity) "stop_readback_failure_child_unexpectedly_stopped" + + $cimFailureChildPidPath = Join-Path $root "cim-failure-child.pid" + function Get-AgentProcessTreeIdentitySnapshot { throw "synthetic_cim_failure" } + $cimFailure = Invoke-AgentBoundedProcess ` + -FilePath $powershell ` + -Arguments @("-NoProfile", "-NonInteractive", "-File", $rootOnlyParentPath, "-ChildPath", $rootOnlyChildPath, "-ChildPidPath", $cimFailureChildPidPath) ` + -TimeoutSeconds 2 ` + -MaximumOutputBytes 4096 + . ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessTreeIdentitySnapshot"))) + . ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessIdentityReadback"))) + Assert-Replay ($cimFailure.timedOut -and $cimFailure.treeReadbackFailed) "cim_failure_not_detected" + Assert-Replay ($cimFailure.rootFallbackTerminationUsed -and $cimFailure.processScopeStopped) "cim_failure_root_not_stopped" + Assert-Replay (-not $cimFailure.processTreeStopped) "cim_failure_tree_false_green" + Assert-Replay (Test-Path -LiteralPath $cimFailureChildPidPath -PathType Leaf) "cim_failure_child_pid_missing" + $cimFailureChildPid = [int]([IO.File]::ReadAllText($cimFailureChildPidPath)) + $cimFailureChildProcess = Get-CimInstance Win32_Process -Filter "ProcessId = $cimFailureChildPid" -ErrorAction Stop + $cimFailureChildIdentity = [pscustomobject]@{ + processId = [int]$cimFailureChildProcess.ProcessId + parentProcessId = [int]$cimFailureChildProcess.ParentProcessId + creationUtc = ([datetime]$cimFailureChildProcess.CreationDate).ToUniversalTime().ToString("o") + name = [string]$cimFailureChildProcess.Name + } + Assert-Replay (Test-AgentProcessIdentityActive $cimFailureChildIdentity) "cim_failure_child_unexpectedly_stopped" +} finally { + . ([ScriptBlock]::Create((Get-ExactFunctionSource $ast "Get-AgentProcessTreeIdentitySnapshot"))) + if ($rootOnlyChildIdentity -and (Test-AgentProcessIdentityActive $rootOnlyChildIdentity)) { + & "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$rootOnlyChildIdentity.processId) /T /F 2>&1 | Out-Null + [void](Wait-AgentProcessTreeIdentityAbsent @($rootOnlyChildIdentity) 8) + } + if ($cimFailureChildIdentity -and (Test-AgentProcessIdentityActive $cimFailureChildIdentity)) { + & "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$cimFailureChildIdentity.processId) /T /F 2>&1 | Out-Null + [void](Wait-AgentProcessTreeIdentityAbsent @($cimFailureChildIdentity) 8) + } + if ($orphanRaceChildIdentity -and (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) { + & "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$orphanRaceChildIdentity.processId) /T /F 2>&1 | Out-Null + [void](Wait-AgentProcessTreeIdentityAbsent @($orphanRaceChildIdentity) 8) + } + if ($stopReadbackFailureChildIdentity -and (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity)) { + & "$env:SystemRoot\System32\taskkill.exe" /PID ([int]$stopReadbackFailureChildIdentity.processId) /T /F 2>&1 | Out-Null + [void](Wait-AgentProcessTreeIdentityAbsent @($stopReadbackFailureChildIdentity) 8) + } + Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue + $cleanupVerified = [bool]( + -not (Test-Path -LiteralPath $root) -and + (-not $rootOnlyChildIdentity -or -not (Test-AgentProcessIdentityActive $rootOnlyChildIdentity)) -and + (-not $orphanRaceChildIdentity -or -not (Test-AgentProcessIdentityActive $orphanRaceChildIdentity)) -and + (-not $stopReadbackFailureChildIdentity -or -not (Test-AgentProcessIdentityActive $stopReadbackFailureChildIdentity)) -and + (-not $cimFailureChildIdentity -or -not (Test-AgentProcessIdentityActive $cimFailureChildIdentity)) + ) +} + +[pscustomobject]@{ + schemaVersion = "agent99_bounded_process_replay_v2" + ok = [bool]($normal.ok -and $timeout.timedOut -and $timeout.processTreeStopped -and $flood.outputLimitExceeded -and $flood.processTreeStopped -and $rootOnly.timedOut -and $rootOnly.processScopeStopped -and -not $rootOnly.processTreeStopped -and $orphanRace.processTreeStopped -and $pidReuseSelection.Count -eq 2 -and $creationBoundSelection.Count -eq 2 -and $stopReadbackFailure.identityReadbackFailed -and -not $stopReadbackFailure.processTreeStopped -and $cimFailure.timedOut -and $cimFailure.treeReadbackFailed -and $cimFailure.rootFallbackTerminationUsed -and $cimFailure.processScopeStopped -and -not $cimFailure.processTreeStopped -and $cleanupVerified) + normal = [pscustomobject]@{ + exitCode = [int]$normal.exitCode + completed = [bool]$normal.completed + } + timeout = [pscustomobject]@{ + exitCode = [int]$timeout.exitCode + timedOut = [bool]$timeout.timedOut + processTreeStopped = [bool]$timeout.processTreeStopped + killedIdentityCount = [int]$timeout.killedIdentityCount + } + flood = [pscustomobject]@{ + exitCode = [int]$flood.exitCode + outputLimitExceeded = [bool]$flood.outputLimitExceeded + capturedBytes = [long]$flood.capturedBytes + maximumOutputBytes = [long]$flood.maximumOutputBytes + processTreeStopped = [bool]$flood.processTreeStopped + } + rootOnly = [pscustomobject]@{ + exitCode = [int]$rootOnly.exitCode + timedOut = [bool]$rootOnly.timedOut + terminationScope = [string]$rootOnly.terminationScope + processScopeStopped = [bool]$rootOnly.processScopeStopped + processTreeStopped = [bool]$rootOnly.processTreeStopped + preservedDescendantCount = [int]$rootOnly.preservedDescendantCount + } + orphanRace = [pscustomobject]@{ + rootExitedBeforeSnapshot = $true + descendantSnapshotCount = [int]$orphanIdentities.Count + processScopeStopped = [bool]$orphanRace.processScopeStopped + processTreeStopped = [bool]$orphanRace.processTreeStopped + } + pidReuseSelection = [pscustomobject]@{ + reusedRootExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9001 }).Count -eq 0) + originalChildIncluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9002 }).Count -eq 1) + originalGrandchildIncluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9003 }).Count -eq 1) + reusedRootChildExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9004 }).Count -eq 0) + intermediateOlderProcessExcluded = [bool](@($pidReuseSelection | Where-Object { [int]$_.processId -eq 9005 }).Count -eq 0) + } + creationBounds = [pscustomobject]@{ + liveRootIncluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9101 }).Count -eq 1) + validChildIncluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9102 }).Count -eq 1) + olderUnrelatedChildExcluded = [bool](@($creationBoundSelection | Where-Object { [int]$_.processId -eq 9103 }).Count -eq 0) + } + stopReadbackFailure = [pscustomobject]@{ + identityReadbackFailed = [bool]$stopReadbackFailure.identityReadbackFailed + treeReadbackFailed = [bool]$stopReadbackFailure.treeReadbackFailed + rootFallbackTerminationUsed = [bool]$stopReadbackFailure.rootFallbackTerminationUsed + processScopeStopped = [bool]$stopReadbackFailure.processScopeStopped + processTreeStopped = [bool]$stopReadbackFailure.processTreeStopped + } + cimFailure = [pscustomobject]@{ + exitCode = [int]$cimFailure.exitCode + timedOut = [bool]$cimFailure.timedOut + treeReadbackFailed = [bool]$cimFailure.treeReadbackFailed + rootFallbackTerminationUsed = [bool]$cimFailure.rootFallbackTerminationUsed + processScopeStopped = [bool]$cimFailure.processScopeStopped + processTreeStopped = [bool]$cimFailure.processTreeStopped + } + cleanupVerified = $cleanupVerified + productionPathWritePerformed = $false + vmPowerChangePerformed = $false + hostRebootPerformed = $false + secretValuesRead = $false +} | ConvertTo-Json -Compress -Depth 5 diff --git a/scripts/reboot-recovery/tests/agent99-breakglass-failure-readback-replay.ps1 b/scripts/reboot-recovery/tests/agent99-breakglass-failure-readback-replay.ps1 new file mode 100644 index 000000000..0ddfe296b --- /dev/null +++ b/scripts/reboot-recovery/tests/agent99-breakglass-failure-readback-replay.ps1 @@ -0,0 +1,55 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$BreakglassSource +) + +$ErrorActionPreference = "Stop" +$tokens = $null +$errors = $null +$ast = [Management.Automation.Language.Parser]::ParseInput( + $BreakglassSource, + [ref]$tokens, + [ref]$errors +) +if ($errors.Count -ne 0) { throw "breakglass_parse_failed" } +$matches = @($ast.FindAll({ + param($node) + $node -is [Management.Automation.Language.FunctionDefinitionAst] -and + $node.Name -eq "Get-FailedMutationReadback" +}, $true)) +if ($matches.Count -ne 1) { throw "failure_readback_function_identity_mismatch" } +. ([ScriptBlock]::Create([string]$matches[0].Extent.Text)) + +function Wait-ExactProcessIdentityAbsent { throw "synthetic_cim_failure" } +function Test-RecoveryMutexActive { throw "synthetic_mutex_readback_failure" } +function Test-Path { param([string]$LiteralPath); throw "synthetic_filesystem_readback_failure" } + +$readback = Get-FailedMutationReadback ` + 1001 ([datetime]::UtcNow.AddHours(-1)) ` + 1002 ([datetime]::UtcNow.AddHours(-1)) ` + "C:\synthetic\recovery-transport-lease.json" +$expected = @( + "parent_identity_unknown", + "child_identity_unknown", + "recovery_mutex_unknown", + "lease_metadata_unknown" +) +if ($readback.complete) { throw "failure_readback_false_green" } +if ($readback.parentAbsent -or $readback.childAbsent -or $readback.recoveryMutexReleased -or $readback.leaseRemoved) { + throw "failure_readback_unknown_projected_true" +} +if ((Compare-Object $expected @($readback.errors)).Count -ne 0) { throw "failure_readback_errors_incomplete" } + +[pscustomobject]@{ + schemaVersion = "agent99_breakglass_failure_readback_replay_v1" + ok = $true + complete = [bool]$readback.complete + errors = @($readback.errors) + terminalReceiptConstructionCanContinue = $true + runtimeWritePerformed = $false + processTerminationPerformed = $false + vmPowerChangePerformed = $false + hostRebootPerformed = $false + secretValuesRead = $false +} | ConvertTo-Json -Compress -Depth 4 diff --git a/scripts/reboot-recovery/tests/agent99-telegram-durable-ingress-replay.ps1 b/scripts/reboot-recovery/tests/agent99-telegram-durable-ingress-replay.ps1 new file mode 100644 index 000000000..d248bc3f6 --- /dev/null +++ b/scripts/reboot-recovery/tests/agent99-telegram-durable-ingress-replay.ps1 @@ -0,0 +1,160 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$TelegramInboxSource, + [Parameter(Mandatory = $true)] + [string]$SubmitRequestSource +) + +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +function Assert-Replay { + param([bool]$Condition, [string]$Reason) + if (-not $Condition) { throw $Reason } +} + +function Invoke-SyntheticCase { + param( + [string]$Name, + [string]$Text + ) + + $root = Join-Path $env:TEMP ("agent99-telegram-ingress-replay-" + $Name + "-" + [guid]::NewGuid().ToString("N")) + $bin = Join-Path $root "bin" + $config = Join-Path $root "config" + $telegramPath = Join-Path $bin "agent99-telegram-inbox.ps1" + $submitPath = Join-Path $bin "agent99-submit-request.ps1" + $stdoutPath = Join-Path $root "telegram.out" + $stderrPath = Join-Path $root "telegram.err" + try { + New-Item -ItemType Directory -Path $bin, $config -Force | Out-Null + $encoding = New-Object Text.UTF8Encoding($true) + [IO.File]::WriteAllText($telegramPath, $TelegramInboxSource, $encoding) + [IO.File]::WriteAllText($submitPath, $SubmitRequestSource, $encoding) + [IO.File]::WriteAllText( + (Join-Path $config "agent99.config.json"), + '{"telegram":{"autoIngestMonitoringAlerts":true}}', + (New-Object Text.UTF8Encoding($false)) + ) + + $powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $output = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $Text -SyntheticChatId "synthetic-chat" 2>$stderrPath) + $exitCode = [int]$LASTEXITCODE + [IO.File]::WriteAllLines($stdoutPath, @($output | ForEach-Object { [string]$_ }), (New-Object Text.UTF8Encoding($false))) + $payload = (($output | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json + $queueFiles = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue) + $evidenceFiles = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-TelegramInbox-*.json" -ErrorAction SilentlyContinue) + $submitOutFile = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-telegram-submit-*.out" -ErrorAction SilentlyContinue | Select-Object -First 1) + $submitErrFile = @(Get-ChildItem -LiteralPath (Join-Path $root "evidence") -File -Filter "agent99-telegram-submit-*.err" -ErrorAction SilentlyContinue | Select-Object -First 1) + [pscustomobject]@{ + name = $Name + exitCode = $exitCode + ok = [bool]$payload.ok + reason = [string]$payload.reason + processedCount = [int]$payload.processedCount + failedUpdateCount = [int]$payload.failedUpdateCount + durableQueued = [bool](@($payload.processed | Where-Object { $_.durableQueued }).Count -gt 0) + submitExitCode = if (@($payload.processed).Count -gt 0) { [int]$payload.processed[0].submitExitCode } else { $null } + failureReason = if (@($payload.processed).Count -gt 0) { [string]$payload.processed[0].failureReason } else { "" } + submitStdoutBytes = if ($submitOutFile.Count -eq 1) { [long]$submitOutFile[0].Length } else { 0 } + submitStderrBytes = if ($submitErrFile.Count -eq 1) { [long]$submitErrFile[0].Length } else { 0 } + offsetBefore = [int64]$payload.offsetBefore + offsetAfter = [int64]$payload.offsetAfter + queueCount = $queueFiles.Count + evidenceCount = $evidenceFiles.Count + offsetFilePresent = Test-Path -LiteralPath (Join-Path $root "state\telegram-inbox-offset.json") + stderrPresent = [bool]((Test-Path -LiteralPath $stderrPath) -and (Get-Item -LiteralPath $stderrPath).Length -gt 0) + } + } finally { + Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue + if (Test-Path -LiteralPath $root) { throw "synthetic_case_cleanup_failed:$Name" } + } +} + +function Invoke-QueueCommitReplayCase { + $root = Join-Path $env:TEMP ("agent99-telegram-ingress-replay-reentry-" + [guid]::NewGuid().ToString("N")) + $bin = Join-Path $root "bin" + $config = Join-Path $root "config" + $telegramPath = Join-Path $bin "agent99-telegram-inbox.ps1" + $submitPath = Join-Path $bin "agent99-submit-request.ps1" + $fixedUpdateId = [int64]1784464999 + $instruction = "/agent99 status readback" + $needle = '$submitReceipt = Write-AgentDurableJson $result $submitEvidence $id $ExternalId' + try { + New-Item -ItemType Directory -Path $bin, $config -Force | Out-Null + $encoding = New-Object Text.UTF8Encoding($true) + [IO.File]::WriteAllText($telegramPath, $TelegramInboxSource, $encoding) + [IO.File]::WriteAllText( + (Join-Path $config "agent99.config.json"), + '{"telegram":{"autoIngestMonitoringAlerts":true}}', + (New-Object Text.UTF8Encoding($false)) + ) + Assert-Replay $SubmitRequestSource.Contains($needle) "queue_commit_fault_marker_missing" + $faultedSubmit = $SubmitRequestSource.Replace($needle, ('throw "synthetic_after_queue_commit"' + "`r`n" + $needle)) + [IO.File]::WriteAllText($submitPath, $faultedSubmit, $encoding) + + $powershell = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $firstOutput = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $instruction -SyntheticUpdateId $fixedUpdateId -SyntheticChatId "synthetic-chat" 2>$null) + $firstExit = [int]$LASTEXITCODE + $firstPayload = (($firstOutput | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json + $firstQueue = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue) + $firstRequests = @(Get-ChildItem -LiteralPath (Join-Path $root "requests") -File -Filter "*.json" -ErrorAction SilentlyContinue) + Assert-Replay ($firstExit -ne 0 -and -not [bool]$firstPayload.ok) "queue_commit_fault_false_green" + Assert-Replay ($firstQueue.Count -eq 1 -and $firstRequests.Count -eq 1) "queue_commit_fault_artifact_count_invalid" + $firstQueuePayload = Get-Content -LiteralPath $firstQueue[0].FullName -Raw | ConvertFrom-Json + + [IO.File]::WriteAllText($submitPath, $SubmitRequestSource, $encoding) + $secondOutput = @(& $powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File $telegramPath -AgentRoot $root -SyntheticUpdateText $instruction -SyntheticUpdateId $fixedUpdateId -SyntheticChatId "synthetic-chat" 2>$null) + $secondExit = [int]$LASTEXITCODE + $secondPayload = (($secondOutput | ForEach-Object { [string]$_ }) -join "`n") | ConvertFrom-Json + $secondQueue = @(Get-ChildItem -LiteralPath (Join-Path $root "queue") -File -Filter "*.json" -ErrorAction SilentlyContinue) + $secondRequests = @(Get-ChildItem -LiteralPath (Join-Path $root "requests") -File -Filter "*.json" -ErrorAction SilentlyContinue) + $processed = @($secondPayload.processed) + Assert-Replay ($secondExit -eq 0 -and [bool]$secondPayload.ok) "queue_commit_retry_failed" + Assert-Replay ($secondQueue.Count -eq 1 -and $secondRequests.Count -eq 1) "queue_commit_retry_duplicated_artifact" + Assert-Replay ($processed.Count -eq 1 -and [bool]$processed[0].durableQueued -and [bool]$processed[0].idempotentReplay) "queue_commit_retry_not_idempotent" + Assert-Replay ([string]$processed[0].queueId -eq [string]$firstQueuePayload.id) "queue_commit_retry_identity_changed" + [pscustomobject]@{ + firstExitCode = $firstExit + firstReason = [string]$firstPayload.reason + secondExitCode = $secondExit + secondReason = [string]$secondPayload.reason + requestId = [string]$firstQueuePayload.id + queueCountAfterRetry = $secondQueue.Count + requestCountAfterRetry = $secondRequests.Count + idempotentReplay = [bool]$processed[0].idempotentReplay + offsetBefore = [int64]$secondPayload.offsetBefore + offsetAfter = [int64]$secondPayload.offsetAfter + } + } finally { + Remove-Item -LiteralPath $root -Recurse -Force -ErrorAction SilentlyContinue + if (Test-Path -LiteralPath $root) { throw "queue_commit_replay_cleanup_failed" } + } +} + +$success = Invoke-SyntheticCase "success" "/agent99 status readback" +$failure = Invoke-SyntheticCase "enqueue-failure" ("/agent99 " + ("A" * 9000)) +$queueCommitReplay = Invoke-QueueCommitReplayCase + +Assert-Replay ($success.exitCode -eq 0 -and $success.ok) ("success_case_failed:" + ($success | ConvertTo-Json -Compress)) +Assert-Replay ($success.processedCount -eq 1 -and $success.durableQueued -and $success.queueCount -eq 1) "success_queue_not_durable" +Assert-Replay ($success.evidenceCount -eq 1 -and -not $success.offsetFilePresent) "success_evidence_or_synthetic_offset_invalid" +Assert-Replay ($failure.exitCode -ne 0 -and -not $failure.ok) ("failure_case_false_green:" + ($failure | ConvertTo-Json -Compress)) +Assert-Replay ($failure.reason -eq "update_processing_failed_no_checkpoint_past_failure") "failure_reason_mismatch" +Assert-Replay ($failure.failedUpdateCount -eq 1 -and $failure.queueCount -eq 0) "failure_queue_or_count_invalid" +Assert-Replay ($failure.offsetBefore -eq $failure.offsetAfter -and -not $failure.offsetFilePresent) "failure_offset_advanced" +Assert-Replay ($failure.evidenceCount -eq 1) "failure_evidence_missing" + +[pscustomobject]@{ + schemaVersion = "agent99_telegram_durable_ingress_replay_v1" + ok = $true + success = $success + enqueueFailure = $failure + queueCommitReplay = $queueCommitReplay + cleanupVerified = $true + productionPathWritePerformed = $false + scheduledTaskStarted = $false + telegramApiCalled = $false + secretValuesRead = $false +} | ConvertTo-Json -Compress -Depth 6 diff --git a/scripts/reboot-recovery/tests/test_agent99_vmrun_timeout_guard.py b/scripts/reboot-recovery/tests/test_agent99_vmrun_timeout_guard.py new file mode 100644 index 000000000..023c53c5b --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_vmrun_timeout_guard.py @@ -0,0 +1,276 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +CONTROL = ROOT / "agent99-control-plane.ps1" +BREAKGLASS = ROOT / "scripts" / "reboot-recovery" / "agent99-stale-recovery-breakglass.ps1" +MAINTENANCE = ROOT / "scripts" / "reboot-recovery" / "agent99-control-loop-maintenance.ps1" +REGISTER_TASKS = ROOT / "agent99-register-tasks.ps1" +SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1" +TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1" +SUBMIT_REQUEST = ROOT / "agent99-submit-request.ps1" +BOUNDED_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-bounded-process-replay.ps1" +TELEGRAM_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-telegram-durable-ingress-replay.ps1" +BREAKGLASS_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-breakglass-failure-readback-replay.ps1" + + +def test_vmrun_start_is_process_aware_and_bounded() -> None: + source = CONTROL.read_text(encoding="utf-8") + + assert "function Get-AgentVmwareVmxProcessReadback" in source + assert "vmx_process_already_running" in source + assert "vmx_process_readback_failed" in source + assert "function Invoke-AgentBoundedVmrunStart" in source + assert "Get-AgentVmwareVmxProcessReadback ([string]$vm.vmx)" in source + assert "Invoke-AgentBoundedVmrunStart" in source + assert '& $vmrun start $vm.vmx nogui' not in source + + +def test_bounded_runner_kills_trees_but_vmrun_preserves_started_vm_descendants() -> None: + source = CONTROL.read_text(encoding="utf-8") + function = source.split("function Invoke-AgentBoundedProcess", 1)[1].split( + "function Test-PublicRoutes", 1 + )[0] + + assert "$stopwatch.Elapsed.TotalSeconds -ge $TimeoutSeconds" in function + assert "StandardOutput.ReadAsync" in function + assert "StandardError.ReadAsync" in function + assert "($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes" in function + assert function.index("($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes") < function.index("$process.WaitForExit()") + assert '[ValidateSet("Tree", "Root")]' in function + assert "processScopeStopped" in function + assert "preservedDescendantCount" in function + assert "Get-AgentProcessTreeIdentitySnapshot $rootProcessIdentity $rootExitedUtc" in function + assert "ConvertTo-AgentProcessIdentitySnapshot $rootProcessInfo" in function + assert "rootIdentityCaptured" in function + assert "Stop-AgentBoundedProcessSnapshot $process $treeIdentities $TerminationScope" in function + assert "processTreeStopped" in function + assert "Read-AgentBoundedTextFile $stdout" in function + assert "Remove-Item -LiteralPath $stdout, $stderr" in function + stopper = source.split("function Stop-AgentBoundedProcessSnapshot", 1)[1].split( + "function Test-AgentProcessIdentityActive", 1 + )[0] + assert 'taskkill.exe" /PID ([string]$rootId) /T /F' in stopper + assert 'taskkill.exe" /PID ([string]$rootId) /F' in stopper + assert 'taskkill.exe" /PID ([string]([int]$identity.processId)) /T /F' in stopper + snapshot = source.split("function Select-AgentProcessTreeIdentitySnapshot", 1)[1].split( + "function Get-AgentProcessTreeIdentitySnapshot", 1 + )[0] + assert "Test-AgentProcessSnapshotMatch $RootIdentity $currentRoot" in snapshot + assert 'throw "bounded_process_root_identity_changed_before_exit_readback"' in snapshot + assert "-le $RootExitedUtc.ToUniversalTime()" in snapshot + assert "Get-AgentCreationBoundedDescendants $Processes @($currentRoot)" in snapshot + descendants = source.split("function Get-AgentCreationBoundedDescendants", 1)[1].split( + "function Test-AgentProcessSnapshotMatch", 1 + )[0] + assert "-ge $parentCreated" in descendants + assert "System.Collections.Generic.HashSet[int]" in descendants + assert "function Get-AgentProcessIdentityReadback" in source + assert 'reason = "identity_readback_failed"' in source + assert 'throw "process_tree_identity_readback_failed"' in source + assert "identityReadbackFailed" in stopper + vmrun = source.split("function Invoke-AgentBoundedVmrunStart", 1)[1].split( + "function Start-ConfiguredVMs", 1 + )[0] + assert "Invoke-AgentBoundedProcess" in vmrun + assert "-TerminationScope Root" in vmrun + assert "processScopeStopped" in vmrun + + +def test_perf_guard_can_release_an_exact_stale_recovery_before_deferral() -> None: + source = CONTROL.read_text(encoding="utf-8") + + assert "function Invoke-AgentStaleRecoveryTransportGuard" in source + assert 'if ($Mode -in @("Perf", "LoadShed") -and $ControlledApply)' in source + invocation = source.index("$staleRecoveryTransportGuard = Invoke-AgentStaleRecoveryTransportGuard") + lease_probe = source.index("$recoveryLeaseObservation = Test-AgentRecoveryTransportLease", invocation) + assert invocation < lease_probe + assert "agent99_stale_recovery_guard_intent_v1" in source + assert "agent99_stale_recovery_guard_receipt_v1" in source + assert "$stream.Flush($true)" in source + assert "stale_recovery_receipt_readback_mismatch" in source + assert "stale_recovery_descendant_identity_unapproved" in source + assert "Get-AgentProcessDescendants $recheckAll ([int]$lease.processId)" in source + assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source + assert "Wait-AgentCimProcessIdentityAbsent $parent 15" in source + assert "Test-AgentCimProcessIdentityMatch $child $childBeforeKill" in source + assert "$leaseAcquiredUtc.Ticks" in source + assert "vmPowerChangePerformed = $false" in source + assert "vmwareVmxProcessTerminated = $false" in source + + +def test_breakglass_targets_only_exact_stale_vmrun_processes() -> None: + source = BREAKGLASS.read_text(encoding="utf-8") + + assert "lease_run_id_mismatch" in source + assert "lease_process_id_mismatch" in source + assert "parent_run_id_mismatch" in source + assert "child_identity_mismatch" in source + assert "child_vmx_mismatch" in source + assert "recovery_mutex_not_active" in source + assert 'taskkill.exe" /PID $ProcessId /T /F' in source + assert "Invoke-ExactTaskkill $ExpectedChildPid" in source + assert "prior_receipt_identity_mismatch" in source + assert '"agent99_stale_recovery_breakglass_v2", "agent99_stale_recovery_breakglass_v3"' in source + assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source + assert "Wait-ExactProcessIdentityAbsent" in source + assert "parent_cleanup_identity_changed" in source + assert "Get-DescendantProcesses $ExpectedParentPid" in source + assert "descendantCount = 0" in source + assert "agent99_stale_recovery_parent_cleanup_intent_v1" in source + assert "Write-AtomicJson $intent $intentPath" in source + assert "Invoke-ExactTaskkill $ExpectedParentPid" in source + assert "vmwareVmxProcessTerminated = $false" in source + assert "vmxLockDeleted = $false" in source + assert "hostRebootPerformed = $false" in source + assert "Remove-Item" in source + assert ".lck" not in source + + +def test_breakglass_apply_is_receipted_and_check_is_no_write() -> None: + source = BREAKGLASS.read_text(encoding="utf-8") + + check_pos = source.index('if ($Mode -eq "Apply" -and $precheckPassed)') + operation_intent_pos = source.index("Write-AtomicJson $operationIntent $operationIntentPath") + child_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedChildPid") + intent_write_pos = source.index("Write-AtomicJson $intent $intentPath") + parent_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedParentPid") + receipt_pos = source.index('if ($Mode -eq "Apply") {', parent_kill_pos) + assert check_pos < operation_intent_pos < child_kill_pos < intent_write_pos < parent_kill_pos < receipt_pos + assert "agent99_stale_recovery_breakglass_intent_v1" in source + assert '"partial_write_failed_closed"' in source + assert "operationFailureType" in source + assert "Write-AtomicJson $result $script:ReceiptPath" in source + assert "function Get-FailedMutationReadback" in source + assert '"parent_identity_unknown"' in source + assert '"child_identity_unknown"' in source + assert '"recovery_mutex_unknown"' in source + assert '"lease_metadata_unknown"' in source + assert "postFailureReadbackComplete" in source + assert "$stream.Flush($true)" in source + assert "receipt_readback_mismatch" in source + assert 'runtimeWritePerformed = $script:RuntimeWritePerformed' in source + assert 'if ($Mode -eq "Check") { exit 0 }' in source + + +def test_control_loop_maintenance_freeze_is_exact_and_receipted() -> None: + source = MAINTENANCE.read_text(encoding="utf-8") + + assert 'if ($env:COMPUTERNAME -ne "WOOO-SUPER")' in source + assert 'agent99-run\\.ps1"?\\s+-Mode\\s+ControlTick' in source + assert 'agent99-run\\.ps1"?\\s+-Mode\\s+Recover\\s+-ControlledApply' in source + assert "Global\\WoooAgent99ControlLoopMaintenanceV1" in source + assert "blocked_single_writer_active" in source + assert "Test-ExactCreationTime $Process $ExpectedControlCreated" in source + assert "Test-ExactCreationTime $Process $ExpectedRecoverCreated" in source + assert "[int]$Process.ParentProcessId -eq $ExpectedRecoverParentPid" in source + assert 'Disable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source + assert 'Stop-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source + assert "control_recheck_identity_mismatch" in source + assert 'taskkill.exe" /PID $ExpectedRecoverPid /T /F' in source + assert "$stream.Flush($true)" in source + assert "control_loop_maintenance_frozen" in source + assert "agent99_control_loop_maintenance_intent_v1" in source + assert source.index("Write-DurableReceipt $operationIntent $operationIntentPath") < source.index( + 'Disable-ScheduledTask -TaskName $taskName' + ) + assert "maintenance_partial_frozen_manual_recovery" in source + assert "taskEnabledBefore" in source + assert "taskEnabledAfter" in source + assert "operationFailureType" in source + assert "vmwareVmxProcessTerminated = $false" in source + assert "hostRebootPerformed = $false" in source + + +def test_control_loop_deadline_and_queue_batch_are_compatible() -> None: + control = CONTROL.read_text(encoding="utf-8") + tasks = REGISTER_TASKS.read_text(encoding="utf-8") + + assert "$script:Agent99QueueOuterDeadlineSeconds = 1680" in control + assert "-ExecutionTimeLimit (New-TimeSpan -Minutes 35)" in tasks + control_registration = tasks.split('$controlArgs = ', 1)[1].split('$telegramInboxScript = ', 1)[0] + assert "-Settings $controlSettings" in control_registration + queue_selection = control.split("function Invoke-AgentQueuedCommands", 1)[1].split("$results = @()", 1)[0] + assert "Select-Object -First 1" in queue_selection + + +def test_ingress_signals_tasks_without_waiting_for_control_execution() -> None: + sre = SRE_INBOX.read_text(encoding="utf-8") + telegram = TELEGRAM_INBOX.read_text(encoding="utf-8") + submit = SUBMIT_REQUEST.read_text(encoding="utf-8") + + assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in sre + assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in submit + assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-SRE-Alert-Inbox"' in telegram + assert '"-InstructionBase64", (Convert-AgentBase64 $instruction)' in telegram + assert '"-RequestedByBase64", (Convert-AgentBase64 ([string]$update.from))' in telegram + assert '"-AgentRoot", $AgentRoot' in telegram + assert "$null = $submit.Handle" in telegram + assert "instruction_transport_ambiguous" in submit + assert "scheduled_task_signaled" in sre + assert "scheduled_task_signaled" in submit + assert "function Write-AgentDurableJson" in submit + assert '"request-" + $ingressDigest.Substring(0, 32)' in submit + assert "Global\\Wooo-Agent99-Submit-" in submit + assert "function Get-AgentIngressLifecycleReceipt" in submit + assert 'throw "ingress_lifecycle_ambiguous"' in submit + assert "idempotentReplay = [bool]$queueReceipt.existing" in submit + assert "durableQueue = [bool]$queueReceipt.ok" in submit + assert "submitEvidenceSha256" in submit + assert "function Get-AgentDurableSubmitReadback" in telegram + assert "Sort-Object update_id" in telegram + assert "lastDurablyHandledUpdateId" in telegram + assert "update_processing_failed_no_checkpoint_past_failure" in telegram + assert "offset_checkpoint_failed_safe_replay_required" in telegram + assert "Write-AgentDurableJson $result $evidencePath" in telegram + assert "Write-AgentDurableJson $offsetState $offsetPath" in telegram + assert "$maxUpdateId + 1" not in telegram + + +def test_windows_native_bounded_process_replay_covers_timeout_tree_and_stream_cap() -> None: + source = BOUNDED_REPLAY.read_text(encoding="utf-8") + + assert "timeout_descendant_not_observed" in source + assert "streaming_output_limit_not_detected" in source + assert "root_only_descendant_not_preserved" in source + assert "-TerminationScope Root" in source + assert "cim_failure_root_not_stopped" in source + assert "cim_failure_tree_false_green" in source + assert "orphan_race_descendant_snapshot_missing" in source + assert "orphan_race_tree_false_green" in source + assert "pid_reuse_selection_not_identity_bounded" in source + assert "reusedRootExcluded" in source + assert "reusedRootChildExcluded" in source + assert "live_root_child_creation_bound_failed" in source + assert "intermediateOlderProcessExcluded" in source + assert "stop_readback_failure_not_detected" in source + assert "stop_readback_failure_not_fail_closed" in source + assert "stop_readback_failure_tree_false_green" in source + assert "captured_output_exceeded_contract" in source + assert "cleanupVerified" in source + assert "productionPathWritePerformed = $false" in source + + +def test_windows_native_telegram_replay_covers_durable_success_and_enqueue_failure() -> None: + source = TELEGRAM_REPLAY.read_text(encoding="utf-8") + + assert "success_queue_not_durable" in source + assert "failure_case_false_green" in source + assert "failure_offset_advanced" in source + assert "synthetic_after_queue_commit" in source + assert "queue_commit_retry_duplicated_artifact" in source + assert "queue_commit_retry_not_idempotent" in source + assert '"A" * 9000' in source + assert "telegramApiCalled = $false" in source + assert "productionPathWritePerformed = $false" in source + + +def test_windows_native_breakglass_failure_readback_replay_is_fail_closed() -> None: + source = BREAKGLASS_REPLAY.read_text(encoding="utf-8") + + assert "synthetic_cim_failure" in source + assert "synthetic_mutex_readback_failure" in source + assert "synthetic_filesystem_readback_failure" in source + assert "failure_readback_false_green" in source + assert "terminalReceiptConstructionCanContinue = $true" in source + assert "runtimeWritePerformed = $false" in source diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py index a17ba8670..92417e16a 100644 --- a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -248,7 +248,12 @@ def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None: assert "executorReserveSeconds -ge 60" in control assert "queueReserveSeconds -ge 300" in control assert "clientReserveSeconds -ge 120" in control - assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' in control + queue_runner = control[ + control.index("$execution = Invoke-AgentBoundedProcess") : + ] + assert "-TimeoutSeconds $script:Agent99QueueOuterDeadlineSeconds" in queue_runner + assert "-MaximumOutputBytes 65536" in queue_runner + assert '$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)' not in control def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> None: @@ -516,7 +521,9 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None: for parameter in ("AutomationRunId", "TraceId", "WorkItemId"): assert f"[string]${parameter} = \"\"" in control assert f'[string]`${parameter} = ""' in bootstrap - assert 'reason = "controlled_dispatch_identity_unsafe"' in control + assert '[string]$dispatchIdentityValidation.reason -eq "identity_control_value_unsafe"' in control + assert '"controlled_dispatch_identity_unsafe"' in control + assert "reason = $identityRejectionReason" in control assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function assert '$dryRun.managerPreflightStatus -eq "ready"' in function assert "-not $dryRun.artifactTransactionBlocked" in function From 5fa97b3d71fb784f5ad35c37f07a2f8875e0ff41 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 13:44:06 +0800 Subject: [PATCH 2/4] fix(agent99): align deploy contract with bounded runner --- agent99-contract-check.ps1 | 4 +++- ..._agent99_transport_recovery_deploy_contract.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 2a4c701e0..26515a026 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -205,7 +205,9 @@ $signozMetadataExecutorContract = [bool]( Add-Check "backup:signoz_metadata_fixed_executor" $signozMetadataExecutorContract "Windows99 dispatches one fixed host110 metadata export contract, consumes only a root-owned credential reference on target, and requires the independent bundle verifier without raw SQLite or secret output" Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("Enter-AgentRecoveryTriggerReservation") -and $control.Contains('"recovery-trigger-reservation.lock"') -and $control.Contains('[System.IO.FileShare]::None') -and $control.Contains('reason = "recovery_trigger_reservation_busy"') -and $control.Contains("recovery_already_queued")) "boot and host-down recovery reserve one atomic single-flight queue decision" Add-Check "recovery:auto_trigger_controlled_identity" ($control.Contains("New-AgentObservedRecoveryDispatchIdentity") -and $control.Contains('-RouteId "agent99_local_observer"') -and $control.Contains('schema_version = "agent99_controlled_dispatch_identity_v1"') -and $control.Contains('correlationKey = $dispatchIdentity.idempotencyKey') -and $control.Contains('canonicalDigest = $dispatchIdentity.canonicalDigest') -and $control.Contains("Invoke-AgentControlledDispatchIdentitySelfTest")) "observer-created Recover requests carry a stable canonical controlled dispatch identity before queue processing" -Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $control.Contains('"controlled_dispatch_identity_invalid"') -and $control.IndexOf('"controlled_dispatch_identity_invalid"') -lt $control.IndexOf('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before any controlled executor starts" +$queueIdentityRejectionIndex = $control.IndexOf('"controlled_dispatch_identity_invalid"') +$queueBoundedExecutorIndex = $control.IndexOf('$execution = Invoke-AgentBoundedProcess') +Add-Check "recovery:queue_identity_fail_closed" ($control.Contains('dispatchIdentitySchemaVersion = $dispatchIdentity.schemaVersion') -and $control.Contains('Test-AgentControlledDispatchIdentity $dispatchIdentityEnvelope') -and $queueIdentityRejectionIndex -ge 0 -and $queueBoundedExecutorIndex -ge 0 -and $queueIdentityRejectionIndex -lt $queueBoundedExecutorIndex -and $control.Contains('-FilePath $powershellPath') -and $control.Contains('-TimeoutSeconds $script:Agent99QueueOuterDeadlineSeconds') -and -not $control.Contains('$process = Start-Process -FilePath "powershell.exe" -ArgumentList $args')) "queue consumer rebuilds and validates canonical identity before the bounded controlled executor starts" Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("Update-AgentBootRecoveryState") -and $control.Contains("recovered_late")) "boot event remains pending until verified recovery terminal" Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index fe86a3f44..714ef754f 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -118,6 +118,21 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None: assert '$process = Start-Process -FilePath "powershell.exe"' not in consumer assert 'Add-Check "recovery:auto_trigger_controlled_identity"' in contract assert 'Add-Check "recovery:queue_identity_fail_closed"' in contract + assert "$queueIdentityRejectionIndex = $control.IndexOf" in contract + assert "$queueBoundedExecutorIndex = $control.IndexOf" in contract + assert "'$execution = Invoke-AgentBoundedProcess'" in contract + assert "$queueIdentityRejectionIndex -ge 0" in contract + assert "$queueBoundedExecutorIndex -ge 0" in contract + assert "$queueIdentityRejectionIndex -lt $queueBoundedExecutorIndex" in contract + assert "-TimeoutSeconds $script:Agent99QueueOuterDeadlineSeconds" in contract + assert ( + '-not $control.Contains(\'$process = Start-Process -FilePath "powershell.exe" ' + "-ArgumentList $args')" + ) in contract + assert ( + '$control.IndexOf(\'$process = Start-Process -FilePath "powershell.exe" ' + "-ArgumentList $args')" + ) not in contract assert 'correlationKey = $dispatchIdentity.idempotencyKey' in contract assert 'canonicalDigest = $dispatchIdentity.canonicalDigest' in contract assert 'Record-AgentEvent "recovery_slo_result"' in source From 3512d0c490da8d4865cddca89e0e0cc770c86dce Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 13:49:57 +0800 Subject: [PATCH 3/4] fix(agent99): align synthetic timeout contract --- agent99-synthetic-tests.ps1 | 2 +- ...est_agent99_transport_recovery_deploy_contract.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/agent99-synthetic-tests.ps1 b/agent99-synthetic-tests.ps1 index 817cc2934..dfb5c691b 100644 --- a/agent99-synthetic-tests.ps1 +++ b/agent99-synthetic-tests.ps1 @@ -363,7 +363,7 @@ Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingR Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed without its fixed-WAL commit, attributed start and immutable receipt/readback" Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility" -Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves" +Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$execution = Invoke-AgentBoundedProcess') -and $control.Contains('-TimeoutSeconds $script:Agent99QueueOuterDeadlineSeconds') -and -not $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines use the bounded process-tree runner with explicit reserves" $deployer = Get-Content (Join-Path $SourceRoot "agent99-deploy.ps1") -Raw $hostConfig = Get-Content (Join-Path $SourceRoot "agent99.config.99.example.json") -Raw Add-SyntheticCheck "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))') -and $hostConfig.Contains('S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx')) "Agent99 normalizes Host112 from one staged VMX identity with a bounded post-verifier" diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 714ef754f..dabc9ac10 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -32,6 +32,7 @@ def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None: def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None: source = CONTROL.read_text(encoding="utf-8") contract = CONTRACT_CHECK.read_text(encoding="utf-8") + synthetic = (ROOT / "agent99-synthetic-tests.ps1").read_text(encoding="utf-8") trigger = source[source.index("function Start-AgentRecoveryFromObservation") :] trigger = trigger[: trigger.index("function Get-AgentRecoverySloConfig")] identity = source[source.index("function New-AgentCanonicalControlledDispatchIdentity") :] @@ -133,6 +134,17 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None: '$control.IndexOf(\'$process = Start-Process -FilePath "powershell.exe" ' "-ArgumentList $args')" ) not in contract + assert 'Add-SyntheticCheck "recovery:host112_timeout_contract"' in synthetic + assert "$control.Contains('$execution = Invoke-AgentBoundedProcess')" in synthetic + assert ( + "$control.Contains('-TimeoutSeconds $script:Agent99QueueOuterDeadlineSeconds')" + in synthetic + ) + assert ( + "-not $control.Contains(" + "'$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')" + in synthetic + ) assert 'correlationKey = $dispatchIdentity.idempotencyKey' in contract assert 'canonicalDigest = $dispatchIdentity.canonicalDigest' in contract assert 'Record-AgentEvent "recovery_slo_result"' in source From 50c7a7f7f5cc88bc0877b510e28fa1527ab1cdc5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 16:04:18 +0800 Subject: [PATCH 4/4] fix(agent99): unblock bounded maintenance deployment --- .../agent99-live-preflight.ps1 | 573 ++++++++++++++++-- .../agent99-remote-atomic-deploy-receiver.ps1 | 182 +++++- .../agent99-stale-queue-quarantine.ps1 | 347 +++++++++++ .../test_agent99_live_preflight_decision.py | 216 ++++++- ...remote_atomic_deploy_transport_contract.py | 45 +- ...agent99_stale_queue_quarantine_contract.py | 44 ++ 6 files changed, 1342 insertions(+), 65 deletions(-) create mode 100644 scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 create mode 100644 scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 92189091a..14b387c08 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -28,7 +28,8 @@ function Get-AgentLivePreflightDecision { [int]$StaleAlertExecutionCount, [int]$AlertIncomingCount, [int]$StaleAlertIncomingCount, - [int]$PendingCommandCount + [int]$PendingCommandBlockingCount, + [int]$PromotionDeferredCommandCount ) $blockingReasons = @() @@ -89,11 +90,14 @@ function Get-AgentLivePreflightDecision { } if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" } - if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) { + if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandBlockingCount -gt 0) { $blockingReasons += "pending_work_before_reboot" } elseif ($AlertIncomingCount -gt 0) { $warningReasons += "fresh_alert_pending_next_inbox_tick" } + if ($PromotionDeferredCommandCount -gt 0) { + $warningReasons += "maintenance_pending_commands_deferred_until_post_promotion" + } return [pscustomobject]@{ deploymentEligible = [bool]($blockingReasons.Count -eq 0) @@ -110,7 +114,8 @@ function Get-AgentLivePreflightDecision { function Get-TaskReadback { param( [string]$TaskPath, - [string]$TaskName + [string]$TaskName, + [string[]]$RequiredArgumentMarkers ) try { @@ -120,8 +125,34 @@ function Get-TaskReadback { $info = Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop $lastResult = [int64]$info.LastTaskResult $state = [string]$task.State + $actions = @($task.Actions) + $actionDefinitionReady = [bool]( + $actions.Count -eq 1 -and + [string]$actions[0].Execute -match '(?i)\\WindowsPowerShell\\v1\.0\\powershell\.exe$' + ) + if ($actionDefinitionReady) { + foreach ($marker in @($RequiredArgumentMarkers)) { + if ( + -not $marker -or + ([string]$actions[0].Arguments).IndexOf( + $marker, + [StringComparison]::OrdinalIgnoreCase + ) -lt 0 + ) { + $actionDefinitionReady = $false + break + } + } + } + $principalReady = [bool]( + [string]$task.Principal.UserId -match '(?i)(^|\\)Administrator$' -and + [string]$task.Principal.LogonType -eq "S4U" -and + [string]$task.Principal.RunLevel -eq "Highest" + ) + $definitionReady = [bool]($actionDefinitionReady -and $principalReady) $healthy = [bool]( $task.Settings.Enabled -and + $definitionReady -and ($lastResult -in @(0, 267009, 267011) -or $state -eq "Running") ) @@ -134,6 +165,9 @@ function Get-TaskReadback { lastTaskResult = $lastResult lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" } nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" } + actionDefinitionReady = $actionDefinitionReady + principalReady = $principalReady + definitionReady = $definitionReady healthy = $healthy } } catch { @@ -146,6 +180,9 @@ function Get-TaskReadback { lastTaskResult = $null lastRunTime = "" nextRunTime = "" + actionDefinitionReady = $false + principalReady = $false + definitionReady = $false healthy = $false } } @@ -158,25 +195,448 @@ function Get-DirectoryReadback { ) $path = Join-Path $AgentRoot $RelativePath - $files = @() - if (Test-Path $path) { - $files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue) - if ($RootQueueFilesOnly) { - $files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" }) + $exists = [bool](Test-Path -LiteralPath $path -PathType Container) + $count = 0 + $staleCount = 0 + $oldest = $null + $newest = $null + if ($exists) { + foreach ($candidatePath in [IO.Directory]::EnumerateFiles( + $path, + "*.json", + [IO.SearchOption]::TopDirectoryOnly + )) { + $name = [IO.Path]::GetFileName($candidatePath) + if ($RootQueueFilesOnly -and $name -match "^(processed|failed|running)-") { + continue + } + $file = [IO.FileInfo]::new($candidatePath) + $count += 1 + if (-not $oldest -or $file.LastWriteTimeUtc -lt $oldest.LastWriteTimeUtc) { + $oldest = $file + } + if (-not $newest -or $file.LastWriteTimeUtc -gt $newest.LastWriteTimeUtc) { + $newest = $file + } + if (((Get-Date) - $file.LastWriteTime).TotalMinutes -gt 5) { + $staleCount += 1 + } } - $files = @($files | Sort-Object LastWriteTime) } return [pscustomobject]@{ relativePath = $RelativePath - exists = [bool](Test-Path $path) - count = [int]$files.Count - oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" } - newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" } - staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count + exists = $exists + count = [int]$count + oldest = if ($oldest) { $oldest.LastWriteTime.ToString("o") } else { "" } + newest = if ($newest) { $newest.LastWriteTime.ToString("o") } else { "" } + staleOverFiveMinutes = [int]$staleCount } } +function Get-AgentBoundedNewestFiles { + param( + [string]$DirectoryPath, + [string]$Pattern, + [ValidateRange(1, 256)] + [int]$CandidateWindow = 32 + ) + + # Agent99 evidence names contain a fixed sortable timestamp and are immutable. + # Keep only the lexically newest bounded window in a native .NET set so a + # large evidence directory never becomes a PowerShell object-sort workload. + $selected = [Collections.Generic.SortedSet[string]]::new( + [StringComparer]::OrdinalIgnoreCase + ) + $enumeratedCount = 0 + if (Test-Path -LiteralPath $DirectoryPath -PathType Container) { + foreach ($candidatePath in [IO.Directory]::EnumerateFiles( + $DirectoryPath, + $Pattern, + [IO.SearchOption]::TopDirectoryOnly + )) { + $enumeratedCount += 1 + $null = $selected.Add($candidatePath) + if ($selected.Count -gt $CandidateWindow) { + $null = $selected.Remove($selected.Min) + } + } + } + + return [pscustomobject]@{ + files = @($selected | Sort-Object -Descending | ForEach-Object { + [IO.FileInfo]::new($_) + }) + enumeratedCount = [int]$enumeratedCount + candidateWindow = [int]$CandidateWindow + windowTruncated = [bool]($enumeratedCount -gt $selected.Count) + } +} + +function Get-AgentControlLoopMaintenanceReadback { + $evidenceDir = Join-Path $AgentRoot "evidence" + $selection = Get-AgentBoundedNewestFiles ` + $evidenceDir ` + "agent99-control-loop-maintenance-2*.json" ` + 8 + $file = @($selection.files | Select-Object -First 1) + $receipt = $null + $intent = $null + $parseError = "" + if ($file.Count -eq 1) { + try { + $receipt = Get-Content -LiteralPath $file[0].FullName -Raw | ConvertFrom-Json + } catch { + $parseError = $_.Exception.GetType().Name + } + } + + $intentRef = if ($receipt -and $receipt.PSObject.Properties["operationIntentRef"]) { + [string]$receipt.operationIntentRef + } else { "" } + $intentRefSafe = [bool]( + $intentRef -match "^agent99-control-loop-maintenance-intent-[0-9]{8}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$" + ) + $intentPath = if ($intentRefSafe) { Join-Path $evidenceDir $intentRef } else { "" } + if ($intentPath -and (Test-Path -LiteralPath $intentPath -PathType Leaf)) { + try { + $intent = Get-Content -LiteralPath $intentPath -Raw | ConvertFrom-Json + } catch { + $parseError = $_.Exception.GetType().Name + } + } + $legacyPropertySet = @( + "blockers", "controlStopped", "expectedControlCreated", "expectedControlPid", + "expectedRecoverCreated", "expectedRecoverParentPid", "expectedRecoverPid", + "hostRebootPerformed", "leaseMetadataRemoved", "mode", "precheckPassed", + "recoverStopped", "runtimeWritePerformed", "schemaVersion", "secretValueRead", + "taskEnabled", "taskName", "terminal", "timestamp", "vmPowerChangePerformed", + "vmwareVmxProcessTerminated", "vmxLockDeleted" + ) + $receiptPropertySet = if ($receipt) { + @($receipt.PSObject.Properties.Name | Sort-Object) + } else { @() } + $legacyPropertySetMatches = [bool]( + $receiptPropertySet.Count -eq $legacyPropertySet.Count -and + (@($receiptPropertySet) -join ",") -eq (@($legacyPropertySet | Sort-Object) -join ",") + ) + $legacyIdentityFieldsValid = [bool]( + $receipt -and + $receipt.expectedControlPid -is [int] -and [int]$receipt.expectedControlPid -gt 0 -and + $receipt.expectedRecoverPid -is [int] -and [int]$receipt.expectedRecoverPid -gt 0 -and + $receipt.expectedRecoverParentPid -is [int] -and [int]$receipt.expectedRecoverParentPid -gt 0 -and + [string]$receipt.expectedControlCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and + [string]$receipt.expectedRecoverCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and + [string]$receipt.timestamp -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" + ) + $legacyReceipt = [bool]( + $receipt -and + -not $receipt.PSObject.Properties["operationIntentRef"] -and + $receipt.PSObject.Properties["taskEnabled"] -and + $legacyPropertySetMatches -and + $legacyIdentityFieldsValid -and + @($receipt.blockers).Count -eq 0 + ) + + $ageMinutes = if ($file.Count -eq 1) { + [math]::Round(((Get-Date) - $file[0].LastWriteTime).TotalMinutes, 1) + } else { $null } + $activeControlProcesses = @(Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction SilentlyContinue | + Where-Object { + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)' + }) + $leasePresent = [bool](Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json") -PathType Leaf) + $recoveryMutexActive = $true + $mutex = $null + $mutexAcquired = $false + try { + $mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1") + try { + $mutexAcquired = [bool]$mutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $mutexAcquired = $true + } + $recoveryMutexActive = [bool](-not $mutexAcquired) + } finally { + if ($mutexAcquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { $mutex.Dispose() } + } + + $commonReceiptFieldsValid = [bool]( + $receipt -and + [string]$receipt.schemaVersion -eq "agent99_control_loop_maintenance_v2" -and + [string]$receipt.mode -eq "Apply" -and + [string]$receipt.taskName -eq "Wooo-Agent99-Control-Loop" -and + [string]$receipt.terminal -eq "control_loop_maintenance_frozen" -and + $receipt.precheckPassed -is [bool] -and $receipt.precheckPassed -and + $receipt.controlStopped -is [bool] -and $receipt.controlStopped -and + $receipt.recoverStopped -is [bool] -and $receipt.recoverStopped -and + $receipt.leaseMetadataRemoved -is [bool] -and $receipt.leaseMetadataRemoved -and + $receipt.runtimeWritePerformed -is [bool] -and $receipt.runtimeWritePerformed -and + $receipt.vmPowerChangePerformed -is [bool] -and -not $receipt.vmPowerChangePerformed -and + $receipt.vmwareVmxProcessTerminated -is [bool] -and -not $receipt.vmwareVmxProcessTerminated -and + $receipt.vmxLockDeleted -is [bool] -and -not $receipt.vmxLockDeleted -and + $receipt.hostRebootPerformed -is [bool] -and -not $receipt.hostRebootPerformed -and + $receipt.secretValueRead -is [bool] -and -not $receipt.secretValueRead + ) + $modernReceiptFieldsValid = [bool]( + $commonReceiptFieldsValid -and + -not $legacyReceipt -and + $receipt.taskEnabledAfter -is [bool] -and -not $receipt.taskEnabledAfter + ) + $legacyReceiptFieldsValid = [bool]( + $commonReceiptFieldsValid -and + $legacyReceipt -and + $receipt.taskEnabled -is [bool] -and $receipt.taskEnabled + ) + $receiptFieldsValid = [bool]($modernReceiptFieldsValid -or $legacyReceiptFieldsValid) + $intentFieldsValid = [bool]( + $legacyReceipt -or + ( + $intent -and + [string]$intent.schemaVersion -eq "agent99_control_loop_maintenance_intent_v1" -and + [string]$intent.taskName -eq "Wooo-Agent99-Control-Loop" -and + $intent.vmPowerChangePerformed -is [bool] -and -not $intent.vmPowerChangePerformed -and + $intent.vmwareVmxProcessTerminated -is [bool] -and -not $intent.vmwareVmxProcessTerminated -and + $intent.secretValueRead -is [bool] -and -not $intent.secretValueRead -and + (@($intent.plannedActions) -join ",") -eq "disable_control_task,stop_exact_control_tree,stop_exact_recover_tree,remove_exact_recovery_lease" + ) + ) + $valid = [bool]( + -not $parseError -and + $file.Count -eq 1 -and + $null -ne $ageMinutes -and + $ageMinutes -ge 0 -and + $ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 }) -and + ($legacyReceipt -or $intentRefSafe) -and + $receiptFieldsValid -and + $intentFieldsValid -and + $activeControlProcesses.Count -eq 0 -and + -not $leasePresent -and + -not $recoveryMutexActive + ) + + return [pscustomobject]@{ + valid = $valid + receiptFile = if ($file.Count -eq 1) { $file[0].Name } else { "" } + ageMinutes = $ageMinutes + parseError = $parseError + intentRefPresent = $intentRefSafe + contractVersion = if ($legacyReceipt) { "legacy_v2_exact_upgrade_bridge" } else { "intent_bound_v2" } + receiptTimestamp = if ($receipt) { [string]$receipt.timestamp } else { "" } + receiptLastWriteTime = if ($file.Count -eq 1) { $file[0].LastWriteTime.ToString("o") } else { "" } + legacyPropertySetMatches = $legacyPropertySetMatches + legacyIdentityFieldsValid = $legacyIdentityFieldsValid + receiptFieldsValid = $receiptFieldsValid + intentFieldsValid = $intentFieldsValid + activeControlProcessCount = [int]$activeControlProcesses.Count + recoveryLeasePresent = $leasePresent + recoveryMutexActive = $recoveryMutexActive + secretValueRead = $false + remoteWritePerformed = $false + } +} + +function Get-AgentPendingCommandPromotionReadback { + param( + [object]$MaintenanceReadback, + [ValidateSet("FullRuntime", "PromotionReserve")] + [string]$Scope + ) + + $queueRoot = Join-Path $AgentRoot "queue" + $files = @() + if (Test-Path -LiteralPath $queueRoot -PathType Container) { + $files = @([IO.Directory]::EnumerateFiles( + $queueRoot, + "*.json", + [IO.SearchOption]::TopDirectoryOnly + ) | Where-Object { + [IO.Path]::GetFileName($_) -notmatch "^(processed|failed|running)-" + }) + } + + $observedCount = [int]$files.Count + $result = [ordered]@{ + scope = $Scope + observedCount = $observedCount + deferredCount = 0 + blockingCount = $observedCount + eligibleMaintenanceAutoRecoverCount = 0 + boundedLimit = 8 + contractValid = [bool]($observedCount -eq 0) + reason = if ($observedCount -eq 0) { "queue_empty" } else { "pending_commands_require_runtime" } + items = @() + rawInstructionStored = $false + secretValueRead = $false + remoteWritePerformed = $false + } + if ($observedCount -eq 0) { + return [pscustomobject]$result + } + if ( + -not $MaintenanceReadback -or + -not $MaintenanceReadback.receiptFieldsValid -or + -not $MaintenanceReadback.intentFieldsValid + ) { + $result.reason = "maintenance_contract_invalid" + return [pscustomobject]$result + } + if ($Scope -eq "PromotionReserve" -and -not $MaintenanceReadback.valid) { + $result.reason = "maintenance_current_state_invalid" + return [pscustomobject]$result + } + if ($observedCount -gt $result.boundedLimit) { + $result.reason = "maintenance_pending_command_limit_exceeded" + return [pscustomobject]$result + } + + $maintenanceTimestamp = [datetimeoffset]::MinValue + $maintenanceTimestampValid = [datetimeoffset]::TryParse( + [string]$MaintenanceReadback.receiptTimestamp, + [ref]$maintenanceTimestamp + ) + if (-not $maintenanceTimestampValid) { + $result.reason = "maintenance_timestamp_invalid" + return [pscustomobject]$result + } + + $requiredPropertySet = @( + "approvalId", "automationRunId", "canonicalDigest", "controlledApply", + "correlationKey", "createdAt", "dispatchIdentitySchemaVersion", + "dispatchRouteId", "executionGeneration", "externalId", "id", + "idempotencyKey", "incidentId", "instruction", "lockOwner", "mode", + "observation", "projectId", "reason", "requestedBy", "singleFlightKey", + "source", "sourceFingerprint", "traceId", "workItemId" + ) | Sort-Object + $safeItems = @() + $allValid = $true + foreach ($path in $files) { + $file = [IO.FileInfo]::new($path) + $payload = $null + $parseError = "" + $sha256 = "" + try { + $bytes = [IO.File]::ReadAllBytes($path) + $sha = [Security.Cryptography.SHA256]::Create() + try { + $sha256 = -join @($sha.ComputeHash($bytes) | ForEach-Object { $_.ToString("x2") }) + } finally { + $sha.Dispose() + } + $jsonOffset = if ( + $bytes.Length -ge 3 -and + $bytes[0] -eq 0xef -and + $bytes[1] -eq 0xbb -and + $bytes[2] -eq 0xbf + ) { 3 } else { 0 } + $jsonText = [Text.Encoding]::UTF8.GetString( + $bytes, + $jsonOffset, + $bytes.Length - $jsonOffset + ) + $payload = $jsonText | ConvertFrom-Json + } catch { + $parseError = $_.Exception.GetType().Name + } + $propertySet = if ($payload) { + @($payload.PSObject.Properties.Name | Sort-Object) + } else { @() } + $propertySetMatches = [bool]( + $propertySet.Count -eq $requiredPropertySet.Count -and + (@($propertySet) -join ",") -eq (@($requiredPropertySet) -join ",") + ) + $createdAt = [datetimeoffset]::MinValue + $createdAtValid = [bool]( + $payload -and + [datetimeoffset]::TryParse([string]$payload.createdAt, [ref]$createdAt) + ) + $ageMinutes = if ($createdAtValid) { + [math]::Round(([datetimeoffset]::Now - $createdAt).TotalMinutes, 1) + } else { $null } + $fileBaseName = [IO.Path]::GetFileNameWithoutExtension($file.Name) + $nativeStringFields = @( + "approvalId", "automationRunId", "canonicalDigest", "correlationKey", + "createdAt", "dispatchIdentitySchemaVersion", "dispatchRouteId", + "executionGeneration", "externalId", "id", "idempotencyKey", + "incidentId", "instruction", "lockOwner", "mode", "projectId", + "reason", "requestedBy", "singleFlightKey", "source", + "sourceFingerprint", "traceId", "workItemId" + ) + $nativeStringTypesValid = [bool]( + $payload -and + @($nativeStringFields | Where-Object { $payload.$_ -isnot [string] }).Count -eq 0 + ) + $identityValid = [bool]( + -not $parseError -and + $propertySetMatches -and + $nativeStringTypesValid -and + $payload.controlledApply -is [bool] -and $payload.controlledApply -and + $payload.observation -is [pscustomobject] -and + [string]$payload.id -eq $fileBaseName -and + [string]$payload.id -match "^auto-recover-[0-9]{8}-[0-9]{6}-[0-9a-f]{8}$" -and + [string]$payload.mode -eq "Recover" -and + [string]$payload.source -eq "agent99-recovery-observer" -and + [string]$payload.dispatchIdentitySchemaVersion -eq "agent99_controlled_dispatch_identity_v1" -and + [string]$payload.dispatchRouteId -eq "agent99_local_observer" -and + [string]$payload.projectId -eq "awoooi" -and + [string]$payload.requestedBy -eq "Agent99" -and + [string]$payload.reason -eq "host_unreachable_detected" -and + [string]$payload.externalId -eq "host_unreachable_detected" -and + [string]$payload.approvalId -eq "" -and + [string]$payload.automationRunId -match "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -and + [string]$payload.lockOwner -eq [string]$payload.automationRunId -and + [string]$payload.executionGeneration -match "^[1-9][0-9]*$" -and + [string]$payload.canonicalDigest -match "^[0-9a-f]{64}$" -and + [string]$payload.sourceFingerprint -match "^[0-9a-f]{64}$" -and + [string]$payload.correlationKey -match "^agent99-controlled:[0-9a-f]{64}$" -and + [string]$payload.idempotencyKey -eq [string]$payload.correlationKey -and + [string]$payload.singleFlightKey -match "^agent99-controlled-flight:[0-9a-f]{64}$" -and + [string]$payload.incidentId -match "^agent99-auto-[0-9a-f]{20}$" -and + [string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):host_unreachable_detected" -and + [string]$payload.traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$" -and + -not [string]::IsNullOrWhiteSpace([string]$payload.instruction) -and + $createdAtValid -and + $createdAt -ge $maintenanceTimestamp -and + $ageMinutes -ge 0 -and + $ageMinutes -le 120 + ) + if (-not $identityValid) { $allValid = $false } + $safeItems += [pscustomobject]@{ + file = $file.Name + id = if ($payload) { [string]$payload.id } else { "" } + mode = if ($payload) { [string]$payload.mode } else { "" } + source = if ($payload) { [string]$payload.source } else { "" } + workItemId = if ($payload) { [string]$payload.workItemId } else { "" } + createdAt = if ($payload) { [string]$payload.createdAt } else { "" } + ageMinutes = $ageMinutes + sha256 = $sha256 + identity = if ($payload -and $sha256) { "$([string]$payload.id)|$sha256" } else { "" } + parseError = $parseError + propertySetMatches = $propertySetMatches + identityValid = $identityValid + instructionContentReadback = $false + } + } + + $result.items = $safeItems + if ($allValid) { + $result.eligibleMaintenanceAutoRecoverCount = $observedCount + $result.contractValid = $true + if ($Scope -eq "PromotionReserve") { + $result.deferredCount = $observedCount + $result.blockingCount = 0 + $result.reason = "bounded_maintenance_auto_recover_deferred_until_post_promotion" + } else { + $result.reason = "full_runtime_requires_maintenance_auto_recover_to_drain" + } + } else { + $result.reason = "maintenance_pending_command_contract_invalid" + } + return [pscustomobject]$result +} + function Get-EvidenceReadback { param( [string]$Name, @@ -187,8 +647,8 @@ function Get-EvidenceReadback { ) $evidenceDir = Join-Path $AgentRoot "evidence" - $candidates = @(Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue | - Sort-Object LastWriteTime -Descending) + $candidateSelection = Get-AgentBoundedNewestFiles $evidenceDir $Pattern + $candidates = @($candidateSelection.files) $deferredCandidates = @() $file = $null foreach ($candidate in $candidates) { @@ -219,6 +679,11 @@ function Get-EvidenceReadback { $file = $candidate break } + $candidateWindowExhausted = [bool]( + -not $file -and + $candidateSelection.windowTruncated -and + $deferredCandidates.Count -eq $candidates.Count + ) $ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null } $fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes) $freshnessHeadroomMinutes = if ($file) { [math]::Round($MaxAgeMinutes - $ageMinutes, 1) } else { $null } @@ -228,7 +693,7 @@ function Get-EvidenceReadback { $freshnessHeadroomMinutes -ge $MinimumFreshnessReserveMinutes ) $safeSummary = $null - $parseError = "" + $parseError = if ($candidateWindowExhausted) { "EvidenceCandidateWindowExhausted" } else { "" } $contentHealthy = $true if ($file) { try { @@ -389,23 +854,52 @@ function Get-EvidenceReadback { parseError = $parseError deferredCandidateCount = [int]$deferredCandidates.Count latestDeferredTime = if ($deferredCandidates.Count -gt 0) { $deferredCandidates[0].LastWriteTime.ToString("o") } else { "" } + enumeratedCandidateCount = [int]$candidateSelection.enumeratedCount + candidateWindowLimit = [int]$candidateSelection.candidateWindow + candidateWindowTruncated = [bool]$candidateSelection.windowTruncated + candidateWindowExhausted = $candidateWindowExhausted safeSummary = $safeSummary } } $rootTaskPath = "\" $requiredTasks = @( - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health" } + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery"; markers = @("agent99-submit-request.ps1", "-Mode Recover", "-Source agent99-startup-recovery") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat"; markers = @("agent99-run.ps1", "-Mode Status") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard"; markers = @("agent99-run.ps1", "-Mode Perf", "-ControlledApply") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop"; markers = @("agent99-run.ps1", "-Mode ControlTick") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox"; markers = @("agent99-telegram-inbox.ps1", "-RunNow") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox"; markers = @("agent99-sre-alert-inbox.ps1", "-RunNow") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay"; markers = @("agent99-sre-alert-relay.ps1") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health"; markers = @("agent99-run.ps1", "-Mode SelfCheck") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health"; markers = @("agent99-run.ps1", "-Mode BackupCheck") } ) -$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback -TaskPath $_.taskPath -TaskName $_.name }) +$tasks = @($requiredTasks | ForEach-Object { + Get-TaskReadback ` + -TaskPath $_.taskPath ` + -TaskName $_.name ` + -RequiredArgumentMarkers $_.markers +}) +$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback +if ($ReadinessScope -eq "PromotionReserve") { + foreach ($task in $tasks) { + $runtimeHealthy = [bool]$task.healthy + $maintenanceFreezeAccepted = [bool]( + $task.name -eq "Wooo-Agent99-Control-Loop" -and + -not $task.enabled -and + $controlLoopMaintenance.valid + ) + $promotionReady = [bool]( + $task.exists -and + $task.definitionReady -and + ($task.enabled -or $maintenanceFreezeAccepted) + ) + $task | Add-Member -NotePropertyName runtimeHealthy -NotePropertyValue $runtimeHealthy -Force + $task | Add-Member -NotePropertyName promotionReady -NotePropertyValue $promotionReady -Force + $task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force + $task.healthy = $promotionReady + } +} $manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json" $manifest = [pscustomobject]@{ @@ -457,14 +951,15 @@ $queues = @( (Get-DirectoryReadback "queue" -RootQueueFilesOnly), (Get-DirectoryReadback "queue\failed") ) +$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime") $evidence = @( - (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0) ) @@ -482,6 +977,9 @@ $selfHealthPerformanceIssueCount = if ($selfCheckEvidence -and $selfCheckEvidenc $alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1 $alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1 $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1 +$pendingCommandPromotion = Get-AgentPendingCommandPromotionReadback ` + -MaintenanceReadback $controlLoopMaintenance ` + -Scope $ReadinessScope $decision = Get-AgentLivePreflightDecision ` -AgentRootPresent ([bool](Test-Path $AgentRoot)) ` -Manifest $manifest ` @@ -498,7 +996,8 @@ $decision = Get-AgentLivePreflightDecision ` -StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes ` -AlertIncomingCount $alertIncoming.count ` -StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes ` - -PendingCommandCount $commandQueue.count + -PendingCommandBlockingCount $pendingCommandPromotion.blockingCount ` + -PromotionDeferredCommandCount $pendingCommandPromotion.deferredCount $blockingReasons = @($decision.blockingReasons) $warningReasons = @($decision.warningReasons) @@ -513,10 +1012,12 @@ $result = [pscustomobject]@{ secretValueRead = $false remoteWritePerformed = $false agentRoot = $AgentRoot + controlLoopMaintenance = $controlLoopMaintenance manifest = $manifest tasks = $tasks relayListeners = $relayListeners queues = $queues + pendingCommandPromotion = $pendingCommandPromotion evidence = $evidence summary = [pscustomobject]@{ requiredTaskCount = $requiredTasks.Count @@ -536,6 +1037,8 @@ $result = [pscustomobject]@{ alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes alertRunningCount = [int]$alertRunning.count pendingCommandCount = [int]$commandQueue.count + pendingCommandBlockingCount = [int]$pendingCommandPromotion.blockingCount + promotionDeferredCommandCount = [int]$pendingCommandPromotion.deferredCount relayListenerCount = $relayListeners.Count preflightGreen = [bool]$decision.deploymentEligible deploymentEligible = [bool]$decision.deploymentEligible diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index af83a6372..2b1e003e0 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -23,7 +23,9 @@ $RelayStartTimeoutSeconds = 360 $RelayPollIntervalSeconds = 1 $ValidateOnlyTimeoutSeconds = 300 $LiveDeployTimeoutSeconds = 300 -$LivePreflightTimeoutSeconds = 120 +$LivePreflightTimeoutSeconds = 60 +$PostVerifyMaxAttempts = 4 +$PostVerifyRetryWaitSeconds = 45 $WindowsControlBaselineTimeoutSeconds = 300 $PromotionReadinessReserveMinutes = 15 $PromotionReadinessMaxAttempts = 2 @@ -434,6 +436,9 @@ function Invoke-AgentLivePreflight { enabled = [bool]$_.enabled state = if ($state -in @("Ready", "Running", "Disabled", "Queued", "Missing")) { $state } else { "Unknown" } lastTaskResult = if ($null -ne $_.lastTaskResult) { [long]$_.lastTaskResult } else { $null } + actionDefinitionReady = [bool]$_.actionDefinitionReady + principalReady = [bool]$_.principalReady + definitionReady = [bool]$_.definitionReady } }) } else { @() } @@ -465,6 +470,14 @@ function Invoke-AgentLivePreflight { $ReadinessScope -eq "PromotionReserve" -or ($payload -and $payload.manifest -and [bool]$payload.manifest.runtimeMatched) ) + $pendingCommandsReadyForScope = [bool]( + $summary -and + [int]$summary.pendingCommandBlockingCount -eq 0 -and + ( + $ReadinessScope -eq "PromotionReserve" -or + [int]$summary.pendingCommandCount -eq 0 + ) + ) $runtimePlaneReady = [bool]( $acceptedExitCode -and -not $processResult.timedOut -and @@ -482,7 +495,7 @@ function Invoke-AgentLivePreflight { [int]$summary.relayListenerCount -gt 0 -and [int]$summary.alertIncomingStaleCount -eq 0 -and [int]$summary.alertRunningCount -eq 0 -and - [int]$summary.pendingCommandCount -eq 0 -and + $pendingCommandsReadyForScope -and $runtimePlaneBlockingReasons.Count -eq 0 ) return [pscustomobject]@{ @@ -521,6 +534,16 @@ function Invoke-AgentLivePreflight { alertIncomingStaleCount = if ($summary) { [int]$summary.alertIncomingStaleCount } else { -1 } alertRunningCount = if ($summary) { [int]$summary.alertRunningCount } else { -1 } pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 } + pendingCommandBlockingCount = if ($summary) { [int]$summary.pendingCommandBlockingCount } else { -1 } + promotionDeferredCommandCount = if ($summary) { [int]$summary.promotionDeferredCommandCount } else { -1 } + pendingCommandDeferralReason = if ($payload -and $payload.pendingCommandPromotion) { + [string]$payload.pendingCommandPromotion.reason + } else { "" } + maintenancePendingCommandIdentities = if ($payload -and $payload.pendingCommandPromotion) { + @($payload.pendingCommandPromotion.items | Where-Object { + [bool]$_.identityValid -and [string]$_.identity -match "^auto-recover-[^|]+\|[0-9a-f]{64}$" + } | ForEach-Object { [string]$_.identity } | Sort-Object -Unique) + } else { @() } preflightGreen = [bool]($summary -and $summary.preflightGreen) overallPreflightGreen = [bool]($summary -and $summary.preflightGreen) runtimePlaneReady = $runtimePlaneReady @@ -579,6 +602,96 @@ function Test-AgentPromotionReadinessRetryable { return $true } +function Test-AgentPostPromotionReadinessRetryable { + param( + [object]$Readiness, + [string[]]$ExpectedMaintenancePendingCommandIdentities = @() + ) + + $pendingCount = if ($Readiness) { [int]$Readiness.pendingCommandCount } else { -1 } + $pendingIdentities = if ($Readiness) { + @($Readiness.maintenancePendingCommandIdentities | ForEach-Object { [string]$_ } | Sort-Object -Unique) + } else { @() } + $expectedIdentities = @($ExpectedMaintenancePendingCommandIdentities | ForEach-Object { [string]$_ } | Sort-Object -Unique) + $pendingCommandsRetryable = [bool]( + $pendingCount -eq 0 -or + ( + $pendingCount -gt 0 -and + [int]$Readiness.pendingCommandBlockingCount -eq $pendingCount -and + [string]$Readiness.pendingCommandDeferralReason -eq "full_runtime_requires_maintenance_auto_recover_to_drain" -and + $pendingIdentities.Count -eq $pendingCount -and + $expectedIdentities.Count -gt 0 -and + @($pendingIdentities | Where-Object { $_ -notin $expectedIdentities }).Count -eq 0 + ) + ) + + if ( + -not $Readiness -or + $Readiness.ok -or + -not $Readiness.parsed -or + -not $Readiness.scopeMatched -or + [string]$Readiness.readinessScope -ne "FullRuntime" -or + $Readiness.timedOut -or + -not $Readiness.acceptedExitCode -or + [int]$Readiness.relayListenerCount -lt 1 -or + [int]$Readiness.alertRunningCount -ne 0 -or + -not $pendingCommandsRetryable + ) { + return $false + } + $allowedReasons = @( + "scheduled_task_unhealthy", + "required_evidence_stale", + "required_evidence_freshness_reserve_insufficient", + "required_evidence_content_unhealthy", + "pending_work_before_reboot" + ) + $reasons = @($Readiness.runtimePlaneBlockingReasons | ForEach-Object { [string]$_ }) + if ( + $reasons.Count -eq 0 -or + @($reasons | Where-Object { $_ -notin $allowedReasons }).Count -gt 0 + ) { + return $false + } + if (@($Readiness.failedTasks | Where-Object { + -not $_.exists -or + -not $_.enabled -or + -not $_.definitionReady -or + $_.taskPath -ne "\" -or + $_.state -notin @("Ready", "Running", "Queued") + }).Count -gt 0) { + return $false + } + if ( + "scheduled_task_unhealthy" -in $reasons -and + @($Readiness.failedTasks).Count -eq 0 + ) { return $false } + if ( + "pending_work_before_reboot" -in $reasons -and + $pendingCount -eq 0 + ) { return $false } + if ( + @($reasons | Where-Object { + $_ -in @( + "required_evidence_stale", + "required_evidence_freshness_reserve_insufficient", + "required_evidence_content_unhealthy" + ) + }).Count -gt 0 -and + @($Readiness.failedEvidence).Count -eq 0 + ) { return $false } + if (@($Readiness.failedEvidence | Where-Object { + -not $_.parsed -or + ( + -not $_.contentHealthy -and + $_.name -notin @("telegram_inbox", "sre_alert_inbox") + ) + }).Count -gt 0) { + return $false + } + return $true +} + function Get-AgentRelayRuntime { $task = $null $taskError = "" @@ -1405,6 +1518,8 @@ try { $relayReload = $null $runtimeManifest = $null $livePreflight = $null + $postVerifyAttempts = @() + $postVerifyRetryCount = 0 $launcherContract = $null $windowsControlBaseline = $null $postVerified = $false @@ -1442,25 +1557,40 @@ try { $script:ReceiverOperationPhase = "independent_post_verifier" $runtimeManifest = Get-AgentSafeRuntimeManifest $launcherContract = Get-AgentLauncherContract - $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") - $postVerified = [bool]( - $runtimeManifest.exists -and - $runtimeManifest.runtimeMatched -and - $runtimeManifest.sourceRevision -eq $sourceRevision -and - $runtimeManifest.fileCount -eq 18 -and - $runtimeManifest.mismatchCount -eq 0 -and - $launcherContract.ok -and - [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and - $relayReload.oldGenerationsGone -and - $relayReload.newGenerationVerified -and - $relayReload.after.taskEnabled -and - $relayReload.after.taskRunning -and - $livePreflight.ok -and - $livePreflight.sourceRevision -eq $sourceRevision -and - $livePreflight.runtimeMatched -and - $livePreflight.relayListenerCount -gt 0 -and - (Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations)) - ) + for ($postVerifyAttempt = 1; $postVerifyAttempt -le $PostVerifyMaxAttempts; $postVerifyAttempt += 1) { + $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") + $livePreflight | Add-Member -NotePropertyName attempt -NotePropertyValue $postVerifyAttempt -Force + $postVerifyAttempts += $livePreflight + $postVerified = [bool]( + $runtimeManifest.exists -and + $runtimeManifest.runtimeMatched -and + $runtimeManifest.sourceRevision -eq $sourceRevision -and + $runtimeManifest.fileCount -eq 18 -and + $runtimeManifest.mismatchCount -eq 0 -and + $launcherContract.ok -and + [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and + $relayReload.oldGenerationsGone -and + $relayReload.newGenerationVerified -and + $relayReload.after.taskEnabled -and + $relayReload.after.taskRunning -and + $livePreflight.ok -and + $livePreflight.sourceRevision -eq $sourceRevision -and + $livePreflight.runtimeMatched -and + $livePreflight.relayListenerCount -gt 0 -and + (Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations)) + ) + if ($postVerified) { break } + if ( + $postVerifyAttempt -ge $PostVerifyMaxAttempts -or + -not (Test-AgentPostPromotionReadinessRetryable ` + $livePreflight ` + -ExpectedMaintenancePendingCommandIdentities @( + $prePromotionReadiness.maintenancePendingCommandIdentities + )) + ) { break } + Start-Sleep -Seconds $PostVerifyRetryWaitSeconds + } + $postVerifyRetryCount = [math]::Max(0, $postVerifyAttempts.Count - 1) if (-not $postVerified) { $failurePhase = "post_verifier" $failureType = "independent_post_verifier_failed" @@ -1504,6 +1634,14 @@ try { runtimeManifest = $runtimeManifest launcherContract = $launcherContract livePreflight = $livePreflight + postVerifyAttempts = $postVerifyAttempts + postVerifyRetryCount = $postVerifyRetryCount + postVerifyPolicy = [pscustomobject]@{ + maxAttempts = $PostVerifyMaxAttempts + retryWaitSeconds = $PostVerifyRetryWaitSeconds + automaticTaskTriggerAllowed = $false + finalAttemptVerified = $postVerified + } windowsControlBaseline = $windowsControlBaseline durableReceiptWritten = $true idempotentReplay = $false @@ -1629,6 +1767,8 @@ try { failedRuntimeManifest = $runtimeManifest failedLauncherContract = $launcherContract failedLivePreflight = $livePreflight + postVerifyAttempts = $postVerifyAttempts + postVerifyRetryCount = $postVerifyRetryCount failedWindowsControlBaseline = $windowsControlBaseline rollback = $rollback rollbackRelayRestart = $rollbackRelayRestart diff --git a/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 b/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 new file mode 100644 index 000000000..317e257a0 --- /dev/null +++ b/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 @@ -0,0 +1,347 @@ +[CmdletBinding()] +param( + [ValidateSet("Check", "Apply")] + [string]$Mode = "Check", + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] + [string]$TraceId, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] + [string]$RunId, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] + [string]$WorkItemId, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._-]{0,180}\.json$")] + [string]$QueueFileName, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-fA-F0-9]{64}$")] + [string]$ExpectedQueueSha256, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] + [string]$ExpectedCommandId, + [Parameter(Mandatory = $true)] + [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}\.json$")] + [string]$MaintenanceReceiptName, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-fA-F0-9]{64}$")] + [string]$ExpectedMaintenanceSha256, + [ValidateRange(30, 1440)] + [int]$MinimumAgeMinutes = 30, + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" +$ExpectedQueueSha256 = $ExpectedQueueSha256.ToLowerInvariant() +$ExpectedMaintenanceSha256 = $ExpectedMaintenanceSha256.ToLowerInvariant() +$queueRoot = Join-Path $AgentRoot "queue" +$failedRoot = Join-Path $queueRoot "failed" +$evidenceRoot = Join-Path $AgentRoot "evidence" +$sourcePath = Join-Path $queueRoot $QueueFileName +$destinationName = "stale-maintenance-$QueueFileName" +$destinationPath = Join-Path $failedRoot $destinationName +$maintenancePath = Join-Path $evidenceRoot $MaintenanceReceiptName +$runtimeWritePerformed = $false + +function Get-AgentSha256 { + param([string]$Path) + (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant() +} + +function Write-AgentDurableJson { + param([object]$Value, [string]$Path) + + $temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))" + $stream = $null + $writer = $null + try { + $encoding = New-Object Text.UTF8Encoding($false) + $stream = [IO.File]::Open( + $temp, + [IO.FileMode]::CreateNew, + [IO.FileAccess]::Write, + [IO.FileShare]::None + ) + $writer = New-Object IO.StreamWriter($stream, $encoding) + $writer.Write(($Value | ConvertTo-Json -Depth 10)) + $writer.Flush() + $stream.Flush($true) + $writer.Dispose(); $writer = $null + $stream.Dispose(); $stream = $null + [IO.File]::Move($temp, $Path) + $hash = Get-AgentSha256 $Path + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "durable_receipt_readback_missing" + } + return $hash + } finally { + if ($writer) { $writer.Dispose() } + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue + } +} + +function Test-AgentRecoveryMutexActive { + $mutex = $null + $acquired = $false + try { + $mutex = [Threading.Mutex]::new( + $false, + "Global\WoooAgent99RecoveryTransportV1" + ) + try { + $acquired = [bool]$mutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $acquired = $true + } + return [bool](-not $acquired) + } finally { + if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { $mutex.Dispose() } + } +} + +function Get-AgentActiveControlProcesses { + @( + Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop | + Where-Object { + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)' + } | + ForEach-Object { + [pscustomobject]@{ + processId = [int]$_.ProcessId + creationDate = ([datetime]$_.CreationDate).ToUniversalTime().ToString("o") + } + } + ) +} + +function Get-AgentQuarantineGate { + if (-not (Test-Path -LiteralPath $sourcePath -PathType Leaf)) { + throw "exact_queue_source_missing" + } + if (Test-Path -LiteralPath $destinationPath) { + throw "quarantine_destination_already_exists" + } + if (-not (Test-Path -LiteralPath $maintenancePath -PathType Leaf)) { + throw "maintenance_receipt_missing" + } + $queueHash = Get-AgentSha256 $sourcePath + $maintenanceHash = Get-AgentSha256 $maintenancePath + if ($queueHash -ne $ExpectedQueueSha256) { throw "queue_hash_mismatch" } + if ($maintenanceHash -ne $ExpectedMaintenanceSha256) { + throw "maintenance_hash_mismatch" + } + + $task = Get-ScheduledTask ` + -TaskName "Wooo-Agent99-Control-Loop" ` + -TaskPath "\" ` + -ErrorAction Stop + if ($task.Settings.Enabled -or [string]$task.State -ne "Disabled") { + throw "control_loop_not_frozen" + } + $activeProcesses = Get-AgentActiveControlProcesses + if ($activeProcesses.Count -ne 0) { throw "active_control_or_recover_process" } + $leasePresent = [bool](Test-Path -LiteralPath ( + Join-Path $AgentRoot "state\recovery-transport-lease.json" + ) -PathType Leaf) + if ($leasePresent) { throw "recovery_lease_present" } + if (Test-AgentRecoveryMutexActive) { throw "recovery_mutex_active" } + + $maintenance = Get-Content -LiteralPath $maintenancePath -Raw | ConvertFrom-Json + if ( + [string]$maintenance.schemaVersion -ne "agent99_control_loop_maintenance_v2" -or + [string]$maintenance.mode -ne "Apply" -or + [string]$maintenance.taskName -ne "Wooo-Agent99-Control-Loop" -or + [string]$maintenance.terminal -ne "control_loop_maintenance_frozen" -or + $maintenance.precheckPassed -isnot [bool] -or -not $maintenance.precheckPassed -or + $maintenance.controlStopped -isnot [bool] -or -not $maintenance.controlStopped -or + $maintenance.recoverStopped -isnot [bool] -or -not $maintenance.recoverStopped -or + $maintenance.leaseMetadataRemoved -isnot [bool] -or -not $maintenance.leaseMetadataRemoved -or + $maintenance.runtimeWritePerformed -isnot [bool] -or -not $maintenance.runtimeWritePerformed -or + $maintenance.vmPowerChangePerformed -isnot [bool] -or $maintenance.vmPowerChangePerformed -or + $maintenance.vmwareVmxProcessTerminated -isnot [bool] -or $maintenance.vmwareVmxProcessTerminated -or + $maintenance.vmxLockDeleted -isnot [bool] -or $maintenance.vmxLockDeleted -or + $maintenance.hostRebootPerformed -isnot [bool] -or $maintenance.hostRebootPerformed -or + $maintenance.secretValueRead -isnot [bool] -or $maintenance.secretValueRead + ) { + throw "maintenance_receipt_contract_invalid" + } + + $command = Get-Content -LiteralPath $sourcePath -Raw | ConvertFrom-Json + if ( + [string]$command.id -ne $ExpectedCommandId -or + [string]$command.mode -ne "Recover" -or + [string]$command.source -ne "agent99-recovery-observer" -or + $command.controlledApply -isnot [bool] -or + -not $command.controlledApply -or + [string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or + [string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$' + ) { + throw "queue_identity_contract_invalid" + } + $sourceItem = Get-Item -LiteralPath $sourcePath + $ageMinutes = [math]::Round(((Get-Date) - $sourceItem.LastWriteTime).TotalMinutes, 1) + if ($ageMinutes -lt $MinimumAgeMinutes) { throw "queue_item_not_stale" } + + [pscustomobject]@{ + queueHash = $queueHash + maintenanceHash = $maintenanceHash + ageMinutes = $ageMinutes + commandId = [string]$command.id + commandMode = [string]$command.mode + commandSource = [string]$command.source + controlTaskEnabled = [bool]$task.Settings.Enabled + controlTaskState = [string]$task.State + activeControlProcessCount = [int]$activeProcesses.Count + recoveryLeasePresent = $leasePresent + recoveryMutexActive = $false + } +} + +$gate = Get-AgentQuarantineGate +$base = [ordered]@{ + schemaVersion = "agent99_stale_queue_quarantine_v1" + timestamp = (Get-Date).ToString("o") + mode = $Mode + traceId = $TraceId + runId = $RunId + workItemId = $WorkItemId + commandId = $gate.commandId + commandMode = $gate.commandMode + commandSource = $gate.commandSource + sourceFile = $QueueFileName + sourceSha256 = $gate.queueHash + destinationFile = $destinationName + ageMinutes = $gate.ageMinutes + minimumAgeMinutes = $MinimumAgeMinutes + maintenanceReceipt = $MaintenanceReceiptName + maintenanceReceiptSha256 = $gate.maintenanceHash + activeControlProcessCount = $gate.activeControlProcessCount + recoveryLeasePresent = $gate.recoveryLeasePresent + recoveryMutexActive = $gate.recoveryMutexActive + commandExecuted = $false + deletePerformed = $false + otherQueueMutationPerformed = $false + rawInstructionStored = $false + secretValueRead = $false + runtimeWritePerformed = $false +} + +if ($Mode -eq "Check") { + $base["terminal"] = "check_passed_apply_allowed" + $base | ConvertTo-Json -Depth 8 -Compress + exit 0 +} + +$writerMutex = [Threading.Mutex]::new( + $false, + "Global\WoooAgent99StaleQueueMaintenanceV1" +) +$writerAcquired = $false +$intentPath = "" +$intentHash = "" +$resultPath = "" +$moved = $false +$rollbackAttempted = $false +$rollbackVerified = $false +try { + try { + $writerAcquired = [bool]$writerMutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $writerAcquired = $true + } + if (-not $writerAcquired) { throw "queue_maintenance_writer_active" } + + $gate = Get-AgentQuarantineGate + $safeRunId = $RunId -replace "[^A-Za-z0-9._-]", "_" + $stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" + $intentPath = Join-Path $evidenceRoot ( + "agent99-stale-queue-quarantine-$stamp-$safeRunId-intent.json" + ) + $resultPath = Join-Path $evidenceRoot ( + "agent99-stale-queue-quarantine-$stamp-$safeRunId-result.json" + ) + $intent = [ordered]@{} + $base + $intent["schemaVersion"] = "agent99_stale_queue_quarantine_intent_v1" + $intent["mode"] = "Apply" + $intent["plannedAction"] = "atomic_move_exact_queue_item_to_failed" + $intent["rollback"] = "atomic_move_back_if_post_move_hash_verifier_fails" + $intent["terminal"] = "intent_durable_apply_pending" + $intentHash = Write-AgentDurableJson $intent $intentPath + $runtimeWritePerformed = $true + + $gate = Get-AgentQuarantineGate + [IO.File]::Move($sourcePath, $destinationPath) + $moved = $true + $sourceAbsent = -not (Test-Path -LiteralPath $sourcePath) + $destinationPresent = Test-Path -LiteralPath $destinationPath -PathType Leaf + $destinationHash = if ($destinationPresent) { + Get-AgentSha256 $destinationPath + } else { "" } + if (-not ($sourceAbsent -and $destinationPresent -and $destinationHash -eq $ExpectedQueueSha256)) { + throw "post_move_verifier_failed" + } + + $result = [ordered]@{} + $base + $result["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1" + $result["mode"] = "Apply" + $result["intentFile"] = Split-Path -Leaf $intentPath + $result["intentSha256"] = $intentHash + $result["sourceAbsent"] = $sourceAbsent + $result["destinationPresent"] = $destinationPresent + $result["destinationSha256"] = $destinationHash + $result["runtimeWritePerformed"] = $true + $result["terminal"] = "verified_quarantined" + $resultHash = Write-AgentDurableJson $result $resultPath + $result["resultFile"] = Split-Path -Leaf $resultPath + $result["resultSha256"] = $resultHash + $result | ConvertTo-Json -Depth 8 -Compress + exit 0 +} catch { + $failureType = $_.Exception.GetType().Name + if ( + $moved -and + -not (Test-Path -LiteralPath $sourcePath) -and + (Test-Path -LiteralPath $destinationPath -PathType Leaf) + ) { + $rollbackAttempted = $true + if ((Get-AgentSha256 $destinationPath) -eq $ExpectedQueueSha256) { + [IO.File]::Move($destinationPath, $sourcePath) + $rollbackVerified = [bool]( + (Test-Path -LiteralPath $sourcePath -PathType Leaf) -and + -not (Test-Path -LiteralPath $destinationPath) -and + (Get-AgentSha256 $sourcePath) -eq $ExpectedQueueSha256 + ) + } + } + $failure = [ordered]@{} + $base + $failure["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1" + $failure["mode"] = "Apply" + $failure["intentFile"] = if ($intentPath) { Split-Path -Leaf $intentPath } else { "" } + $failure["intentSha256"] = $intentHash + $failure["failureType"] = $failureType + $failure["rollbackAttempted"] = $rollbackAttempted + $failure["rollbackVerified"] = $rollbackVerified + $failure["runtimeWritePerformed"] = $runtimeWritePerformed + $failure["terminal"] = if ($rollbackAttempted -and $rollbackVerified) { + "failed_rolled_back" + } elseif ($rollbackAttempted) { + "rollback_unverified" + } else { + "failed_no_queue_move" + } + if ($resultPath -and -not (Test-Path -LiteralPath $resultPath)) { + try { + $failureHash = Write-AgentDurableJson $failure $resultPath + $failure["resultFile"] = Split-Path -Leaf $resultPath + $failure["resultSha256"] = $failureHash + } catch {} + } + $failure | ConvertTo-Json -Depth 8 -Compress + exit 2 +} finally { + if ($writerAcquired) { try { $writerMutex.ReleaseMutex() } catch {} } + $writerMutex.Dispose() +} diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index cdf725aef..f63812070 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -5,6 +5,7 @@ import os import shutil import subprocess from copy import deepcopy +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -24,7 +25,7 @@ def _function_source() -> str: def _evidence_function_source() -> str: source = PREFLIGHT.read_text(encoding="utf-8") - start = source.index("function Get-EvidenceReadback") + start = source.index("function Get-AgentBoundedNewestFiles") end = source.index("$requiredTasks = @(", start) return source[start:end] @@ -52,7 +53,8 @@ def _base_case() -> dict[str, Any]: "staleAlertExecutionCount": 0, "alertIncomingCount": 0, "staleAlertIncomingCount": 0, - "pendingCommandCount": 0, + "pendingCommandBlockingCount": 0, + "promotionDeferredCommandCount": 0, } @@ -96,7 +98,8 @@ $parameters = @{{ StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount AlertIncomingCount = [int]$case.alertIncomingCount StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount - PendingCommandCount = [int]$case.pendingCommandCount + PendingCommandBlockingCount = [int]$case.pendingCommandBlockingCount + PromotionDeferredCommandCount = [int]$case.promotionDeferredCommandCount }} Get-AgentLivePreflightDecision @parameters | ConvertTo-Json -Depth 6 -Compress @@ -121,6 +124,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '$warningReasons += "self_health_warning"' in decision assert '$warningReasons += "performance_warning"' in decision assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision + assert ( + '$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"' + in decision + ) assert '"critical" { $blockingReasons += "self_health_not_ok" }' in decision assert 'default { $blockingReasons += "self_health_unclassified" }' in decision assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision @@ -147,9 +154,45 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '"agent99_background_mode_deferred_v1"' in source assert '"agent99_recovery_transport_deferred_v1"' in source assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source + assert "[IO.Directory]::EnumerateFiles(" in source + assert "[Collections.Generic.SortedSet[string]]::new(" in source + assert "$null = $selected.Remove($selected.Min)" in source + assert "candidateWindowExhausted = $candidateWindowExhausted" in source + assert "function Get-AgentControlLoopMaintenanceReadback" in source + assert '"agent99-control-loop-maintenance-2*.json"' in source + assert '$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 })' in source + assert '$legacyPropertySetMatches' in source + assert '$legacyIdentityFieldsValid' in source + assert '$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled' in source + assert '"legacy_v2_exact_upgrade_bridge"' in source + assert '$receipt.precheckPassed -is [bool]' in source + assert '$receipt.taskEnabledAfter -is [bool]' in source + assert '$receipt.runtimeWritePerformed -is [bool]' in source + assert '$intent.secretValueRead -is [bool]' in source + assert '$activeControlProcesses.Count -eq 0' in source + assert '-not $leasePresent' in source + assert '-not $recoveryMutexActive' in source + assert '$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")' in source + assert '$task | Add-Member -NotePropertyName runtimeHealthy' in source + assert '$task | Add-Member -NotePropertyName promotionReady' in source + assert '$controlLoopMaintenance.valid' in source + assert "function Get-AgentPendingCommandPromotionReadback" in source + assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source + assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source + assert '$payload.controlledApply -is [bool] -and $payload.controlledApply' in source + assert '$payload.observation -is [pscustomobject]' in source + assert 'instructionContentReadback = $false' in source assert '$rootTaskPath = "\\"' in source assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source + assert "[string[]]$RequiredArgumentMarkers" in source + assert "$actions.Count -eq 1" in source + assert "actionDefinitionReady = $actionDefinitionReady" in source + assert "principalReady = $principalReady" in source + assert "definitionReady = $definitionReady" in source + assert '$task.Principal.LogonType -eq "S4U"' in source + assert '$task.Principal.RunLevel -eq "Highest"' in source + assert "$task.definitionReady" in source def test_evidence_selection_skips_deferred_but_not_malformed_candidates( @@ -274,6 +317,165 @@ Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 | assert payload["healthy"] is False +def test_evidence_candidate_window_fails_closed_when_recent_files_are_all_deferred( + tmp_path: Path, +) -> None: + powershell = shutil.which("pwsh") or shutil.which("powershell") + if powershell is None: + pytest.skip("PowerShell runtime is not installed on this macOS verifier") + + evidence = tmp_path / "evidence" + evidence.mkdir() + canonical = evidence / "agent99-SelfCheck-20260715-190000.json" + canonical.write_text( + json.dumps( + { + "mode": "SelfCheck", + "selfHealth": {"severity": "ok"}, + } + ), + encoding="utf-8", + ) + base_time = canonical.stat().st_mtime + os.utime(canonical, (base_time - 1000, base_time - 1000)) + for index in range(40): + deferred = evidence / f"agent99-SelfCheck-20260715-20{index:04d}.json" + deferred.write_text( + json.dumps( + { + "schemaVersion": "agent99_background_mode_deferred_v1", + "deferred": True, + } + ), + encoding="utf-8", + ) + candidate_time = base_time + index + os.utime(deferred, (candidate_time, candidate_time)) + + agent_root = str(tmp_path).replace("'", "''") + command = f""" +{_evidence_function_source()} +$AgentRoot = '{agent_root}' +Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true | + ConvertTo-Json -Depth 8 -Compress +""" + result = subprocess.run( + [powershell, "-NoProfile", "-NonInteractive", "-Command", command], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + payload = json.loads( + next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{")) + ) + assert payload["enumeratedCandidateCount"] == 41 + assert payload["candidateWindowLimit"] == 32 + assert payload["candidateWindowTruncated"] is True + assert payload["candidateWindowExhausted"] is True + assert payload["deferredCandidateCount"] == 32 + assert payload["parseError"] == "EvidenceCandidateWindowExhausted" + assert payload["healthy"] is False + + +def test_promotion_reserve_defers_only_exact_maintenance_auto_recover( + tmp_path: Path, +) -> None: + powershell = shutil.which("pwsh") or shutil.which("powershell") + if powershell is None: + pytest.skip("PowerShell runtime is not installed on this macOS verifier") + + queue = tmp_path / "queue" + queue.mkdir() + now = datetime.now(timezone.utc) + incident_id = "agent99-auto-" + ("a" * 20) + command_id = "auto-recover-20260719-152746-39ca4a72" + command_path = queue / f"{command_id}.json" + payload = { + "approvalId": "", + "automationRunId": "096c8173-96b9-5659-94d3-5679c3fb770f", + "canonicalDigest": "b" * 64, + "controlledApply": True, + "correlationKey": "agent99-controlled:" + ("c" * 64), + "createdAt": now.isoformat(), + "dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1", + "dispatchRouteId": "agent99_local_observer", + "executionGeneration": "1", + "externalId": "host_unreachable_detected", + "id": command_id, + "idempotencyKey": "agent99-controlled:" + ("c" * 64), + "incidentId": incident_id, + "instruction": "recover exact observed unreachable host through controlled playbook", + "lockOwner": "096c8173-96b9-5659-94d3-5679c3fb770f", + "mode": "Recover", + "observation": {}, + "projectId": "awoooi", + "reason": "host_unreachable_detected", + "requestedBy": "Agent99", + "singleFlightKey": "agent99-controlled-flight:" + ("d" * 64), + "source": "agent99-recovery-observer", + "sourceFingerprint": "e" * 64, + "traceId": "00-" + ("f" * 32) + "-" + ("1" * 16) + "-01", + "workItemId": f"agent99-auto:{incident_id}:host_unreachable_detected", + } + + function_source = _evidence_function_source() + agent_root = str(tmp_path).replace("'", "''") + maintenance_timestamp = (now - timedelta(minutes=10)).isoformat() + + def run(scope: str) -> dict[str, Any]: + command = f""" +{function_source} +$AgentRoot = '{agent_root}' +$maintenance = [pscustomobject]@{{ + valid = $true + receiptFieldsValid = $true + intentFieldsValid = $true + receiptTimestamp = '{maintenance_timestamp}' +}} +Get-AgentPendingCommandPromotionReadback -MaintenanceReadback $maintenance -Scope {scope} | + ConvertTo-Json -Depth 8 -Compress +""" + result = subprocess.run( + [powershell, "-NoProfile", "-NonInteractive", "-Command", command], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + return json.loads( + next( + line + for line in reversed(result.stdout.splitlines()) + if line.startswith("{") + ) + ) + + command_path.write_bytes(b"\xef\xbb\xbf" + json.dumps(payload).encode("utf-8")) + reserve = run("PromotionReserve") + assert reserve["observedCount"] == 1 + assert reserve["deferredCount"] == 1 + assert reserve["blockingCount"] == 0 + assert reserve["contractValid"] is True + assert reserve["items"][0]["instructionContentReadback"] is False + assert reserve["items"][0]["identity"].startswith(f"{command_id}|") + + full_runtime = run("FullRuntime") + assert full_runtime["deferredCount"] == 0 + assert full_runtime["blockingCount"] == 1 + assert full_runtime["eligibleMaintenanceAutoRecoverCount"] == 1 + assert full_runtime["reason"] == ( + "full_runtime_requires_maintenance_auto_recover_to_drain" + ) + + payload["mode"] = "Status" + command_path.write_text(json.dumps(payload), encoding="utf-8") + invalid = run("PromotionReserve") + assert invalid["deferredCount"] == 0 + assert invalid["blockingCount"] == 1 + assert invalid["contractValid"] is False + + def test_decision_behavior_when_powershell_is_available() -> None: powershell = shutil.which("pwsh") or shutil.which("powershell") if powershell is None: @@ -348,11 +550,17 @@ def test_decision_behavior_when_powershell_is_available() -> None: (), ), ( - _with_overrides(pendingCommandCount=1), + _with_overrides(pendingCommandBlockingCount=1), False, ("pending_work_before_reboot",), (), ), + ( + _with_overrides(promotionDeferredCommandCount=1), + True, + (), + ("maintenance_pending_commands_deferred_until_post_promotion",), + ), ) for case, expected_eligible, expected_blockers, expected_warnings in cases: diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index c00631130..aee8f27e2 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -391,11 +391,39 @@ def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() -> assert "failedTasks" in source assert "failedEvidenceNames" in source assert "failedEvidence" in source + assert "actionDefinitionReady = [bool]$_.actionDefinitionReady" in source + assert "principalReady = [bool]$_.principalReady" in source + assert "definitionReady = [bool]$_.definitionReady" in source assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source +def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + assert "$LivePreflightTimeoutSeconds = 60" in source + assert "$PostVerifyMaxAttempts = 4" in source + assert "$PostVerifyRetryWaitSeconds = 45" in source + assert "function Test-AgentPostPromotionReadinessRetryable" in source + assert '[string]$Readiness.readinessScope -ne "FullRuntime"' in source + assert '"required_evidence_stale"' in source + assert '"required_evidence_content_unhealthy"' in source + assert '$_.name -notin @("telegram_inbox", "sre_alert_inbox")' in source + assert "-not $_.definitionReady" in source + assert "[int]$Readiness.alertRunningCount -ne 0" in source + assert "$pendingCommandsRetryable" in source + assert "$ExpectedMaintenancePendingCommandIdentities" in source + assert '"pending_work_before_reboot"' in source + assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source + assert "maintenancePendingCommandIdentities" in source + assert "$postVerifyAttempt -le $PostVerifyMaxAttempts" in source + assert "Start-Sleep -Seconds $PostVerifyRetryWaitSeconds" in source + assert "automaticTaskTriggerAllowed = $false" in source + assert "postVerifyAttempts = $postVerifyAttempts" in source + assert "postVerifyRetryCount = $postVerifyRetryCount" in source + + def test_check_and_apply_receipts_project_the_same_run_identity() -> None: sender = SENDER.read_text(encoding="utf-8") projector = sender[ @@ -607,7 +635,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert "$RelayPollIntervalSeconds = 1" in source assert "$ValidateOnlyTimeoutSeconds = 300" in source assert "$LiveDeployTimeoutSeconds = 300" in source - assert "$LivePreflightTimeoutSeconds = 120" in source + assert "$LivePreflightTimeoutSeconds = 60" in source assert "$PromotionReadinessMaxAttempts = 2" in source assert "$PromotionReadinessRetryWaitSeconds = 30" in source assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source @@ -648,18 +676,23 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati promotion_readiness_attempts = 2 promotion_readiness_retry_wait_seconds = 30 deploy_seconds = 300 - live_preflight_seconds = 120 + live_preflight_seconds = 60 + post_verify_attempts = 4 + post_verify_retry_wait_seconds = 45 + windows_control_baseline_seconds = 300 worst_case_seconds = ( validate_seconds + (promotion_readiness_attempts * live_preflight_seconds) + promotion_readiness_retry_wait_seconds + deploy_seconds - + live_preflight_seconds + + (post_verify_attempts * live_preflight_seconds) + + ((post_verify_attempts - 1) * post_verify_retry_wait_seconds) + (2 * (60 + 360)) + + windows_control_baseline_seconds ) assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source - assert worst_case_seconds == 1830 - assert wrapper_budget_seconds - worst_case_seconds == 570 + assert worst_case_seconds == 2265 + assert wrapper_budget_seconds - worst_case_seconds == 135 deploy = source.index( "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath " @@ -781,6 +814,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() - assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight assert '[int]$summary.alertIncomingCount -eq 0' not in preflight assert '[int]$summary.alertRunningCount -eq 0' in preflight + assert '[int]$summary.pendingCommandBlockingCount -eq 0' in preflight + assert '$ReadinessScope -eq "PromotionReserve"' in preflight assert '[int]$summary.pendingCommandCount -eq 0' in preflight assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight assert '[string[]]$blockingReasons = @()' in preflight diff --git a/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py b/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py new file mode 100644 index 000000000..49eb93bb7 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py @@ -0,0 +1,44 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1" + + +def test_stale_queue_quarantine_is_exact_recoverable_and_receipted() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert '[ValidateSet("Check", "Apply")]' in source + assert "ExpectedQueueSha256" in source + assert "ExpectedMaintenanceSha256" in source + assert '"Global\\WoooAgent99StaleQueueMaintenanceV1"' in source + assert '"Global\\WoooAgent99RecoveryTransportV1"' in source + assert 'Get-ScheduledTask `' in source + assert '"Wooo-Agent99-Control-Loop"' in source + assert 'throw "control_loop_not_frozen"' in source + assert 'throw "active_control_or_recover_process"' in source + assert 'throw "recovery_lease_present"' in source + assert 'throw "recovery_mutex_active"' in source + assert '[string]$command.mode -ne "Recover"' in source + assert '[string]$command.source -ne "agent99-recovery-observer"' in source + assert "$command.controlledApply -isnot [bool]" in source + assert '"atomic_move_exact_queue_item_to_failed"' in source + assert "[IO.File]::Move($sourcePath, $destinationPath)" in source + assert "[IO.File]::Move($destinationPath, $sourcePath)" in source + assert '"verified_quarantined"' in source + assert '"failed_rolled_back"' in source + assert '"rollback_unverified"' in source + + +def test_stale_queue_quarantine_does_not_execute_or_delete_command() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert "commandExecuted = $false" in source + assert "deletePerformed = $false" in source + assert "otherQueueMutationPerformed = $false" in source + assert "rawInstructionStored = $false" in source + assert "secretValueRead = $false" in source + assert "Remove-Item -LiteralPath $sourcePath" not in source + assert "Start-ScheduledTask" not in source + assert "Invoke-Expression" not in source + assert "& $command" not in source