from pathlib import Path ROOT = Path(__file__).resolve().parents[3] CONTROL = ROOT / "agent99-control-plane.ps1" BREAKGLASS = ROOT / "scripts" / "reboot-recovery" / "agent99-stale-recovery-breakglass.ps1" MAINTENANCE = ROOT / "scripts" / "reboot-recovery" / "agent99-control-loop-maintenance.ps1" MAINTENANCE_RESTORE = ( ROOT / "scripts" / "reboot-recovery" / "agent99-control-loop-maintenance-restore.ps1" ) REGISTER_TASKS = ROOT / "agent99-register-tasks.ps1" SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1" TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1" SUBMIT_REQUEST = ROOT / "agent99-submit-request.ps1" BOUNDED_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-bounded-process-replay.ps1" TELEGRAM_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-telegram-durable-ingress-replay.ps1" BREAKGLASS_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-breakglass-failure-readback-replay.ps1" def test_vmrun_start_is_process_aware_and_bounded() -> None: source = CONTROL.read_text(encoding="utf-8") assert "function Get-AgentVmwareVmxProcessReadback" in source assert "vmx_process_already_running" in source assert "vmx_process_readback_failed" in source assert "function Invoke-AgentBoundedVmrunStart" in source assert "Get-AgentVmwareVmxProcessReadback ([string]$vm.vmx)" in source assert "Invoke-AgentBoundedVmrunStart" in source assert '& $vmrun start $vm.vmx nogui' not in source def test_bounded_runner_kills_trees_but_vmrun_preserves_started_vm_descendants() -> None: source = CONTROL.read_text(encoding="utf-8") function = source.split("function Invoke-AgentBoundedProcess", 1)[1].split( "function Test-PublicRoutes", 1 )[0] assert "$stopwatch.Elapsed.TotalSeconds -ge $TimeoutSeconds" in function assert "StandardOutput.ReadAsync" in function assert "StandardError.ReadAsync" in function assert "($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes" in function assert function.index("($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes") < function.index("$process.WaitForExit()") assert '[ValidateSet("Tree", "Root")]' in function assert "processScopeStopped" in function assert "preservedDescendantCount" in function assert "Get-AgentProcessTreeIdentitySnapshot $rootProcessIdentity $rootExitedUtc" in function assert "ConvertTo-AgentProcessIdentitySnapshot $rootProcessInfo" in function assert "rootIdentityCaptured" in function assert "Stop-AgentBoundedProcessSnapshot $process $treeIdentities $TerminationScope" in function assert "processTreeStopped" in function assert "Read-AgentBoundedTextFile $stdout" in function assert "Remove-Item -LiteralPath $stdout, $stderr" in function stopper = source.split("function Stop-AgentBoundedProcessSnapshot", 1)[1].split( "function Test-AgentProcessIdentityActive", 1 )[0] assert 'taskkill.exe" /PID ([string]$rootId) /T /F' in stopper assert 'taskkill.exe" /PID ([string]$rootId) /F' in stopper assert 'taskkill.exe" /PID ([string]([int]$identity.processId)) /T /F' in stopper snapshot = source.split("function Select-AgentProcessTreeIdentitySnapshot", 1)[1].split( "function Get-AgentProcessTreeIdentitySnapshot", 1 )[0] assert "Test-AgentProcessSnapshotMatch $RootIdentity $currentRoot" in snapshot assert 'throw "bounded_process_root_identity_changed_before_exit_readback"' in snapshot assert "-le $RootExitedUtc.ToUniversalTime()" in snapshot assert "Get-AgentCreationBoundedDescendants $Processes @($currentRoot)" in snapshot descendants = source.split("function Get-AgentCreationBoundedDescendants", 1)[1].split( "function Test-AgentProcessSnapshotMatch", 1 )[0] assert "-ge $parentCreated" in descendants assert "System.Collections.Generic.HashSet[int]" in descendants assert "function Get-AgentProcessIdentityReadback" in source assert 'reason = "identity_readback_failed"' in source assert 'throw "process_tree_identity_readback_failed"' in source assert "identityReadbackFailed" in stopper vmrun = source.split("function Invoke-AgentBoundedVmrunStart", 1)[1].split( "function Start-ConfiguredVMs", 1 )[0] assert "Invoke-AgentBoundedProcess" in vmrun assert "-TerminationScope Root" in vmrun assert "processScopeStopped" in vmrun def test_perf_guard_can_release_an_exact_stale_recovery_before_deferral() -> None: source = CONTROL.read_text(encoding="utf-8") assert "function Invoke-AgentStaleRecoveryTransportGuard" in source assert 'if ($Mode -in @("Perf", "LoadShed") -and $ControlledApply)' in source invocation = source.index("$staleRecoveryTransportGuard = Invoke-AgentStaleRecoveryTransportGuard") lease_probe = source.index("$recoveryLeaseObservation = Test-AgentRecoveryTransportLease", invocation) assert invocation < lease_probe assert "agent99_stale_recovery_guard_intent_v1" in source assert "agent99_stale_recovery_guard_receipt_v1" in source assert "$stream.Flush($true)" in source assert "stale_recovery_receipt_readback_mismatch" in source assert "stale_recovery_descendant_identity_unapproved" in source assert "Get-AgentProcessDescendants $recheckAll ([int]$lease.processId)" in source assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source assert "Wait-AgentCimProcessIdentityAbsent $parent 15" in source assert "Test-AgentCimProcessIdentityMatch $child $childBeforeKill" in source assert "$leaseAcquiredUtc.Ticks" in source assert "vmPowerChangePerformed = $false" in source assert "vmwareVmxProcessTerminated = $false" in source def test_breakglass_targets_only_exact_stale_vmrun_processes() -> None: source = BREAKGLASS.read_text(encoding="utf-8") assert "lease_run_id_mismatch" in source assert "lease_process_id_mismatch" in source assert "parent_run_id_mismatch" in source assert "child_identity_mismatch" in source assert "child_vmx_mismatch" in source assert "recovery_mutex_not_active" in source assert 'taskkill.exe" /PID $ProcessId /T /F' in source assert "Invoke-ExactTaskkill $ExpectedChildPid" in source assert "prior_receipt_identity_mismatch" in source assert '"agent99_stale_recovery_breakglass_v2", "agent99_stale_recovery_breakglass_v3"' in source assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source assert "Wait-ExactProcessIdentityAbsent" in source assert "parent_cleanup_identity_changed" in source assert "Get-DescendantProcesses $ExpectedParentPid" in source assert "descendantCount = 0" in source assert "agent99_stale_recovery_parent_cleanup_intent_v1" in source assert "Write-AtomicJson $intent $intentPath" in source assert "Invoke-ExactTaskkill $ExpectedParentPid" in source assert "vmwareVmxProcessTerminated = $false" in source assert "vmxLockDeleted = $false" in source assert "hostRebootPerformed = $false" in source assert "Remove-Item" in source assert ".lck" not in source def test_breakglass_apply_is_receipted_and_check_is_no_write() -> None: source = BREAKGLASS.read_text(encoding="utf-8") check_pos = source.index('if ($Mode -eq "Apply" -and $precheckPassed)') operation_intent_pos = source.index("Write-AtomicJson $operationIntent $operationIntentPath") child_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedChildPid") intent_write_pos = source.index("Write-AtomicJson $intent $intentPath") parent_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedParentPid") receipt_pos = source.index('if ($Mode -eq "Apply") {', parent_kill_pos) assert check_pos < operation_intent_pos < child_kill_pos < intent_write_pos < parent_kill_pos < receipt_pos assert "agent99_stale_recovery_breakglass_intent_v1" in source assert '"partial_write_failed_closed"' in source assert "operationFailureType" in source assert "Write-AtomicJson $result $script:ReceiptPath" in source assert "function Get-FailedMutationReadback" in source assert '"parent_identity_unknown"' in source assert '"child_identity_unknown"' in source assert '"recovery_mutex_unknown"' in source assert '"lease_metadata_unknown"' in source assert "postFailureReadbackComplete" in source assert "$stream.Flush($true)" in source assert "receipt_readback_mismatch" in source assert 'runtimeWritePerformed = $script:RuntimeWritePerformed' in source assert 'if ($Mode -eq "Check") { exit 0 }' in source def test_control_loop_maintenance_freeze_is_exact_and_receipted() -> None: source = MAINTENANCE.read_text(encoding="utf-8") assert 'if ($env:COMPUTERNAME -ne "WOOO-SUPER")' in source assert 'agent99-run\\.ps1"?\\s+-Mode\\s+ControlTick' in source assert 'agent99-run\\.ps1"?\\s+-Mode\\s+Recover\\s+-ControlledApply' in source assert "Global\\WoooAgent99ControlLoopMaintenanceV1" in source assert "blocked_single_writer_active" in source assert "Test-ExactCreationTime $Process $ExpectedControlCreated" in source assert "Test-ExactCreationTime $Process $ExpectedRecoverCreated" in source assert "[int]$Process.ParentProcessId -eq $ExpectedRecoverParentPid" in source assert 'Disable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source assert 'Stop-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source assert "control_recheck_identity_mismatch" in source assert 'taskkill.exe" /PID $ExpectedRecoverPid /T /F' in source assert "$stream.Flush($true)" in source assert "control_loop_maintenance_frozen" in source assert "agent99_control_loop_maintenance_intent_v1" in source assert source.index("Write-DurableReceipt $operationIntent $operationIntentPath") < source.index( 'Disable-ScheduledTask -TaskName $taskName' ) assert "maintenance_partial_frozen_manual_recovery" in source assert "taskEnabledBefore" in source assert "taskEnabledAfter" in source assert "operationFailureType" in source assert "vmwareVmxProcessTerminated = $false" in source assert "hostRebootPerformed = $false" in source def test_control_loop_maintenance_restore_is_bounded_and_receipted() -> None: source = MAINTENANCE_RESTORE.read_text(encoding="utf-8") assert 'if ($env:COMPUTERNAME -ne "WOOO-SUPER")' in source assert "Global\\WoooAgent99ControlLoopMaintenanceV1" in source assert 'ValidateSet("Check", "Apply")' in source assert "ValidateRange(1, 180)" in source assert "ExpectedMaintenanceReceiptSha256" in source assert "extended_receipt_window_requires_exact_sha256" in source assert "maintenance_receipt_sha256_mismatch" in source assert "[IO.File]::ReadAllBytes($maintenancePath)" in source assert "ComputeHash($maintenanceBytes)" in source assert "maintenance_receipt_shape_invalid" in source assert "maintenance_intent_shape_invalid" in source assert "Get-Content -LiteralPath $maintenancePath" not in source extended_guard = source.index("extended_receipt_window_requires_exact_sha256") maintenance_contract = source.index('schemaVersion -ne "agent99_control_loop_maintenance_v2"') assert extended_guard < maintenance_contract assert "maintenanceReceiptSha256" in source assert 'schemaVersion -ne "agent99_control_loop_maintenance_v2"' in source assert 'terminal -ne "control_loop_maintenance_frozen"' in source assert 'schemaVersion -ne "agent99_control_loop_maintenance_intent_v1"' in source assert "maintenance_intent_contract_mismatch" in source assert 'if ([bool]$task.Settings.Enabled)' in source assert "StringComparison]::OrdinalIgnoreCase" in source assert "StringComparison]::Ordinal" in source assert 'Join-Path $AgentRoot "agent99-run.ps1"' in source assert 'Enable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source assert 'Start-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source assert 'Stop-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source assert 'Disable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source assert source.index("Write-DurableReceipt $operationIntent $operationIntentPath") < source.index( 'Enable-ScheduledTask -TaskName $taskName' ) assert "control_loop_maintenance_restored" in source assert "maintenance_restore_failed_refrozen" in source assert "maintenance_restore_failed_manual_recovery" in source assert "maintenance_restore_receipt_failed_refrozen" in source assert "maintenance_restore_receipt_failed_manual_recovery" in source assert "Get-RefreezeReadback" in source assert "rollbackReadbackErrors" in source assert "recovery_lease_readback_failed" in source assert "$leaseAbsent" in source assert 'recovery-transport-lease.json") -PathType Leaf' not in source assert "$stream.Flush($true)" in source assert "vmPowerChangePerformed = $false" in source assert "hostRebootPerformed = $false" in source assert "secretValueRead = $false" in source def test_control_loop_deadline_and_queue_batch_are_compatible() -> None: control = CONTROL.read_text(encoding="utf-8") tasks = REGISTER_TASKS.read_text(encoding="utf-8") assert "$script:Agent99QueueOuterDeadlineSeconds = 1680" in control assert "-ExecutionTimeLimit (New-TimeSpan -Minutes 35)" in tasks control_registration = tasks.split('$controlArgs = ', 1)[1].split('$telegramInboxScript = ', 1)[0] assert "-Settings $controlSettings" in control_registration queue_selection = control.split("function Invoke-AgentQueuedCommands", 1)[1].split("$results = @()", 1)[0] assert "Select-Object -First 1" in queue_selection def test_ingress_signals_tasks_without_waiting_for_control_execution() -> None: sre = SRE_INBOX.read_text(encoding="utf-8") telegram = TELEGRAM_INBOX.read_text(encoding="utf-8") submit = SUBMIT_REQUEST.read_text(encoding="utf-8") assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in sre assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in submit assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-SRE-Alert-Inbox"' in telegram assert '"-InstructionBase64", (Convert-AgentBase64 $instruction)' in telegram assert '"-RequestedByBase64", (Convert-AgentBase64 ([string]$update.from))' in telegram assert '"-AgentRoot", $AgentRoot' in telegram assert "$null = $submit.Handle" in telegram assert "instruction_transport_ambiguous" in submit assert "scheduled_task_signaled" in sre assert "scheduled_task_signaled" in submit assert "function Write-AgentDurableJson" in submit assert '"request-" + $ingressDigest.Substring(0, 32)' in submit assert "Global\\Wooo-Agent99-Submit-" in submit assert "function Get-AgentIngressLifecycleReceipt" in submit assert 'throw "ingress_lifecycle_ambiguous"' in submit assert "idempotentReplay = [bool]$queueReceipt.existing" in submit assert "durableQueue = [bool]$queueReceipt.ok" in submit assert "submitEvidenceSha256" in submit assert "function Get-AgentDurableSubmitReadback" in telegram assert "Sort-Object update_id" in telegram assert "lastDurablyHandledUpdateId" in telegram assert "update_processing_failed_no_checkpoint_past_failure" in telegram assert "offset_checkpoint_failed_safe_replay_required" in telegram assert "Write-AgentDurableJson $result $evidencePath" in telegram assert "Write-AgentDurableJson $offsetState $offsetPath" in telegram assert "$maxUpdateId + 1" not in telegram def test_windows_native_bounded_process_replay_covers_timeout_tree_and_stream_cap() -> None: source = BOUNDED_REPLAY.read_text(encoding="utf-8") assert "timeout_descendant_not_observed" in source assert "streaming_output_limit_not_detected" in source assert "root_only_descendant_not_preserved" in source assert "-TerminationScope Root" in source assert "cim_failure_root_not_stopped" in source assert "cim_failure_tree_false_green" in source assert "orphan_race_descendant_snapshot_missing" in source assert "orphan_race_tree_false_green" in source assert "pid_reuse_selection_not_identity_bounded" in source assert "reusedRootExcluded" in source assert "reusedRootChildExcluded" in source assert "live_root_child_creation_bound_failed" in source assert "intermediateOlderProcessExcluded" in source assert "stop_readback_failure_not_detected" in source assert "stop_readback_failure_not_fail_closed" in source assert "stop_readback_failure_tree_false_green" in source assert "captured_output_exceeded_contract" in source assert "cleanupVerified" in source assert "productionPathWritePerformed = $false" in source def test_windows_native_telegram_replay_covers_durable_success_and_enqueue_failure() -> None: source = TELEGRAM_REPLAY.read_text(encoding="utf-8") assert "success_queue_not_durable" in source assert "failure_case_false_green" in source assert "failure_offset_advanced" in source assert "synthetic_after_queue_commit" in source assert "queue_commit_retry_duplicated_artifact" in source assert "queue_commit_retry_not_idempotent" in source assert '"A" * 9000' in source assert "telegramApiCalled = $false" in source assert "productionPathWritePerformed = $false" in source def test_windows_native_breakglass_failure_readback_replay_is_fail_closed() -> None: source = BREAKGLASS_REPLAY.read_text(encoding="utf-8") assert "synthetic_cim_failure" in source assert "synthetic_mutex_readback_failure" in source assert "synthetic_filesystem_readback_failure" in source assert "failure_readback_false_green" in source assert "terminalReceiptConstructionCanContinue = $true" in source assert "runtimeWritePerformed = $false" in source