3a81d9bedc
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 2m2s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 6m29s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 29s
CD Pipeline / build-and-deploy (push) Failing after 42s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
277 lines
15 KiB
Python
277 lines
15 KiB
Python
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
CONTROL = ROOT / "agent99-control-plane.ps1"
|
|
BREAKGLASS = ROOT / "scripts" / "reboot-recovery" / "agent99-stale-recovery-breakglass.ps1"
|
|
MAINTENANCE = ROOT / "scripts" / "reboot-recovery" / "agent99-control-loop-maintenance.ps1"
|
|
REGISTER_TASKS = ROOT / "agent99-register-tasks.ps1"
|
|
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
|
|
TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1"
|
|
SUBMIT_REQUEST = ROOT / "agent99-submit-request.ps1"
|
|
BOUNDED_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-bounded-process-replay.ps1"
|
|
TELEGRAM_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-telegram-durable-ingress-replay.ps1"
|
|
BREAKGLASS_REPLAY = ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-breakglass-failure-readback-replay.ps1"
|
|
|
|
|
|
def test_vmrun_start_is_process_aware_and_bounded() -> None:
|
|
source = CONTROL.read_text(encoding="utf-8")
|
|
|
|
assert "function Get-AgentVmwareVmxProcessReadback" in source
|
|
assert "vmx_process_already_running" in source
|
|
assert "vmx_process_readback_failed" in source
|
|
assert "function Invoke-AgentBoundedVmrunStart" in source
|
|
assert "Get-AgentVmwareVmxProcessReadback ([string]$vm.vmx)" in source
|
|
assert "Invoke-AgentBoundedVmrunStart" in source
|
|
assert '& $vmrun start $vm.vmx nogui' not in source
|
|
|
|
|
|
def test_bounded_runner_kills_trees_but_vmrun_preserves_started_vm_descendants() -> None:
|
|
source = CONTROL.read_text(encoding="utf-8")
|
|
function = source.split("function Invoke-AgentBoundedProcess", 1)[1].split(
|
|
"function Test-PublicRoutes", 1
|
|
)[0]
|
|
|
|
assert "$stopwatch.Elapsed.TotalSeconds -ge $TimeoutSeconds" in function
|
|
assert "StandardOutput.ReadAsync" in function
|
|
assert "StandardError.ReadAsync" in function
|
|
assert "($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes" in function
|
|
assert function.index("($capturedBytes + $chunkBytes) -gt $MaximumOutputBytes") < function.index("$process.WaitForExit()")
|
|
assert '[ValidateSet("Tree", "Root")]' in function
|
|
assert "processScopeStopped" in function
|
|
assert "preservedDescendantCount" in function
|
|
assert "Get-AgentProcessTreeIdentitySnapshot $rootProcessIdentity $rootExitedUtc" in function
|
|
assert "ConvertTo-AgentProcessIdentitySnapshot $rootProcessInfo" in function
|
|
assert "rootIdentityCaptured" in function
|
|
assert "Stop-AgentBoundedProcessSnapshot $process $treeIdentities $TerminationScope" in function
|
|
assert "processTreeStopped" in function
|
|
assert "Read-AgentBoundedTextFile $stdout" in function
|
|
assert "Remove-Item -LiteralPath $stdout, $stderr" in function
|
|
stopper = source.split("function Stop-AgentBoundedProcessSnapshot", 1)[1].split(
|
|
"function Test-AgentProcessIdentityActive", 1
|
|
)[0]
|
|
assert 'taskkill.exe" /PID ([string]$rootId) /T /F' in stopper
|
|
assert 'taskkill.exe" /PID ([string]$rootId) /F' in stopper
|
|
assert 'taskkill.exe" /PID ([string]([int]$identity.processId)) /T /F' in stopper
|
|
snapshot = source.split("function Select-AgentProcessTreeIdentitySnapshot", 1)[1].split(
|
|
"function Get-AgentProcessTreeIdentitySnapshot", 1
|
|
)[0]
|
|
assert "Test-AgentProcessSnapshotMatch $RootIdentity $currentRoot" in snapshot
|
|
assert 'throw "bounded_process_root_identity_changed_before_exit_readback"' in snapshot
|
|
assert "-le $RootExitedUtc.ToUniversalTime()" in snapshot
|
|
assert "Get-AgentCreationBoundedDescendants $Processes @($currentRoot)" in snapshot
|
|
descendants = source.split("function Get-AgentCreationBoundedDescendants", 1)[1].split(
|
|
"function Test-AgentProcessSnapshotMatch", 1
|
|
)[0]
|
|
assert "-ge $parentCreated" in descendants
|
|
assert "System.Collections.Generic.HashSet[int]" in descendants
|
|
assert "function Get-AgentProcessIdentityReadback" in source
|
|
assert 'reason = "identity_readback_failed"' in source
|
|
assert 'throw "process_tree_identity_readback_failed"' in source
|
|
assert "identityReadbackFailed" in stopper
|
|
vmrun = source.split("function Invoke-AgentBoundedVmrunStart", 1)[1].split(
|
|
"function Start-ConfiguredVMs", 1
|
|
)[0]
|
|
assert "Invoke-AgentBoundedProcess" in vmrun
|
|
assert "-TerminationScope Root" in vmrun
|
|
assert "processScopeStopped" in vmrun
|
|
|
|
|
|
def test_perf_guard_can_release_an_exact_stale_recovery_before_deferral() -> None:
|
|
source = CONTROL.read_text(encoding="utf-8")
|
|
|
|
assert "function Invoke-AgentStaleRecoveryTransportGuard" in source
|
|
assert 'if ($Mode -in @("Perf", "LoadShed") -and $ControlledApply)' in source
|
|
invocation = source.index("$staleRecoveryTransportGuard = Invoke-AgentStaleRecoveryTransportGuard")
|
|
lease_probe = source.index("$recoveryLeaseObservation = Test-AgentRecoveryTransportLease", invocation)
|
|
assert invocation < lease_probe
|
|
assert "agent99_stale_recovery_guard_intent_v1" in source
|
|
assert "agent99_stale_recovery_guard_receipt_v1" in source
|
|
assert "$stream.Flush($true)" in source
|
|
assert "stale_recovery_receipt_readback_mismatch" in source
|
|
assert "stale_recovery_descendant_identity_unapproved" in source
|
|
assert "Get-AgentProcessDescendants $recheckAll ([int]$lease.processId)" in source
|
|
assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source
|
|
assert "Wait-AgentCimProcessIdentityAbsent $parent 15" in source
|
|
assert "Test-AgentCimProcessIdentityMatch $child $childBeforeKill" in source
|
|
assert "$leaseAcquiredUtc.Ticks" in source
|
|
assert "vmPowerChangePerformed = $false" in source
|
|
assert "vmwareVmxProcessTerminated = $false" in source
|
|
|
|
|
|
def test_breakglass_targets_only_exact_stale_vmrun_processes() -> None:
|
|
source = BREAKGLASS.read_text(encoding="utf-8")
|
|
|
|
assert "lease_run_id_mismatch" in source
|
|
assert "lease_process_id_mismatch" in source
|
|
assert "parent_run_id_mismatch" in source
|
|
assert "child_identity_mismatch" in source
|
|
assert "child_vmx_mismatch" in source
|
|
assert "recovery_mutex_not_active" in source
|
|
assert 'taskkill.exe" /PID $ProcessId /T /F' in source
|
|
assert "Invoke-ExactTaskkill $ExpectedChildPid" in source
|
|
assert "prior_receipt_identity_mismatch" in source
|
|
assert '"agent99_stale_recovery_breakglass_v2", "agent99_stale_recovery_breakglass_v3"' in source
|
|
assert "Global\\WoooAgent99StaleRecoveryReconcilerV1" in source
|
|
assert "Wait-ExactProcessIdentityAbsent" in source
|
|
assert "parent_cleanup_identity_changed" in source
|
|
assert "Get-DescendantProcesses $ExpectedParentPid" in source
|
|
assert "descendantCount = 0" in source
|
|
assert "agent99_stale_recovery_parent_cleanup_intent_v1" in source
|
|
assert "Write-AtomicJson $intent $intentPath" in source
|
|
assert "Invoke-ExactTaskkill $ExpectedParentPid" in source
|
|
assert "vmwareVmxProcessTerminated = $false" in source
|
|
assert "vmxLockDeleted = $false" in source
|
|
assert "hostRebootPerformed = $false" in source
|
|
assert "Remove-Item" in source
|
|
assert ".lck" not in source
|
|
|
|
|
|
def test_breakglass_apply_is_receipted_and_check_is_no_write() -> None:
|
|
source = BREAKGLASS.read_text(encoding="utf-8")
|
|
|
|
check_pos = source.index('if ($Mode -eq "Apply" -and $precheckPassed)')
|
|
operation_intent_pos = source.index("Write-AtomicJson $operationIntent $operationIntentPath")
|
|
child_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedChildPid")
|
|
intent_write_pos = source.index("Write-AtomicJson $intent $intentPath")
|
|
parent_kill_pos = source.index("Invoke-ExactTaskkill $ExpectedParentPid")
|
|
receipt_pos = source.index('if ($Mode -eq "Apply") {', parent_kill_pos)
|
|
assert check_pos < operation_intent_pos < child_kill_pos < intent_write_pos < parent_kill_pos < receipt_pos
|
|
assert "agent99_stale_recovery_breakglass_intent_v1" in source
|
|
assert '"partial_write_failed_closed"' in source
|
|
assert "operationFailureType" in source
|
|
assert "Write-AtomicJson $result $script:ReceiptPath" in source
|
|
assert "function Get-FailedMutationReadback" in source
|
|
assert '"parent_identity_unknown"' in source
|
|
assert '"child_identity_unknown"' in source
|
|
assert '"recovery_mutex_unknown"' in source
|
|
assert '"lease_metadata_unknown"' in source
|
|
assert "postFailureReadbackComplete" in source
|
|
assert "$stream.Flush($true)" in source
|
|
assert "receipt_readback_mismatch" in source
|
|
assert 'runtimeWritePerformed = $script:RuntimeWritePerformed' in source
|
|
assert 'if ($Mode -eq "Check") { exit 0 }' in source
|
|
|
|
|
|
def test_control_loop_maintenance_freeze_is_exact_and_receipted() -> None:
|
|
source = MAINTENANCE.read_text(encoding="utf-8")
|
|
|
|
assert 'if ($env:COMPUTERNAME -ne "WOOO-SUPER")' in source
|
|
assert 'agent99-run\\.ps1"?\\s+-Mode\\s+ControlTick' in source
|
|
assert 'agent99-run\\.ps1"?\\s+-Mode\\s+Recover\\s+-ControlledApply' in source
|
|
assert "Global\\WoooAgent99ControlLoopMaintenanceV1" in source
|
|
assert "blocked_single_writer_active" in source
|
|
assert "Test-ExactCreationTime $Process $ExpectedControlCreated" in source
|
|
assert "Test-ExactCreationTime $Process $ExpectedRecoverCreated" in source
|
|
assert "[int]$Process.ParentProcessId -eq $ExpectedRecoverParentPid" in source
|
|
assert 'Disable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source
|
|
assert 'Stop-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source
|
|
assert "control_recheck_identity_mismatch" in source
|
|
assert 'taskkill.exe" /PID $ExpectedRecoverPid /T /F' in source
|
|
assert "$stream.Flush($true)" in source
|
|
assert "control_loop_maintenance_frozen" in source
|
|
assert "agent99_control_loop_maintenance_intent_v1" in source
|
|
assert source.index("Write-DurableReceipt $operationIntent $operationIntentPath") < source.index(
|
|
'Disable-ScheduledTask -TaskName $taskName'
|
|
)
|
|
assert "maintenance_partial_frozen_manual_recovery" in source
|
|
assert "taskEnabledBefore" in source
|
|
assert "taskEnabledAfter" in source
|
|
assert "operationFailureType" in source
|
|
assert "vmwareVmxProcessTerminated = $false" in source
|
|
assert "hostRebootPerformed = $false" in source
|
|
|
|
|
|
def test_control_loop_deadline_and_queue_batch_are_compatible() -> None:
|
|
control = CONTROL.read_text(encoding="utf-8")
|
|
tasks = REGISTER_TASKS.read_text(encoding="utf-8")
|
|
|
|
assert "$script:Agent99QueueOuterDeadlineSeconds = 1680" in control
|
|
assert "-ExecutionTimeLimit (New-TimeSpan -Minutes 35)" in tasks
|
|
control_registration = tasks.split('$controlArgs = ', 1)[1].split('$telegramInboxScript = ', 1)[0]
|
|
assert "-Settings $controlSettings" in control_registration
|
|
queue_selection = control.split("function Invoke-AgentQueuedCommands", 1)[1].split("$results = @()", 1)[0]
|
|
assert "Select-Object -First 1" in queue_selection
|
|
|
|
|
|
def test_ingress_signals_tasks_without_waiting_for_control_execution() -> None:
|
|
sre = SRE_INBOX.read_text(encoding="utf-8")
|
|
telegram = TELEGRAM_INBOX.read_text(encoding="utf-8")
|
|
submit = SUBMIT_REQUEST.read_text(encoding="utf-8")
|
|
|
|
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in sre
|
|
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-Control-Loop"' in submit
|
|
assert 'Start-ScheduledTask -TaskPath "\\" -TaskName "Wooo-Agent99-SRE-Alert-Inbox"' in telegram
|
|
assert '"-InstructionBase64", (Convert-AgentBase64 $instruction)' in telegram
|
|
assert '"-RequestedByBase64", (Convert-AgentBase64 ([string]$update.from))' in telegram
|
|
assert '"-AgentRoot", $AgentRoot' in telegram
|
|
assert "$null = $submit.Handle" in telegram
|
|
assert "instruction_transport_ambiguous" in submit
|
|
assert "scheduled_task_signaled" in sre
|
|
assert "scheduled_task_signaled" in submit
|
|
assert "function Write-AgentDurableJson" in submit
|
|
assert '"request-" + $ingressDigest.Substring(0, 32)' in submit
|
|
assert "Global\\Wooo-Agent99-Submit-" in submit
|
|
assert "function Get-AgentIngressLifecycleReceipt" in submit
|
|
assert 'throw "ingress_lifecycle_ambiguous"' in submit
|
|
assert "idempotentReplay = [bool]$queueReceipt.existing" in submit
|
|
assert "durableQueue = [bool]$queueReceipt.ok" in submit
|
|
assert "submitEvidenceSha256" in submit
|
|
assert "function Get-AgentDurableSubmitReadback" in telegram
|
|
assert "Sort-Object update_id" in telegram
|
|
assert "lastDurablyHandledUpdateId" in telegram
|
|
assert "update_processing_failed_no_checkpoint_past_failure" in telegram
|
|
assert "offset_checkpoint_failed_safe_replay_required" in telegram
|
|
assert "Write-AgentDurableJson $result $evidencePath" in telegram
|
|
assert "Write-AgentDurableJson $offsetState $offsetPath" in telegram
|
|
assert "$maxUpdateId + 1" not in telegram
|
|
|
|
|
|
def test_windows_native_bounded_process_replay_covers_timeout_tree_and_stream_cap() -> None:
|
|
source = BOUNDED_REPLAY.read_text(encoding="utf-8")
|
|
|
|
assert "timeout_descendant_not_observed" in source
|
|
assert "streaming_output_limit_not_detected" in source
|
|
assert "root_only_descendant_not_preserved" in source
|
|
assert "-TerminationScope Root" in source
|
|
assert "cim_failure_root_not_stopped" in source
|
|
assert "cim_failure_tree_false_green" in source
|
|
assert "orphan_race_descendant_snapshot_missing" in source
|
|
assert "orphan_race_tree_false_green" in source
|
|
assert "pid_reuse_selection_not_identity_bounded" in source
|
|
assert "reusedRootExcluded" in source
|
|
assert "reusedRootChildExcluded" in source
|
|
assert "live_root_child_creation_bound_failed" in source
|
|
assert "intermediateOlderProcessExcluded" in source
|
|
assert "stop_readback_failure_not_detected" in source
|
|
assert "stop_readback_failure_not_fail_closed" in source
|
|
assert "stop_readback_failure_tree_false_green" in source
|
|
assert "captured_output_exceeded_contract" in source
|
|
assert "cleanupVerified" in source
|
|
assert "productionPathWritePerformed = $false" in source
|
|
|
|
|
|
def test_windows_native_telegram_replay_covers_durable_success_and_enqueue_failure() -> None:
|
|
source = TELEGRAM_REPLAY.read_text(encoding="utf-8")
|
|
|
|
assert "success_queue_not_durable" in source
|
|
assert "failure_case_false_green" in source
|
|
assert "failure_offset_advanced" in source
|
|
assert "synthetic_after_queue_commit" in source
|
|
assert "queue_commit_retry_duplicated_artifact" in source
|
|
assert "queue_commit_retry_not_idempotent" in source
|
|
assert '"A" * 9000' in source
|
|
assert "telegramApiCalled = $false" in source
|
|
assert "productionPathWritePerformed = $false" in source
|
|
|
|
|
|
def test_windows_native_breakglass_failure_readback_replay_is_fail_closed() -> None:
|
|
source = BREAKGLASS_REPLAY.read_text(encoding="utf-8")
|
|
|
|
assert "synthetic_cim_failure" in source
|
|
assert "synthetic_mutex_readback_failure" in source
|
|
assert "synthetic_filesystem_readback_failure" in source
|
|
assert "failure_readback_false_green" in source
|
|
assert "terminalReceiptConstructionCanContinue = $true" in source
|
|
assert "runtimeWritePerformed = $false" in source
|