From f802d64fafbd6fa7b6e109c66d5ebcce481749f6 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 15:58:13 +0800 Subject: [PATCH] fix(agent99): add bounded control loop restore --- ...ent99-control-loop-maintenance-restore.ps1 | 365 ++++++++++++++++++ .../tests/test_agent99_vmrun_timeout_guard.py | 41 ++ 2 files changed, 406 insertions(+) create mode 100644 scripts/reboot-recovery/agent99-control-loop-maintenance-restore.ps1 diff --git a/scripts/reboot-recovery/agent99-control-loop-maintenance-restore.ps1 b/scripts/reboot-recovery/agent99-control-loop-maintenance-restore.ps1 new file mode 100644 index 000000000..b3998d582 --- /dev/null +++ b/scripts/reboot-recovery/agent99-control-loop-maintenance-restore.ps1 @@ -0,0 +1,365 @@ +[CmdletBinding()] +param( + [ValidateSet("Check", "Apply")] + [string]$Mode = "Check", + [Parameter(Mandatory = $true)] + [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$")] + [string]$MaintenanceReceiptRef, + [ValidateRange(1, 60)] + [int]$MaxReceiptAgeMinutes = 30, + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" +$taskName = "Wooo-Agent99-Control-Loop" +$runtimeWritePerformed = $false + +function Test-RecoveryMutexActive { + $mutex = $null + $acquired = $false + try { + $mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1") + try { $acquired = [bool]$mutex.WaitOne(0) } catch [Threading.AbandonedMutexException] { $acquired = $true } + [bool](-not $acquired) + } finally { + if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { $mutex.Dispose() } + } +} + +function Get-ControlProcesses { + @( + Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop | + Where-Object { + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' + } + ) +} + +function Get-RecoverProcesses { + @( + Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop | + Where-Object { + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)' + } + ) +} + +function Write-DurableReceipt { + param([object]$Value, [string]$Path) + $temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))" + $stream = $null + $writer = $null + try { + $encoding = New-Object Text.UTF8Encoding($false) + $stream = [IO.File]::Open($temp, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) + $writer = New-Object IO.StreamWriter($stream, $encoding) + $writer.Write(($Value | ConvertTo-Json -Depth 8)) + $writer.Flush() + $stream.Flush($true) + $writer.Dispose(); $writer = $null + $stream.Dispose(); $stream = $null + $beforeHash = (Get-FileHash -LiteralPath $temp -Algorithm SHA256).Hash.ToLowerInvariant() + Move-Item -LiteralPath $temp -Destination $Path -Force + $afterHash = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($beforeHash -ne $afterHash) { throw "receipt_readback_mismatch" } + } finally { + if ($writer) { $writer.Dispose() } + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue + } +} + +function Get-RefreezeReadback { + $errors = New-Object System.Collections.Generic.List[string] + $controlCount = $null + $recoverCount = $null + $taskEnabled = $null + $mutexInactive = $false + $leaseAbsent = $false + try { $controlCount = @(Get-ControlProcesses).Count } catch { $errors.Add("control_process_readback_failed") } + try { $recoverCount = @(Get-RecoverProcesses).Count } catch { $errors.Add("recover_process_readback_failed") } + try { + $taskReadback = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop + $taskEnabled = [bool]$taskReadback.Settings.Enabled + } catch { + $errors.Add("control_task_readback_failed") + } + try { + $leaseAbsent = [bool](-not (Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json") -ErrorAction Stop)) + } catch { + $errors.Add("recovery_lease_readback_failed") + } + try { $mutexInactive = [bool](-not (Test-RecoveryMutexActive)) } catch { $errors.Add("recovery_mutex_readback_failed") } + [pscustomobject]@{ + complete = [bool]($errors.Count -eq 0) + errors = @($errors) + taskEnabled = $taskEnabled + controlCount = $controlCount + recoverCount = $recoverCount + leaseAbsent = $leaseAbsent + mutexInactive = $mutexInactive + verified = [bool]( + $errors.Count -eq 0 -and + $taskEnabled -is [bool] -and -not $taskEnabled -and + $controlCount -eq 0 -and + $recoverCount -eq 0 -and + $leaseAbsent -and + $mutexInactive + ) + } +} + +function Wait-RefreezeReadback { + param([int]$TimeoutSeconds) + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + $readback = Get-RefreezeReadback + while (-not $readback.verified -and $readback.complete -and $stopwatch.Elapsed.TotalSeconds -lt $TimeoutSeconds) { + Start-Sleep -Milliseconds 250 + $readback = Get-RefreezeReadback + } + $readback +} + +$maintenanceMutex = [Threading.Mutex]::new($false, "Global\WoooAgent99ControlLoopMaintenanceV1") +$maintenanceMutexAcquired = $false +try { + try { + $maintenanceMutexAcquired = [bool]$maintenanceMutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $maintenanceMutexAcquired = $true + } + if (-not $maintenanceMutexAcquired) { + [pscustomobject]@{ + schemaVersion = "agent99_control_loop_maintenance_restore_v1" + timestamp = (Get-Date).ToString("o") + mode = $Mode + precheckPassed = $false + blockers = @("maintenance_writer_active") + terminal = "blocked_single_writer_active" + runtimeWritePerformed = $false + secretValueRead = $false + } | ConvertTo-Json -Depth 5 -Compress + exit 75 + } + + $blockers = New-Object System.Collections.Generic.List[string] + $evidenceDir = Join-Path $AgentRoot "evidence" + $maintenancePath = Join-Path $evidenceDir $MaintenanceReceiptRef + $maintenance = $null + $maintenanceIntent = $null + $maintenanceSha256 = "" + $maintenanceAgeMinutes = $null + if ($env:COMPUTERNAME -ne "WOOO-SUPER") { $blockers.Add("computer_identity_mismatch") } + if (-not (Test-Path -LiteralPath $maintenancePath -PathType Leaf)) { + $blockers.Add("maintenance_receipt_missing") + } else { + try { + $maintenance = Get-Content -LiteralPath $maintenancePath -Raw | ConvertFrom-Json + $maintenanceSha256 = (Get-FileHash -LiteralPath $maintenancePath -Algorithm SHA256).Hash.ToLowerInvariant() + $maintenanceAgeMinutes = ((Get-Date) - (Get-Item -LiteralPath $maintenancePath).LastWriteTime).TotalMinutes + } catch { + $blockers.Add("maintenance_receipt_invalid") + } + } + if ($maintenance) { + if ( + [string]$maintenance.schemaVersion -ne "agent99_control_loop_maintenance_v2" -or + [string]$maintenance.mode -ne "Apply" -or + [string]$maintenance.taskName -ne $taskName -or + [string]$maintenance.terminal -ne "control_loop_maintenance_frozen" -or + -not ($maintenance.precheckPassed -is [bool] -and $maintenance.precheckPassed) -or + -not ($maintenance.controlStopped -is [bool] -and $maintenance.controlStopped) -or + -not ($maintenance.recoverStopped -is [bool] -and $maintenance.recoverStopped) -or + -not ($maintenance.leaseMetadataRemoved -is [bool] -and $maintenance.leaseMetadataRemoved) -or + -not ($maintenance.runtimeWritePerformed -is [bool] -and $maintenance.runtimeWritePerformed) -or + -not ($maintenance.taskEnabledBefore -is [bool] -and $maintenance.taskEnabledBefore) -or + -not ($maintenance.taskEnabledAfter -is [bool] -and -not $maintenance.taskEnabledAfter) -or + -not ($maintenance.vmPowerChangePerformed -is [bool] -and -not $maintenance.vmPowerChangePerformed) -or + -not ($maintenance.hostRebootPerformed -is [bool] -and -not $maintenance.hostRebootPerformed) -or + -not ($maintenance.secretValueRead -is [bool] -and -not $maintenance.secretValueRead) + ) { + $blockers.Add("maintenance_receipt_contract_mismatch") + } + if ($null -eq $maintenanceAgeMinutes -or $maintenanceAgeMinutes -lt 0 -or $maintenanceAgeMinutes -gt $MaxReceiptAgeMinutes) { + $blockers.Add("maintenance_receipt_stale") + } + $intentRef = [string]$maintenance.operationIntentRef + if ($intentRef -notmatch '^agent99-control-loop-maintenance-intent-[0-9]{8}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$') { + $blockers.Add("maintenance_intent_ref_invalid") + } else { + $intentPath = Join-Path $evidenceDir $intentRef + try { + $maintenanceIntent = Get-Content -LiteralPath $intentPath -Raw -ErrorAction Stop | ConvertFrom-Json + } catch { + $blockers.Add("maintenance_intent_invalid") + } + if ($maintenanceIntent -and ( + [string]$maintenanceIntent.schemaVersion -ne "agent99_control_loop_maintenance_intent_v1" -or + [string]$maintenanceIntent.taskName -ne $taskName -or + (@($maintenanceIntent.plannedActions) -join ",") -ne "disable_control_task,stop_exact_control_tree,stop_exact_recover_tree,remove_exact_recovery_lease" -or + -not ($maintenanceIntent.taskEnabledBefore -is [bool] -and $maintenanceIntent.taskEnabledBefore) -or + -not ($maintenanceIntent.vmPowerChangePerformed -is [bool] -and -not $maintenanceIntent.vmPowerChangePerformed) -or + -not ($maintenanceIntent.secretValueRead -is [bool] -and -not $maintenanceIntent.secretValueRead) + )) { + $blockers.Add("maintenance_intent_contract_mismatch") + } + } + } + + $task = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue + if (-not $task) { + $blockers.Add("control_task_missing") + } else { + $expectedExecute = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" + $expectedLauncher = Join-Path $AgentRoot "agent99-run.ps1" + $expectedArguments = "-NoProfile -ExecutionPolicy Bypass -File `"$expectedLauncher`" -Mode ControlTick" + $actions = @($task.Actions) + if ($actions.Count -ne 1) { $blockers.Add("control_task_action_count_mismatch") } + if ($actions.Count -eq 1) { + if (-not [string]::Equals([string]$actions[0].Execute, $expectedExecute, [StringComparison]::OrdinalIgnoreCase)) { + $blockers.Add("control_task_executable_mismatch") + } + if (-not [string]::Equals([string]$actions[0].Arguments, $expectedArguments, [StringComparison]::Ordinal)) { + $blockers.Add("control_task_arguments_mismatch") + } + } + if ([bool]$task.Settings.Enabled) { $blockers.Add("control_task_not_frozen") } + } + + $controlBefore = @(Get-ControlProcesses) + $recoverBefore = @(Get-RecoverProcesses) + if ($controlBefore.Count -ne 0) { $blockers.Add("control_process_present") } + if ($recoverBefore.Count -ne 0) { $blockers.Add("recover_process_present") } + if (Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json")) { + $blockers.Add("recovery_lease_present") + } + if (Test-RecoveryMutexActive) { $blockers.Add("recovery_mutex_active") } + + $precheckPassed = [bool]($blockers.Count -eq 0) + $terminal = if ($precheckPassed) { "maintenance_restore_check_verified" } else { "blocked_restore_precheck" } + $deviceMutationAttempted = $false + $operationFailureType = "" + $taskEnabledAfter = if ($task) { [bool]$task.Settings.Enabled } else { $null } + $controlProcessObserved = $false + $taskRunObserved = $false + $rollbackStatus = "not_required" + $rollbackReadbackErrors = @() + $operationIntentRef = "" + + if ($Mode -eq "Apply" -and $precheckPassed) { + $intentStamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" + $operationIntentRef = "agent99-control-loop-maintenance-restore-intent-$intentStamp-$([guid]::NewGuid().ToString('N').Substring(0, 8)).json" + $operationIntentPath = Join-Path $evidenceDir $operationIntentRef + $operationIntent = [pscustomobject]@{ + schemaVersion = "agent99_control_loop_maintenance_restore_intent_v1" + timestamp = (Get-Date).ToString("o") + taskName = $taskName + maintenanceReceiptRef = $MaintenanceReceiptRef + maintenanceReceiptSha256 = $maintenanceSha256 + plannedActions = @("enable_exact_control_task", "start_exact_control_task", "verify_control_task_started") + rollback = "stop and disable the exact control task if postcheck fails" + vmPowerChangePerformed = $false + secretValueRead = $false + } + Write-DurableReceipt $operationIntent $operationIntentPath + $runtimeWritePerformed = $true + + $startedAt = Get-Date + try { + $deviceMutationAttempted = $true + Enable-ScheduledTask -TaskName $taskName -TaskPath "\" | Out-Null + Start-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($stopwatch.Elapsed.TotalSeconds -lt 20) { + $taskAfter = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop + $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName -TaskPath "\" -ErrorAction Stop + $controlsAfter = @(Get-ControlProcesses) + if ($controlsAfter.Count -gt 0) { $controlProcessObserved = $true } + $taskRunObserved = [bool]($taskInfo.LastRunTime -ge $startedAt.AddSeconds(-2)) + if ([bool]$taskAfter.Settings.Enabled -and $taskRunObserved -and ($controlProcessObserved -or [string]$taskAfter.State -eq "Running")) { + break + } + Start-Sleep -Milliseconds 250 + } + $taskAfter = Get-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop + $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName -TaskPath "\" -ErrorAction Stop + $taskEnabledAfter = [bool]$taskAfter.Settings.Enabled + $taskRunObserved = [bool]($taskInfo.LastRunTime -ge $startedAt.AddSeconds(-2)) + $verified = [bool]($taskEnabledAfter -and $taskRunObserved -and ($controlProcessObserved -or [string]$taskAfter.State -eq "Running")) + if (-not $verified) { throw "control_task_restore_postcheck_failed" } + $terminal = "control_loop_maintenance_restored" + } catch { + $operationFailureType = $_.Exception.GetType().Name + try { Stop-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue } catch {} + try { Disable-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop | Out-Null } catch {} + $rollbackReadback = Wait-RefreezeReadback 10 + $taskEnabledAfter = $rollbackReadback.taskEnabled + $rollbackReadbackErrors = @($rollbackReadback.errors) + $rollbackVerified = [bool]$rollbackReadback.verified + $rollbackStatus = if ($rollbackVerified) { "task_refrozen" } else { "manual_recovery_required" } + $terminal = if ($rollbackVerified) { "maintenance_restore_failed_refrozen" } else { "maintenance_restore_failed_manual_recovery" } + } + } + + $result = [pscustomobject]@{ + schemaVersion = "agent99_control_loop_maintenance_restore_v1" + timestamp = (Get-Date).ToString("o") + mode = $Mode + taskName = $taskName + maintenanceReceiptRef = $MaintenanceReceiptRef + maintenanceReceiptSha256 = $maintenanceSha256 + maintenanceAgeMinutes = if ($null -ne $maintenanceAgeMinutes) { [math]::Round($maintenanceAgeMinutes, 2) } else { $null } + precheckPassed = $precheckPassed + blockers = @($blockers) + operationIntentRef = $operationIntentRef + deviceMutationAttempted = $deviceMutationAttempted + operationFailureType = $operationFailureType + taskEnabledAfter = $taskEnabledAfter + taskRunObserved = $taskRunObserved + controlProcessObserved = $controlProcessObserved + rollbackStatus = $rollbackStatus + rollbackReadbackErrors = @($rollbackReadbackErrors) + terminal = $terminal + runtimeWritePerformed = $runtimeWritePerformed + vmPowerChangePerformed = $false + hostRebootPerformed = $false + secretValueRead = $false + } + + if ($Mode -eq "Apply") { + $path = Join-Path $evidenceDir ("agent99-control-loop-maintenance-restore-" + (Get-Date -Format "yyyyMMdd-HHmmss-fff") + "-" + ([guid]::NewGuid().ToString('N').Substring(0, 8)) + ".json") + try { + Write-DurableReceipt $result $path + $result | Add-Member -NotePropertyName receiptRef -NotePropertyValue (Split-Path -Leaf $path) -Force + } catch { + $receiptFailureType = $_.Exception.GetType().Name + if ($deviceMutationAttempted) { + try { Stop-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction SilentlyContinue } catch {} + try { Disable-ScheduledTask -TaskName $taskName -TaskPath "\" -ErrorAction Stop | Out-Null } catch {} + $receiptRollbackReadback = Wait-RefreezeReadback 10 + $receiptRollbackVerified = [bool]$receiptRollbackReadback.verified + $result.taskEnabledAfter = $receiptRollbackReadback.taskEnabled + $result.rollbackReadbackErrors = @($receiptRollbackReadback.errors) + $result.rollbackStatus = if ($receiptRollbackVerified) { "task_refrozen" } else { "manual_recovery_required" } + $terminal = if ($receiptRollbackVerified) { "maintenance_restore_receipt_failed_refrozen" } else { "maintenance_restore_receipt_failed_manual_recovery" } + $result.terminal = $terminal + } else { + $terminal = "maintenance_restore_receipt_failed_no_mutation" + $result.terminal = $terminal + } + $result.operationFailureType = $receiptFailureType + $result | Add-Member -NotePropertyName receiptRef -NotePropertyValue "" -Force + } + } + $result | ConvertTo-Json -Depth 8 -Compress + if (-not $precheckPassed) { exit 64 } + if ($Mode -eq "Check") { exit 0 } + if ($terminal -eq "control_loop_maintenance_restored") { exit 0 } + exit 70 +} finally { + if ($maintenanceMutexAcquired -and $maintenanceMutex) { + try { $maintenanceMutex.ReleaseMutex() } catch {} + } + if ($maintenanceMutex) { $maintenanceMutex.Dispose() } +} diff --git a/scripts/reboot-recovery/tests/test_agent99_vmrun_timeout_guard.py b/scripts/reboot-recovery/tests/test_agent99_vmrun_timeout_guard.py index 023c53c5b..b317e26f0 100644 --- a/scripts/reboot-recovery/tests/test_agent99_vmrun_timeout_guard.py +++ b/scripts/reboot-recovery/tests/test_agent99_vmrun_timeout_guard.py @@ -5,6 +5,9 @@ ROOT = Path(__file__).resolve().parents[3] CONTROL = ROOT / "agent99-control-plane.ps1" BREAKGLASS = ROOT / "scripts" / "reboot-recovery" / "agent99-stale-recovery-breakglass.ps1" MAINTENANCE = ROOT / "scripts" / "reboot-recovery" / "agent99-control-loop-maintenance.ps1" +MAINTENANCE_RESTORE = ( + ROOT / "scripts" / "reboot-recovery" / "agent99-control-loop-maintenance-restore.ps1" +) REGISTER_TASKS = ROOT / "agent99-register-tasks.ps1" SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1" TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1" @@ -182,6 +185,44 @@ def test_control_loop_maintenance_freeze_is_exact_and_receipted() -> None: assert "hostRebootPerformed = $false" in source +def test_control_loop_maintenance_restore_is_bounded_and_receipted() -> None: + source = MAINTENANCE_RESTORE.read_text(encoding="utf-8") + + assert 'if ($env:COMPUTERNAME -ne "WOOO-SUPER")' in source + assert "Global\\WoooAgent99ControlLoopMaintenanceV1" in source + assert 'ValidateSet("Check", "Apply")' in source + assert "maintenanceReceiptSha256" in source + assert 'schemaVersion -ne "agent99_control_loop_maintenance_v2"' in source + assert 'terminal -ne "control_loop_maintenance_frozen"' in source + assert 'schemaVersion -ne "agent99_control_loop_maintenance_intent_v1"' in source + assert "maintenance_intent_contract_mismatch" in source + assert 'if ([bool]$task.Settings.Enabled)' in source + assert "StringComparison]::OrdinalIgnoreCase" in source + assert "StringComparison]::Ordinal" in source + assert 'Join-Path $AgentRoot "agent99-run.ps1"' in source + assert 'Enable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source + assert 'Start-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source + assert 'Stop-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source + assert 'Disable-ScheduledTask -TaskName $taskName -TaskPath "\\"' in source + assert source.index("Write-DurableReceipt $operationIntent $operationIntentPath") < source.index( + 'Enable-ScheduledTask -TaskName $taskName' + ) + assert "control_loop_maintenance_restored" in source + assert "maintenance_restore_failed_refrozen" in source + assert "maintenance_restore_failed_manual_recovery" in source + assert "maintenance_restore_receipt_failed_refrozen" in source + assert "maintenance_restore_receipt_failed_manual_recovery" in source + assert "Get-RefreezeReadback" in source + assert "rollbackReadbackErrors" in source + assert "recovery_lease_readback_failed" in source + assert "$leaseAbsent" in source + assert 'recovery-transport-lease.json") -PathType Leaf' not in source + assert "$stream.Flush($true)" in source + assert "vmPowerChangePerformed = $false" in source + assert "hostRebootPerformed = $false" in source + assert "secretValueRead = $false" in source + + def test_control_loop_deadline_and_queue_batch_are_compatible() -> None: control = CONTROL.read_text(encoding="utf-8") tasks = REGISTER_TASKS.read_text(encoding="utf-8")