From 50c7a7f7f5cc88bc0877b510e28fa1527ab1cdc5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 16:04:18 +0800 Subject: [PATCH] fix(agent99): unblock bounded maintenance deployment --- .../agent99-live-preflight.ps1 | 573 ++++++++++++++++-- .../agent99-remote-atomic-deploy-receiver.ps1 | 182 +++++- .../agent99-stale-queue-quarantine.ps1 | 347 +++++++++++ .../test_agent99_live_preflight_decision.py | 216 ++++++- ...remote_atomic_deploy_transport_contract.py | 45 +- ...agent99_stale_queue_quarantine_contract.py | 44 ++ 6 files changed, 1342 insertions(+), 65 deletions(-) create mode 100644 scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 create mode 100644 scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 92189091a..14b387c08 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -28,7 +28,8 @@ function Get-AgentLivePreflightDecision { [int]$StaleAlertExecutionCount, [int]$AlertIncomingCount, [int]$StaleAlertIncomingCount, - [int]$PendingCommandCount + [int]$PendingCommandBlockingCount, + [int]$PromotionDeferredCommandCount ) $blockingReasons = @() @@ -89,11 +90,14 @@ function Get-AgentLivePreflightDecision { } if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" } - if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) { + if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandBlockingCount -gt 0) { $blockingReasons += "pending_work_before_reboot" } elseif ($AlertIncomingCount -gt 0) { $warningReasons += "fresh_alert_pending_next_inbox_tick" } + if ($PromotionDeferredCommandCount -gt 0) { + $warningReasons += "maintenance_pending_commands_deferred_until_post_promotion" + } return [pscustomobject]@{ deploymentEligible = [bool]($blockingReasons.Count -eq 0) @@ -110,7 +114,8 @@ function Get-AgentLivePreflightDecision { function Get-TaskReadback { param( [string]$TaskPath, - [string]$TaskName + [string]$TaskName, + [string[]]$RequiredArgumentMarkers ) try { @@ -120,8 +125,34 @@ function Get-TaskReadback { $info = Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop $lastResult = [int64]$info.LastTaskResult $state = [string]$task.State + $actions = @($task.Actions) + $actionDefinitionReady = [bool]( + $actions.Count -eq 1 -and + [string]$actions[0].Execute -match '(?i)\\WindowsPowerShell\\v1\.0\\powershell\.exe$' + ) + if ($actionDefinitionReady) { + foreach ($marker in @($RequiredArgumentMarkers)) { + if ( + -not $marker -or + ([string]$actions[0].Arguments).IndexOf( + $marker, + [StringComparison]::OrdinalIgnoreCase + ) -lt 0 + ) { + $actionDefinitionReady = $false + break + } + } + } + $principalReady = [bool]( + [string]$task.Principal.UserId -match '(?i)(^|\\)Administrator$' -and + [string]$task.Principal.LogonType -eq "S4U" -and + [string]$task.Principal.RunLevel -eq "Highest" + ) + $definitionReady = [bool]($actionDefinitionReady -and $principalReady) $healthy = [bool]( $task.Settings.Enabled -and + $definitionReady -and ($lastResult -in @(0, 267009, 267011) -or $state -eq "Running") ) @@ -134,6 +165,9 @@ function Get-TaskReadback { lastTaskResult = $lastResult lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" } nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" } + actionDefinitionReady = $actionDefinitionReady + principalReady = $principalReady + definitionReady = $definitionReady healthy = $healthy } } catch { @@ -146,6 +180,9 @@ function Get-TaskReadback { lastTaskResult = $null lastRunTime = "" nextRunTime = "" + actionDefinitionReady = $false + principalReady = $false + definitionReady = $false healthy = $false } } @@ -158,25 +195,448 @@ function Get-DirectoryReadback { ) $path = Join-Path $AgentRoot $RelativePath - $files = @() - if (Test-Path $path) { - $files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue) - if ($RootQueueFilesOnly) { - $files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" }) + $exists = [bool](Test-Path -LiteralPath $path -PathType Container) + $count = 0 + $staleCount = 0 + $oldest = $null + $newest = $null + if ($exists) { + foreach ($candidatePath in [IO.Directory]::EnumerateFiles( + $path, + "*.json", + [IO.SearchOption]::TopDirectoryOnly + )) { + $name = [IO.Path]::GetFileName($candidatePath) + if ($RootQueueFilesOnly -and $name -match "^(processed|failed|running)-") { + continue + } + $file = [IO.FileInfo]::new($candidatePath) + $count += 1 + if (-not $oldest -or $file.LastWriteTimeUtc -lt $oldest.LastWriteTimeUtc) { + $oldest = $file + } + if (-not $newest -or $file.LastWriteTimeUtc -gt $newest.LastWriteTimeUtc) { + $newest = $file + } + if (((Get-Date) - $file.LastWriteTime).TotalMinutes -gt 5) { + $staleCount += 1 + } } - $files = @($files | Sort-Object LastWriteTime) } return [pscustomobject]@{ relativePath = $RelativePath - exists = [bool](Test-Path $path) - count = [int]$files.Count - oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" } - newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" } - staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count + exists = $exists + count = [int]$count + oldest = if ($oldest) { $oldest.LastWriteTime.ToString("o") } else { "" } + newest = if ($newest) { $newest.LastWriteTime.ToString("o") } else { "" } + staleOverFiveMinutes = [int]$staleCount } } +function Get-AgentBoundedNewestFiles { + param( + [string]$DirectoryPath, + [string]$Pattern, + [ValidateRange(1, 256)] + [int]$CandidateWindow = 32 + ) + + # Agent99 evidence names contain a fixed sortable timestamp and are immutable. + # Keep only the lexically newest bounded window in a native .NET set so a + # large evidence directory never becomes a PowerShell object-sort workload. + $selected = [Collections.Generic.SortedSet[string]]::new( + [StringComparer]::OrdinalIgnoreCase + ) + $enumeratedCount = 0 + if (Test-Path -LiteralPath $DirectoryPath -PathType Container) { + foreach ($candidatePath in [IO.Directory]::EnumerateFiles( + $DirectoryPath, + $Pattern, + [IO.SearchOption]::TopDirectoryOnly + )) { + $enumeratedCount += 1 + $null = $selected.Add($candidatePath) + if ($selected.Count -gt $CandidateWindow) { + $null = $selected.Remove($selected.Min) + } + } + } + + return [pscustomobject]@{ + files = @($selected | Sort-Object -Descending | ForEach-Object { + [IO.FileInfo]::new($_) + }) + enumeratedCount = [int]$enumeratedCount + candidateWindow = [int]$CandidateWindow + windowTruncated = [bool]($enumeratedCount -gt $selected.Count) + } +} + +function Get-AgentControlLoopMaintenanceReadback { + $evidenceDir = Join-Path $AgentRoot "evidence" + $selection = Get-AgentBoundedNewestFiles ` + $evidenceDir ` + "agent99-control-loop-maintenance-2*.json" ` + 8 + $file = @($selection.files | Select-Object -First 1) + $receipt = $null + $intent = $null + $parseError = "" + if ($file.Count -eq 1) { + try { + $receipt = Get-Content -LiteralPath $file[0].FullName -Raw | ConvertFrom-Json + } catch { + $parseError = $_.Exception.GetType().Name + } + } + + $intentRef = if ($receipt -and $receipt.PSObject.Properties["operationIntentRef"]) { + [string]$receipt.operationIntentRef + } else { "" } + $intentRefSafe = [bool]( + $intentRef -match "^agent99-control-loop-maintenance-intent-[0-9]{8}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$" + ) + $intentPath = if ($intentRefSafe) { Join-Path $evidenceDir $intentRef } else { "" } + if ($intentPath -and (Test-Path -LiteralPath $intentPath -PathType Leaf)) { + try { + $intent = Get-Content -LiteralPath $intentPath -Raw | ConvertFrom-Json + } catch { + $parseError = $_.Exception.GetType().Name + } + } + $legacyPropertySet = @( + "blockers", "controlStopped", "expectedControlCreated", "expectedControlPid", + "expectedRecoverCreated", "expectedRecoverParentPid", "expectedRecoverPid", + "hostRebootPerformed", "leaseMetadataRemoved", "mode", "precheckPassed", + "recoverStopped", "runtimeWritePerformed", "schemaVersion", "secretValueRead", + "taskEnabled", "taskName", "terminal", "timestamp", "vmPowerChangePerformed", + "vmwareVmxProcessTerminated", "vmxLockDeleted" + ) + $receiptPropertySet = if ($receipt) { + @($receipt.PSObject.Properties.Name | Sort-Object) + } else { @() } + $legacyPropertySetMatches = [bool]( + $receiptPropertySet.Count -eq $legacyPropertySet.Count -and + (@($receiptPropertySet) -join ",") -eq (@($legacyPropertySet | Sort-Object) -join ",") + ) + $legacyIdentityFieldsValid = [bool]( + $receipt -and + $receipt.expectedControlPid -is [int] -and [int]$receipt.expectedControlPid -gt 0 -and + $receipt.expectedRecoverPid -is [int] -and [int]$receipt.expectedRecoverPid -gt 0 -and + $receipt.expectedRecoverParentPid -is [int] -and [int]$receipt.expectedRecoverParentPid -gt 0 -and + [string]$receipt.expectedControlCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and + [string]$receipt.expectedRecoverCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and + [string]$receipt.timestamp -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" + ) + $legacyReceipt = [bool]( + $receipt -and + -not $receipt.PSObject.Properties["operationIntentRef"] -and + $receipt.PSObject.Properties["taskEnabled"] -and + $legacyPropertySetMatches -and + $legacyIdentityFieldsValid -and + @($receipt.blockers).Count -eq 0 + ) + + $ageMinutes = if ($file.Count -eq 1) { + [math]::Round(((Get-Date) - $file[0].LastWriteTime).TotalMinutes, 1) + } else { $null } + $activeControlProcesses = @(Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction SilentlyContinue | + Where-Object { + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)' + }) + $leasePresent = [bool](Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json") -PathType Leaf) + $recoveryMutexActive = $true + $mutex = $null + $mutexAcquired = $false + try { + $mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1") + try { + $mutexAcquired = [bool]$mutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $mutexAcquired = $true + } + $recoveryMutexActive = [bool](-not $mutexAcquired) + } finally { + if ($mutexAcquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { $mutex.Dispose() } + } + + $commonReceiptFieldsValid = [bool]( + $receipt -and + [string]$receipt.schemaVersion -eq "agent99_control_loop_maintenance_v2" -and + [string]$receipt.mode -eq "Apply" -and + [string]$receipt.taskName -eq "Wooo-Agent99-Control-Loop" -and + [string]$receipt.terminal -eq "control_loop_maintenance_frozen" -and + $receipt.precheckPassed -is [bool] -and $receipt.precheckPassed -and + $receipt.controlStopped -is [bool] -and $receipt.controlStopped -and + $receipt.recoverStopped -is [bool] -and $receipt.recoverStopped -and + $receipt.leaseMetadataRemoved -is [bool] -and $receipt.leaseMetadataRemoved -and + $receipt.runtimeWritePerformed -is [bool] -and $receipt.runtimeWritePerformed -and + $receipt.vmPowerChangePerformed -is [bool] -and -not $receipt.vmPowerChangePerformed -and + $receipt.vmwareVmxProcessTerminated -is [bool] -and -not $receipt.vmwareVmxProcessTerminated -and + $receipt.vmxLockDeleted -is [bool] -and -not $receipt.vmxLockDeleted -and + $receipt.hostRebootPerformed -is [bool] -and -not $receipt.hostRebootPerformed -and + $receipt.secretValueRead -is [bool] -and -not $receipt.secretValueRead + ) + $modernReceiptFieldsValid = [bool]( + $commonReceiptFieldsValid -and + -not $legacyReceipt -and + $receipt.taskEnabledAfter -is [bool] -and -not $receipt.taskEnabledAfter + ) + $legacyReceiptFieldsValid = [bool]( + $commonReceiptFieldsValid -and + $legacyReceipt -and + $receipt.taskEnabled -is [bool] -and $receipt.taskEnabled + ) + $receiptFieldsValid = [bool]($modernReceiptFieldsValid -or $legacyReceiptFieldsValid) + $intentFieldsValid = [bool]( + $legacyReceipt -or + ( + $intent -and + [string]$intent.schemaVersion -eq "agent99_control_loop_maintenance_intent_v1" -and + [string]$intent.taskName -eq "Wooo-Agent99-Control-Loop" -and + $intent.vmPowerChangePerformed -is [bool] -and -not $intent.vmPowerChangePerformed -and + $intent.vmwareVmxProcessTerminated -is [bool] -and -not $intent.vmwareVmxProcessTerminated -and + $intent.secretValueRead -is [bool] -and -not $intent.secretValueRead -and + (@($intent.plannedActions) -join ",") -eq "disable_control_task,stop_exact_control_tree,stop_exact_recover_tree,remove_exact_recovery_lease" + ) + ) + $valid = [bool]( + -not $parseError -and + $file.Count -eq 1 -and + $null -ne $ageMinutes -and + $ageMinutes -ge 0 -and + $ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 }) -and + ($legacyReceipt -or $intentRefSafe) -and + $receiptFieldsValid -and + $intentFieldsValid -and + $activeControlProcesses.Count -eq 0 -and + -not $leasePresent -and + -not $recoveryMutexActive + ) + + return [pscustomobject]@{ + valid = $valid + receiptFile = if ($file.Count -eq 1) { $file[0].Name } else { "" } + ageMinutes = $ageMinutes + parseError = $parseError + intentRefPresent = $intentRefSafe + contractVersion = if ($legacyReceipt) { "legacy_v2_exact_upgrade_bridge" } else { "intent_bound_v2" } + receiptTimestamp = if ($receipt) { [string]$receipt.timestamp } else { "" } + receiptLastWriteTime = if ($file.Count -eq 1) { $file[0].LastWriteTime.ToString("o") } else { "" } + legacyPropertySetMatches = $legacyPropertySetMatches + legacyIdentityFieldsValid = $legacyIdentityFieldsValid + receiptFieldsValid = $receiptFieldsValid + intentFieldsValid = $intentFieldsValid + activeControlProcessCount = [int]$activeControlProcesses.Count + recoveryLeasePresent = $leasePresent + recoveryMutexActive = $recoveryMutexActive + secretValueRead = $false + remoteWritePerformed = $false + } +} + +function Get-AgentPendingCommandPromotionReadback { + param( + [object]$MaintenanceReadback, + [ValidateSet("FullRuntime", "PromotionReserve")] + [string]$Scope + ) + + $queueRoot = Join-Path $AgentRoot "queue" + $files = @() + if (Test-Path -LiteralPath $queueRoot -PathType Container) { + $files = @([IO.Directory]::EnumerateFiles( + $queueRoot, + "*.json", + [IO.SearchOption]::TopDirectoryOnly + ) | Where-Object { + [IO.Path]::GetFileName($_) -notmatch "^(processed|failed|running)-" + }) + } + + $observedCount = [int]$files.Count + $result = [ordered]@{ + scope = $Scope + observedCount = $observedCount + deferredCount = 0 + blockingCount = $observedCount + eligibleMaintenanceAutoRecoverCount = 0 + boundedLimit = 8 + contractValid = [bool]($observedCount -eq 0) + reason = if ($observedCount -eq 0) { "queue_empty" } else { "pending_commands_require_runtime" } + items = @() + rawInstructionStored = $false + secretValueRead = $false + remoteWritePerformed = $false + } + if ($observedCount -eq 0) { + return [pscustomobject]$result + } + if ( + -not $MaintenanceReadback -or + -not $MaintenanceReadback.receiptFieldsValid -or + -not $MaintenanceReadback.intentFieldsValid + ) { + $result.reason = "maintenance_contract_invalid" + return [pscustomobject]$result + } + if ($Scope -eq "PromotionReserve" -and -not $MaintenanceReadback.valid) { + $result.reason = "maintenance_current_state_invalid" + return [pscustomobject]$result + } + if ($observedCount -gt $result.boundedLimit) { + $result.reason = "maintenance_pending_command_limit_exceeded" + return [pscustomobject]$result + } + + $maintenanceTimestamp = [datetimeoffset]::MinValue + $maintenanceTimestampValid = [datetimeoffset]::TryParse( + [string]$MaintenanceReadback.receiptTimestamp, + [ref]$maintenanceTimestamp + ) + if (-not $maintenanceTimestampValid) { + $result.reason = "maintenance_timestamp_invalid" + return [pscustomobject]$result + } + + $requiredPropertySet = @( + "approvalId", "automationRunId", "canonicalDigest", "controlledApply", + "correlationKey", "createdAt", "dispatchIdentitySchemaVersion", + "dispatchRouteId", "executionGeneration", "externalId", "id", + "idempotencyKey", "incidentId", "instruction", "lockOwner", "mode", + "observation", "projectId", "reason", "requestedBy", "singleFlightKey", + "source", "sourceFingerprint", "traceId", "workItemId" + ) | Sort-Object + $safeItems = @() + $allValid = $true + foreach ($path in $files) { + $file = [IO.FileInfo]::new($path) + $payload = $null + $parseError = "" + $sha256 = "" + try { + $bytes = [IO.File]::ReadAllBytes($path) + $sha = [Security.Cryptography.SHA256]::Create() + try { + $sha256 = -join @($sha.ComputeHash($bytes) | ForEach-Object { $_.ToString("x2") }) + } finally { + $sha.Dispose() + } + $jsonOffset = if ( + $bytes.Length -ge 3 -and + $bytes[0] -eq 0xef -and + $bytes[1] -eq 0xbb -and + $bytes[2] -eq 0xbf + ) { 3 } else { 0 } + $jsonText = [Text.Encoding]::UTF8.GetString( + $bytes, + $jsonOffset, + $bytes.Length - $jsonOffset + ) + $payload = $jsonText | ConvertFrom-Json + } catch { + $parseError = $_.Exception.GetType().Name + } + $propertySet = if ($payload) { + @($payload.PSObject.Properties.Name | Sort-Object) + } else { @() } + $propertySetMatches = [bool]( + $propertySet.Count -eq $requiredPropertySet.Count -and + (@($propertySet) -join ",") -eq (@($requiredPropertySet) -join ",") + ) + $createdAt = [datetimeoffset]::MinValue + $createdAtValid = [bool]( + $payload -and + [datetimeoffset]::TryParse([string]$payload.createdAt, [ref]$createdAt) + ) + $ageMinutes = if ($createdAtValid) { + [math]::Round(([datetimeoffset]::Now - $createdAt).TotalMinutes, 1) + } else { $null } + $fileBaseName = [IO.Path]::GetFileNameWithoutExtension($file.Name) + $nativeStringFields = @( + "approvalId", "automationRunId", "canonicalDigest", "correlationKey", + "createdAt", "dispatchIdentitySchemaVersion", "dispatchRouteId", + "executionGeneration", "externalId", "id", "idempotencyKey", + "incidentId", "instruction", "lockOwner", "mode", "projectId", + "reason", "requestedBy", "singleFlightKey", "source", + "sourceFingerprint", "traceId", "workItemId" + ) + $nativeStringTypesValid = [bool]( + $payload -and + @($nativeStringFields | Where-Object { $payload.$_ -isnot [string] }).Count -eq 0 + ) + $identityValid = [bool]( + -not $parseError -and + $propertySetMatches -and + $nativeStringTypesValid -and + $payload.controlledApply -is [bool] -and $payload.controlledApply -and + $payload.observation -is [pscustomobject] -and + [string]$payload.id -eq $fileBaseName -and + [string]$payload.id -match "^auto-recover-[0-9]{8}-[0-9]{6}-[0-9a-f]{8}$" -and + [string]$payload.mode -eq "Recover" -and + [string]$payload.source -eq "agent99-recovery-observer" -and + [string]$payload.dispatchIdentitySchemaVersion -eq "agent99_controlled_dispatch_identity_v1" -and + [string]$payload.dispatchRouteId -eq "agent99_local_observer" -and + [string]$payload.projectId -eq "awoooi" -and + [string]$payload.requestedBy -eq "Agent99" -and + [string]$payload.reason -eq "host_unreachable_detected" -and + [string]$payload.externalId -eq "host_unreachable_detected" -and + [string]$payload.approvalId -eq "" -and + [string]$payload.automationRunId -match "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -and + [string]$payload.lockOwner -eq [string]$payload.automationRunId -and + [string]$payload.executionGeneration -match "^[1-9][0-9]*$" -and + [string]$payload.canonicalDigest -match "^[0-9a-f]{64}$" -and + [string]$payload.sourceFingerprint -match "^[0-9a-f]{64}$" -and + [string]$payload.correlationKey -match "^agent99-controlled:[0-9a-f]{64}$" -and + [string]$payload.idempotencyKey -eq [string]$payload.correlationKey -and + [string]$payload.singleFlightKey -match "^agent99-controlled-flight:[0-9a-f]{64}$" -and + [string]$payload.incidentId -match "^agent99-auto-[0-9a-f]{20}$" -and + [string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):host_unreachable_detected" -and + [string]$payload.traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$" -and + -not [string]::IsNullOrWhiteSpace([string]$payload.instruction) -and + $createdAtValid -and + $createdAt -ge $maintenanceTimestamp -and + $ageMinutes -ge 0 -and + $ageMinutes -le 120 + ) + if (-not $identityValid) { $allValid = $false } + $safeItems += [pscustomobject]@{ + file = $file.Name + id = if ($payload) { [string]$payload.id } else { "" } + mode = if ($payload) { [string]$payload.mode } else { "" } + source = if ($payload) { [string]$payload.source } else { "" } + workItemId = if ($payload) { [string]$payload.workItemId } else { "" } + createdAt = if ($payload) { [string]$payload.createdAt } else { "" } + ageMinutes = $ageMinutes + sha256 = $sha256 + identity = if ($payload -and $sha256) { "$([string]$payload.id)|$sha256" } else { "" } + parseError = $parseError + propertySetMatches = $propertySetMatches + identityValid = $identityValid + instructionContentReadback = $false + } + } + + $result.items = $safeItems + if ($allValid) { + $result.eligibleMaintenanceAutoRecoverCount = $observedCount + $result.contractValid = $true + if ($Scope -eq "PromotionReserve") { + $result.deferredCount = $observedCount + $result.blockingCount = 0 + $result.reason = "bounded_maintenance_auto_recover_deferred_until_post_promotion" + } else { + $result.reason = "full_runtime_requires_maintenance_auto_recover_to_drain" + } + } else { + $result.reason = "maintenance_pending_command_contract_invalid" + } + return [pscustomobject]$result +} + function Get-EvidenceReadback { param( [string]$Name, @@ -187,8 +647,8 @@ function Get-EvidenceReadback { ) $evidenceDir = Join-Path $AgentRoot "evidence" - $candidates = @(Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue | - Sort-Object LastWriteTime -Descending) + $candidateSelection = Get-AgentBoundedNewestFiles $evidenceDir $Pattern + $candidates = @($candidateSelection.files) $deferredCandidates = @() $file = $null foreach ($candidate in $candidates) { @@ -219,6 +679,11 @@ function Get-EvidenceReadback { $file = $candidate break } + $candidateWindowExhausted = [bool]( + -not $file -and + $candidateSelection.windowTruncated -and + $deferredCandidates.Count -eq $candidates.Count + ) $ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null } $fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes) $freshnessHeadroomMinutes = if ($file) { [math]::Round($MaxAgeMinutes - $ageMinutes, 1) } else { $null } @@ -228,7 +693,7 @@ function Get-EvidenceReadback { $freshnessHeadroomMinutes -ge $MinimumFreshnessReserveMinutes ) $safeSummary = $null - $parseError = "" + $parseError = if ($candidateWindowExhausted) { "EvidenceCandidateWindowExhausted" } else { "" } $contentHealthy = $true if ($file) { try { @@ -389,23 +854,52 @@ function Get-EvidenceReadback { parseError = $parseError deferredCandidateCount = [int]$deferredCandidates.Count latestDeferredTime = if ($deferredCandidates.Count -gt 0) { $deferredCandidates[0].LastWriteTime.ToString("o") } else { "" } + enumeratedCandidateCount = [int]$candidateSelection.enumeratedCount + candidateWindowLimit = [int]$candidateSelection.candidateWindow + candidateWindowTruncated = [bool]$candidateSelection.windowTruncated + candidateWindowExhausted = $candidateWindowExhausted safeSummary = $safeSummary } } $rootTaskPath = "\" $requiredTasks = @( - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health" }, - [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health" } + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery"; markers = @("agent99-submit-request.ps1", "-Mode Recover", "-Source agent99-startup-recovery") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat"; markers = @("agent99-run.ps1", "-Mode Status") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard"; markers = @("agent99-run.ps1", "-Mode Perf", "-ControlledApply") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop"; markers = @("agent99-run.ps1", "-Mode ControlTick") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox"; markers = @("agent99-telegram-inbox.ps1", "-RunNow") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox"; markers = @("agent99-sre-alert-inbox.ps1", "-RunNow") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay"; markers = @("agent99-sre-alert-relay.ps1") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health"; markers = @("agent99-run.ps1", "-Mode SelfCheck") }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health"; markers = @("agent99-run.ps1", "-Mode BackupCheck") } ) -$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback -TaskPath $_.taskPath -TaskName $_.name }) +$tasks = @($requiredTasks | ForEach-Object { + Get-TaskReadback ` + -TaskPath $_.taskPath ` + -TaskName $_.name ` + -RequiredArgumentMarkers $_.markers +}) +$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback +if ($ReadinessScope -eq "PromotionReserve") { + foreach ($task in $tasks) { + $runtimeHealthy = [bool]$task.healthy + $maintenanceFreezeAccepted = [bool]( + $task.name -eq "Wooo-Agent99-Control-Loop" -and + -not $task.enabled -and + $controlLoopMaintenance.valid + ) + $promotionReady = [bool]( + $task.exists -and + $task.definitionReady -and + ($task.enabled -or $maintenanceFreezeAccepted) + ) + $task | Add-Member -NotePropertyName runtimeHealthy -NotePropertyValue $runtimeHealthy -Force + $task | Add-Member -NotePropertyName promotionReady -NotePropertyValue $promotionReady -Force + $task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force + $task.healthy = $promotionReady + } +} $manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json" $manifest = [pscustomobject]@{ @@ -457,14 +951,15 @@ $queues = @( (Get-DirectoryReadback "queue" -RootQueueFilesOnly), (Get-DirectoryReadback "queue\failed") ) +$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime") $evidence = @( - (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0) ) @@ -482,6 +977,9 @@ $selfHealthPerformanceIssueCount = if ($selfCheckEvidence -and $selfCheckEvidenc $alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1 $alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1 $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1 +$pendingCommandPromotion = Get-AgentPendingCommandPromotionReadback ` + -MaintenanceReadback $controlLoopMaintenance ` + -Scope $ReadinessScope $decision = Get-AgentLivePreflightDecision ` -AgentRootPresent ([bool](Test-Path $AgentRoot)) ` -Manifest $manifest ` @@ -498,7 +996,8 @@ $decision = Get-AgentLivePreflightDecision ` -StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes ` -AlertIncomingCount $alertIncoming.count ` -StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes ` - -PendingCommandCount $commandQueue.count + -PendingCommandBlockingCount $pendingCommandPromotion.blockingCount ` + -PromotionDeferredCommandCount $pendingCommandPromotion.deferredCount $blockingReasons = @($decision.blockingReasons) $warningReasons = @($decision.warningReasons) @@ -513,10 +1012,12 @@ $result = [pscustomobject]@{ secretValueRead = $false remoteWritePerformed = $false agentRoot = $AgentRoot + controlLoopMaintenance = $controlLoopMaintenance manifest = $manifest tasks = $tasks relayListeners = $relayListeners queues = $queues + pendingCommandPromotion = $pendingCommandPromotion evidence = $evidence summary = [pscustomobject]@{ requiredTaskCount = $requiredTasks.Count @@ -536,6 +1037,8 @@ $result = [pscustomobject]@{ alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes alertRunningCount = [int]$alertRunning.count pendingCommandCount = [int]$commandQueue.count + pendingCommandBlockingCount = [int]$pendingCommandPromotion.blockingCount + promotionDeferredCommandCount = [int]$pendingCommandPromotion.deferredCount relayListenerCount = $relayListeners.Count preflightGreen = [bool]$decision.deploymentEligible deploymentEligible = [bool]$decision.deploymentEligible diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index af83a6372..2b1e003e0 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -23,7 +23,9 @@ $RelayStartTimeoutSeconds = 360 $RelayPollIntervalSeconds = 1 $ValidateOnlyTimeoutSeconds = 300 $LiveDeployTimeoutSeconds = 300 -$LivePreflightTimeoutSeconds = 120 +$LivePreflightTimeoutSeconds = 60 +$PostVerifyMaxAttempts = 4 +$PostVerifyRetryWaitSeconds = 45 $WindowsControlBaselineTimeoutSeconds = 300 $PromotionReadinessReserveMinutes = 15 $PromotionReadinessMaxAttempts = 2 @@ -434,6 +436,9 @@ function Invoke-AgentLivePreflight { enabled = [bool]$_.enabled state = if ($state -in @("Ready", "Running", "Disabled", "Queued", "Missing")) { $state } else { "Unknown" } lastTaskResult = if ($null -ne $_.lastTaskResult) { [long]$_.lastTaskResult } else { $null } + actionDefinitionReady = [bool]$_.actionDefinitionReady + principalReady = [bool]$_.principalReady + definitionReady = [bool]$_.definitionReady } }) } else { @() } @@ -465,6 +470,14 @@ function Invoke-AgentLivePreflight { $ReadinessScope -eq "PromotionReserve" -or ($payload -and $payload.manifest -and [bool]$payload.manifest.runtimeMatched) ) + $pendingCommandsReadyForScope = [bool]( + $summary -and + [int]$summary.pendingCommandBlockingCount -eq 0 -and + ( + $ReadinessScope -eq "PromotionReserve" -or + [int]$summary.pendingCommandCount -eq 0 + ) + ) $runtimePlaneReady = [bool]( $acceptedExitCode -and -not $processResult.timedOut -and @@ -482,7 +495,7 @@ function Invoke-AgentLivePreflight { [int]$summary.relayListenerCount -gt 0 -and [int]$summary.alertIncomingStaleCount -eq 0 -and [int]$summary.alertRunningCount -eq 0 -and - [int]$summary.pendingCommandCount -eq 0 -and + $pendingCommandsReadyForScope -and $runtimePlaneBlockingReasons.Count -eq 0 ) return [pscustomobject]@{ @@ -521,6 +534,16 @@ function Invoke-AgentLivePreflight { alertIncomingStaleCount = if ($summary) { [int]$summary.alertIncomingStaleCount } else { -1 } alertRunningCount = if ($summary) { [int]$summary.alertRunningCount } else { -1 } pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 } + pendingCommandBlockingCount = if ($summary) { [int]$summary.pendingCommandBlockingCount } else { -1 } + promotionDeferredCommandCount = if ($summary) { [int]$summary.promotionDeferredCommandCount } else { -1 } + pendingCommandDeferralReason = if ($payload -and $payload.pendingCommandPromotion) { + [string]$payload.pendingCommandPromotion.reason + } else { "" } + maintenancePendingCommandIdentities = if ($payload -and $payload.pendingCommandPromotion) { + @($payload.pendingCommandPromotion.items | Where-Object { + [bool]$_.identityValid -and [string]$_.identity -match "^auto-recover-[^|]+\|[0-9a-f]{64}$" + } | ForEach-Object { [string]$_.identity } | Sort-Object -Unique) + } else { @() } preflightGreen = [bool]($summary -and $summary.preflightGreen) overallPreflightGreen = [bool]($summary -and $summary.preflightGreen) runtimePlaneReady = $runtimePlaneReady @@ -579,6 +602,96 @@ function Test-AgentPromotionReadinessRetryable { return $true } +function Test-AgentPostPromotionReadinessRetryable { + param( + [object]$Readiness, + [string[]]$ExpectedMaintenancePendingCommandIdentities = @() + ) + + $pendingCount = if ($Readiness) { [int]$Readiness.pendingCommandCount } else { -1 } + $pendingIdentities = if ($Readiness) { + @($Readiness.maintenancePendingCommandIdentities | ForEach-Object { [string]$_ } | Sort-Object -Unique) + } else { @() } + $expectedIdentities = @($ExpectedMaintenancePendingCommandIdentities | ForEach-Object { [string]$_ } | Sort-Object -Unique) + $pendingCommandsRetryable = [bool]( + $pendingCount -eq 0 -or + ( + $pendingCount -gt 0 -and + [int]$Readiness.pendingCommandBlockingCount -eq $pendingCount -and + [string]$Readiness.pendingCommandDeferralReason -eq "full_runtime_requires_maintenance_auto_recover_to_drain" -and + $pendingIdentities.Count -eq $pendingCount -and + $expectedIdentities.Count -gt 0 -and + @($pendingIdentities | Where-Object { $_ -notin $expectedIdentities }).Count -eq 0 + ) + ) + + if ( + -not $Readiness -or + $Readiness.ok -or + -not $Readiness.parsed -or + -not $Readiness.scopeMatched -or + [string]$Readiness.readinessScope -ne "FullRuntime" -or + $Readiness.timedOut -or + -not $Readiness.acceptedExitCode -or + [int]$Readiness.relayListenerCount -lt 1 -or + [int]$Readiness.alertRunningCount -ne 0 -or + -not $pendingCommandsRetryable + ) { + return $false + } + $allowedReasons = @( + "scheduled_task_unhealthy", + "required_evidence_stale", + "required_evidence_freshness_reserve_insufficient", + "required_evidence_content_unhealthy", + "pending_work_before_reboot" + ) + $reasons = @($Readiness.runtimePlaneBlockingReasons | ForEach-Object { [string]$_ }) + if ( + $reasons.Count -eq 0 -or + @($reasons | Where-Object { $_ -notin $allowedReasons }).Count -gt 0 + ) { + return $false + } + if (@($Readiness.failedTasks | Where-Object { + -not $_.exists -or + -not $_.enabled -or + -not $_.definitionReady -or + $_.taskPath -ne "\" -or + $_.state -notin @("Ready", "Running", "Queued") + }).Count -gt 0) { + return $false + } + if ( + "scheduled_task_unhealthy" -in $reasons -and + @($Readiness.failedTasks).Count -eq 0 + ) { return $false } + if ( + "pending_work_before_reboot" -in $reasons -and + $pendingCount -eq 0 + ) { return $false } + if ( + @($reasons | Where-Object { + $_ -in @( + "required_evidence_stale", + "required_evidence_freshness_reserve_insufficient", + "required_evidence_content_unhealthy" + ) + }).Count -gt 0 -and + @($Readiness.failedEvidence).Count -eq 0 + ) { return $false } + if (@($Readiness.failedEvidence | Where-Object { + -not $_.parsed -or + ( + -not $_.contentHealthy -and + $_.name -notin @("telegram_inbox", "sre_alert_inbox") + ) + }).Count -gt 0) { + return $false + } + return $true +} + function Get-AgentRelayRuntime { $task = $null $taskError = "" @@ -1405,6 +1518,8 @@ try { $relayReload = $null $runtimeManifest = $null $livePreflight = $null + $postVerifyAttempts = @() + $postVerifyRetryCount = 0 $launcherContract = $null $windowsControlBaseline = $null $postVerified = $false @@ -1442,25 +1557,40 @@ try { $script:ReceiverOperationPhase = "independent_post_verifier" $runtimeManifest = Get-AgentSafeRuntimeManifest $launcherContract = Get-AgentLauncherContract - $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") - $postVerified = [bool]( - $runtimeManifest.exists -and - $runtimeManifest.runtimeMatched -and - $runtimeManifest.sourceRevision -eq $sourceRevision -and - $runtimeManifest.fileCount -eq 18 -and - $runtimeManifest.mismatchCount -eq 0 -and - $launcherContract.ok -and - [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and - $relayReload.oldGenerationsGone -and - $relayReload.newGenerationVerified -and - $relayReload.after.taskEnabled -and - $relayReload.after.taskRunning -and - $livePreflight.ok -and - $livePreflight.sourceRevision -eq $sourceRevision -and - $livePreflight.runtimeMatched -and - $livePreflight.relayListenerCount -gt 0 -and - (Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations)) - ) + for ($postVerifyAttempt = 1; $postVerifyAttempt -le $PostVerifyMaxAttempts; $postVerifyAttempt += 1) { + $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") + $livePreflight | Add-Member -NotePropertyName attempt -NotePropertyValue $postVerifyAttempt -Force + $postVerifyAttempts += $livePreflight + $postVerified = [bool]( + $runtimeManifest.exists -and + $runtimeManifest.runtimeMatched -and + $runtimeManifest.sourceRevision -eq $sourceRevision -and + $runtimeManifest.fileCount -eq 18 -and + $runtimeManifest.mismatchCount -eq 0 -and + $launcherContract.ok -and + [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and + $relayReload.oldGenerationsGone -and + $relayReload.newGenerationVerified -and + $relayReload.after.taskEnabled -and + $relayReload.after.taskRunning -and + $livePreflight.ok -and + $livePreflight.sourceRevision -eq $sourceRevision -and + $livePreflight.runtimeMatched -and + $livePreflight.relayListenerCount -gt 0 -and + (Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations)) + ) + if ($postVerified) { break } + if ( + $postVerifyAttempt -ge $PostVerifyMaxAttempts -or + -not (Test-AgentPostPromotionReadinessRetryable ` + $livePreflight ` + -ExpectedMaintenancePendingCommandIdentities @( + $prePromotionReadiness.maintenancePendingCommandIdentities + )) + ) { break } + Start-Sleep -Seconds $PostVerifyRetryWaitSeconds + } + $postVerifyRetryCount = [math]::Max(0, $postVerifyAttempts.Count - 1) if (-not $postVerified) { $failurePhase = "post_verifier" $failureType = "independent_post_verifier_failed" @@ -1504,6 +1634,14 @@ try { runtimeManifest = $runtimeManifest launcherContract = $launcherContract livePreflight = $livePreflight + postVerifyAttempts = $postVerifyAttempts + postVerifyRetryCount = $postVerifyRetryCount + postVerifyPolicy = [pscustomobject]@{ + maxAttempts = $PostVerifyMaxAttempts + retryWaitSeconds = $PostVerifyRetryWaitSeconds + automaticTaskTriggerAllowed = $false + finalAttemptVerified = $postVerified + } windowsControlBaseline = $windowsControlBaseline durableReceiptWritten = $true idempotentReplay = $false @@ -1629,6 +1767,8 @@ try { failedRuntimeManifest = $runtimeManifest failedLauncherContract = $launcherContract failedLivePreflight = $livePreflight + postVerifyAttempts = $postVerifyAttempts + postVerifyRetryCount = $postVerifyRetryCount failedWindowsControlBaseline = $windowsControlBaseline rollback = $rollback rollbackRelayRestart = $rollbackRelayRestart diff --git a/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 b/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 new file mode 100644 index 000000000..317e257a0 --- /dev/null +++ b/scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1 @@ -0,0 +1,347 @@ +[CmdletBinding()] +param( + [ValidateSet("Check", "Apply")] + [string]$Mode = "Check", + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] + [string]$TraceId, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] + [string]$RunId, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] + [string]$WorkItemId, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._-]{0,180}\.json$")] + [string]$QueueFileName, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-fA-F0-9]{64}$")] + [string]$ExpectedQueueSha256, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")] + [string]$ExpectedCommandId, + [Parameter(Mandatory = $true)] + [ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}\.json$")] + [string]$MaintenanceReceiptName, + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-fA-F0-9]{64}$")] + [string]$ExpectedMaintenanceSha256, + [ValidateRange(30, 1440)] + [int]$MinimumAgeMinutes = 30, + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" +$ExpectedQueueSha256 = $ExpectedQueueSha256.ToLowerInvariant() +$ExpectedMaintenanceSha256 = $ExpectedMaintenanceSha256.ToLowerInvariant() +$queueRoot = Join-Path $AgentRoot "queue" +$failedRoot = Join-Path $queueRoot "failed" +$evidenceRoot = Join-Path $AgentRoot "evidence" +$sourcePath = Join-Path $queueRoot $QueueFileName +$destinationName = "stale-maintenance-$QueueFileName" +$destinationPath = Join-Path $failedRoot $destinationName +$maintenancePath = Join-Path $evidenceRoot $MaintenanceReceiptName +$runtimeWritePerformed = $false + +function Get-AgentSha256 { + param([string]$Path) + (Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant() +} + +function Write-AgentDurableJson { + param([object]$Value, [string]$Path) + + $temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))" + $stream = $null + $writer = $null + try { + $encoding = New-Object Text.UTF8Encoding($false) + $stream = [IO.File]::Open( + $temp, + [IO.FileMode]::CreateNew, + [IO.FileAccess]::Write, + [IO.FileShare]::None + ) + $writer = New-Object IO.StreamWriter($stream, $encoding) + $writer.Write(($Value | ConvertTo-Json -Depth 10)) + $writer.Flush() + $stream.Flush($true) + $writer.Dispose(); $writer = $null + $stream.Dispose(); $stream = $null + [IO.File]::Move($temp, $Path) + $hash = Get-AgentSha256 $Path + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "durable_receipt_readback_missing" + } + return $hash + } finally { + if ($writer) { $writer.Dispose() } + if ($stream) { $stream.Dispose() } + Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue + } +} + +function Test-AgentRecoveryMutexActive { + $mutex = $null + $acquired = $false + try { + $mutex = [Threading.Mutex]::new( + $false, + "Global\WoooAgent99RecoveryTransportV1" + ) + try { + $acquired = [bool]$mutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $acquired = $true + } + return [bool](-not $acquired) + } finally { + if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { $mutex.Dispose() } + } +} + +function Get-AgentActiveControlProcesses { + @( + Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop | + Where-Object { + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or + ([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)' + } | + ForEach-Object { + [pscustomobject]@{ + processId = [int]$_.ProcessId + creationDate = ([datetime]$_.CreationDate).ToUniversalTime().ToString("o") + } + } + ) +} + +function Get-AgentQuarantineGate { + if (-not (Test-Path -LiteralPath $sourcePath -PathType Leaf)) { + throw "exact_queue_source_missing" + } + if (Test-Path -LiteralPath $destinationPath) { + throw "quarantine_destination_already_exists" + } + if (-not (Test-Path -LiteralPath $maintenancePath -PathType Leaf)) { + throw "maintenance_receipt_missing" + } + $queueHash = Get-AgentSha256 $sourcePath + $maintenanceHash = Get-AgentSha256 $maintenancePath + if ($queueHash -ne $ExpectedQueueSha256) { throw "queue_hash_mismatch" } + if ($maintenanceHash -ne $ExpectedMaintenanceSha256) { + throw "maintenance_hash_mismatch" + } + + $task = Get-ScheduledTask ` + -TaskName "Wooo-Agent99-Control-Loop" ` + -TaskPath "\" ` + -ErrorAction Stop + if ($task.Settings.Enabled -or [string]$task.State -ne "Disabled") { + throw "control_loop_not_frozen" + } + $activeProcesses = Get-AgentActiveControlProcesses + if ($activeProcesses.Count -ne 0) { throw "active_control_or_recover_process" } + $leasePresent = [bool](Test-Path -LiteralPath ( + Join-Path $AgentRoot "state\recovery-transport-lease.json" + ) -PathType Leaf) + if ($leasePresent) { throw "recovery_lease_present" } + if (Test-AgentRecoveryMutexActive) { throw "recovery_mutex_active" } + + $maintenance = Get-Content -LiteralPath $maintenancePath -Raw | ConvertFrom-Json + if ( + [string]$maintenance.schemaVersion -ne "agent99_control_loop_maintenance_v2" -or + [string]$maintenance.mode -ne "Apply" -or + [string]$maintenance.taskName -ne "Wooo-Agent99-Control-Loop" -or + [string]$maintenance.terminal -ne "control_loop_maintenance_frozen" -or + $maintenance.precheckPassed -isnot [bool] -or -not $maintenance.precheckPassed -or + $maintenance.controlStopped -isnot [bool] -or -not $maintenance.controlStopped -or + $maintenance.recoverStopped -isnot [bool] -or -not $maintenance.recoverStopped -or + $maintenance.leaseMetadataRemoved -isnot [bool] -or -not $maintenance.leaseMetadataRemoved -or + $maintenance.runtimeWritePerformed -isnot [bool] -or -not $maintenance.runtimeWritePerformed -or + $maintenance.vmPowerChangePerformed -isnot [bool] -or $maintenance.vmPowerChangePerformed -or + $maintenance.vmwareVmxProcessTerminated -isnot [bool] -or $maintenance.vmwareVmxProcessTerminated -or + $maintenance.vmxLockDeleted -isnot [bool] -or $maintenance.vmxLockDeleted -or + $maintenance.hostRebootPerformed -isnot [bool] -or $maintenance.hostRebootPerformed -or + $maintenance.secretValueRead -isnot [bool] -or $maintenance.secretValueRead + ) { + throw "maintenance_receipt_contract_invalid" + } + + $command = Get-Content -LiteralPath $sourcePath -Raw | ConvertFrom-Json + if ( + [string]$command.id -ne $ExpectedCommandId -or + [string]$command.mode -ne "Recover" -or + [string]$command.source -ne "agent99-recovery-observer" -or + $command.controlledApply -isnot [bool] -or + -not $command.controlledApply -or + [string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or + [string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$' + ) { + throw "queue_identity_contract_invalid" + } + $sourceItem = Get-Item -LiteralPath $sourcePath + $ageMinutes = [math]::Round(((Get-Date) - $sourceItem.LastWriteTime).TotalMinutes, 1) + if ($ageMinutes -lt $MinimumAgeMinutes) { throw "queue_item_not_stale" } + + [pscustomobject]@{ + queueHash = $queueHash + maintenanceHash = $maintenanceHash + ageMinutes = $ageMinutes + commandId = [string]$command.id + commandMode = [string]$command.mode + commandSource = [string]$command.source + controlTaskEnabled = [bool]$task.Settings.Enabled + controlTaskState = [string]$task.State + activeControlProcessCount = [int]$activeProcesses.Count + recoveryLeasePresent = $leasePresent + recoveryMutexActive = $false + } +} + +$gate = Get-AgentQuarantineGate +$base = [ordered]@{ + schemaVersion = "agent99_stale_queue_quarantine_v1" + timestamp = (Get-Date).ToString("o") + mode = $Mode + traceId = $TraceId + runId = $RunId + workItemId = $WorkItemId + commandId = $gate.commandId + commandMode = $gate.commandMode + commandSource = $gate.commandSource + sourceFile = $QueueFileName + sourceSha256 = $gate.queueHash + destinationFile = $destinationName + ageMinutes = $gate.ageMinutes + minimumAgeMinutes = $MinimumAgeMinutes + maintenanceReceipt = $MaintenanceReceiptName + maintenanceReceiptSha256 = $gate.maintenanceHash + activeControlProcessCount = $gate.activeControlProcessCount + recoveryLeasePresent = $gate.recoveryLeasePresent + recoveryMutexActive = $gate.recoveryMutexActive + commandExecuted = $false + deletePerformed = $false + otherQueueMutationPerformed = $false + rawInstructionStored = $false + secretValueRead = $false + runtimeWritePerformed = $false +} + +if ($Mode -eq "Check") { + $base["terminal"] = "check_passed_apply_allowed" + $base | ConvertTo-Json -Depth 8 -Compress + exit 0 +} + +$writerMutex = [Threading.Mutex]::new( + $false, + "Global\WoooAgent99StaleQueueMaintenanceV1" +) +$writerAcquired = $false +$intentPath = "" +$intentHash = "" +$resultPath = "" +$moved = $false +$rollbackAttempted = $false +$rollbackVerified = $false +try { + try { + $writerAcquired = [bool]$writerMutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $writerAcquired = $true + } + if (-not $writerAcquired) { throw "queue_maintenance_writer_active" } + + $gate = Get-AgentQuarantineGate + $safeRunId = $RunId -replace "[^A-Za-z0-9._-]", "_" + $stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff" + $intentPath = Join-Path $evidenceRoot ( + "agent99-stale-queue-quarantine-$stamp-$safeRunId-intent.json" + ) + $resultPath = Join-Path $evidenceRoot ( + "agent99-stale-queue-quarantine-$stamp-$safeRunId-result.json" + ) + $intent = [ordered]@{} + $base + $intent["schemaVersion"] = "agent99_stale_queue_quarantine_intent_v1" + $intent["mode"] = "Apply" + $intent["plannedAction"] = "atomic_move_exact_queue_item_to_failed" + $intent["rollback"] = "atomic_move_back_if_post_move_hash_verifier_fails" + $intent["terminal"] = "intent_durable_apply_pending" + $intentHash = Write-AgentDurableJson $intent $intentPath + $runtimeWritePerformed = $true + + $gate = Get-AgentQuarantineGate + [IO.File]::Move($sourcePath, $destinationPath) + $moved = $true + $sourceAbsent = -not (Test-Path -LiteralPath $sourcePath) + $destinationPresent = Test-Path -LiteralPath $destinationPath -PathType Leaf + $destinationHash = if ($destinationPresent) { + Get-AgentSha256 $destinationPath + } else { "" } + if (-not ($sourceAbsent -and $destinationPresent -and $destinationHash -eq $ExpectedQueueSha256)) { + throw "post_move_verifier_failed" + } + + $result = [ordered]@{} + $base + $result["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1" + $result["mode"] = "Apply" + $result["intentFile"] = Split-Path -Leaf $intentPath + $result["intentSha256"] = $intentHash + $result["sourceAbsent"] = $sourceAbsent + $result["destinationPresent"] = $destinationPresent + $result["destinationSha256"] = $destinationHash + $result["runtimeWritePerformed"] = $true + $result["terminal"] = "verified_quarantined" + $resultHash = Write-AgentDurableJson $result $resultPath + $result["resultFile"] = Split-Path -Leaf $resultPath + $result["resultSha256"] = $resultHash + $result | ConvertTo-Json -Depth 8 -Compress + exit 0 +} catch { + $failureType = $_.Exception.GetType().Name + if ( + $moved -and + -not (Test-Path -LiteralPath $sourcePath) -and + (Test-Path -LiteralPath $destinationPath -PathType Leaf) + ) { + $rollbackAttempted = $true + if ((Get-AgentSha256 $destinationPath) -eq $ExpectedQueueSha256) { + [IO.File]::Move($destinationPath, $sourcePath) + $rollbackVerified = [bool]( + (Test-Path -LiteralPath $sourcePath -PathType Leaf) -and + -not (Test-Path -LiteralPath $destinationPath) -and + (Get-AgentSha256 $sourcePath) -eq $ExpectedQueueSha256 + ) + } + } + $failure = [ordered]@{} + $base + $failure["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1" + $failure["mode"] = "Apply" + $failure["intentFile"] = if ($intentPath) { Split-Path -Leaf $intentPath } else { "" } + $failure["intentSha256"] = $intentHash + $failure["failureType"] = $failureType + $failure["rollbackAttempted"] = $rollbackAttempted + $failure["rollbackVerified"] = $rollbackVerified + $failure["runtimeWritePerformed"] = $runtimeWritePerformed + $failure["terminal"] = if ($rollbackAttempted -and $rollbackVerified) { + "failed_rolled_back" + } elseif ($rollbackAttempted) { + "rollback_unverified" + } else { + "failed_no_queue_move" + } + if ($resultPath -and -not (Test-Path -LiteralPath $resultPath)) { + try { + $failureHash = Write-AgentDurableJson $failure $resultPath + $failure["resultFile"] = Split-Path -Leaf $resultPath + $failure["resultSha256"] = $failureHash + } catch {} + } + $failure | ConvertTo-Json -Depth 8 -Compress + exit 2 +} finally { + if ($writerAcquired) { try { $writerMutex.ReleaseMutex() } catch {} } + $writerMutex.Dispose() +} diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index cdf725aef..f63812070 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -5,6 +5,7 @@ import os import shutil import subprocess from copy import deepcopy +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any @@ -24,7 +25,7 @@ def _function_source() -> str: def _evidence_function_source() -> str: source = PREFLIGHT.read_text(encoding="utf-8") - start = source.index("function Get-EvidenceReadback") + start = source.index("function Get-AgentBoundedNewestFiles") end = source.index("$requiredTasks = @(", start) return source[start:end] @@ -52,7 +53,8 @@ def _base_case() -> dict[str, Any]: "staleAlertExecutionCount": 0, "alertIncomingCount": 0, "staleAlertIncomingCount": 0, - "pendingCommandCount": 0, + "pendingCommandBlockingCount": 0, + "promotionDeferredCommandCount": 0, } @@ -96,7 +98,8 @@ $parameters = @{{ StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount AlertIncomingCount = [int]$case.alertIncomingCount StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount - PendingCommandCount = [int]$case.pendingCommandCount + PendingCommandBlockingCount = [int]$case.pendingCommandBlockingCount + PromotionDeferredCommandCount = [int]$case.promotionDeferredCommandCount }} Get-AgentLivePreflightDecision @parameters | ConvertTo-Json -Depth 6 -Compress @@ -121,6 +124,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '$warningReasons += "self_health_warning"' in decision assert '$warningReasons += "performance_warning"' in decision assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision + assert ( + '$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"' + in decision + ) assert '"critical" { $blockingReasons += "self_health_not_ok" }' in decision assert 'default { $blockingReasons += "self_health_unclassified" }' in decision assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision @@ -147,9 +154,45 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '"agent99_background_mode_deferred_v1"' in source assert '"agent99_recovery_transport_deferred_v1"' in source assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source + assert "[IO.Directory]::EnumerateFiles(" in source + assert "[Collections.Generic.SortedSet[string]]::new(" in source + assert "$null = $selected.Remove($selected.Min)" in source + assert "candidateWindowExhausted = $candidateWindowExhausted" in source + assert "function Get-AgentControlLoopMaintenanceReadback" in source + assert '"agent99-control-loop-maintenance-2*.json"' in source + assert '$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 })' in source + assert '$legacyPropertySetMatches' in source + assert '$legacyIdentityFieldsValid' in source + assert '$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled' in source + assert '"legacy_v2_exact_upgrade_bridge"' in source + assert '$receipt.precheckPassed -is [bool]' in source + assert '$receipt.taskEnabledAfter -is [bool]' in source + assert '$receipt.runtimeWritePerformed -is [bool]' in source + assert '$intent.secretValueRead -is [bool]' in source + assert '$activeControlProcesses.Count -eq 0' in source + assert '-not $leasePresent' in source + assert '-not $recoveryMutexActive' in source + assert '$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")' in source + assert '$task | Add-Member -NotePropertyName runtimeHealthy' in source + assert '$task | Add-Member -NotePropertyName promotionReady' in source + assert '$controlLoopMaintenance.valid' in source + assert "function Get-AgentPendingCommandPromotionReadback" in source + assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source + assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source + assert '$payload.controlledApply -is [bool] -and $payload.controlledApply' in source + assert '$payload.observation -is [pscustomobject]' in source + assert 'instructionContentReadback = $false' in source assert '$rootTaskPath = "\\"' in source assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source + assert "[string[]]$RequiredArgumentMarkers" in source + assert "$actions.Count -eq 1" in source + assert "actionDefinitionReady = $actionDefinitionReady" in source + assert "principalReady = $principalReady" in source + assert "definitionReady = $definitionReady" in source + assert '$task.Principal.LogonType -eq "S4U"' in source + assert '$task.Principal.RunLevel -eq "Highest"' in source + assert "$task.definitionReady" in source def test_evidence_selection_skips_deferred_but_not_malformed_candidates( @@ -274,6 +317,165 @@ Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 | assert payload["healthy"] is False +def test_evidence_candidate_window_fails_closed_when_recent_files_are_all_deferred( + tmp_path: Path, +) -> None: + powershell = shutil.which("pwsh") or shutil.which("powershell") + if powershell is None: + pytest.skip("PowerShell runtime is not installed on this macOS verifier") + + evidence = tmp_path / "evidence" + evidence.mkdir() + canonical = evidence / "agent99-SelfCheck-20260715-190000.json" + canonical.write_text( + json.dumps( + { + "mode": "SelfCheck", + "selfHealth": {"severity": "ok"}, + } + ), + encoding="utf-8", + ) + base_time = canonical.stat().st_mtime + os.utime(canonical, (base_time - 1000, base_time - 1000)) + for index in range(40): + deferred = evidence / f"agent99-SelfCheck-20260715-20{index:04d}.json" + deferred.write_text( + json.dumps( + { + "schemaVersion": "agent99_background_mode_deferred_v1", + "deferred": True, + } + ), + encoding="utf-8", + ) + candidate_time = base_time + index + os.utime(deferred, (candidate_time, candidate_time)) + + agent_root = str(tmp_path).replace("'", "''") + command = f""" +{_evidence_function_source()} +$AgentRoot = '{agent_root}' +Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true | + ConvertTo-Json -Depth 8 -Compress +""" + result = subprocess.run( + [powershell, "-NoProfile", "-NonInteractive", "-Command", command], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + payload = json.loads( + next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{")) + ) + assert payload["enumeratedCandidateCount"] == 41 + assert payload["candidateWindowLimit"] == 32 + assert payload["candidateWindowTruncated"] is True + assert payload["candidateWindowExhausted"] is True + assert payload["deferredCandidateCount"] == 32 + assert payload["parseError"] == "EvidenceCandidateWindowExhausted" + assert payload["healthy"] is False + + +def test_promotion_reserve_defers_only_exact_maintenance_auto_recover( + tmp_path: Path, +) -> None: + powershell = shutil.which("pwsh") or shutil.which("powershell") + if powershell is None: + pytest.skip("PowerShell runtime is not installed on this macOS verifier") + + queue = tmp_path / "queue" + queue.mkdir() + now = datetime.now(timezone.utc) + incident_id = "agent99-auto-" + ("a" * 20) + command_id = "auto-recover-20260719-152746-39ca4a72" + command_path = queue / f"{command_id}.json" + payload = { + "approvalId": "", + "automationRunId": "096c8173-96b9-5659-94d3-5679c3fb770f", + "canonicalDigest": "b" * 64, + "controlledApply": True, + "correlationKey": "agent99-controlled:" + ("c" * 64), + "createdAt": now.isoformat(), + "dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1", + "dispatchRouteId": "agent99_local_observer", + "executionGeneration": "1", + "externalId": "host_unreachable_detected", + "id": command_id, + "idempotencyKey": "agent99-controlled:" + ("c" * 64), + "incidentId": incident_id, + "instruction": "recover exact observed unreachable host through controlled playbook", + "lockOwner": "096c8173-96b9-5659-94d3-5679c3fb770f", + "mode": "Recover", + "observation": {}, + "projectId": "awoooi", + "reason": "host_unreachable_detected", + "requestedBy": "Agent99", + "singleFlightKey": "agent99-controlled-flight:" + ("d" * 64), + "source": "agent99-recovery-observer", + "sourceFingerprint": "e" * 64, + "traceId": "00-" + ("f" * 32) + "-" + ("1" * 16) + "-01", + "workItemId": f"agent99-auto:{incident_id}:host_unreachable_detected", + } + + function_source = _evidence_function_source() + agent_root = str(tmp_path).replace("'", "''") + maintenance_timestamp = (now - timedelta(minutes=10)).isoformat() + + def run(scope: str) -> dict[str, Any]: + command = f""" +{function_source} +$AgentRoot = '{agent_root}' +$maintenance = [pscustomobject]@{{ + valid = $true + receiptFieldsValid = $true + intentFieldsValid = $true + receiptTimestamp = '{maintenance_timestamp}' +}} +Get-AgentPendingCommandPromotionReadback -MaintenanceReadback $maintenance -Scope {scope} | + ConvertTo-Json -Depth 8 -Compress +""" + result = subprocess.run( + [powershell, "-NoProfile", "-NonInteractive", "-Command", command], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + return json.loads( + next( + line + for line in reversed(result.stdout.splitlines()) + if line.startswith("{") + ) + ) + + command_path.write_bytes(b"\xef\xbb\xbf" + json.dumps(payload).encode("utf-8")) + reserve = run("PromotionReserve") + assert reserve["observedCount"] == 1 + assert reserve["deferredCount"] == 1 + assert reserve["blockingCount"] == 0 + assert reserve["contractValid"] is True + assert reserve["items"][0]["instructionContentReadback"] is False + assert reserve["items"][0]["identity"].startswith(f"{command_id}|") + + full_runtime = run("FullRuntime") + assert full_runtime["deferredCount"] == 0 + assert full_runtime["blockingCount"] == 1 + assert full_runtime["eligibleMaintenanceAutoRecoverCount"] == 1 + assert full_runtime["reason"] == ( + "full_runtime_requires_maintenance_auto_recover_to_drain" + ) + + payload["mode"] = "Status" + command_path.write_text(json.dumps(payload), encoding="utf-8") + invalid = run("PromotionReserve") + assert invalid["deferredCount"] == 0 + assert invalid["blockingCount"] == 1 + assert invalid["contractValid"] is False + + def test_decision_behavior_when_powershell_is_available() -> None: powershell = shutil.which("pwsh") or shutil.which("powershell") if powershell is None: @@ -348,11 +550,17 @@ def test_decision_behavior_when_powershell_is_available() -> None: (), ), ( - _with_overrides(pendingCommandCount=1), + _with_overrides(pendingCommandBlockingCount=1), False, ("pending_work_before_reboot",), (), ), + ( + _with_overrides(promotionDeferredCommandCount=1), + True, + (), + ("maintenance_pending_commands_deferred_until_post_promotion",), + ), ) for case, expected_eligible, expected_blockers, expected_warnings in cases: diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index c00631130..aee8f27e2 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -391,11 +391,39 @@ def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() -> assert "failedTasks" in source assert "failedEvidenceNames" in source assert "failedEvidence" in source + assert "actionDefinitionReady = [bool]$_.actionDefinitionReady" in source + assert "principalReady = [bool]$_.principalReady" in source + assert "definitionReady = [bool]$_.definitionReady" in source assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source +def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + assert "$LivePreflightTimeoutSeconds = 60" in source + assert "$PostVerifyMaxAttempts = 4" in source + assert "$PostVerifyRetryWaitSeconds = 45" in source + assert "function Test-AgentPostPromotionReadinessRetryable" in source + assert '[string]$Readiness.readinessScope -ne "FullRuntime"' in source + assert '"required_evidence_stale"' in source + assert '"required_evidence_content_unhealthy"' in source + assert '$_.name -notin @("telegram_inbox", "sre_alert_inbox")' in source + assert "-not $_.definitionReady" in source + assert "[int]$Readiness.alertRunningCount -ne 0" in source + assert "$pendingCommandsRetryable" in source + assert "$ExpectedMaintenancePendingCommandIdentities" in source + assert '"pending_work_before_reboot"' in source + assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source + assert "maintenancePendingCommandIdentities" in source + assert "$postVerifyAttempt -le $PostVerifyMaxAttempts" in source + assert "Start-Sleep -Seconds $PostVerifyRetryWaitSeconds" in source + assert "automaticTaskTriggerAllowed = $false" in source + assert "postVerifyAttempts = $postVerifyAttempts" in source + assert "postVerifyRetryCount = $postVerifyRetryCount" in source + + def test_check_and_apply_receipts_project_the_same_run_identity() -> None: sender = SENDER.read_text(encoding="utf-8") projector = sender[ @@ -607,7 +635,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert "$RelayPollIntervalSeconds = 1" in source assert "$ValidateOnlyTimeoutSeconds = 300" in source assert "$LiveDeployTimeoutSeconds = 300" in source - assert "$LivePreflightTimeoutSeconds = 120" in source + assert "$LivePreflightTimeoutSeconds = 60" in source assert "$PromotionReadinessMaxAttempts = 2" in source assert "$PromotionReadinessRetryWaitSeconds = 30" in source assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source @@ -648,18 +676,23 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati promotion_readiness_attempts = 2 promotion_readiness_retry_wait_seconds = 30 deploy_seconds = 300 - live_preflight_seconds = 120 + live_preflight_seconds = 60 + post_verify_attempts = 4 + post_verify_retry_wait_seconds = 45 + windows_control_baseline_seconds = 300 worst_case_seconds = ( validate_seconds + (promotion_readiness_attempts * live_preflight_seconds) + promotion_readiness_retry_wait_seconds + deploy_seconds - + live_preflight_seconds + + (post_verify_attempts * live_preflight_seconds) + + ((post_verify_attempts - 1) * post_verify_retry_wait_seconds) + (2 * (60 + 360)) + + windows_control_baseline_seconds ) assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source - assert worst_case_seconds == 1830 - assert wrapper_budget_seconds - worst_case_seconds == 570 + assert worst_case_seconds == 2265 + assert wrapper_budget_seconds - worst_case_seconds == 135 deploy = source.index( "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath " @@ -781,6 +814,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() - assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight assert '[int]$summary.alertIncomingCount -eq 0' not in preflight assert '[int]$summary.alertRunningCount -eq 0' in preflight + assert '[int]$summary.pendingCommandBlockingCount -eq 0' in preflight + assert '$ReadinessScope -eq "PromotionReserve"' in preflight assert '[int]$summary.pendingCommandCount -eq 0' in preflight assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight assert '[string[]]$blockingReasons = @()' in preflight diff --git a/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py b/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py new file mode 100644 index 000000000..49eb93bb7 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py @@ -0,0 +1,44 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1" + + +def test_stale_queue_quarantine_is_exact_recoverable_and_receipted() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert '[ValidateSet("Check", "Apply")]' in source + assert "ExpectedQueueSha256" in source + assert "ExpectedMaintenanceSha256" in source + assert '"Global\\WoooAgent99StaleQueueMaintenanceV1"' in source + assert '"Global\\WoooAgent99RecoveryTransportV1"' in source + assert 'Get-ScheduledTask `' in source + assert '"Wooo-Agent99-Control-Loop"' in source + assert 'throw "control_loop_not_frozen"' in source + assert 'throw "active_control_or_recover_process"' in source + assert 'throw "recovery_lease_present"' in source + assert 'throw "recovery_mutex_active"' in source + assert '[string]$command.mode -ne "Recover"' in source + assert '[string]$command.source -ne "agent99-recovery-observer"' in source + assert "$command.controlledApply -isnot [bool]" in source + assert '"atomic_move_exact_queue_item_to_failed"' in source + assert "[IO.File]::Move($sourcePath, $destinationPath)" in source + assert "[IO.File]::Move($destinationPath, $sourcePath)" in source + assert '"verified_quarantined"' in source + assert '"failed_rolled_back"' in source + assert '"rollback_unverified"' in source + + +def test_stale_queue_quarantine_does_not_execute_or_delete_command() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert "commandExecuted = $false" in source + assert "deletePerformed = $false" in source + assert "otherQueueMutationPerformed = $false" in source + assert "rawInstructionStored = $false" in source + assert "secretValueRead = $false" in source + assert "Remove-Item -LiteralPath $sourcePath" not in source + assert "Start-ScheduledTask" not in source + assert "Invoke-Expression" not in source + assert "& $command" not in source