fix(agent99): unblock bounded maintenance deployment
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 1m9s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 10m26s
CD Pipeline / revalidate-deploy-carrier (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / revalidate-post-deploy-carrier (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
MCP External Protocol Replay / replay-and-verify (push) Successful in 44s
MCP External Artifact Mirror / mirror-and-verify (push) Successful in 33s
E2E Health Check / e2e-health (push) Failing after 48s
AI 技術雷達監控 / ai-technology-watch (push) Successful in 34s
Agent Version Lifecycle Watch / version-lifecycle-watch (push) Successful in 7s
Agent Market Watch / market-watch (push) Failing after 26s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 23s

This commit is contained in:
Your Name
2026-07-19 16:04:18 +08:00
parent 3512d0c490
commit 50c7a7f7f5
6 changed files with 1342 additions and 65 deletions

View File

@@ -28,7 +28,8 @@ function Get-AgentLivePreflightDecision {
[int]$StaleAlertExecutionCount,
[int]$AlertIncomingCount,
[int]$StaleAlertIncomingCount,
[int]$PendingCommandCount
[int]$PendingCommandBlockingCount,
[int]$PromotionDeferredCommandCount
)
$blockingReasons = @()
@@ -89,11 +90,14 @@ function Get-AgentLivePreflightDecision {
}
if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" }
if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) {
if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandBlockingCount -gt 0) {
$blockingReasons += "pending_work_before_reboot"
} elseif ($AlertIncomingCount -gt 0) {
$warningReasons += "fresh_alert_pending_next_inbox_tick"
}
if ($PromotionDeferredCommandCount -gt 0) {
$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"
}
return [pscustomobject]@{
deploymentEligible = [bool]($blockingReasons.Count -eq 0)
@@ -110,7 +114,8 @@ function Get-AgentLivePreflightDecision {
function Get-TaskReadback {
param(
[string]$TaskPath,
[string]$TaskName
[string]$TaskName,
[string[]]$RequiredArgumentMarkers
)
try {
@@ -120,8 +125,34 @@ function Get-TaskReadback {
$info = Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop
$lastResult = [int64]$info.LastTaskResult
$state = [string]$task.State
$actions = @($task.Actions)
$actionDefinitionReady = [bool](
$actions.Count -eq 1 -and
[string]$actions[0].Execute -match '(?i)\\WindowsPowerShell\\v1\.0\\powershell\.exe$'
)
if ($actionDefinitionReady) {
foreach ($marker in @($RequiredArgumentMarkers)) {
if (
-not $marker -or
([string]$actions[0].Arguments).IndexOf(
$marker,
[StringComparison]::OrdinalIgnoreCase
) -lt 0
) {
$actionDefinitionReady = $false
break
}
}
}
$principalReady = [bool](
[string]$task.Principal.UserId -match '(?i)(^|\\)Administrator$' -and
[string]$task.Principal.LogonType -eq "S4U" -and
[string]$task.Principal.RunLevel -eq "Highest"
)
$definitionReady = [bool]($actionDefinitionReady -and $principalReady)
$healthy = [bool](
$task.Settings.Enabled -and
$definitionReady -and
($lastResult -in @(0, 267009, 267011) -or $state -eq "Running")
)
@@ -134,6 +165,9 @@ function Get-TaskReadback {
lastTaskResult = $lastResult
lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" }
nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" }
actionDefinitionReady = $actionDefinitionReady
principalReady = $principalReady
definitionReady = $definitionReady
healthy = $healthy
}
} catch {
@@ -146,6 +180,9 @@ function Get-TaskReadback {
lastTaskResult = $null
lastRunTime = ""
nextRunTime = ""
actionDefinitionReady = $false
principalReady = $false
definitionReady = $false
healthy = $false
}
}
@@ -158,25 +195,448 @@ function Get-DirectoryReadback {
)
$path = Join-Path $AgentRoot $RelativePath
$files = @()
if (Test-Path $path) {
$files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue)
if ($RootQueueFilesOnly) {
$files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" })
$exists = [bool](Test-Path -LiteralPath $path -PathType Container)
$count = 0
$staleCount = 0
$oldest = $null
$newest = $null
if ($exists) {
foreach ($candidatePath in [IO.Directory]::EnumerateFiles(
$path,
"*.json",
[IO.SearchOption]::TopDirectoryOnly
)) {
$name = [IO.Path]::GetFileName($candidatePath)
if ($RootQueueFilesOnly -and $name -match "^(processed|failed|running)-") {
continue
}
$file = [IO.FileInfo]::new($candidatePath)
$count += 1
if (-not $oldest -or $file.LastWriteTimeUtc -lt $oldest.LastWriteTimeUtc) {
$oldest = $file
}
if (-not $newest -or $file.LastWriteTimeUtc -gt $newest.LastWriteTimeUtc) {
$newest = $file
}
if (((Get-Date) - $file.LastWriteTime).TotalMinutes -gt 5) {
$staleCount += 1
}
}
$files = @($files | Sort-Object LastWriteTime)
}
return [pscustomobject]@{
relativePath = $RelativePath
exists = [bool](Test-Path $path)
count = [int]$files.Count
oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" }
newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" }
staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count
exists = $exists
count = [int]$count
oldest = if ($oldest) { $oldest.LastWriteTime.ToString("o") } else { "" }
newest = if ($newest) { $newest.LastWriteTime.ToString("o") } else { "" }
staleOverFiveMinutes = [int]$staleCount
}
}
function Get-AgentBoundedNewestFiles {
param(
[string]$DirectoryPath,
[string]$Pattern,
[ValidateRange(1, 256)]
[int]$CandidateWindow = 32
)
# Agent99 evidence names contain a fixed sortable timestamp and are immutable.
# Keep only the lexically newest bounded window in a native .NET set so a
# large evidence directory never becomes a PowerShell object-sort workload.
$selected = [Collections.Generic.SortedSet[string]]::new(
[StringComparer]::OrdinalIgnoreCase
)
$enumeratedCount = 0
if (Test-Path -LiteralPath $DirectoryPath -PathType Container) {
foreach ($candidatePath in [IO.Directory]::EnumerateFiles(
$DirectoryPath,
$Pattern,
[IO.SearchOption]::TopDirectoryOnly
)) {
$enumeratedCount += 1
$null = $selected.Add($candidatePath)
if ($selected.Count -gt $CandidateWindow) {
$null = $selected.Remove($selected.Min)
}
}
}
return [pscustomobject]@{
files = @($selected | Sort-Object -Descending | ForEach-Object {
[IO.FileInfo]::new($_)
})
enumeratedCount = [int]$enumeratedCount
candidateWindow = [int]$CandidateWindow
windowTruncated = [bool]($enumeratedCount -gt $selected.Count)
}
}
function Get-AgentControlLoopMaintenanceReadback {
$evidenceDir = Join-Path $AgentRoot "evidence"
$selection = Get-AgentBoundedNewestFiles `
$evidenceDir `
"agent99-control-loop-maintenance-2*.json" `
8
$file = @($selection.files | Select-Object -First 1)
$receipt = $null
$intent = $null
$parseError = ""
if ($file.Count -eq 1) {
try {
$receipt = Get-Content -LiteralPath $file[0].FullName -Raw | ConvertFrom-Json
} catch {
$parseError = $_.Exception.GetType().Name
}
}
$intentRef = if ($receipt -and $receipt.PSObject.Properties["operationIntentRef"]) {
[string]$receipt.operationIntentRef
} else { "" }
$intentRefSafe = [bool](
$intentRef -match "^agent99-control-loop-maintenance-intent-[0-9]{8}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$"
)
$intentPath = if ($intentRefSafe) { Join-Path $evidenceDir $intentRef } else { "" }
if ($intentPath -and (Test-Path -LiteralPath $intentPath -PathType Leaf)) {
try {
$intent = Get-Content -LiteralPath $intentPath -Raw | ConvertFrom-Json
} catch {
$parseError = $_.Exception.GetType().Name
}
}
$legacyPropertySet = @(
"blockers", "controlStopped", "expectedControlCreated", "expectedControlPid",
"expectedRecoverCreated", "expectedRecoverParentPid", "expectedRecoverPid",
"hostRebootPerformed", "leaseMetadataRemoved", "mode", "precheckPassed",
"recoverStopped", "runtimeWritePerformed", "schemaVersion", "secretValueRead",
"taskEnabled", "taskName", "terminal", "timestamp", "vmPowerChangePerformed",
"vmwareVmxProcessTerminated", "vmxLockDeleted"
)
$receiptPropertySet = if ($receipt) {
@($receipt.PSObject.Properties.Name | Sort-Object)
} else { @() }
$legacyPropertySetMatches = [bool](
$receiptPropertySet.Count -eq $legacyPropertySet.Count -and
(@($receiptPropertySet) -join ",") -eq (@($legacyPropertySet | Sort-Object) -join ",")
)
$legacyIdentityFieldsValid = [bool](
$receipt -and
$receipt.expectedControlPid -is [int] -and [int]$receipt.expectedControlPid -gt 0 -and
$receipt.expectedRecoverPid -is [int] -and [int]$receipt.expectedRecoverPid -gt 0 -and
$receipt.expectedRecoverParentPid -is [int] -and [int]$receipt.expectedRecoverParentPid -gt 0 -and
[string]$receipt.expectedControlCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and
[string]$receipt.expectedRecoverCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and
[string]$receipt.timestamp -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T"
)
$legacyReceipt = [bool](
$receipt -and
-not $receipt.PSObject.Properties["operationIntentRef"] -and
$receipt.PSObject.Properties["taskEnabled"] -and
$legacyPropertySetMatches -and
$legacyIdentityFieldsValid -and
@($receipt.blockers).Count -eq 0
)
$ageMinutes = if ($file.Count -eq 1) {
[math]::Round(((Get-Date) - $file[0].LastWriteTime).TotalMinutes, 1)
} else { $null }
$activeControlProcesses = @(Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction SilentlyContinue |
Where-Object {
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)'
})
$leasePresent = [bool](Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json") -PathType Leaf)
$recoveryMutexActive = $true
$mutex = $null
$mutexAcquired = $false
try {
$mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1")
try {
$mutexAcquired = [bool]$mutex.WaitOne(0)
} catch [Threading.AbandonedMutexException] {
$mutexAcquired = $true
}
$recoveryMutexActive = [bool](-not $mutexAcquired)
} finally {
if ($mutexAcquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
if ($mutex) { $mutex.Dispose() }
}
$commonReceiptFieldsValid = [bool](
$receipt -and
[string]$receipt.schemaVersion -eq "agent99_control_loop_maintenance_v2" -and
[string]$receipt.mode -eq "Apply" -and
[string]$receipt.taskName -eq "Wooo-Agent99-Control-Loop" -and
[string]$receipt.terminal -eq "control_loop_maintenance_frozen" -and
$receipt.precheckPassed -is [bool] -and $receipt.precheckPassed -and
$receipt.controlStopped -is [bool] -and $receipt.controlStopped -and
$receipt.recoverStopped -is [bool] -and $receipt.recoverStopped -and
$receipt.leaseMetadataRemoved -is [bool] -and $receipt.leaseMetadataRemoved -and
$receipt.runtimeWritePerformed -is [bool] -and $receipt.runtimeWritePerformed -and
$receipt.vmPowerChangePerformed -is [bool] -and -not $receipt.vmPowerChangePerformed -and
$receipt.vmwareVmxProcessTerminated -is [bool] -and -not $receipt.vmwareVmxProcessTerminated -and
$receipt.vmxLockDeleted -is [bool] -and -not $receipt.vmxLockDeleted -and
$receipt.hostRebootPerformed -is [bool] -and -not $receipt.hostRebootPerformed -and
$receipt.secretValueRead -is [bool] -and -not $receipt.secretValueRead
)
$modernReceiptFieldsValid = [bool](
$commonReceiptFieldsValid -and
-not $legacyReceipt -and
$receipt.taskEnabledAfter -is [bool] -and -not $receipt.taskEnabledAfter
)
$legacyReceiptFieldsValid = [bool](
$commonReceiptFieldsValid -and
$legacyReceipt -and
$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled
)
$receiptFieldsValid = [bool]($modernReceiptFieldsValid -or $legacyReceiptFieldsValid)
$intentFieldsValid = [bool](
$legacyReceipt -or
(
$intent -and
[string]$intent.schemaVersion -eq "agent99_control_loop_maintenance_intent_v1" -and
[string]$intent.taskName -eq "Wooo-Agent99-Control-Loop" -and
$intent.vmPowerChangePerformed -is [bool] -and -not $intent.vmPowerChangePerformed -and
$intent.vmwareVmxProcessTerminated -is [bool] -and -not $intent.vmwareVmxProcessTerminated -and
$intent.secretValueRead -is [bool] -and -not $intent.secretValueRead -and
(@($intent.plannedActions) -join ",") -eq "disable_control_task,stop_exact_control_tree,stop_exact_recover_tree,remove_exact_recovery_lease"
)
)
$valid = [bool](
-not $parseError -and
$file.Count -eq 1 -and
$null -ne $ageMinutes -and
$ageMinutes -ge 0 -and
$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 }) -and
($legacyReceipt -or $intentRefSafe) -and
$receiptFieldsValid -and
$intentFieldsValid -and
$activeControlProcesses.Count -eq 0 -and
-not $leasePresent -and
-not $recoveryMutexActive
)
return [pscustomobject]@{
valid = $valid
receiptFile = if ($file.Count -eq 1) { $file[0].Name } else { "" }
ageMinutes = $ageMinutes
parseError = $parseError
intentRefPresent = $intentRefSafe
contractVersion = if ($legacyReceipt) { "legacy_v2_exact_upgrade_bridge" } else { "intent_bound_v2" }
receiptTimestamp = if ($receipt) { [string]$receipt.timestamp } else { "" }
receiptLastWriteTime = if ($file.Count -eq 1) { $file[0].LastWriteTime.ToString("o") } else { "" }
legacyPropertySetMatches = $legacyPropertySetMatches
legacyIdentityFieldsValid = $legacyIdentityFieldsValid
receiptFieldsValid = $receiptFieldsValid
intentFieldsValid = $intentFieldsValid
activeControlProcessCount = [int]$activeControlProcesses.Count
recoveryLeasePresent = $leasePresent
recoveryMutexActive = $recoveryMutexActive
secretValueRead = $false
remoteWritePerformed = $false
}
}
function Get-AgentPendingCommandPromotionReadback {
param(
[object]$MaintenanceReadback,
[ValidateSet("FullRuntime", "PromotionReserve")]
[string]$Scope
)
$queueRoot = Join-Path $AgentRoot "queue"
$files = @()
if (Test-Path -LiteralPath $queueRoot -PathType Container) {
$files = @([IO.Directory]::EnumerateFiles(
$queueRoot,
"*.json",
[IO.SearchOption]::TopDirectoryOnly
) | Where-Object {
[IO.Path]::GetFileName($_) -notmatch "^(processed|failed|running)-"
})
}
$observedCount = [int]$files.Count
$result = [ordered]@{
scope = $Scope
observedCount = $observedCount
deferredCount = 0
blockingCount = $observedCount
eligibleMaintenanceAutoRecoverCount = 0
boundedLimit = 8
contractValid = [bool]($observedCount -eq 0)
reason = if ($observedCount -eq 0) { "queue_empty" } else { "pending_commands_require_runtime" }
items = @()
rawInstructionStored = $false
secretValueRead = $false
remoteWritePerformed = $false
}
if ($observedCount -eq 0) {
return [pscustomobject]$result
}
if (
-not $MaintenanceReadback -or
-not $MaintenanceReadback.receiptFieldsValid -or
-not $MaintenanceReadback.intentFieldsValid
) {
$result.reason = "maintenance_contract_invalid"
return [pscustomobject]$result
}
if ($Scope -eq "PromotionReserve" -and -not $MaintenanceReadback.valid) {
$result.reason = "maintenance_current_state_invalid"
return [pscustomobject]$result
}
if ($observedCount -gt $result.boundedLimit) {
$result.reason = "maintenance_pending_command_limit_exceeded"
return [pscustomobject]$result
}
$maintenanceTimestamp = [datetimeoffset]::MinValue
$maintenanceTimestampValid = [datetimeoffset]::TryParse(
[string]$MaintenanceReadback.receiptTimestamp,
[ref]$maintenanceTimestamp
)
if (-not $maintenanceTimestampValid) {
$result.reason = "maintenance_timestamp_invalid"
return [pscustomobject]$result
}
$requiredPropertySet = @(
"approvalId", "automationRunId", "canonicalDigest", "controlledApply",
"correlationKey", "createdAt", "dispatchIdentitySchemaVersion",
"dispatchRouteId", "executionGeneration", "externalId", "id",
"idempotencyKey", "incidentId", "instruction", "lockOwner", "mode",
"observation", "projectId", "reason", "requestedBy", "singleFlightKey",
"source", "sourceFingerprint", "traceId", "workItemId"
) | Sort-Object
$safeItems = @()
$allValid = $true
foreach ($path in $files) {
$file = [IO.FileInfo]::new($path)
$payload = $null
$parseError = ""
$sha256 = ""
try {
$bytes = [IO.File]::ReadAllBytes($path)
$sha = [Security.Cryptography.SHA256]::Create()
try {
$sha256 = -join @($sha.ComputeHash($bytes) | ForEach-Object { $_.ToString("x2") })
} finally {
$sha.Dispose()
}
$jsonOffset = if (
$bytes.Length -ge 3 -and
$bytes[0] -eq 0xef -and
$bytes[1] -eq 0xbb -and
$bytes[2] -eq 0xbf
) { 3 } else { 0 }
$jsonText = [Text.Encoding]::UTF8.GetString(
$bytes,
$jsonOffset,
$bytes.Length - $jsonOffset
)
$payload = $jsonText | ConvertFrom-Json
} catch {
$parseError = $_.Exception.GetType().Name
}
$propertySet = if ($payload) {
@($payload.PSObject.Properties.Name | Sort-Object)
} else { @() }
$propertySetMatches = [bool](
$propertySet.Count -eq $requiredPropertySet.Count -and
(@($propertySet) -join ",") -eq (@($requiredPropertySet) -join ",")
)
$createdAt = [datetimeoffset]::MinValue
$createdAtValid = [bool](
$payload -and
[datetimeoffset]::TryParse([string]$payload.createdAt, [ref]$createdAt)
)
$ageMinutes = if ($createdAtValid) {
[math]::Round(([datetimeoffset]::Now - $createdAt).TotalMinutes, 1)
} else { $null }
$fileBaseName = [IO.Path]::GetFileNameWithoutExtension($file.Name)
$nativeStringFields = @(
"approvalId", "automationRunId", "canonicalDigest", "correlationKey",
"createdAt", "dispatchIdentitySchemaVersion", "dispatchRouteId",
"executionGeneration", "externalId", "id", "idempotencyKey",
"incidentId", "instruction", "lockOwner", "mode", "projectId",
"reason", "requestedBy", "singleFlightKey", "source",
"sourceFingerprint", "traceId", "workItemId"
)
$nativeStringTypesValid = [bool](
$payload -and
@($nativeStringFields | Where-Object { $payload.$_ -isnot [string] }).Count -eq 0
)
$identityValid = [bool](
-not $parseError -and
$propertySetMatches -and
$nativeStringTypesValid -and
$payload.controlledApply -is [bool] -and $payload.controlledApply -and
$payload.observation -is [pscustomobject] -and
[string]$payload.id -eq $fileBaseName -and
[string]$payload.id -match "^auto-recover-[0-9]{8}-[0-9]{6}-[0-9a-f]{8}$" -and
[string]$payload.mode -eq "Recover" -and
[string]$payload.source -eq "agent99-recovery-observer" -and
[string]$payload.dispatchIdentitySchemaVersion -eq "agent99_controlled_dispatch_identity_v1" -and
[string]$payload.dispatchRouteId -eq "agent99_local_observer" -and
[string]$payload.projectId -eq "awoooi" -and
[string]$payload.requestedBy -eq "Agent99" -and
[string]$payload.reason -eq "host_unreachable_detected" -and
[string]$payload.externalId -eq "host_unreachable_detected" -and
[string]$payload.approvalId -eq "" -and
[string]$payload.automationRunId -match "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -and
[string]$payload.lockOwner -eq [string]$payload.automationRunId -and
[string]$payload.executionGeneration -match "^[1-9][0-9]*$" -and
[string]$payload.canonicalDigest -match "^[0-9a-f]{64}$" -and
[string]$payload.sourceFingerprint -match "^[0-9a-f]{64}$" -and
[string]$payload.correlationKey -match "^agent99-controlled:[0-9a-f]{64}$" -and
[string]$payload.idempotencyKey -eq [string]$payload.correlationKey -and
[string]$payload.singleFlightKey -match "^agent99-controlled-flight:[0-9a-f]{64}$" -and
[string]$payload.incidentId -match "^agent99-auto-[0-9a-f]{20}$" -and
[string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):host_unreachable_detected" -and
[string]$payload.traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$" -and
-not [string]::IsNullOrWhiteSpace([string]$payload.instruction) -and
$createdAtValid -and
$createdAt -ge $maintenanceTimestamp -and
$ageMinutes -ge 0 -and
$ageMinutes -le 120
)
if (-not $identityValid) { $allValid = $false }
$safeItems += [pscustomobject]@{
file = $file.Name
id = if ($payload) { [string]$payload.id } else { "" }
mode = if ($payload) { [string]$payload.mode } else { "" }
source = if ($payload) { [string]$payload.source } else { "" }
workItemId = if ($payload) { [string]$payload.workItemId } else { "" }
createdAt = if ($payload) { [string]$payload.createdAt } else { "" }
ageMinutes = $ageMinutes
sha256 = $sha256
identity = if ($payload -and $sha256) { "$([string]$payload.id)|$sha256" } else { "" }
parseError = $parseError
propertySetMatches = $propertySetMatches
identityValid = $identityValid
instructionContentReadback = $false
}
}
$result.items = $safeItems
if ($allValid) {
$result.eligibleMaintenanceAutoRecoverCount = $observedCount
$result.contractValid = $true
if ($Scope -eq "PromotionReserve") {
$result.deferredCount = $observedCount
$result.blockingCount = 0
$result.reason = "bounded_maintenance_auto_recover_deferred_until_post_promotion"
} else {
$result.reason = "full_runtime_requires_maintenance_auto_recover_to_drain"
}
} else {
$result.reason = "maintenance_pending_command_contract_invalid"
}
return [pscustomobject]$result
}
function Get-EvidenceReadback {
param(
[string]$Name,
@@ -187,8 +647,8 @@ function Get-EvidenceReadback {
)
$evidenceDir = Join-Path $AgentRoot "evidence"
$candidates = @(Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue |
Sort-Object LastWriteTime -Descending)
$candidateSelection = Get-AgentBoundedNewestFiles $evidenceDir $Pattern
$candidates = @($candidateSelection.files)
$deferredCandidates = @()
$file = $null
foreach ($candidate in $candidates) {
@@ -219,6 +679,11 @@ function Get-EvidenceReadback {
$file = $candidate
break
}
$candidateWindowExhausted = [bool](
-not $file -and
$candidateSelection.windowTruncated -and
$deferredCandidates.Count -eq $candidates.Count
)
$ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null }
$fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes)
$freshnessHeadroomMinutes = if ($file) { [math]::Round($MaxAgeMinutes - $ageMinutes, 1) } else { $null }
@@ -228,7 +693,7 @@ function Get-EvidenceReadback {
$freshnessHeadroomMinutes -ge $MinimumFreshnessReserveMinutes
)
$safeSummary = $null
$parseError = ""
$parseError = if ($candidateWindowExhausted) { "EvidenceCandidateWindowExhausted" } else { "" }
$contentHealthy = $true
if ($file) {
try {
@@ -389,23 +854,52 @@ function Get-EvidenceReadback {
parseError = $parseError
deferredCandidateCount = [int]$deferredCandidates.Count
latestDeferredTime = if ($deferredCandidates.Count -gt 0) { $deferredCandidates[0].LastWriteTime.ToString("o") } else { "" }
enumeratedCandidateCount = [int]$candidateSelection.enumeratedCount
candidateWindowLimit = [int]$candidateSelection.candidateWindow
candidateWindowTruncated = [bool]$candidateSelection.windowTruncated
candidateWindowExhausted = $candidateWindowExhausted
safeSummary = $safeSummary
}
}
$rootTaskPath = "\"
$requiredTasks = @(
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery" },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat" },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard" },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop" },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox" },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox" },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay" },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health" },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health" }
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery"; markers = @("agent99-submit-request.ps1", "-Mode Recover", "-Source agent99-startup-recovery") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat"; markers = @("agent99-run.ps1", "-Mode Status") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard"; markers = @("agent99-run.ps1", "-Mode Perf", "-ControlledApply") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop"; markers = @("agent99-run.ps1", "-Mode ControlTick") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox"; markers = @("agent99-telegram-inbox.ps1", "-RunNow") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox"; markers = @("agent99-sre-alert-inbox.ps1", "-RunNow") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay"; markers = @("agent99-sre-alert-relay.ps1") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health"; markers = @("agent99-run.ps1", "-Mode SelfCheck") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health"; markers = @("agent99-run.ps1", "-Mode BackupCheck") }
)
$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback -TaskPath $_.taskPath -TaskName $_.name })
$tasks = @($requiredTasks | ForEach-Object {
Get-TaskReadback `
-TaskPath $_.taskPath `
-TaskName $_.name `
-RequiredArgumentMarkers $_.markers
})
$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback
if ($ReadinessScope -eq "PromotionReserve") {
foreach ($task in $tasks) {
$runtimeHealthy = [bool]$task.healthy
$maintenanceFreezeAccepted = [bool](
$task.name -eq "Wooo-Agent99-Control-Loop" -and
-not $task.enabled -and
$controlLoopMaintenance.valid
)
$promotionReady = [bool](
$task.exists -and
$task.definitionReady -and
($task.enabled -or $maintenanceFreezeAccepted)
)
$task | Add-Member -NotePropertyName runtimeHealthy -NotePropertyValue $runtimeHealthy -Force
$task | Add-Member -NotePropertyName promotionReady -NotePropertyValue $promotionReady -Force
$task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force
$task.healthy = $promotionReady
}
}
$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
$manifest = [pscustomobject]@{
@@ -457,14 +951,15 @@ $queues = @(
(Get-DirectoryReadback "queue" -RootQueueFilesOnly),
(Get-DirectoryReadback "queue\failed")
)
$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")
$evidence = @(
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0)
)
@@ -482,6 +977,9 @@ $selfHealthPerformanceIssueCount = if ($selfCheckEvidence -and $selfCheckEvidenc
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
$pendingCommandPromotion = Get-AgentPendingCommandPromotionReadback `
-MaintenanceReadback $controlLoopMaintenance `
-Scope $ReadinessScope
$decision = Get-AgentLivePreflightDecision `
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
-Manifest $manifest `
@@ -498,7 +996,8 @@ $decision = Get-AgentLivePreflightDecision `
-StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes `
-AlertIncomingCount $alertIncoming.count `
-StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes `
-PendingCommandCount $commandQueue.count
-PendingCommandBlockingCount $pendingCommandPromotion.blockingCount `
-PromotionDeferredCommandCount $pendingCommandPromotion.deferredCount
$blockingReasons = @($decision.blockingReasons)
$warningReasons = @($decision.warningReasons)
@@ -513,10 +1012,12 @@ $result = [pscustomobject]@{
secretValueRead = $false
remoteWritePerformed = $false
agentRoot = $AgentRoot
controlLoopMaintenance = $controlLoopMaintenance
manifest = $manifest
tasks = $tasks
relayListeners = $relayListeners
queues = $queues
pendingCommandPromotion = $pendingCommandPromotion
evidence = $evidence
summary = [pscustomobject]@{
requiredTaskCount = $requiredTasks.Count
@@ -536,6 +1037,8 @@ $result = [pscustomobject]@{
alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes
alertRunningCount = [int]$alertRunning.count
pendingCommandCount = [int]$commandQueue.count
pendingCommandBlockingCount = [int]$pendingCommandPromotion.blockingCount
promotionDeferredCommandCount = [int]$pendingCommandPromotion.deferredCount
relayListenerCount = $relayListeners.Count
preflightGreen = [bool]$decision.deploymentEligible
deploymentEligible = [bool]$decision.deploymentEligible

View File

@@ -23,7 +23,9 @@ $RelayStartTimeoutSeconds = 360
$RelayPollIntervalSeconds = 1
$ValidateOnlyTimeoutSeconds = 300
$LiveDeployTimeoutSeconds = 300
$LivePreflightTimeoutSeconds = 120
$LivePreflightTimeoutSeconds = 60
$PostVerifyMaxAttempts = 4
$PostVerifyRetryWaitSeconds = 45
$WindowsControlBaselineTimeoutSeconds = 300
$PromotionReadinessReserveMinutes = 15
$PromotionReadinessMaxAttempts = 2
@@ -434,6 +436,9 @@ function Invoke-AgentLivePreflight {
enabled = [bool]$_.enabled
state = if ($state -in @("Ready", "Running", "Disabled", "Queued", "Missing")) { $state } else { "Unknown" }
lastTaskResult = if ($null -ne $_.lastTaskResult) { [long]$_.lastTaskResult } else { $null }
actionDefinitionReady = [bool]$_.actionDefinitionReady
principalReady = [bool]$_.principalReady
definitionReady = [bool]$_.definitionReady
}
})
} else { @() }
@@ -465,6 +470,14 @@ function Invoke-AgentLivePreflight {
$ReadinessScope -eq "PromotionReserve" -or
($payload -and $payload.manifest -and [bool]$payload.manifest.runtimeMatched)
)
$pendingCommandsReadyForScope = [bool](
$summary -and
[int]$summary.pendingCommandBlockingCount -eq 0 -and
(
$ReadinessScope -eq "PromotionReserve" -or
[int]$summary.pendingCommandCount -eq 0
)
)
$runtimePlaneReady = [bool](
$acceptedExitCode -and
-not $processResult.timedOut -and
@@ -482,7 +495,7 @@ function Invoke-AgentLivePreflight {
[int]$summary.relayListenerCount -gt 0 -and
[int]$summary.alertIncomingStaleCount -eq 0 -and
[int]$summary.alertRunningCount -eq 0 -and
[int]$summary.pendingCommandCount -eq 0 -and
$pendingCommandsReadyForScope -and
$runtimePlaneBlockingReasons.Count -eq 0
)
return [pscustomobject]@{
@@ -521,6 +534,16 @@ function Invoke-AgentLivePreflight {
alertIncomingStaleCount = if ($summary) { [int]$summary.alertIncomingStaleCount } else { -1 }
alertRunningCount = if ($summary) { [int]$summary.alertRunningCount } else { -1 }
pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 }
pendingCommandBlockingCount = if ($summary) { [int]$summary.pendingCommandBlockingCount } else { -1 }
promotionDeferredCommandCount = if ($summary) { [int]$summary.promotionDeferredCommandCount } else { -1 }
pendingCommandDeferralReason = if ($payload -and $payload.pendingCommandPromotion) {
[string]$payload.pendingCommandPromotion.reason
} else { "" }
maintenancePendingCommandIdentities = if ($payload -and $payload.pendingCommandPromotion) {
@($payload.pendingCommandPromotion.items | Where-Object {
[bool]$_.identityValid -and [string]$_.identity -match "^auto-recover-[^|]+\|[0-9a-f]{64}$"
} | ForEach-Object { [string]$_.identity } | Sort-Object -Unique)
} else { @() }
preflightGreen = [bool]($summary -and $summary.preflightGreen)
overallPreflightGreen = [bool]($summary -and $summary.preflightGreen)
runtimePlaneReady = $runtimePlaneReady
@@ -579,6 +602,96 @@ function Test-AgentPromotionReadinessRetryable {
return $true
}
function Test-AgentPostPromotionReadinessRetryable {
param(
[object]$Readiness,
[string[]]$ExpectedMaintenancePendingCommandIdentities = @()
)
$pendingCount = if ($Readiness) { [int]$Readiness.pendingCommandCount } else { -1 }
$pendingIdentities = if ($Readiness) {
@($Readiness.maintenancePendingCommandIdentities | ForEach-Object { [string]$_ } | Sort-Object -Unique)
} else { @() }
$expectedIdentities = @($ExpectedMaintenancePendingCommandIdentities | ForEach-Object { [string]$_ } | Sort-Object -Unique)
$pendingCommandsRetryable = [bool](
$pendingCount -eq 0 -or
(
$pendingCount -gt 0 -and
[int]$Readiness.pendingCommandBlockingCount -eq $pendingCount -and
[string]$Readiness.pendingCommandDeferralReason -eq "full_runtime_requires_maintenance_auto_recover_to_drain" -and
$pendingIdentities.Count -eq $pendingCount -and
$expectedIdentities.Count -gt 0 -and
@($pendingIdentities | Where-Object { $_ -notin $expectedIdentities }).Count -eq 0
)
)
if (
-not $Readiness -or
$Readiness.ok -or
-not $Readiness.parsed -or
-not $Readiness.scopeMatched -or
[string]$Readiness.readinessScope -ne "FullRuntime" -or
$Readiness.timedOut -or
-not $Readiness.acceptedExitCode -or
[int]$Readiness.relayListenerCount -lt 1 -or
[int]$Readiness.alertRunningCount -ne 0 -or
-not $pendingCommandsRetryable
) {
return $false
}
$allowedReasons = @(
"scheduled_task_unhealthy",
"required_evidence_stale",
"required_evidence_freshness_reserve_insufficient",
"required_evidence_content_unhealthy",
"pending_work_before_reboot"
)
$reasons = @($Readiness.runtimePlaneBlockingReasons | ForEach-Object { [string]$_ })
if (
$reasons.Count -eq 0 -or
@($reasons | Where-Object { $_ -notin $allowedReasons }).Count -gt 0
) {
return $false
}
if (@($Readiness.failedTasks | Where-Object {
-not $_.exists -or
-not $_.enabled -or
-not $_.definitionReady -or
$_.taskPath -ne "\" -or
$_.state -notin @("Ready", "Running", "Queued")
}).Count -gt 0) {
return $false
}
if (
"scheduled_task_unhealthy" -in $reasons -and
@($Readiness.failedTasks).Count -eq 0
) { return $false }
if (
"pending_work_before_reboot" -in $reasons -and
$pendingCount -eq 0
) { return $false }
if (
@($reasons | Where-Object {
$_ -in @(
"required_evidence_stale",
"required_evidence_freshness_reserve_insufficient",
"required_evidence_content_unhealthy"
)
}).Count -gt 0 -and
@($Readiness.failedEvidence).Count -eq 0
) { return $false }
if (@($Readiness.failedEvidence | Where-Object {
-not $_.parsed -or
(
-not $_.contentHealthy -and
$_.name -notin @("telegram_inbox", "sre_alert_inbox")
)
}).Count -gt 0) {
return $false
}
return $true
}
function Get-AgentRelayRuntime {
$task = $null
$taskError = ""
@@ -1405,6 +1518,8 @@ try {
$relayReload = $null
$runtimeManifest = $null
$livePreflight = $null
$postVerifyAttempts = @()
$postVerifyRetryCount = 0
$launcherContract = $null
$windowsControlBaseline = $null
$postVerified = $false
@@ -1442,25 +1557,40 @@ try {
$script:ReceiverOperationPhase = "independent_post_verifier"
$runtimeManifest = Get-AgentSafeRuntimeManifest
$launcherContract = Get-AgentLauncherContract
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$postVerified = [bool](
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 18 -and
$runtimeManifest.mismatchCount -eq 0 -and
$launcherContract.ok -and
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
$relayReload.oldGenerationsGone -and
$relayReload.newGenerationVerified -and
$relayReload.after.taskEnabled -and
$relayReload.after.taskRunning -and
$livePreflight.ok -and
$livePreflight.sourceRevision -eq $sourceRevision -and
$livePreflight.runtimeMatched -and
$livePreflight.relayListenerCount -gt 0 -and
(Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations))
)
for ($postVerifyAttempt = 1; $postVerifyAttempt -le $PostVerifyMaxAttempts; $postVerifyAttempt += 1) {
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$livePreflight | Add-Member -NotePropertyName attempt -NotePropertyValue $postVerifyAttempt -Force
$postVerifyAttempts += $livePreflight
$postVerified = [bool](
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 18 -and
$runtimeManifest.mismatchCount -eq 0 -and
$launcherContract.ok -and
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
$relayReload.oldGenerationsGone -and
$relayReload.newGenerationVerified -and
$relayReload.after.taskEnabled -and
$relayReload.after.taskRunning -and
$livePreflight.ok -and
$livePreflight.sourceRevision -eq $sourceRevision -and
$livePreflight.runtimeMatched -and
$livePreflight.relayListenerCount -gt 0 -and
(Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations))
)
if ($postVerified) { break }
if (
$postVerifyAttempt -ge $PostVerifyMaxAttempts -or
-not (Test-AgentPostPromotionReadinessRetryable `
$livePreflight `
-ExpectedMaintenancePendingCommandIdentities @(
$prePromotionReadiness.maintenancePendingCommandIdentities
))
) { break }
Start-Sleep -Seconds $PostVerifyRetryWaitSeconds
}
$postVerifyRetryCount = [math]::Max(0, $postVerifyAttempts.Count - 1)
if (-not $postVerified) {
$failurePhase = "post_verifier"
$failureType = "independent_post_verifier_failed"
@@ -1504,6 +1634,14 @@ try {
runtimeManifest = $runtimeManifest
launcherContract = $launcherContract
livePreflight = $livePreflight
postVerifyAttempts = $postVerifyAttempts
postVerifyRetryCount = $postVerifyRetryCount
postVerifyPolicy = [pscustomobject]@{
maxAttempts = $PostVerifyMaxAttempts
retryWaitSeconds = $PostVerifyRetryWaitSeconds
automaticTaskTriggerAllowed = $false
finalAttemptVerified = $postVerified
}
windowsControlBaseline = $windowsControlBaseline
durableReceiptWritten = $true
idempotentReplay = $false
@@ -1629,6 +1767,8 @@ try {
failedRuntimeManifest = $runtimeManifest
failedLauncherContract = $launcherContract
failedLivePreflight = $livePreflight
postVerifyAttempts = $postVerifyAttempts
postVerifyRetryCount = $postVerifyRetryCount
failedWindowsControlBaseline = $windowsControlBaseline
rollback = $rollback
rollbackRelayRestart = $rollbackRelayRestart

View File

@@ -0,0 +1,347 @@
[CmdletBinding()]
param(
[ValidateSet("Check", "Apply")]
[string]$Mode = "Check",
[Parameter(Mandatory = $true)]
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
[string]$TraceId,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
[string]$RunId,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
[string]$WorkItemId,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._-]{0,180}\.json$")]
[string]$QueueFileName,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-fA-F0-9]{64}$")]
[string]$ExpectedQueueSha256,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
[string]$ExpectedCommandId,
[Parameter(Mandatory = $true)]
[ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}\.json$")]
[string]$MaintenanceReceiptName,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-fA-F0-9]{64}$")]
[string]$ExpectedMaintenanceSha256,
[ValidateRange(30, 1440)]
[int]$MinimumAgeMinutes = 30,
[string]$AgentRoot = "C:\Wooo\Agent99"
)
$ErrorActionPreference = "Stop"
$ExpectedQueueSha256 = $ExpectedQueueSha256.ToLowerInvariant()
$ExpectedMaintenanceSha256 = $ExpectedMaintenanceSha256.ToLowerInvariant()
$queueRoot = Join-Path $AgentRoot "queue"
$failedRoot = Join-Path $queueRoot "failed"
$evidenceRoot = Join-Path $AgentRoot "evidence"
$sourcePath = Join-Path $queueRoot $QueueFileName
$destinationName = "stale-maintenance-$QueueFileName"
$destinationPath = Join-Path $failedRoot $destinationName
$maintenancePath = Join-Path $evidenceRoot $MaintenanceReceiptName
$runtimeWritePerformed = $false
function Get-AgentSha256 {
param([string]$Path)
(Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant()
}
function Write-AgentDurableJson {
param([object]$Value, [string]$Path)
$temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
$stream = $null
$writer = $null
try {
$encoding = New-Object Text.UTF8Encoding($false)
$stream = [IO.File]::Open(
$temp,
[IO.FileMode]::CreateNew,
[IO.FileAccess]::Write,
[IO.FileShare]::None
)
$writer = New-Object IO.StreamWriter($stream, $encoding)
$writer.Write(($Value | ConvertTo-Json -Depth 10))
$writer.Flush()
$stream.Flush($true)
$writer.Dispose(); $writer = $null
$stream.Dispose(); $stream = $null
[IO.File]::Move($temp, $Path)
$hash = Get-AgentSha256 $Path
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
throw "durable_receipt_readback_missing"
}
return $hash
} finally {
if ($writer) { $writer.Dispose() }
if ($stream) { $stream.Dispose() }
Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue
}
}
function Test-AgentRecoveryMutexActive {
$mutex = $null
$acquired = $false
try {
$mutex = [Threading.Mutex]::new(
$false,
"Global\WoooAgent99RecoveryTransportV1"
)
try {
$acquired = [bool]$mutex.WaitOne(0)
} catch [Threading.AbandonedMutexException] {
$acquired = $true
}
return [bool](-not $acquired)
} finally {
if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
if ($mutex) { $mutex.Dispose() }
}
}
function Get-AgentActiveControlProcesses {
@(
Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop |
Where-Object {
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)'
} |
ForEach-Object {
[pscustomobject]@{
processId = [int]$_.ProcessId
creationDate = ([datetime]$_.CreationDate).ToUniversalTime().ToString("o")
}
}
)
}
function Get-AgentQuarantineGate {
if (-not (Test-Path -LiteralPath $sourcePath -PathType Leaf)) {
throw "exact_queue_source_missing"
}
if (Test-Path -LiteralPath $destinationPath) {
throw "quarantine_destination_already_exists"
}
if (-not (Test-Path -LiteralPath $maintenancePath -PathType Leaf)) {
throw "maintenance_receipt_missing"
}
$queueHash = Get-AgentSha256 $sourcePath
$maintenanceHash = Get-AgentSha256 $maintenancePath
if ($queueHash -ne $ExpectedQueueSha256) { throw "queue_hash_mismatch" }
if ($maintenanceHash -ne $ExpectedMaintenanceSha256) {
throw "maintenance_hash_mismatch"
}
$task = Get-ScheduledTask `
-TaskName "Wooo-Agent99-Control-Loop" `
-TaskPath "\" `
-ErrorAction Stop
if ($task.Settings.Enabled -or [string]$task.State -ne "Disabled") {
throw "control_loop_not_frozen"
}
$activeProcesses = Get-AgentActiveControlProcesses
if ($activeProcesses.Count -ne 0) { throw "active_control_or_recover_process" }
$leasePresent = [bool](Test-Path -LiteralPath (
Join-Path $AgentRoot "state\recovery-transport-lease.json"
) -PathType Leaf)
if ($leasePresent) { throw "recovery_lease_present" }
if (Test-AgentRecoveryMutexActive) { throw "recovery_mutex_active" }
$maintenance = Get-Content -LiteralPath $maintenancePath -Raw | ConvertFrom-Json
if (
[string]$maintenance.schemaVersion -ne "agent99_control_loop_maintenance_v2" -or
[string]$maintenance.mode -ne "Apply" -or
[string]$maintenance.taskName -ne "Wooo-Agent99-Control-Loop" -or
[string]$maintenance.terminal -ne "control_loop_maintenance_frozen" -or
$maintenance.precheckPassed -isnot [bool] -or -not $maintenance.precheckPassed -or
$maintenance.controlStopped -isnot [bool] -or -not $maintenance.controlStopped -or
$maintenance.recoverStopped -isnot [bool] -or -not $maintenance.recoverStopped -or
$maintenance.leaseMetadataRemoved -isnot [bool] -or -not $maintenance.leaseMetadataRemoved -or
$maintenance.runtimeWritePerformed -isnot [bool] -or -not $maintenance.runtimeWritePerformed -or
$maintenance.vmPowerChangePerformed -isnot [bool] -or $maintenance.vmPowerChangePerformed -or
$maintenance.vmwareVmxProcessTerminated -isnot [bool] -or $maintenance.vmwareVmxProcessTerminated -or
$maintenance.vmxLockDeleted -isnot [bool] -or $maintenance.vmxLockDeleted -or
$maintenance.hostRebootPerformed -isnot [bool] -or $maintenance.hostRebootPerformed -or
$maintenance.secretValueRead -isnot [bool] -or $maintenance.secretValueRead
) {
throw "maintenance_receipt_contract_invalid"
}
$command = Get-Content -LiteralPath $sourcePath -Raw | ConvertFrom-Json
if (
[string]$command.id -ne $ExpectedCommandId -or
[string]$command.mode -ne "Recover" -or
[string]$command.source -ne "agent99-recovery-observer" -or
$command.controlledApply -isnot [bool] -or
-not $command.controlledApply -or
[string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or
[string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$'
) {
throw "queue_identity_contract_invalid"
}
$sourceItem = Get-Item -LiteralPath $sourcePath
$ageMinutes = [math]::Round(((Get-Date) - $sourceItem.LastWriteTime).TotalMinutes, 1)
if ($ageMinutes -lt $MinimumAgeMinutes) { throw "queue_item_not_stale" }
[pscustomobject]@{
queueHash = $queueHash
maintenanceHash = $maintenanceHash
ageMinutes = $ageMinutes
commandId = [string]$command.id
commandMode = [string]$command.mode
commandSource = [string]$command.source
controlTaskEnabled = [bool]$task.Settings.Enabled
controlTaskState = [string]$task.State
activeControlProcessCount = [int]$activeProcesses.Count
recoveryLeasePresent = $leasePresent
recoveryMutexActive = $false
}
}
$gate = Get-AgentQuarantineGate
$base = [ordered]@{
schemaVersion = "agent99_stale_queue_quarantine_v1"
timestamp = (Get-Date).ToString("o")
mode = $Mode
traceId = $TraceId
runId = $RunId
workItemId = $WorkItemId
commandId = $gate.commandId
commandMode = $gate.commandMode
commandSource = $gate.commandSource
sourceFile = $QueueFileName
sourceSha256 = $gate.queueHash
destinationFile = $destinationName
ageMinutes = $gate.ageMinutes
minimumAgeMinutes = $MinimumAgeMinutes
maintenanceReceipt = $MaintenanceReceiptName
maintenanceReceiptSha256 = $gate.maintenanceHash
activeControlProcessCount = $gate.activeControlProcessCount
recoveryLeasePresent = $gate.recoveryLeasePresent
recoveryMutexActive = $gate.recoveryMutexActive
commandExecuted = $false
deletePerformed = $false
otherQueueMutationPerformed = $false
rawInstructionStored = $false
secretValueRead = $false
runtimeWritePerformed = $false
}
if ($Mode -eq "Check") {
$base["terminal"] = "check_passed_apply_allowed"
$base | ConvertTo-Json -Depth 8 -Compress
exit 0
}
$writerMutex = [Threading.Mutex]::new(
$false,
"Global\WoooAgent99StaleQueueMaintenanceV1"
)
$writerAcquired = $false
$intentPath = ""
$intentHash = ""
$resultPath = ""
$moved = $false
$rollbackAttempted = $false
$rollbackVerified = $false
try {
try {
$writerAcquired = [bool]$writerMutex.WaitOne(0)
} catch [Threading.AbandonedMutexException] {
$writerAcquired = $true
}
if (-not $writerAcquired) { throw "queue_maintenance_writer_active" }
$gate = Get-AgentQuarantineGate
$safeRunId = $RunId -replace "[^A-Za-z0-9._-]", "_"
$stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
$intentPath = Join-Path $evidenceRoot (
"agent99-stale-queue-quarantine-$stamp-$safeRunId-intent.json"
)
$resultPath = Join-Path $evidenceRoot (
"agent99-stale-queue-quarantine-$stamp-$safeRunId-result.json"
)
$intent = [ordered]@{} + $base
$intent["schemaVersion"] = "agent99_stale_queue_quarantine_intent_v1"
$intent["mode"] = "Apply"
$intent["plannedAction"] = "atomic_move_exact_queue_item_to_failed"
$intent["rollback"] = "atomic_move_back_if_post_move_hash_verifier_fails"
$intent["terminal"] = "intent_durable_apply_pending"
$intentHash = Write-AgentDurableJson $intent $intentPath
$runtimeWritePerformed = $true
$gate = Get-AgentQuarantineGate
[IO.File]::Move($sourcePath, $destinationPath)
$moved = $true
$sourceAbsent = -not (Test-Path -LiteralPath $sourcePath)
$destinationPresent = Test-Path -LiteralPath $destinationPath -PathType Leaf
$destinationHash = if ($destinationPresent) {
Get-AgentSha256 $destinationPath
} else { "" }
if (-not ($sourceAbsent -and $destinationPresent -and $destinationHash -eq $ExpectedQueueSha256)) {
throw "post_move_verifier_failed"
}
$result = [ordered]@{} + $base
$result["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1"
$result["mode"] = "Apply"
$result["intentFile"] = Split-Path -Leaf $intentPath
$result["intentSha256"] = $intentHash
$result["sourceAbsent"] = $sourceAbsent
$result["destinationPresent"] = $destinationPresent
$result["destinationSha256"] = $destinationHash
$result["runtimeWritePerformed"] = $true
$result["terminal"] = "verified_quarantined"
$resultHash = Write-AgentDurableJson $result $resultPath
$result["resultFile"] = Split-Path -Leaf $resultPath
$result["resultSha256"] = $resultHash
$result | ConvertTo-Json -Depth 8 -Compress
exit 0
} catch {
$failureType = $_.Exception.GetType().Name
if (
$moved -and
-not (Test-Path -LiteralPath $sourcePath) -and
(Test-Path -LiteralPath $destinationPath -PathType Leaf)
) {
$rollbackAttempted = $true
if ((Get-AgentSha256 $destinationPath) -eq $ExpectedQueueSha256) {
[IO.File]::Move($destinationPath, $sourcePath)
$rollbackVerified = [bool](
(Test-Path -LiteralPath $sourcePath -PathType Leaf) -and
-not (Test-Path -LiteralPath $destinationPath) -and
(Get-AgentSha256 $sourcePath) -eq $ExpectedQueueSha256
)
}
}
$failure = [ordered]@{} + $base
$failure["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1"
$failure["mode"] = "Apply"
$failure["intentFile"] = if ($intentPath) { Split-Path -Leaf $intentPath } else { "" }
$failure["intentSha256"] = $intentHash
$failure["failureType"] = $failureType
$failure["rollbackAttempted"] = $rollbackAttempted
$failure["rollbackVerified"] = $rollbackVerified
$failure["runtimeWritePerformed"] = $runtimeWritePerformed
$failure["terminal"] = if ($rollbackAttempted -and $rollbackVerified) {
"failed_rolled_back"
} elseif ($rollbackAttempted) {
"rollback_unverified"
} else {
"failed_no_queue_move"
}
if ($resultPath -and -not (Test-Path -LiteralPath $resultPath)) {
try {
$failureHash = Write-AgentDurableJson $failure $resultPath
$failure["resultFile"] = Split-Path -Leaf $resultPath
$failure["resultSha256"] = $failureHash
} catch {}
}
$failure | ConvertTo-Json -Depth 8 -Compress
exit 2
} finally {
if ($writerAcquired) { try { $writerMutex.ReleaseMutex() } catch {} }
$writerMutex.Dispose()
}

View File

@@ -5,6 +5,7 @@ import os
import shutil
import subprocess
from copy import deepcopy
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any
@@ -24,7 +25,7 @@ def _function_source() -> str:
def _evidence_function_source() -> str:
source = PREFLIGHT.read_text(encoding="utf-8")
start = source.index("function Get-EvidenceReadback")
start = source.index("function Get-AgentBoundedNewestFiles")
end = source.index("$requiredTasks = @(", start)
return source[start:end]
@@ -52,7 +53,8 @@ def _base_case() -> dict[str, Any]:
"staleAlertExecutionCount": 0,
"alertIncomingCount": 0,
"staleAlertIncomingCount": 0,
"pendingCommandCount": 0,
"pendingCommandBlockingCount": 0,
"promotionDeferredCommandCount": 0,
}
@@ -96,7 +98,8 @@ $parameters = @{{
StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
AlertIncomingCount = [int]$case.alertIncomingCount
StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount
PendingCommandCount = [int]$case.pendingCommandCount
PendingCommandBlockingCount = [int]$case.pendingCommandBlockingCount
PromotionDeferredCommandCount = [int]$case.promotionDeferredCommandCount
}}
Get-AgentLivePreflightDecision @parameters |
ConvertTo-Json -Depth 6 -Compress
@@ -121,6 +124,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
assert '$warningReasons += "self_health_warning"' in decision
assert '$warningReasons += "performance_warning"' in decision
assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision
assert (
'$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"'
in decision
)
assert '"critical" { $blockingReasons += "self_health_not_ok" }' in decision
assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
@@ -147,9 +154,45 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
assert '"agent99_background_mode_deferred_v1"' in source
assert '"agent99_recovery_transport_deferred_v1"' in source
assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source
assert "[IO.Directory]::EnumerateFiles(" in source
assert "[Collections.Generic.SortedSet[string]]::new(" in source
assert "$null = $selected.Remove($selected.Min)" in source
assert "candidateWindowExhausted = $candidateWindowExhausted" in source
assert "function Get-AgentControlLoopMaintenanceReadback" in source
assert '"agent99-control-loop-maintenance-2*.json"' in source
assert '$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 })' in source
assert '$legacyPropertySetMatches' in source
assert '$legacyIdentityFieldsValid' in source
assert '$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled' in source
assert '"legacy_v2_exact_upgrade_bridge"' in source
assert '$receipt.precheckPassed -is [bool]' in source
assert '$receipt.taskEnabledAfter -is [bool]' in source
assert '$receipt.runtimeWritePerformed -is [bool]' in source
assert '$intent.secretValueRead -is [bool]' in source
assert '$activeControlProcesses.Count -eq 0' in source
assert '-not $leasePresent' in source
assert '-not $recoveryMutexActive' in source
assert '$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")' in source
assert '$task | Add-Member -NotePropertyName runtimeHealthy' in source
assert '$task | Add-Member -NotePropertyName promotionReady' in source
assert '$controlLoopMaintenance.valid' in source
assert "function Get-AgentPendingCommandPromotionReadback" in source
assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source
assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
assert '$payload.controlledApply -is [bool] -and $payload.controlledApply' in source
assert '$payload.observation -is [pscustomobject]' in source
assert 'instructionContentReadback = $false' in source
assert '$rootTaskPath = "\\"' in source
assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source
assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source
assert "[string[]]$RequiredArgumentMarkers" in source
assert "$actions.Count -eq 1" in source
assert "actionDefinitionReady = $actionDefinitionReady" in source
assert "principalReady = $principalReady" in source
assert "definitionReady = $definitionReady" in source
assert '$task.Principal.LogonType -eq "S4U"' in source
assert '$task.Principal.RunLevel -eq "Highest"' in source
assert "$task.definitionReady" in source
def test_evidence_selection_skips_deferred_but_not_malformed_candidates(
@@ -274,6 +317,165 @@ Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 |
assert payload["healthy"] is False
def test_evidence_candidate_window_fails_closed_when_recent_files_are_all_deferred(
tmp_path: Path,
) -> None:
powershell = shutil.which("pwsh") or shutil.which("powershell")
if powershell is None:
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
evidence = tmp_path / "evidence"
evidence.mkdir()
canonical = evidence / "agent99-SelfCheck-20260715-190000.json"
canonical.write_text(
json.dumps(
{
"mode": "SelfCheck",
"selfHealth": {"severity": "ok"},
}
),
encoding="utf-8",
)
base_time = canonical.stat().st_mtime
os.utime(canonical, (base_time - 1000, base_time - 1000))
for index in range(40):
deferred = evidence / f"agent99-SelfCheck-20260715-20{index:04d}.json"
deferred.write_text(
json.dumps(
{
"schemaVersion": "agent99_background_mode_deferred_v1",
"deferred": True,
}
),
encoding="utf-8",
)
candidate_time = base_time + index
os.utime(deferred, (candidate_time, candidate_time))
agent_root = str(tmp_path).replace("'", "''")
command = f"""
{_evidence_function_source()}
$AgentRoot = '{agent_root}'
Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true |
ConvertTo-Json -Depth 8 -Compress
"""
result = subprocess.run(
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stdout + result.stderr
payload = json.loads(
next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{"))
)
assert payload["enumeratedCandidateCount"] == 41
assert payload["candidateWindowLimit"] == 32
assert payload["candidateWindowTruncated"] is True
assert payload["candidateWindowExhausted"] is True
assert payload["deferredCandidateCount"] == 32
assert payload["parseError"] == "EvidenceCandidateWindowExhausted"
assert payload["healthy"] is False
def test_promotion_reserve_defers_only_exact_maintenance_auto_recover(
tmp_path: Path,
) -> None:
powershell = shutil.which("pwsh") or shutil.which("powershell")
if powershell is None:
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
queue = tmp_path / "queue"
queue.mkdir()
now = datetime.now(timezone.utc)
incident_id = "agent99-auto-" + ("a" * 20)
command_id = "auto-recover-20260719-152746-39ca4a72"
command_path = queue / f"{command_id}.json"
payload = {
"approvalId": "",
"automationRunId": "096c8173-96b9-5659-94d3-5679c3fb770f",
"canonicalDigest": "b" * 64,
"controlledApply": True,
"correlationKey": "agent99-controlled:" + ("c" * 64),
"createdAt": now.isoformat(),
"dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1",
"dispatchRouteId": "agent99_local_observer",
"executionGeneration": "1",
"externalId": "host_unreachable_detected",
"id": command_id,
"idempotencyKey": "agent99-controlled:" + ("c" * 64),
"incidentId": incident_id,
"instruction": "recover exact observed unreachable host through controlled playbook",
"lockOwner": "096c8173-96b9-5659-94d3-5679c3fb770f",
"mode": "Recover",
"observation": {},
"projectId": "awoooi",
"reason": "host_unreachable_detected",
"requestedBy": "Agent99",
"singleFlightKey": "agent99-controlled-flight:" + ("d" * 64),
"source": "agent99-recovery-observer",
"sourceFingerprint": "e" * 64,
"traceId": "00-" + ("f" * 32) + "-" + ("1" * 16) + "-01",
"workItemId": f"agent99-auto:{incident_id}:host_unreachable_detected",
}
function_source = _evidence_function_source()
agent_root = str(tmp_path).replace("'", "''")
maintenance_timestamp = (now - timedelta(minutes=10)).isoformat()
def run(scope: str) -> dict[str, Any]:
command = f"""
{function_source}
$AgentRoot = '{agent_root}'
$maintenance = [pscustomobject]@{{
valid = $true
receiptFieldsValid = $true
intentFieldsValid = $true
receiptTimestamp = '{maintenance_timestamp}'
}}
Get-AgentPendingCommandPromotionReadback -MaintenanceReadback $maintenance -Scope {scope} |
ConvertTo-Json -Depth 8 -Compress
"""
result = subprocess.run(
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stdout + result.stderr
return json.loads(
next(
line
for line in reversed(result.stdout.splitlines())
if line.startswith("{")
)
)
command_path.write_bytes(b"\xef\xbb\xbf" + json.dumps(payload).encode("utf-8"))
reserve = run("PromotionReserve")
assert reserve["observedCount"] == 1
assert reserve["deferredCount"] == 1
assert reserve["blockingCount"] == 0
assert reserve["contractValid"] is True
assert reserve["items"][0]["instructionContentReadback"] is False
assert reserve["items"][0]["identity"].startswith(f"{command_id}|")
full_runtime = run("FullRuntime")
assert full_runtime["deferredCount"] == 0
assert full_runtime["blockingCount"] == 1
assert full_runtime["eligibleMaintenanceAutoRecoverCount"] == 1
assert full_runtime["reason"] == (
"full_runtime_requires_maintenance_auto_recover_to_drain"
)
payload["mode"] = "Status"
command_path.write_text(json.dumps(payload), encoding="utf-8")
invalid = run("PromotionReserve")
assert invalid["deferredCount"] == 0
assert invalid["blockingCount"] == 1
assert invalid["contractValid"] is False
def test_decision_behavior_when_powershell_is_available() -> None:
powershell = shutil.which("pwsh") or shutil.which("powershell")
if powershell is None:
@@ -348,11 +550,17 @@ def test_decision_behavior_when_powershell_is_available() -> None:
(),
),
(
_with_overrides(pendingCommandCount=1),
_with_overrides(pendingCommandBlockingCount=1),
False,
("pending_work_before_reboot",),
(),
),
(
_with_overrides(promotionDeferredCommandCount=1),
True,
(),
("maintenance_pending_commands_deferred_until_post_promotion",),
),
)
for case, expected_eligible, expected_blockers, expected_warnings in cases:

View File

@@ -391,11 +391,39 @@ def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() ->
assert "failedTasks" in source
assert "failedEvidenceNames" in source
assert "failedEvidence" in source
assert "actionDefinitionReady = [bool]$_.actionDefinitionReady" in source
assert "principalReady = [bool]$_.principalReady" in source
assert "definitionReady = [bool]$_.definitionReady" in source
assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source
assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source
assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source
def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> None:
source = RECEIVER.read_text(encoding="utf-8")
assert "$LivePreflightTimeoutSeconds = 60" in source
assert "$PostVerifyMaxAttempts = 4" in source
assert "$PostVerifyRetryWaitSeconds = 45" in source
assert "function Test-AgentPostPromotionReadinessRetryable" in source
assert '[string]$Readiness.readinessScope -ne "FullRuntime"' in source
assert '"required_evidence_stale"' in source
assert '"required_evidence_content_unhealthy"' in source
assert '$_.name -notin @("telegram_inbox", "sre_alert_inbox")' in source
assert "-not $_.definitionReady" in source
assert "[int]$Readiness.alertRunningCount -ne 0" in source
assert "$pendingCommandsRetryable" in source
assert "$ExpectedMaintenancePendingCommandIdentities" in source
assert '"pending_work_before_reboot"' in source
assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
assert "maintenancePendingCommandIdentities" in source
assert "$postVerifyAttempt -le $PostVerifyMaxAttempts" in source
assert "Start-Sleep -Seconds $PostVerifyRetryWaitSeconds" in source
assert "automaticTaskTriggerAllowed = $false" in source
assert "postVerifyAttempts = $postVerifyAttempts" in source
assert "postVerifyRetryCount = $postVerifyRetryCount" in source
def test_check_and_apply_receipts_project_the_same_run_identity() -> None:
sender = SENDER.read_text(encoding="utf-8")
projector = sender[
@@ -607,7 +635,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert "$RelayPollIntervalSeconds = 1" in source
assert "$ValidateOnlyTimeoutSeconds = 300" in source
assert "$LiveDeployTimeoutSeconds = 300" in source
assert "$LivePreflightTimeoutSeconds = 120" in source
assert "$LivePreflightTimeoutSeconds = 60" in source
assert "$PromotionReadinessMaxAttempts = 2" in source
assert "$PromotionReadinessRetryWaitSeconds = 30" in source
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
@@ -648,18 +676,23 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
promotion_readiness_attempts = 2
promotion_readiness_retry_wait_seconds = 30
deploy_seconds = 300
live_preflight_seconds = 120
live_preflight_seconds = 60
post_verify_attempts = 4
post_verify_retry_wait_seconds = 45
windows_control_baseline_seconds = 300
worst_case_seconds = (
validate_seconds
+ (promotion_readiness_attempts * live_preflight_seconds)
+ promotion_readiness_retry_wait_seconds
+ deploy_seconds
+ live_preflight_seconds
+ (post_verify_attempts * live_preflight_seconds)
+ ((post_verify_attempts - 1) * post_verify_retry_wait_seconds)
+ (2 * (60 + 360))
+ windows_control_baseline_seconds
)
assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source
assert worst_case_seconds == 1830
assert wrapper_budget_seconds - worst_case_seconds == 570
assert worst_case_seconds == 2265
assert wrapper_budget_seconds - worst_case_seconds == 135
deploy = source.index(
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
@@ -781,6 +814,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() -
assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight
assert '[int]$summary.alertIncomingCount -eq 0' not in preflight
assert '[int]$summary.alertRunningCount -eq 0' in preflight
assert '[int]$summary.pendingCommandBlockingCount -eq 0' in preflight
assert '$ReadinessScope -eq "PromotionReserve"' in preflight
assert '[int]$summary.pendingCommandCount -eq 0' in preflight
assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight
assert '[string[]]$blockingReasons = @()' in preflight

View File

@@ -0,0 +1,44 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1"
def test_stale_queue_quarantine_is_exact_recoverable_and_receipted() -> None:
source = SCRIPT.read_text(encoding="utf-8")
assert '[ValidateSet("Check", "Apply")]' in source
assert "ExpectedQueueSha256" in source
assert "ExpectedMaintenanceSha256" in source
assert '"Global\\WoooAgent99StaleQueueMaintenanceV1"' in source
assert '"Global\\WoooAgent99RecoveryTransportV1"' in source
assert 'Get-ScheduledTask `' in source
assert '"Wooo-Agent99-Control-Loop"' in source
assert 'throw "control_loop_not_frozen"' in source
assert 'throw "active_control_or_recover_process"' in source
assert 'throw "recovery_lease_present"' in source
assert 'throw "recovery_mutex_active"' in source
assert '[string]$command.mode -ne "Recover"' in source
assert '[string]$command.source -ne "agent99-recovery-observer"' in source
assert "$command.controlledApply -isnot [bool]" in source
assert '"atomic_move_exact_queue_item_to_failed"' in source
assert "[IO.File]::Move($sourcePath, $destinationPath)" in source
assert "[IO.File]::Move($destinationPath, $sourcePath)" in source
assert '"verified_quarantined"' in source
assert '"failed_rolled_back"' in source
assert '"rollback_unverified"' in source
def test_stale_queue_quarantine_does_not_execute_or_delete_command() -> None:
source = SCRIPT.read_text(encoding="utf-8")
assert "commandExecuted = $false" in source
assert "deletePerformed = $false" in source
assert "otherQueueMutationPerformed = $false" in source
assert "rawInstructionStored = $false" in source
assert "secretValueRead = $false" in source
assert "Remove-Item -LiteralPath $sourcePath" not in source
assert "Start-ScheduledTask" not in source
assert "Invoke-Expression" not in source
assert "& $command" not in source