fix(agent99): unblock bounded maintenance deployment
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 1m9s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 10m26s
CD Pipeline / revalidate-deploy-carrier (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / revalidate-post-deploy-carrier (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
MCP External Protocol Replay / replay-and-verify (push) Successful in 44s
MCP External Artifact Mirror / mirror-and-verify (push) Successful in 33s
E2E Health Check / e2e-health (push) Failing after 48s
AI 技術雷達監控 / ai-technology-watch (push) Successful in 34s
Agent Version Lifecycle Watch / version-lifecycle-watch (push) Successful in 7s
Agent Market Watch / market-watch (push) Failing after 26s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 23s
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 1m9s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 10m26s
CD Pipeline / revalidate-deploy-carrier (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / revalidate-post-deploy-carrier (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
MCP External Protocol Replay / replay-and-verify (push) Successful in 44s
MCP External Artifact Mirror / mirror-and-verify (push) Successful in 33s
E2E Health Check / e2e-health (push) Failing after 48s
AI 技術雷達監控 / ai-technology-watch (push) Successful in 34s
Agent Version Lifecycle Watch / version-lifecycle-watch (push) Successful in 7s
Agent Market Watch / market-watch (push) Failing after 26s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 23s
This commit is contained in:
@@ -28,7 +28,8 @@ function Get-AgentLivePreflightDecision {
|
||||
[int]$StaleAlertExecutionCount,
|
||||
[int]$AlertIncomingCount,
|
||||
[int]$StaleAlertIncomingCount,
|
||||
[int]$PendingCommandCount
|
||||
[int]$PendingCommandBlockingCount,
|
||||
[int]$PromotionDeferredCommandCount
|
||||
)
|
||||
|
||||
$blockingReasons = @()
|
||||
@@ -89,11 +90,14 @@ function Get-AgentLivePreflightDecision {
|
||||
}
|
||||
|
||||
if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" }
|
||||
if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) {
|
||||
if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandBlockingCount -gt 0) {
|
||||
$blockingReasons += "pending_work_before_reboot"
|
||||
} elseif ($AlertIncomingCount -gt 0) {
|
||||
$warningReasons += "fresh_alert_pending_next_inbox_tick"
|
||||
}
|
||||
if ($PromotionDeferredCommandCount -gt 0) {
|
||||
$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
deploymentEligible = [bool]($blockingReasons.Count -eq 0)
|
||||
@@ -110,7 +114,8 @@ function Get-AgentLivePreflightDecision {
|
||||
function Get-TaskReadback {
|
||||
param(
|
||||
[string]$TaskPath,
|
||||
[string]$TaskName
|
||||
[string]$TaskName,
|
||||
[string[]]$RequiredArgumentMarkers
|
||||
)
|
||||
|
||||
try {
|
||||
@@ -120,8 +125,34 @@ function Get-TaskReadback {
|
||||
$info = Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop
|
||||
$lastResult = [int64]$info.LastTaskResult
|
||||
$state = [string]$task.State
|
||||
$actions = @($task.Actions)
|
||||
$actionDefinitionReady = [bool](
|
||||
$actions.Count -eq 1 -and
|
||||
[string]$actions[0].Execute -match '(?i)\\WindowsPowerShell\\v1\.0\\powershell\.exe$'
|
||||
)
|
||||
if ($actionDefinitionReady) {
|
||||
foreach ($marker in @($RequiredArgumentMarkers)) {
|
||||
if (
|
||||
-not $marker -or
|
||||
([string]$actions[0].Arguments).IndexOf(
|
||||
$marker,
|
||||
[StringComparison]::OrdinalIgnoreCase
|
||||
) -lt 0
|
||||
) {
|
||||
$actionDefinitionReady = $false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
$principalReady = [bool](
|
||||
[string]$task.Principal.UserId -match '(?i)(^|\\)Administrator$' -and
|
||||
[string]$task.Principal.LogonType -eq "S4U" -and
|
||||
[string]$task.Principal.RunLevel -eq "Highest"
|
||||
)
|
||||
$definitionReady = [bool]($actionDefinitionReady -and $principalReady)
|
||||
$healthy = [bool](
|
||||
$task.Settings.Enabled -and
|
||||
$definitionReady -and
|
||||
($lastResult -in @(0, 267009, 267011) -or $state -eq "Running")
|
||||
)
|
||||
|
||||
@@ -134,6 +165,9 @@ function Get-TaskReadback {
|
||||
lastTaskResult = $lastResult
|
||||
lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" }
|
||||
nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" }
|
||||
actionDefinitionReady = $actionDefinitionReady
|
||||
principalReady = $principalReady
|
||||
definitionReady = $definitionReady
|
||||
healthy = $healthy
|
||||
}
|
||||
} catch {
|
||||
@@ -146,6 +180,9 @@ function Get-TaskReadback {
|
||||
lastTaskResult = $null
|
||||
lastRunTime = ""
|
||||
nextRunTime = ""
|
||||
actionDefinitionReady = $false
|
||||
principalReady = $false
|
||||
definitionReady = $false
|
||||
healthy = $false
|
||||
}
|
||||
}
|
||||
@@ -158,25 +195,448 @@ function Get-DirectoryReadback {
|
||||
)
|
||||
|
||||
$path = Join-Path $AgentRoot $RelativePath
|
||||
$files = @()
|
||||
if (Test-Path $path) {
|
||||
$files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue)
|
||||
if ($RootQueueFilesOnly) {
|
||||
$files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" })
|
||||
$exists = [bool](Test-Path -LiteralPath $path -PathType Container)
|
||||
$count = 0
|
||||
$staleCount = 0
|
||||
$oldest = $null
|
||||
$newest = $null
|
||||
if ($exists) {
|
||||
foreach ($candidatePath in [IO.Directory]::EnumerateFiles(
|
||||
$path,
|
||||
"*.json",
|
||||
[IO.SearchOption]::TopDirectoryOnly
|
||||
)) {
|
||||
$name = [IO.Path]::GetFileName($candidatePath)
|
||||
if ($RootQueueFilesOnly -and $name -match "^(processed|failed|running)-") {
|
||||
continue
|
||||
}
|
||||
$file = [IO.FileInfo]::new($candidatePath)
|
||||
$count += 1
|
||||
if (-not $oldest -or $file.LastWriteTimeUtc -lt $oldest.LastWriteTimeUtc) {
|
||||
$oldest = $file
|
||||
}
|
||||
if (-not $newest -or $file.LastWriteTimeUtc -gt $newest.LastWriteTimeUtc) {
|
||||
$newest = $file
|
||||
}
|
||||
if (((Get-Date) - $file.LastWriteTime).TotalMinutes -gt 5) {
|
||||
$staleCount += 1
|
||||
}
|
||||
}
|
||||
$files = @($files | Sort-Object LastWriteTime)
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
relativePath = $RelativePath
|
||||
exists = [bool](Test-Path $path)
|
||||
count = [int]$files.Count
|
||||
oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" }
|
||||
newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" }
|
||||
staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count
|
||||
exists = $exists
|
||||
count = [int]$count
|
||||
oldest = if ($oldest) { $oldest.LastWriteTime.ToString("o") } else { "" }
|
||||
newest = if ($newest) { $newest.LastWriteTime.ToString("o") } else { "" }
|
||||
staleOverFiveMinutes = [int]$staleCount
|
||||
}
|
||||
}
|
||||
|
||||
function Get-AgentBoundedNewestFiles {
|
||||
param(
|
||||
[string]$DirectoryPath,
|
||||
[string]$Pattern,
|
||||
[ValidateRange(1, 256)]
|
||||
[int]$CandidateWindow = 32
|
||||
)
|
||||
|
||||
# Agent99 evidence names contain a fixed sortable timestamp and are immutable.
|
||||
# Keep only the lexically newest bounded window in a native .NET set so a
|
||||
# large evidence directory never becomes a PowerShell object-sort workload.
|
||||
$selected = [Collections.Generic.SortedSet[string]]::new(
|
||||
[StringComparer]::OrdinalIgnoreCase
|
||||
)
|
||||
$enumeratedCount = 0
|
||||
if (Test-Path -LiteralPath $DirectoryPath -PathType Container) {
|
||||
foreach ($candidatePath in [IO.Directory]::EnumerateFiles(
|
||||
$DirectoryPath,
|
||||
$Pattern,
|
||||
[IO.SearchOption]::TopDirectoryOnly
|
||||
)) {
|
||||
$enumeratedCount += 1
|
||||
$null = $selected.Add($candidatePath)
|
||||
if ($selected.Count -gt $CandidateWindow) {
|
||||
$null = $selected.Remove($selected.Min)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
files = @($selected | Sort-Object -Descending | ForEach-Object {
|
||||
[IO.FileInfo]::new($_)
|
||||
})
|
||||
enumeratedCount = [int]$enumeratedCount
|
||||
candidateWindow = [int]$CandidateWindow
|
||||
windowTruncated = [bool]($enumeratedCount -gt $selected.Count)
|
||||
}
|
||||
}
|
||||
|
||||
function Get-AgentControlLoopMaintenanceReadback {
|
||||
$evidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$selection = Get-AgentBoundedNewestFiles `
|
||||
$evidenceDir `
|
||||
"agent99-control-loop-maintenance-2*.json" `
|
||||
8
|
||||
$file = @($selection.files | Select-Object -First 1)
|
||||
$receipt = $null
|
||||
$intent = $null
|
||||
$parseError = ""
|
||||
if ($file.Count -eq 1) {
|
||||
try {
|
||||
$receipt = Get-Content -LiteralPath $file[0].FullName -Raw | ConvertFrom-Json
|
||||
} catch {
|
||||
$parseError = $_.Exception.GetType().Name
|
||||
}
|
||||
}
|
||||
|
||||
$intentRef = if ($receipt -and $receipt.PSObject.Properties["operationIntentRef"]) {
|
||||
[string]$receipt.operationIntentRef
|
||||
} else { "" }
|
||||
$intentRefSafe = [bool](
|
||||
$intentRef -match "^agent99-control-loop-maintenance-intent-[0-9]{8}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$"
|
||||
)
|
||||
$intentPath = if ($intentRefSafe) { Join-Path $evidenceDir $intentRef } else { "" }
|
||||
if ($intentPath -and (Test-Path -LiteralPath $intentPath -PathType Leaf)) {
|
||||
try {
|
||||
$intent = Get-Content -LiteralPath $intentPath -Raw | ConvertFrom-Json
|
||||
} catch {
|
||||
$parseError = $_.Exception.GetType().Name
|
||||
}
|
||||
}
|
||||
$legacyPropertySet = @(
|
||||
"blockers", "controlStopped", "expectedControlCreated", "expectedControlPid",
|
||||
"expectedRecoverCreated", "expectedRecoverParentPid", "expectedRecoverPid",
|
||||
"hostRebootPerformed", "leaseMetadataRemoved", "mode", "precheckPassed",
|
||||
"recoverStopped", "runtimeWritePerformed", "schemaVersion", "secretValueRead",
|
||||
"taskEnabled", "taskName", "terminal", "timestamp", "vmPowerChangePerformed",
|
||||
"vmwareVmxProcessTerminated", "vmxLockDeleted"
|
||||
)
|
||||
$receiptPropertySet = if ($receipt) {
|
||||
@($receipt.PSObject.Properties.Name | Sort-Object)
|
||||
} else { @() }
|
||||
$legacyPropertySetMatches = [bool](
|
||||
$receiptPropertySet.Count -eq $legacyPropertySet.Count -and
|
||||
(@($receiptPropertySet) -join ",") -eq (@($legacyPropertySet | Sort-Object) -join ",")
|
||||
)
|
||||
$legacyIdentityFieldsValid = [bool](
|
||||
$receipt -and
|
||||
$receipt.expectedControlPid -is [int] -and [int]$receipt.expectedControlPid -gt 0 -and
|
||||
$receipt.expectedRecoverPid -is [int] -and [int]$receipt.expectedRecoverPid -gt 0 -and
|
||||
$receipt.expectedRecoverParentPid -is [int] -and [int]$receipt.expectedRecoverParentPid -gt 0 -and
|
||||
[string]$receipt.expectedControlCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and
|
||||
[string]$receipt.expectedRecoverCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and
|
||||
[string]$receipt.timestamp -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T"
|
||||
)
|
||||
$legacyReceipt = [bool](
|
||||
$receipt -and
|
||||
-not $receipt.PSObject.Properties["operationIntentRef"] -and
|
||||
$receipt.PSObject.Properties["taskEnabled"] -and
|
||||
$legacyPropertySetMatches -and
|
||||
$legacyIdentityFieldsValid -and
|
||||
@($receipt.blockers).Count -eq 0
|
||||
)
|
||||
|
||||
$ageMinutes = if ($file.Count -eq 1) {
|
||||
[math]::Round(((Get-Date) - $file[0].LastWriteTime).TotalMinutes, 1)
|
||||
} else { $null }
|
||||
$activeControlProcesses = @(Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction SilentlyContinue |
|
||||
Where-Object {
|
||||
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or
|
||||
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)'
|
||||
})
|
||||
$leasePresent = [bool](Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json") -PathType Leaf)
|
||||
$recoveryMutexActive = $true
|
||||
$mutex = $null
|
||||
$mutexAcquired = $false
|
||||
try {
|
||||
$mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1")
|
||||
try {
|
||||
$mutexAcquired = [bool]$mutex.WaitOne(0)
|
||||
} catch [Threading.AbandonedMutexException] {
|
||||
$mutexAcquired = $true
|
||||
}
|
||||
$recoveryMutexActive = [bool](-not $mutexAcquired)
|
||||
} finally {
|
||||
if ($mutexAcquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
|
||||
if ($mutex) { $mutex.Dispose() }
|
||||
}
|
||||
|
||||
$commonReceiptFieldsValid = [bool](
|
||||
$receipt -and
|
||||
[string]$receipt.schemaVersion -eq "agent99_control_loop_maintenance_v2" -and
|
||||
[string]$receipt.mode -eq "Apply" -and
|
||||
[string]$receipt.taskName -eq "Wooo-Agent99-Control-Loop" -and
|
||||
[string]$receipt.terminal -eq "control_loop_maintenance_frozen" -and
|
||||
$receipt.precheckPassed -is [bool] -and $receipt.precheckPassed -and
|
||||
$receipt.controlStopped -is [bool] -and $receipt.controlStopped -and
|
||||
$receipt.recoverStopped -is [bool] -and $receipt.recoverStopped -and
|
||||
$receipt.leaseMetadataRemoved -is [bool] -and $receipt.leaseMetadataRemoved -and
|
||||
$receipt.runtimeWritePerformed -is [bool] -and $receipt.runtimeWritePerformed -and
|
||||
$receipt.vmPowerChangePerformed -is [bool] -and -not $receipt.vmPowerChangePerformed -and
|
||||
$receipt.vmwareVmxProcessTerminated -is [bool] -and -not $receipt.vmwareVmxProcessTerminated -and
|
||||
$receipt.vmxLockDeleted -is [bool] -and -not $receipt.vmxLockDeleted -and
|
||||
$receipt.hostRebootPerformed -is [bool] -and -not $receipt.hostRebootPerformed -and
|
||||
$receipt.secretValueRead -is [bool] -and -not $receipt.secretValueRead
|
||||
)
|
||||
$modernReceiptFieldsValid = [bool](
|
||||
$commonReceiptFieldsValid -and
|
||||
-not $legacyReceipt -and
|
||||
$receipt.taskEnabledAfter -is [bool] -and -not $receipt.taskEnabledAfter
|
||||
)
|
||||
$legacyReceiptFieldsValid = [bool](
|
||||
$commonReceiptFieldsValid -and
|
||||
$legacyReceipt -and
|
||||
$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled
|
||||
)
|
||||
$receiptFieldsValid = [bool]($modernReceiptFieldsValid -or $legacyReceiptFieldsValid)
|
||||
$intentFieldsValid = [bool](
|
||||
$legacyReceipt -or
|
||||
(
|
||||
$intent -and
|
||||
[string]$intent.schemaVersion -eq "agent99_control_loop_maintenance_intent_v1" -and
|
||||
[string]$intent.taskName -eq "Wooo-Agent99-Control-Loop" -and
|
||||
$intent.vmPowerChangePerformed -is [bool] -and -not $intent.vmPowerChangePerformed -and
|
||||
$intent.vmwareVmxProcessTerminated -is [bool] -and -not $intent.vmwareVmxProcessTerminated -and
|
||||
$intent.secretValueRead -is [bool] -and -not $intent.secretValueRead -and
|
||||
(@($intent.plannedActions) -join ",") -eq "disable_control_task,stop_exact_control_tree,stop_exact_recover_tree,remove_exact_recovery_lease"
|
||||
)
|
||||
)
|
||||
$valid = [bool](
|
||||
-not $parseError -and
|
||||
$file.Count -eq 1 -and
|
||||
$null -ne $ageMinutes -and
|
||||
$ageMinutes -ge 0 -and
|
||||
$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 }) -and
|
||||
($legacyReceipt -or $intentRefSafe) -and
|
||||
$receiptFieldsValid -and
|
||||
$intentFieldsValid -and
|
||||
$activeControlProcesses.Count -eq 0 -and
|
||||
-not $leasePresent -and
|
||||
-not $recoveryMutexActive
|
||||
)
|
||||
|
||||
return [pscustomobject]@{
|
||||
valid = $valid
|
||||
receiptFile = if ($file.Count -eq 1) { $file[0].Name } else { "" }
|
||||
ageMinutes = $ageMinutes
|
||||
parseError = $parseError
|
||||
intentRefPresent = $intentRefSafe
|
||||
contractVersion = if ($legacyReceipt) { "legacy_v2_exact_upgrade_bridge" } else { "intent_bound_v2" }
|
||||
receiptTimestamp = if ($receipt) { [string]$receipt.timestamp } else { "" }
|
||||
receiptLastWriteTime = if ($file.Count -eq 1) { $file[0].LastWriteTime.ToString("o") } else { "" }
|
||||
legacyPropertySetMatches = $legacyPropertySetMatches
|
||||
legacyIdentityFieldsValid = $legacyIdentityFieldsValid
|
||||
receiptFieldsValid = $receiptFieldsValid
|
||||
intentFieldsValid = $intentFieldsValid
|
||||
activeControlProcessCount = [int]$activeControlProcesses.Count
|
||||
recoveryLeasePresent = $leasePresent
|
||||
recoveryMutexActive = $recoveryMutexActive
|
||||
secretValueRead = $false
|
||||
remoteWritePerformed = $false
|
||||
}
|
||||
}
|
||||
|
||||
function Get-AgentPendingCommandPromotionReadback {
|
||||
param(
|
||||
[object]$MaintenanceReadback,
|
||||
[ValidateSet("FullRuntime", "PromotionReserve")]
|
||||
[string]$Scope
|
||||
)
|
||||
|
||||
$queueRoot = Join-Path $AgentRoot "queue"
|
||||
$files = @()
|
||||
if (Test-Path -LiteralPath $queueRoot -PathType Container) {
|
||||
$files = @([IO.Directory]::EnumerateFiles(
|
||||
$queueRoot,
|
||||
"*.json",
|
||||
[IO.SearchOption]::TopDirectoryOnly
|
||||
) | Where-Object {
|
||||
[IO.Path]::GetFileName($_) -notmatch "^(processed|failed|running)-"
|
||||
})
|
||||
}
|
||||
|
||||
$observedCount = [int]$files.Count
|
||||
$result = [ordered]@{
|
||||
scope = $Scope
|
||||
observedCount = $observedCount
|
||||
deferredCount = 0
|
||||
blockingCount = $observedCount
|
||||
eligibleMaintenanceAutoRecoverCount = 0
|
||||
boundedLimit = 8
|
||||
contractValid = [bool]($observedCount -eq 0)
|
||||
reason = if ($observedCount -eq 0) { "queue_empty" } else { "pending_commands_require_runtime" }
|
||||
items = @()
|
||||
rawInstructionStored = $false
|
||||
secretValueRead = $false
|
||||
remoteWritePerformed = $false
|
||||
}
|
||||
if ($observedCount -eq 0) {
|
||||
return [pscustomobject]$result
|
||||
}
|
||||
if (
|
||||
-not $MaintenanceReadback -or
|
||||
-not $MaintenanceReadback.receiptFieldsValid -or
|
||||
-not $MaintenanceReadback.intentFieldsValid
|
||||
) {
|
||||
$result.reason = "maintenance_contract_invalid"
|
||||
return [pscustomobject]$result
|
||||
}
|
||||
if ($Scope -eq "PromotionReserve" -and -not $MaintenanceReadback.valid) {
|
||||
$result.reason = "maintenance_current_state_invalid"
|
||||
return [pscustomobject]$result
|
||||
}
|
||||
if ($observedCount -gt $result.boundedLimit) {
|
||||
$result.reason = "maintenance_pending_command_limit_exceeded"
|
||||
return [pscustomobject]$result
|
||||
}
|
||||
|
||||
$maintenanceTimestamp = [datetimeoffset]::MinValue
|
||||
$maintenanceTimestampValid = [datetimeoffset]::TryParse(
|
||||
[string]$MaintenanceReadback.receiptTimestamp,
|
||||
[ref]$maintenanceTimestamp
|
||||
)
|
||||
if (-not $maintenanceTimestampValid) {
|
||||
$result.reason = "maintenance_timestamp_invalid"
|
||||
return [pscustomobject]$result
|
||||
}
|
||||
|
||||
$requiredPropertySet = @(
|
||||
"approvalId", "automationRunId", "canonicalDigest", "controlledApply",
|
||||
"correlationKey", "createdAt", "dispatchIdentitySchemaVersion",
|
||||
"dispatchRouteId", "executionGeneration", "externalId", "id",
|
||||
"idempotencyKey", "incidentId", "instruction", "lockOwner", "mode",
|
||||
"observation", "projectId", "reason", "requestedBy", "singleFlightKey",
|
||||
"source", "sourceFingerprint", "traceId", "workItemId"
|
||||
) | Sort-Object
|
||||
$safeItems = @()
|
||||
$allValid = $true
|
||||
foreach ($path in $files) {
|
||||
$file = [IO.FileInfo]::new($path)
|
||||
$payload = $null
|
||||
$parseError = ""
|
||||
$sha256 = ""
|
||||
try {
|
||||
$bytes = [IO.File]::ReadAllBytes($path)
|
||||
$sha = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$sha256 = -join @($sha.ComputeHash($bytes) | ForEach-Object { $_.ToString("x2") })
|
||||
} finally {
|
||||
$sha.Dispose()
|
||||
}
|
||||
$jsonOffset = if (
|
||||
$bytes.Length -ge 3 -and
|
||||
$bytes[0] -eq 0xef -and
|
||||
$bytes[1] -eq 0xbb -and
|
||||
$bytes[2] -eq 0xbf
|
||||
) { 3 } else { 0 }
|
||||
$jsonText = [Text.Encoding]::UTF8.GetString(
|
||||
$bytes,
|
||||
$jsonOffset,
|
||||
$bytes.Length - $jsonOffset
|
||||
)
|
||||
$payload = $jsonText | ConvertFrom-Json
|
||||
} catch {
|
||||
$parseError = $_.Exception.GetType().Name
|
||||
}
|
||||
$propertySet = if ($payload) {
|
||||
@($payload.PSObject.Properties.Name | Sort-Object)
|
||||
} else { @() }
|
||||
$propertySetMatches = [bool](
|
||||
$propertySet.Count -eq $requiredPropertySet.Count -and
|
||||
(@($propertySet) -join ",") -eq (@($requiredPropertySet) -join ",")
|
||||
)
|
||||
$createdAt = [datetimeoffset]::MinValue
|
||||
$createdAtValid = [bool](
|
||||
$payload -and
|
||||
[datetimeoffset]::TryParse([string]$payload.createdAt, [ref]$createdAt)
|
||||
)
|
||||
$ageMinutes = if ($createdAtValid) {
|
||||
[math]::Round(([datetimeoffset]::Now - $createdAt).TotalMinutes, 1)
|
||||
} else { $null }
|
||||
$fileBaseName = [IO.Path]::GetFileNameWithoutExtension($file.Name)
|
||||
$nativeStringFields = @(
|
||||
"approvalId", "automationRunId", "canonicalDigest", "correlationKey",
|
||||
"createdAt", "dispatchIdentitySchemaVersion", "dispatchRouteId",
|
||||
"executionGeneration", "externalId", "id", "idempotencyKey",
|
||||
"incidentId", "instruction", "lockOwner", "mode", "projectId",
|
||||
"reason", "requestedBy", "singleFlightKey", "source",
|
||||
"sourceFingerprint", "traceId", "workItemId"
|
||||
)
|
||||
$nativeStringTypesValid = [bool](
|
||||
$payload -and
|
||||
@($nativeStringFields | Where-Object { $payload.$_ -isnot [string] }).Count -eq 0
|
||||
)
|
||||
$identityValid = [bool](
|
||||
-not $parseError -and
|
||||
$propertySetMatches -and
|
||||
$nativeStringTypesValid -and
|
||||
$payload.controlledApply -is [bool] -and $payload.controlledApply -and
|
||||
$payload.observation -is [pscustomobject] -and
|
||||
[string]$payload.id -eq $fileBaseName -and
|
||||
[string]$payload.id -match "^auto-recover-[0-9]{8}-[0-9]{6}-[0-9a-f]{8}$" -and
|
||||
[string]$payload.mode -eq "Recover" -and
|
||||
[string]$payload.source -eq "agent99-recovery-observer" -and
|
||||
[string]$payload.dispatchIdentitySchemaVersion -eq "agent99_controlled_dispatch_identity_v1" -and
|
||||
[string]$payload.dispatchRouteId -eq "agent99_local_observer" -and
|
||||
[string]$payload.projectId -eq "awoooi" -and
|
||||
[string]$payload.requestedBy -eq "Agent99" -and
|
||||
[string]$payload.reason -eq "host_unreachable_detected" -and
|
||||
[string]$payload.externalId -eq "host_unreachable_detected" -and
|
||||
[string]$payload.approvalId -eq "" -and
|
||||
[string]$payload.automationRunId -match "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -and
|
||||
[string]$payload.lockOwner -eq [string]$payload.automationRunId -and
|
||||
[string]$payload.executionGeneration -match "^[1-9][0-9]*$" -and
|
||||
[string]$payload.canonicalDigest -match "^[0-9a-f]{64}$" -and
|
||||
[string]$payload.sourceFingerprint -match "^[0-9a-f]{64}$" -and
|
||||
[string]$payload.correlationKey -match "^agent99-controlled:[0-9a-f]{64}$" -and
|
||||
[string]$payload.idempotencyKey -eq [string]$payload.correlationKey -and
|
||||
[string]$payload.singleFlightKey -match "^agent99-controlled-flight:[0-9a-f]{64}$" -and
|
||||
[string]$payload.incidentId -match "^agent99-auto-[0-9a-f]{20}$" -and
|
||||
[string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):host_unreachable_detected" -and
|
||||
[string]$payload.traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$" -and
|
||||
-not [string]::IsNullOrWhiteSpace([string]$payload.instruction) -and
|
||||
$createdAtValid -and
|
||||
$createdAt -ge $maintenanceTimestamp -and
|
||||
$ageMinutes -ge 0 -and
|
||||
$ageMinutes -le 120
|
||||
)
|
||||
if (-not $identityValid) { $allValid = $false }
|
||||
$safeItems += [pscustomobject]@{
|
||||
file = $file.Name
|
||||
id = if ($payload) { [string]$payload.id } else { "" }
|
||||
mode = if ($payload) { [string]$payload.mode } else { "" }
|
||||
source = if ($payload) { [string]$payload.source } else { "" }
|
||||
workItemId = if ($payload) { [string]$payload.workItemId } else { "" }
|
||||
createdAt = if ($payload) { [string]$payload.createdAt } else { "" }
|
||||
ageMinutes = $ageMinutes
|
||||
sha256 = $sha256
|
||||
identity = if ($payload -and $sha256) { "$([string]$payload.id)|$sha256" } else { "" }
|
||||
parseError = $parseError
|
||||
propertySetMatches = $propertySetMatches
|
||||
identityValid = $identityValid
|
||||
instructionContentReadback = $false
|
||||
}
|
||||
}
|
||||
|
||||
$result.items = $safeItems
|
||||
if ($allValid) {
|
||||
$result.eligibleMaintenanceAutoRecoverCount = $observedCount
|
||||
$result.contractValid = $true
|
||||
if ($Scope -eq "PromotionReserve") {
|
||||
$result.deferredCount = $observedCount
|
||||
$result.blockingCount = 0
|
||||
$result.reason = "bounded_maintenance_auto_recover_deferred_until_post_promotion"
|
||||
} else {
|
||||
$result.reason = "full_runtime_requires_maintenance_auto_recover_to_drain"
|
||||
}
|
||||
} else {
|
||||
$result.reason = "maintenance_pending_command_contract_invalid"
|
||||
}
|
||||
return [pscustomobject]$result
|
||||
}
|
||||
|
||||
function Get-EvidenceReadback {
|
||||
param(
|
||||
[string]$Name,
|
||||
@@ -187,8 +647,8 @@ function Get-EvidenceReadback {
|
||||
)
|
||||
|
||||
$evidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$candidates = @(Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTime -Descending)
|
||||
$candidateSelection = Get-AgentBoundedNewestFiles $evidenceDir $Pattern
|
||||
$candidates = @($candidateSelection.files)
|
||||
$deferredCandidates = @()
|
||||
$file = $null
|
||||
foreach ($candidate in $candidates) {
|
||||
@@ -219,6 +679,11 @@ function Get-EvidenceReadback {
|
||||
$file = $candidate
|
||||
break
|
||||
}
|
||||
$candidateWindowExhausted = [bool](
|
||||
-not $file -and
|
||||
$candidateSelection.windowTruncated -and
|
||||
$deferredCandidates.Count -eq $candidates.Count
|
||||
)
|
||||
$ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null }
|
||||
$fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes)
|
||||
$freshnessHeadroomMinutes = if ($file) { [math]::Round($MaxAgeMinutes - $ageMinutes, 1) } else { $null }
|
||||
@@ -228,7 +693,7 @@ function Get-EvidenceReadback {
|
||||
$freshnessHeadroomMinutes -ge $MinimumFreshnessReserveMinutes
|
||||
)
|
||||
$safeSummary = $null
|
||||
$parseError = ""
|
||||
$parseError = if ($candidateWindowExhausted) { "EvidenceCandidateWindowExhausted" } else { "" }
|
||||
$contentHealthy = $true
|
||||
if ($file) {
|
||||
try {
|
||||
@@ -389,23 +854,52 @@ function Get-EvidenceReadback {
|
||||
parseError = $parseError
|
||||
deferredCandidateCount = [int]$deferredCandidates.Count
|
||||
latestDeferredTime = if ($deferredCandidates.Count -gt 0) { $deferredCandidates[0].LastWriteTime.ToString("o") } else { "" }
|
||||
enumeratedCandidateCount = [int]$candidateSelection.enumeratedCount
|
||||
candidateWindowLimit = [int]$candidateSelection.candidateWindow
|
||||
candidateWindowTruncated = [bool]$candidateSelection.windowTruncated
|
||||
candidateWindowExhausted = $candidateWindowExhausted
|
||||
safeSummary = $safeSummary
|
||||
}
|
||||
}
|
||||
|
||||
$rootTaskPath = "\"
|
||||
$requiredTasks = @(
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health" }
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery"; markers = @("agent99-submit-request.ps1", "-Mode Recover", "-Source agent99-startup-recovery") },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat"; markers = @("agent99-run.ps1", "-Mode Status") },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard"; markers = @("agent99-run.ps1", "-Mode Perf", "-ControlledApply") },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop"; markers = @("agent99-run.ps1", "-Mode ControlTick") },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox"; markers = @("agent99-telegram-inbox.ps1", "-RunNow") },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox"; markers = @("agent99-sre-alert-inbox.ps1", "-RunNow") },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay"; markers = @("agent99-sre-alert-relay.ps1") },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health"; markers = @("agent99-run.ps1", "-Mode SelfCheck") },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health"; markers = @("agent99-run.ps1", "-Mode BackupCheck") }
|
||||
)
|
||||
$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback -TaskPath $_.taskPath -TaskName $_.name })
|
||||
$tasks = @($requiredTasks | ForEach-Object {
|
||||
Get-TaskReadback `
|
||||
-TaskPath $_.taskPath `
|
||||
-TaskName $_.name `
|
||||
-RequiredArgumentMarkers $_.markers
|
||||
})
|
||||
$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback
|
||||
if ($ReadinessScope -eq "PromotionReserve") {
|
||||
foreach ($task in $tasks) {
|
||||
$runtimeHealthy = [bool]$task.healthy
|
||||
$maintenanceFreezeAccepted = [bool](
|
||||
$task.name -eq "Wooo-Agent99-Control-Loop" -and
|
||||
-not $task.enabled -and
|
||||
$controlLoopMaintenance.valid
|
||||
)
|
||||
$promotionReady = [bool](
|
||||
$task.exists -and
|
||||
$task.definitionReady -and
|
||||
($task.enabled -or $maintenanceFreezeAccepted)
|
||||
)
|
||||
$task | Add-Member -NotePropertyName runtimeHealthy -NotePropertyValue $runtimeHealthy -Force
|
||||
$task | Add-Member -NotePropertyName promotionReady -NotePropertyValue $promotionReady -Force
|
||||
$task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force
|
||||
$task.healthy = $promotionReady
|
||||
}
|
||||
}
|
||||
|
||||
$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
||||
$manifest = [pscustomobject]@{
|
||||
@@ -457,14 +951,15 @@ $queues = @(
|
||||
(Get-DirectoryReadback "queue" -RootQueueFilesOnly),
|
||||
(Get-DirectoryReadback "queue\failed")
|
||||
)
|
||||
$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")
|
||||
$evidence = @(
|
||||
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0)
|
||||
)
|
||||
|
||||
@@ -482,6 +977,9 @@ $selfHealthPerformanceIssueCount = if ($selfCheckEvidence -and $selfCheckEvidenc
|
||||
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
|
||||
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
|
||||
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
|
||||
$pendingCommandPromotion = Get-AgentPendingCommandPromotionReadback `
|
||||
-MaintenanceReadback $controlLoopMaintenance `
|
||||
-Scope $ReadinessScope
|
||||
$decision = Get-AgentLivePreflightDecision `
|
||||
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
|
||||
-Manifest $manifest `
|
||||
@@ -498,7 +996,8 @@ $decision = Get-AgentLivePreflightDecision `
|
||||
-StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes `
|
||||
-AlertIncomingCount $alertIncoming.count `
|
||||
-StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes `
|
||||
-PendingCommandCount $commandQueue.count
|
||||
-PendingCommandBlockingCount $pendingCommandPromotion.blockingCount `
|
||||
-PromotionDeferredCommandCount $pendingCommandPromotion.deferredCount
|
||||
$blockingReasons = @($decision.blockingReasons)
|
||||
$warningReasons = @($decision.warningReasons)
|
||||
|
||||
@@ -513,10 +1012,12 @@ $result = [pscustomobject]@{
|
||||
secretValueRead = $false
|
||||
remoteWritePerformed = $false
|
||||
agentRoot = $AgentRoot
|
||||
controlLoopMaintenance = $controlLoopMaintenance
|
||||
manifest = $manifest
|
||||
tasks = $tasks
|
||||
relayListeners = $relayListeners
|
||||
queues = $queues
|
||||
pendingCommandPromotion = $pendingCommandPromotion
|
||||
evidence = $evidence
|
||||
summary = [pscustomobject]@{
|
||||
requiredTaskCount = $requiredTasks.Count
|
||||
@@ -536,6 +1037,8 @@ $result = [pscustomobject]@{
|
||||
alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes
|
||||
alertRunningCount = [int]$alertRunning.count
|
||||
pendingCommandCount = [int]$commandQueue.count
|
||||
pendingCommandBlockingCount = [int]$pendingCommandPromotion.blockingCount
|
||||
promotionDeferredCommandCount = [int]$pendingCommandPromotion.deferredCount
|
||||
relayListenerCount = $relayListeners.Count
|
||||
preflightGreen = [bool]$decision.deploymentEligible
|
||||
deploymentEligible = [bool]$decision.deploymentEligible
|
||||
|
||||
@@ -23,7 +23,9 @@ $RelayStartTimeoutSeconds = 360
|
||||
$RelayPollIntervalSeconds = 1
|
||||
$ValidateOnlyTimeoutSeconds = 300
|
||||
$LiveDeployTimeoutSeconds = 300
|
||||
$LivePreflightTimeoutSeconds = 120
|
||||
$LivePreflightTimeoutSeconds = 60
|
||||
$PostVerifyMaxAttempts = 4
|
||||
$PostVerifyRetryWaitSeconds = 45
|
||||
$WindowsControlBaselineTimeoutSeconds = 300
|
||||
$PromotionReadinessReserveMinutes = 15
|
||||
$PromotionReadinessMaxAttempts = 2
|
||||
@@ -434,6 +436,9 @@ function Invoke-AgentLivePreflight {
|
||||
enabled = [bool]$_.enabled
|
||||
state = if ($state -in @("Ready", "Running", "Disabled", "Queued", "Missing")) { $state } else { "Unknown" }
|
||||
lastTaskResult = if ($null -ne $_.lastTaskResult) { [long]$_.lastTaskResult } else { $null }
|
||||
actionDefinitionReady = [bool]$_.actionDefinitionReady
|
||||
principalReady = [bool]$_.principalReady
|
||||
definitionReady = [bool]$_.definitionReady
|
||||
}
|
||||
})
|
||||
} else { @() }
|
||||
@@ -465,6 +470,14 @@ function Invoke-AgentLivePreflight {
|
||||
$ReadinessScope -eq "PromotionReserve" -or
|
||||
($payload -and $payload.manifest -and [bool]$payload.manifest.runtimeMatched)
|
||||
)
|
||||
$pendingCommandsReadyForScope = [bool](
|
||||
$summary -and
|
||||
[int]$summary.pendingCommandBlockingCount -eq 0 -and
|
||||
(
|
||||
$ReadinessScope -eq "PromotionReserve" -or
|
||||
[int]$summary.pendingCommandCount -eq 0
|
||||
)
|
||||
)
|
||||
$runtimePlaneReady = [bool](
|
||||
$acceptedExitCode -and
|
||||
-not $processResult.timedOut -and
|
||||
@@ -482,7 +495,7 @@ function Invoke-AgentLivePreflight {
|
||||
[int]$summary.relayListenerCount -gt 0 -and
|
||||
[int]$summary.alertIncomingStaleCount -eq 0 -and
|
||||
[int]$summary.alertRunningCount -eq 0 -and
|
||||
[int]$summary.pendingCommandCount -eq 0 -and
|
||||
$pendingCommandsReadyForScope -and
|
||||
$runtimePlaneBlockingReasons.Count -eq 0
|
||||
)
|
||||
return [pscustomobject]@{
|
||||
@@ -521,6 +534,16 @@ function Invoke-AgentLivePreflight {
|
||||
alertIncomingStaleCount = if ($summary) { [int]$summary.alertIncomingStaleCount } else { -1 }
|
||||
alertRunningCount = if ($summary) { [int]$summary.alertRunningCount } else { -1 }
|
||||
pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 }
|
||||
pendingCommandBlockingCount = if ($summary) { [int]$summary.pendingCommandBlockingCount } else { -1 }
|
||||
promotionDeferredCommandCount = if ($summary) { [int]$summary.promotionDeferredCommandCount } else { -1 }
|
||||
pendingCommandDeferralReason = if ($payload -and $payload.pendingCommandPromotion) {
|
||||
[string]$payload.pendingCommandPromotion.reason
|
||||
} else { "" }
|
||||
maintenancePendingCommandIdentities = if ($payload -and $payload.pendingCommandPromotion) {
|
||||
@($payload.pendingCommandPromotion.items | Where-Object {
|
||||
[bool]$_.identityValid -and [string]$_.identity -match "^auto-recover-[^|]+\|[0-9a-f]{64}$"
|
||||
} | ForEach-Object { [string]$_.identity } | Sort-Object -Unique)
|
||||
} else { @() }
|
||||
preflightGreen = [bool]($summary -and $summary.preflightGreen)
|
||||
overallPreflightGreen = [bool]($summary -and $summary.preflightGreen)
|
||||
runtimePlaneReady = $runtimePlaneReady
|
||||
@@ -579,6 +602,96 @@ function Test-AgentPromotionReadinessRetryable {
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-AgentPostPromotionReadinessRetryable {
|
||||
param(
|
||||
[object]$Readiness,
|
||||
[string[]]$ExpectedMaintenancePendingCommandIdentities = @()
|
||||
)
|
||||
|
||||
$pendingCount = if ($Readiness) { [int]$Readiness.pendingCommandCount } else { -1 }
|
||||
$pendingIdentities = if ($Readiness) {
|
||||
@($Readiness.maintenancePendingCommandIdentities | ForEach-Object { [string]$_ } | Sort-Object -Unique)
|
||||
} else { @() }
|
||||
$expectedIdentities = @($ExpectedMaintenancePendingCommandIdentities | ForEach-Object { [string]$_ } | Sort-Object -Unique)
|
||||
$pendingCommandsRetryable = [bool](
|
||||
$pendingCount -eq 0 -or
|
||||
(
|
||||
$pendingCount -gt 0 -and
|
||||
[int]$Readiness.pendingCommandBlockingCount -eq $pendingCount -and
|
||||
[string]$Readiness.pendingCommandDeferralReason -eq "full_runtime_requires_maintenance_auto_recover_to_drain" -and
|
||||
$pendingIdentities.Count -eq $pendingCount -and
|
||||
$expectedIdentities.Count -gt 0 -and
|
||||
@($pendingIdentities | Where-Object { $_ -notin $expectedIdentities }).Count -eq 0
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
-not $Readiness -or
|
||||
$Readiness.ok -or
|
||||
-not $Readiness.parsed -or
|
||||
-not $Readiness.scopeMatched -or
|
||||
[string]$Readiness.readinessScope -ne "FullRuntime" -or
|
||||
$Readiness.timedOut -or
|
||||
-not $Readiness.acceptedExitCode -or
|
||||
[int]$Readiness.relayListenerCount -lt 1 -or
|
||||
[int]$Readiness.alertRunningCount -ne 0 -or
|
||||
-not $pendingCommandsRetryable
|
||||
) {
|
||||
return $false
|
||||
}
|
||||
$allowedReasons = @(
|
||||
"scheduled_task_unhealthy",
|
||||
"required_evidence_stale",
|
||||
"required_evidence_freshness_reserve_insufficient",
|
||||
"required_evidence_content_unhealthy",
|
||||
"pending_work_before_reboot"
|
||||
)
|
||||
$reasons = @($Readiness.runtimePlaneBlockingReasons | ForEach-Object { [string]$_ })
|
||||
if (
|
||||
$reasons.Count -eq 0 -or
|
||||
@($reasons | Where-Object { $_ -notin $allowedReasons }).Count -gt 0
|
||||
) {
|
||||
return $false
|
||||
}
|
||||
if (@($Readiness.failedTasks | Where-Object {
|
||||
-not $_.exists -or
|
||||
-not $_.enabled -or
|
||||
-not $_.definitionReady -or
|
||||
$_.taskPath -ne "\" -or
|
||||
$_.state -notin @("Ready", "Running", "Queued")
|
||||
}).Count -gt 0) {
|
||||
return $false
|
||||
}
|
||||
if (
|
||||
"scheduled_task_unhealthy" -in $reasons -and
|
||||
@($Readiness.failedTasks).Count -eq 0
|
||||
) { return $false }
|
||||
if (
|
||||
"pending_work_before_reboot" -in $reasons -and
|
||||
$pendingCount -eq 0
|
||||
) { return $false }
|
||||
if (
|
||||
@($reasons | Where-Object {
|
||||
$_ -in @(
|
||||
"required_evidence_stale",
|
||||
"required_evidence_freshness_reserve_insufficient",
|
||||
"required_evidence_content_unhealthy"
|
||||
)
|
||||
}).Count -gt 0 -and
|
||||
@($Readiness.failedEvidence).Count -eq 0
|
||||
) { return $false }
|
||||
if (@($Readiness.failedEvidence | Where-Object {
|
||||
-not $_.parsed -or
|
||||
(
|
||||
-not $_.contentHealthy -and
|
||||
$_.name -notin @("telegram_inbox", "sre_alert_inbox")
|
||||
)
|
||||
}).Count -gt 0) {
|
||||
return $false
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Get-AgentRelayRuntime {
|
||||
$task = $null
|
||||
$taskError = ""
|
||||
@@ -1405,6 +1518,8 @@ try {
|
||||
$relayReload = $null
|
||||
$runtimeManifest = $null
|
||||
$livePreflight = $null
|
||||
$postVerifyAttempts = @()
|
||||
$postVerifyRetryCount = 0
|
||||
$launcherContract = $null
|
||||
$windowsControlBaseline = $null
|
||||
$postVerified = $false
|
||||
@@ -1442,25 +1557,40 @@ try {
|
||||
$script:ReceiverOperationPhase = "independent_post_verifier"
|
||||
$runtimeManifest = Get-AgentSafeRuntimeManifest
|
||||
$launcherContract = Get-AgentLauncherContract
|
||||
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
|
||||
$postVerified = [bool](
|
||||
$runtimeManifest.exists -and
|
||||
$runtimeManifest.runtimeMatched -and
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 18 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$launcherContract.ok -and
|
||||
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
|
||||
$relayReload.oldGenerationsGone -and
|
||||
$relayReload.newGenerationVerified -and
|
||||
$relayReload.after.taskEnabled -and
|
||||
$relayReload.after.taskRunning -and
|
||||
$livePreflight.ok -and
|
||||
$livePreflight.sourceRevision -eq $sourceRevision -and
|
||||
$livePreflight.runtimeMatched -and
|
||||
$livePreflight.relayListenerCount -gt 0 -and
|
||||
(Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations))
|
||||
)
|
||||
for ($postVerifyAttempt = 1; $postVerifyAttempt -le $PostVerifyMaxAttempts; $postVerifyAttempt += 1) {
|
||||
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
|
||||
$livePreflight | Add-Member -NotePropertyName attempt -NotePropertyValue $postVerifyAttempt -Force
|
||||
$postVerifyAttempts += $livePreflight
|
||||
$postVerified = [bool](
|
||||
$runtimeManifest.exists -and
|
||||
$runtimeManifest.runtimeMatched -and
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 18 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$launcherContract.ok -and
|
||||
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
|
||||
$relayReload.oldGenerationsGone -and
|
||||
$relayReload.newGenerationVerified -and
|
||||
$relayReload.after.taskEnabled -and
|
||||
$relayReload.after.taskRunning -and
|
||||
$livePreflight.ok -and
|
||||
$livePreflight.sourceRevision -eq $sourceRevision -and
|
||||
$livePreflight.runtimeMatched -and
|
||||
$livePreflight.relayListenerCount -gt 0 -and
|
||||
(Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations))
|
||||
)
|
||||
if ($postVerified) { break }
|
||||
if (
|
||||
$postVerifyAttempt -ge $PostVerifyMaxAttempts -or
|
||||
-not (Test-AgentPostPromotionReadinessRetryable `
|
||||
$livePreflight `
|
||||
-ExpectedMaintenancePendingCommandIdentities @(
|
||||
$prePromotionReadiness.maintenancePendingCommandIdentities
|
||||
))
|
||||
) { break }
|
||||
Start-Sleep -Seconds $PostVerifyRetryWaitSeconds
|
||||
}
|
||||
$postVerifyRetryCount = [math]::Max(0, $postVerifyAttempts.Count - 1)
|
||||
if (-not $postVerified) {
|
||||
$failurePhase = "post_verifier"
|
||||
$failureType = "independent_post_verifier_failed"
|
||||
@@ -1504,6 +1634,14 @@ try {
|
||||
runtimeManifest = $runtimeManifest
|
||||
launcherContract = $launcherContract
|
||||
livePreflight = $livePreflight
|
||||
postVerifyAttempts = $postVerifyAttempts
|
||||
postVerifyRetryCount = $postVerifyRetryCount
|
||||
postVerifyPolicy = [pscustomobject]@{
|
||||
maxAttempts = $PostVerifyMaxAttempts
|
||||
retryWaitSeconds = $PostVerifyRetryWaitSeconds
|
||||
automaticTaskTriggerAllowed = $false
|
||||
finalAttemptVerified = $postVerified
|
||||
}
|
||||
windowsControlBaseline = $windowsControlBaseline
|
||||
durableReceiptWritten = $true
|
||||
idempotentReplay = $false
|
||||
@@ -1629,6 +1767,8 @@ try {
|
||||
failedRuntimeManifest = $runtimeManifest
|
||||
failedLauncherContract = $launcherContract
|
||||
failedLivePreflight = $livePreflight
|
||||
postVerifyAttempts = $postVerifyAttempts
|
||||
postVerifyRetryCount = $postVerifyRetryCount
|
||||
failedWindowsControlBaseline = $windowsControlBaseline
|
||||
rollback = $rollback
|
||||
rollbackRelayRestart = $rollbackRelayRestart
|
||||
|
||||
347
scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1
Normal file
347
scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1
Normal file
@@ -0,0 +1,347 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("Check", "Apply")]
|
||||
[string]$Mode = "Check",
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
|
||||
[string]$TraceId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
|
||||
[string]$RunId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
|
||||
[string]$WorkItemId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._-]{0,180}\.json$")]
|
||||
[string]$QueueFileName,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-fA-F0-9]{64}$")]
|
||||
[string]$ExpectedQueueSha256,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")]
|
||||
[string]$ExpectedCommandId,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^agent99-control-loop-maintenance-2[0-9]{7}-[0-9]{6}\.json$")]
|
||||
[string]$MaintenanceReceiptName,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-fA-F0-9]{64}$")]
|
||||
[string]$ExpectedMaintenanceSha256,
|
||||
[ValidateRange(30, 1440)]
|
||||
[int]$MinimumAgeMinutes = 30,
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ExpectedQueueSha256 = $ExpectedQueueSha256.ToLowerInvariant()
|
||||
$ExpectedMaintenanceSha256 = $ExpectedMaintenanceSha256.ToLowerInvariant()
|
||||
$queueRoot = Join-Path $AgentRoot "queue"
|
||||
$failedRoot = Join-Path $queueRoot "failed"
|
||||
$evidenceRoot = Join-Path $AgentRoot "evidence"
|
||||
$sourcePath = Join-Path $queueRoot $QueueFileName
|
||||
$destinationName = "stale-maintenance-$QueueFileName"
|
||||
$destinationPath = Join-Path $failedRoot $destinationName
|
||||
$maintenancePath = Join-Path $evidenceRoot $MaintenanceReceiptName
|
||||
$runtimeWritePerformed = $false
|
||||
|
||||
function Get-AgentSha256 {
|
||||
param([string]$Path)
|
||||
(Get-FileHash -Algorithm SHA256 -LiteralPath $Path).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Write-AgentDurableJson {
|
||||
param([object]$Value, [string]$Path)
|
||||
|
||||
$temp = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
|
||||
$stream = $null
|
||||
$writer = $null
|
||||
try {
|
||||
$encoding = New-Object Text.UTF8Encoding($false)
|
||||
$stream = [IO.File]::Open(
|
||||
$temp,
|
||||
[IO.FileMode]::CreateNew,
|
||||
[IO.FileAccess]::Write,
|
||||
[IO.FileShare]::None
|
||||
)
|
||||
$writer = New-Object IO.StreamWriter($stream, $encoding)
|
||||
$writer.Write(($Value | ConvertTo-Json -Depth 10))
|
||||
$writer.Flush()
|
||||
$stream.Flush($true)
|
||||
$writer.Dispose(); $writer = $null
|
||||
$stream.Dispose(); $stream = $null
|
||||
[IO.File]::Move($temp, $Path)
|
||||
$hash = Get-AgentSha256 $Path
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
throw "durable_receipt_readback_missing"
|
||||
}
|
||||
return $hash
|
||||
} finally {
|
||||
if ($writer) { $writer.Dispose() }
|
||||
if ($stream) { $stream.Dispose() }
|
||||
Remove-Item -LiteralPath $temp -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
function Test-AgentRecoveryMutexActive {
|
||||
$mutex = $null
|
||||
$acquired = $false
|
||||
try {
|
||||
$mutex = [Threading.Mutex]::new(
|
||||
$false,
|
||||
"Global\WoooAgent99RecoveryTransportV1"
|
||||
)
|
||||
try {
|
||||
$acquired = [bool]$mutex.WaitOne(0)
|
||||
} catch [Threading.AbandonedMutexException] {
|
||||
$acquired = $true
|
||||
}
|
||||
return [bool](-not $acquired)
|
||||
} finally {
|
||||
if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
|
||||
if ($mutex) { $mutex.Dispose() }
|
||||
}
|
||||
}
|
||||
|
||||
function Get-AgentActiveControlProcesses {
|
||||
@(
|
||||
Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction Stop |
|
||||
Where-Object {
|
||||
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or
|
||||
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)'
|
||||
} |
|
||||
ForEach-Object {
|
||||
[pscustomobject]@{
|
||||
processId = [int]$_.ProcessId
|
||||
creationDate = ([datetime]$_.CreationDate).ToUniversalTime().ToString("o")
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function Get-AgentQuarantineGate {
|
||||
if (-not (Test-Path -LiteralPath $sourcePath -PathType Leaf)) {
|
||||
throw "exact_queue_source_missing"
|
||||
}
|
||||
if (Test-Path -LiteralPath $destinationPath) {
|
||||
throw "quarantine_destination_already_exists"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $maintenancePath -PathType Leaf)) {
|
||||
throw "maintenance_receipt_missing"
|
||||
}
|
||||
$queueHash = Get-AgentSha256 $sourcePath
|
||||
$maintenanceHash = Get-AgentSha256 $maintenancePath
|
||||
if ($queueHash -ne $ExpectedQueueSha256) { throw "queue_hash_mismatch" }
|
||||
if ($maintenanceHash -ne $ExpectedMaintenanceSha256) {
|
||||
throw "maintenance_hash_mismatch"
|
||||
}
|
||||
|
||||
$task = Get-ScheduledTask `
|
||||
-TaskName "Wooo-Agent99-Control-Loop" `
|
||||
-TaskPath "\" `
|
||||
-ErrorAction Stop
|
||||
if ($task.Settings.Enabled -or [string]$task.State -ne "Disabled") {
|
||||
throw "control_loop_not_frozen"
|
||||
}
|
||||
$activeProcesses = Get-AgentActiveControlProcesses
|
||||
if ($activeProcesses.Count -ne 0) { throw "active_control_or_recover_process" }
|
||||
$leasePresent = [bool](Test-Path -LiteralPath (
|
||||
Join-Path $AgentRoot "state\recovery-transport-lease.json"
|
||||
) -PathType Leaf)
|
||||
if ($leasePresent) { throw "recovery_lease_present" }
|
||||
if (Test-AgentRecoveryMutexActive) { throw "recovery_mutex_active" }
|
||||
|
||||
$maintenance = Get-Content -LiteralPath $maintenancePath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
[string]$maintenance.schemaVersion -ne "agent99_control_loop_maintenance_v2" -or
|
||||
[string]$maintenance.mode -ne "Apply" -or
|
||||
[string]$maintenance.taskName -ne "Wooo-Agent99-Control-Loop" -or
|
||||
[string]$maintenance.terminal -ne "control_loop_maintenance_frozen" -or
|
||||
$maintenance.precheckPassed -isnot [bool] -or -not $maintenance.precheckPassed -or
|
||||
$maintenance.controlStopped -isnot [bool] -or -not $maintenance.controlStopped -or
|
||||
$maintenance.recoverStopped -isnot [bool] -or -not $maintenance.recoverStopped -or
|
||||
$maintenance.leaseMetadataRemoved -isnot [bool] -or -not $maintenance.leaseMetadataRemoved -or
|
||||
$maintenance.runtimeWritePerformed -isnot [bool] -or -not $maintenance.runtimeWritePerformed -or
|
||||
$maintenance.vmPowerChangePerformed -isnot [bool] -or $maintenance.vmPowerChangePerformed -or
|
||||
$maintenance.vmwareVmxProcessTerminated -isnot [bool] -or $maintenance.vmwareVmxProcessTerminated -or
|
||||
$maintenance.vmxLockDeleted -isnot [bool] -or $maintenance.vmxLockDeleted -or
|
||||
$maintenance.hostRebootPerformed -isnot [bool] -or $maintenance.hostRebootPerformed -or
|
||||
$maintenance.secretValueRead -isnot [bool] -or $maintenance.secretValueRead
|
||||
) {
|
||||
throw "maintenance_receipt_contract_invalid"
|
||||
}
|
||||
|
||||
$command = Get-Content -LiteralPath $sourcePath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
[string]$command.id -ne $ExpectedCommandId -or
|
||||
[string]$command.mode -ne "Recover" -or
|
||||
[string]$command.source -ne "agent99-recovery-observer" -or
|
||||
$command.controlledApply -isnot [bool] -or
|
||||
-not $command.controlledApply -or
|
||||
[string]$command.traceId -notmatch '^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$' -or
|
||||
[string]$command.workItemId -notmatch '^agent99-auto:[A-Za-z0-9._:@+-]{1,160}$'
|
||||
) {
|
||||
throw "queue_identity_contract_invalid"
|
||||
}
|
||||
$sourceItem = Get-Item -LiteralPath $sourcePath
|
||||
$ageMinutes = [math]::Round(((Get-Date) - $sourceItem.LastWriteTime).TotalMinutes, 1)
|
||||
if ($ageMinutes -lt $MinimumAgeMinutes) { throw "queue_item_not_stale" }
|
||||
|
||||
[pscustomobject]@{
|
||||
queueHash = $queueHash
|
||||
maintenanceHash = $maintenanceHash
|
||||
ageMinutes = $ageMinutes
|
||||
commandId = [string]$command.id
|
||||
commandMode = [string]$command.mode
|
||||
commandSource = [string]$command.source
|
||||
controlTaskEnabled = [bool]$task.Settings.Enabled
|
||||
controlTaskState = [string]$task.State
|
||||
activeControlProcessCount = [int]$activeProcesses.Count
|
||||
recoveryLeasePresent = $leasePresent
|
||||
recoveryMutexActive = $false
|
||||
}
|
||||
}
|
||||
|
||||
$gate = Get-AgentQuarantineGate
|
||||
$base = [ordered]@{
|
||||
schemaVersion = "agent99_stale_queue_quarantine_v1"
|
||||
timestamp = (Get-Date).ToString("o")
|
||||
mode = $Mode
|
||||
traceId = $TraceId
|
||||
runId = $RunId
|
||||
workItemId = $WorkItemId
|
||||
commandId = $gate.commandId
|
||||
commandMode = $gate.commandMode
|
||||
commandSource = $gate.commandSource
|
||||
sourceFile = $QueueFileName
|
||||
sourceSha256 = $gate.queueHash
|
||||
destinationFile = $destinationName
|
||||
ageMinutes = $gate.ageMinutes
|
||||
minimumAgeMinutes = $MinimumAgeMinutes
|
||||
maintenanceReceipt = $MaintenanceReceiptName
|
||||
maintenanceReceiptSha256 = $gate.maintenanceHash
|
||||
activeControlProcessCount = $gate.activeControlProcessCount
|
||||
recoveryLeasePresent = $gate.recoveryLeasePresent
|
||||
recoveryMutexActive = $gate.recoveryMutexActive
|
||||
commandExecuted = $false
|
||||
deletePerformed = $false
|
||||
otherQueueMutationPerformed = $false
|
||||
rawInstructionStored = $false
|
||||
secretValueRead = $false
|
||||
runtimeWritePerformed = $false
|
||||
}
|
||||
|
||||
if ($Mode -eq "Check") {
|
||||
$base["terminal"] = "check_passed_apply_allowed"
|
||||
$base | ConvertTo-Json -Depth 8 -Compress
|
||||
exit 0
|
||||
}
|
||||
|
||||
$writerMutex = [Threading.Mutex]::new(
|
||||
$false,
|
||||
"Global\WoooAgent99StaleQueueMaintenanceV1"
|
||||
)
|
||||
$writerAcquired = $false
|
||||
$intentPath = ""
|
||||
$intentHash = ""
|
||||
$resultPath = ""
|
||||
$moved = $false
|
||||
$rollbackAttempted = $false
|
||||
$rollbackVerified = $false
|
||||
try {
|
||||
try {
|
||||
$writerAcquired = [bool]$writerMutex.WaitOne(0)
|
||||
} catch [Threading.AbandonedMutexException] {
|
||||
$writerAcquired = $true
|
||||
}
|
||||
if (-not $writerAcquired) { throw "queue_maintenance_writer_active" }
|
||||
|
||||
$gate = Get-AgentQuarantineGate
|
||||
$safeRunId = $RunId -replace "[^A-Za-z0-9._-]", "_"
|
||||
$stamp = Get-Date -Format "yyyyMMdd-HHmmss-fff"
|
||||
$intentPath = Join-Path $evidenceRoot (
|
||||
"agent99-stale-queue-quarantine-$stamp-$safeRunId-intent.json"
|
||||
)
|
||||
$resultPath = Join-Path $evidenceRoot (
|
||||
"agent99-stale-queue-quarantine-$stamp-$safeRunId-result.json"
|
||||
)
|
||||
$intent = [ordered]@{} + $base
|
||||
$intent["schemaVersion"] = "agent99_stale_queue_quarantine_intent_v1"
|
||||
$intent["mode"] = "Apply"
|
||||
$intent["plannedAction"] = "atomic_move_exact_queue_item_to_failed"
|
||||
$intent["rollback"] = "atomic_move_back_if_post_move_hash_verifier_fails"
|
||||
$intent["terminal"] = "intent_durable_apply_pending"
|
||||
$intentHash = Write-AgentDurableJson $intent $intentPath
|
||||
$runtimeWritePerformed = $true
|
||||
|
||||
$gate = Get-AgentQuarantineGate
|
||||
[IO.File]::Move($sourcePath, $destinationPath)
|
||||
$moved = $true
|
||||
$sourceAbsent = -not (Test-Path -LiteralPath $sourcePath)
|
||||
$destinationPresent = Test-Path -LiteralPath $destinationPath -PathType Leaf
|
||||
$destinationHash = if ($destinationPresent) {
|
||||
Get-AgentSha256 $destinationPath
|
||||
} else { "" }
|
||||
if (-not ($sourceAbsent -and $destinationPresent -and $destinationHash -eq $ExpectedQueueSha256)) {
|
||||
throw "post_move_verifier_failed"
|
||||
}
|
||||
|
||||
$result = [ordered]@{} + $base
|
||||
$result["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1"
|
||||
$result["mode"] = "Apply"
|
||||
$result["intentFile"] = Split-Path -Leaf $intentPath
|
||||
$result["intentSha256"] = $intentHash
|
||||
$result["sourceAbsent"] = $sourceAbsent
|
||||
$result["destinationPresent"] = $destinationPresent
|
||||
$result["destinationSha256"] = $destinationHash
|
||||
$result["runtimeWritePerformed"] = $true
|
||||
$result["terminal"] = "verified_quarantined"
|
||||
$resultHash = Write-AgentDurableJson $result $resultPath
|
||||
$result["resultFile"] = Split-Path -Leaf $resultPath
|
||||
$result["resultSha256"] = $resultHash
|
||||
$result | ConvertTo-Json -Depth 8 -Compress
|
||||
exit 0
|
||||
} catch {
|
||||
$failureType = $_.Exception.GetType().Name
|
||||
if (
|
||||
$moved -and
|
||||
-not (Test-Path -LiteralPath $sourcePath) -and
|
||||
(Test-Path -LiteralPath $destinationPath -PathType Leaf)
|
||||
) {
|
||||
$rollbackAttempted = $true
|
||||
if ((Get-AgentSha256 $destinationPath) -eq $ExpectedQueueSha256) {
|
||||
[IO.File]::Move($destinationPath, $sourcePath)
|
||||
$rollbackVerified = [bool](
|
||||
(Test-Path -LiteralPath $sourcePath -PathType Leaf) -and
|
||||
-not (Test-Path -LiteralPath $destinationPath) -and
|
||||
(Get-AgentSha256 $sourcePath) -eq $ExpectedQueueSha256
|
||||
)
|
||||
}
|
||||
}
|
||||
$failure = [ordered]@{} + $base
|
||||
$failure["schemaVersion"] = "agent99_stale_queue_quarantine_result_v1"
|
||||
$failure["mode"] = "Apply"
|
||||
$failure["intentFile"] = if ($intentPath) { Split-Path -Leaf $intentPath } else { "" }
|
||||
$failure["intentSha256"] = $intentHash
|
||||
$failure["failureType"] = $failureType
|
||||
$failure["rollbackAttempted"] = $rollbackAttempted
|
||||
$failure["rollbackVerified"] = $rollbackVerified
|
||||
$failure["runtimeWritePerformed"] = $runtimeWritePerformed
|
||||
$failure["terminal"] = if ($rollbackAttempted -and $rollbackVerified) {
|
||||
"failed_rolled_back"
|
||||
} elseif ($rollbackAttempted) {
|
||||
"rollback_unverified"
|
||||
} else {
|
||||
"failed_no_queue_move"
|
||||
}
|
||||
if ($resultPath -and -not (Test-Path -LiteralPath $resultPath)) {
|
||||
try {
|
||||
$failureHash = Write-AgentDurableJson $failure $resultPath
|
||||
$failure["resultFile"] = Split-Path -Leaf $resultPath
|
||||
$failure["resultSha256"] = $failureHash
|
||||
} catch {}
|
||||
}
|
||||
$failure | ConvertTo-Json -Depth 8 -Compress
|
||||
exit 2
|
||||
} finally {
|
||||
if ($writerAcquired) { try { $writerMutex.ReleaseMutex() } catch {} }
|
||||
$writerMutex.Dispose()
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -24,7 +25,7 @@ def _function_source() -> str:
|
||||
|
||||
def _evidence_function_source() -> str:
|
||||
source = PREFLIGHT.read_text(encoding="utf-8")
|
||||
start = source.index("function Get-EvidenceReadback")
|
||||
start = source.index("function Get-AgentBoundedNewestFiles")
|
||||
end = source.index("$requiredTasks = @(", start)
|
||||
return source[start:end]
|
||||
|
||||
@@ -52,7 +53,8 @@ def _base_case() -> dict[str, Any]:
|
||||
"staleAlertExecutionCount": 0,
|
||||
"alertIncomingCount": 0,
|
||||
"staleAlertIncomingCount": 0,
|
||||
"pendingCommandCount": 0,
|
||||
"pendingCommandBlockingCount": 0,
|
||||
"promotionDeferredCommandCount": 0,
|
||||
}
|
||||
|
||||
|
||||
@@ -96,7 +98,8 @@ $parameters = @{{
|
||||
StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
|
||||
AlertIncomingCount = [int]$case.alertIncomingCount
|
||||
StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount
|
||||
PendingCommandCount = [int]$case.pendingCommandCount
|
||||
PendingCommandBlockingCount = [int]$case.pendingCommandBlockingCount
|
||||
PromotionDeferredCommandCount = [int]$case.promotionDeferredCommandCount
|
||||
}}
|
||||
Get-AgentLivePreflightDecision @parameters |
|
||||
ConvertTo-Json -Depth 6 -Compress
|
||||
@@ -121,6 +124,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
assert '$warningReasons += "self_health_warning"' in decision
|
||||
assert '$warningReasons += "performance_warning"' in decision
|
||||
assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision
|
||||
assert (
|
||||
'$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"'
|
||||
in decision
|
||||
)
|
||||
assert '"critical" { $blockingReasons += "self_health_not_ok" }' in decision
|
||||
assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
|
||||
assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
|
||||
@@ -147,9 +154,45 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
assert '"agent99_background_mode_deferred_v1"' in source
|
||||
assert '"agent99_recovery_transport_deferred_v1"' in source
|
||||
assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source
|
||||
assert "[IO.Directory]::EnumerateFiles(" in source
|
||||
assert "[Collections.Generic.SortedSet[string]]::new(" in source
|
||||
assert "$null = $selected.Remove($selected.Min)" in source
|
||||
assert "candidateWindowExhausted = $candidateWindowExhausted" in source
|
||||
assert "function Get-AgentControlLoopMaintenanceReadback" in source
|
||||
assert '"agent99-control-loop-maintenance-2*.json"' in source
|
||||
assert '$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 })' in source
|
||||
assert '$legacyPropertySetMatches' in source
|
||||
assert '$legacyIdentityFieldsValid' in source
|
||||
assert '$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled' in source
|
||||
assert '"legacy_v2_exact_upgrade_bridge"' in source
|
||||
assert '$receipt.precheckPassed -is [bool]' in source
|
||||
assert '$receipt.taskEnabledAfter -is [bool]' in source
|
||||
assert '$receipt.runtimeWritePerformed -is [bool]' in source
|
||||
assert '$intent.secretValueRead -is [bool]' in source
|
||||
assert '$activeControlProcesses.Count -eq 0' in source
|
||||
assert '-not $leasePresent' in source
|
||||
assert '-not $recoveryMutexActive' in source
|
||||
assert '$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")' in source
|
||||
assert '$task | Add-Member -NotePropertyName runtimeHealthy' in source
|
||||
assert '$task | Add-Member -NotePropertyName promotionReady' in source
|
||||
assert '$controlLoopMaintenance.valid' in source
|
||||
assert "function Get-AgentPendingCommandPromotionReadback" in source
|
||||
assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source
|
||||
assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
|
||||
assert '$payload.controlledApply -is [bool] -and $payload.controlledApply' in source
|
||||
assert '$payload.observation -is [pscustomobject]' in source
|
||||
assert 'instructionContentReadback = $false' in source
|
||||
assert '$rootTaskPath = "\\"' in source
|
||||
assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source
|
||||
assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source
|
||||
assert "[string[]]$RequiredArgumentMarkers" in source
|
||||
assert "$actions.Count -eq 1" in source
|
||||
assert "actionDefinitionReady = $actionDefinitionReady" in source
|
||||
assert "principalReady = $principalReady" in source
|
||||
assert "definitionReady = $definitionReady" in source
|
||||
assert '$task.Principal.LogonType -eq "S4U"' in source
|
||||
assert '$task.Principal.RunLevel -eq "Highest"' in source
|
||||
assert "$task.definitionReady" in source
|
||||
|
||||
|
||||
def test_evidence_selection_skips_deferred_but_not_malformed_candidates(
|
||||
@@ -274,6 +317,165 @@ Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 |
|
||||
assert payload["healthy"] is False
|
||||
|
||||
|
||||
def test_evidence_candidate_window_fails_closed_when_recent_files_are_all_deferred(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
||||
|
||||
evidence = tmp_path / "evidence"
|
||||
evidence.mkdir()
|
||||
canonical = evidence / "agent99-SelfCheck-20260715-190000.json"
|
||||
canonical.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"mode": "SelfCheck",
|
||||
"selfHealth": {"severity": "ok"},
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
base_time = canonical.stat().st_mtime
|
||||
os.utime(canonical, (base_time - 1000, base_time - 1000))
|
||||
for index in range(40):
|
||||
deferred = evidence / f"agent99-SelfCheck-20260715-20{index:04d}.json"
|
||||
deferred.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schemaVersion": "agent99_background_mode_deferred_v1",
|
||||
"deferred": True,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
candidate_time = base_time + index
|
||||
os.utime(deferred, (candidate_time, candidate_time))
|
||||
|
||||
agent_root = str(tmp_path).replace("'", "''")
|
||||
command = f"""
|
||||
{_evidence_function_source()}
|
||||
$AgentRoot = '{agent_root}'
|
||||
Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true |
|
||||
ConvertTo-Json -Depth 8 -Compress
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
payload = json.loads(
|
||||
next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{"))
|
||||
)
|
||||
assert payload["enumeratedCandidateCount"] == 41
|
||||
assert payload["candidateWindowLimit"] == 32
|
||||
assert payload["candidateWindowTruncated"] is True
|
||||
assert payload["candidateWindowExhausted"] is True
|
||||
assert payload["deferredCandidateCount"] == 32
|
||||
assert payload["parseError"] == "EvidenceCandidateWindowExhausted"
|
||||
assert payload["healthy"] is False
|
||||
|
||||
|
||||
def test_promotion_reserve_defers_only_exact_maintenance_auto_recover(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
||||
|
||||
queue = tmp_path / "queue"
|
||||
queue.mkdir()
|
||||
now = datetime.now(timezone.utc)
|
||||
incident_id = "agent99-auto-" + ("a" * 20)
|
||||
command_id = "auto-recover-20260719-152746-39ca4a72"
|
||||
command_path = queue / f"{command_id}.json"
|
||||
payload = {
|
||||
"approvalId": "",
|
||||
"automationRunId": "096c8173-96b9-5659-94d3-5679c3fb770f",
|
||||
"canonicalDigest": "b" * 64,
|
||||
"controlledApply": True,
|
||||
"correlationKey": "agent99-controlled:" + ("c" * 64),
|
||||
"createdAt": now.isoformat(),
|
||||
"dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1",
|
||||
"dispatchRouteId": "agent99_local_observer",
|
||||
"executionGeneration": "1",
|
||||
"externalId": "host_unreachable_detected",
|
||||
"id": command_id,
|
||||
"idempotencyKey": "agent99-controlled:" + ("c" * 64),
|
||||
"incidentId": incident_id,
|
||||
"instruction": "recover exact observed unreachable host through controlled playbook",
|
||||
"lockOwner": "096c8173-96b9-5659-94d3-5679c3fb770f",
|
||||
"mode": "Recover",
|
||||
"observation": {},
|
||||
"projectId": "awoooi",
|
||||
"reason": "host_unreachable_detected",
|
||||
"requestedBy": "Agent99",
|
||||
"singleFlightKey": "agent99-controlled-flight:" + ("d" * 64),
|
||||
"source": "agent99-recovery-observer",
|
||||
"sourceFingerprint": "e" * 64,
|
||||
"traceId": "00-" + ("f" * 32) + "-" + ("1" * 16) + "-01",
|
||||
"workItemId": f"agent99-auto:{incident_id}:host_unreachable_detected",
|
||||
}
|
||||
|
||||
function_source = _evidence_function_source()
|
||||
agent_root = str(tmp_path).replace("'", "''")
|
||||
maintenance_timestamp = (now - timedelta(minutes=10)).isoformat()
|
||||
|
||||
def run(scope: str) -> dict[str, Any]:
|
||||
command = f"""
|
||||
{function_source}
|
||||
$AgentRoot = '{agent_root}'
|
||||
$maintenance = [pscustomobject]@{{
|
||||
valid = $true
|
||||
receiptFieldsValid = $true
|
||||
intentFieldsValid = $true
|
||||
receiptTimestamp = '{maintenance_timestamp}'
|
||||
}}
|
||||
Get-AgentPendingCommandPromotionReadback -MaintenanceReadback $maintenance -Scope {scope} |
|
||||
ConvertTo-Json -Depth 8 -Compress
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
return json.loads(
|
||||
next(
|
||||
line
|
||||
for line in reversed(result.stdout.splitlines())
|
||||
if line.startswith("{")
|
||||
)
|
||||
)
|
||||
|
||||
command_path.write_bytes(b"\xef\xbb\xbf" + json.dumps(payload).encode("utf-8"))
|
||||
reserve = run("PromotionReserve")
|
||||
assert reserve["observedCount"] == 1
|
||||
assert reserve["deferredCount"] == 1
|
||||
assert reserve["blockingCount"] == 0
|
||||
assert reserve["contractValid"] is True
|
||||
assert reserve["items"][0]["instructionContentReadback"] is False
|
||||
assert reserve["items"][0]["identity"].startswith(f"{command_id}|")
|
||||
|
||||
full_runtime = run("FullRuntime")
|
||||
assert full_runtime["deferredCount"] == 0
|
||||
assert full_runtime["blockingCount"] == 1
|
||||
assert full_runtime["eligibleMaintenanceAutoRecoverCount"] == 1
|
||||
assert full_runtime["reason"] == (
|
||||
"full_runtime_requires_maintenance_auto_recover_to_drain"
|
||||
)
|
||||
|
||||
payload["mode"] = "Status"
|
||||
command_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
invalid = run("PromotionReserve")
|
||||
assert invalid["deferredCount"] == 0
|
||||
assert invalid["blockingCount"] == 1
|
||||
assert invalid["contractValid"] is False
|
||||
|
||||
|
||||
def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
@@ -348,11 +550,17 @@ def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(pendingCommandCount=1),
|
||||
_with_overrides(pendingCommandBlockingCount=1),
|
||||
False,
|
||||
("pending_work_before_reboot",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(promotionDeferredCommandCount=1),
|
||||
True,
|
||||
(),
|
||||
("maintenance_pending_commands_deferred_until_post_promotion",),
|
||||
),
|
||||
)
|
||||
|
||||
for case, expected_eligible, expected_blockers, expected_warnings in cases:
|
||||
|
||||
@@ -391,11 +391,39 @@ def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() ->
|
||||
assert "failedTasks" in source
|
||||
assert "failedEvidenceNames" in source
|
||||
assert "failedEvidence" in source
|
||||
assert "actionDefinitionReady = [bool]$_.actionDefinitionReady" in source
|
||||
assert "principalReady = [bool]$_.principalReady" in source
|
||||
assert "definitionReady = [bool]$_.definitionReady" in source
|
||||
assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source
|
||||
assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source
|
||||
assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source
|
||||
|
||||
|
||||
def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
assert "$LivePreflightTimeoutSeconds = 60" in source
|
||||
assert "$PostVerifyMaxAttempts = 4" in source
|
||||
assert "$PostVerifyRetryWaitSeconds = 45" in source
|
||||
assert "function Test-AgentPostPromotionReadinessRetryable" in source
|
||||
assert '[string]$Readiness.readinessScope -ne "FullRuntime"' in source
|
||||
assert '"required_evidence_stale"' in source
|
||||
assert '"required_evidence_content_unhealthy"' in source
|
||||
assert '$_.name -notin @("telegram_inbox", "sre_alert_inbox")' in source
|
||||
assert "-not $_.definitionReady" in source
|
||||
assert "[int]$Readiness.alertRunningCount -ne 0" in source
|
||||
assert "$pendingCommandsRetryable" in source
|
||||
assert "$ExpectedMaintenancePendingCommandIdentities" in source
|
||||
assert '"pending_work_before_reboot"' in source
|
||||
assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
|
||||
assert "maintenancePendingCommandIdentities" in source
|
||||
assert "$postVerifyAttempt -le $PostVerifyMaxAttempts" in source
|
||||
assert "Start-Sleep -Seconds $PostVerifyRetryWaitSeconds" in source
|
||||
assert "automaticTaskTriggerAllowed = $false" in source
|
||||
assert "postVerifyAttempts = $postVerifyAttempts" in source
|
||||
assert "postVerifyRetryCount = $postVerifyRetryCount" in source
|
||||
|
||||
|
||||
def test_check_and_apply_receipts_project_the_same_run_identity() -> None:
|
||||
sender = SENDER.read_text(encoding="utf-8")
|
||||
projector = sender[
|
||||
@@ -607,7 +635,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
assert "$RelayPollIntervalSeconds = 1" in source
|
||||
assert "$ValidateOnlyTimeoutSeconds = 300" in source
|
||||
assert "$LiveDeployTimeoutSeconds = 300" in source
|
||||
assert "$LivePreflightTimeoutSeconds = 120" in source
|
||||
assert "$LivePreflightTimeoutSeconds = 60" in source
|
||||
assert "$PromotionReadinessMaxAttempts = 2" in source
|
||||
assert "$PromotionReadinessRetryWaitSeconds = 30" in source
|
||||
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
@@ -648,18 +676,23 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
promotion_readiness_attempts = 2
|
||||
promotion_readiness_retry_wait_seconds = 30
|
||||
deploy_seconds = 300
|
||||
live_preflight_seconds = 120
|
||||
live_preflight_seconds = 60
|
||||
post_verify_attempts = 4
|
||||
post_verify_retry_wait_seconds = 45
|
||||
windows_control_baseline_seconds = 300
|
||||
worst_case_seconds = (
|
||||
validate_seconds
|
||||
+ (promotion_readiness_attempts * live_preflight_seconds)
|
||||
+ promotion_readiness_retry_wait_seconds
|
||||
+ deploy_seconds
|
||||
+ live_preflight_seconds
|
||||
+ (post_verify_attempts * live_preflight_seconds)
|
||||
+ ((post_verify_attempts - 1) * post_verify_retry_wait_seconds)
|
||||
+ (2 * (60 + 360))
|
||||
+ windows_control_baseline_seconds
|
||||
)
|
||||
assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source
|
||||
assert worst_case_seconds == 1830
|
||||
assert wrapper_budget_seconds - worst_case_seconds == 570
|
||||
assert worst_case_seconds == 2265
|
||||
assert wrapper_budget_seconds - worst_case_seconds == 135
|
||||
|
||||
deploy = source.index(
|
||||
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
|
||||
@@ -781,6 +814,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() -
|
||||
assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight
|
||||
assert '[int]$summary.alertIncomingCount -eq 0' not in preflight
|
||||
assert '[int]$summary.alertRunningCount -eq 0' in preflight
|
||||
assert '[int]$summary.pendingCommandBlockingCount -eq 0' in preflight
|
||||
assert '$ReadinessScope -eq "PromotionReserve"' in preflight
|
||||
assert '[int]$summary.pendingCommandCount -eq 0' in preflight
|
||||
assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight
|
||||
assert '[string[]]$blockingReasons = @()' in preflight
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1"
|
||||
|
||||
|
||||
def test_stale_queue_quarantine_is_exact_recoverable_and_receipted() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert '[ValidateSet("Check", "Apply")]' in source
|
||||
assert "ExpectedQueueSha256" in source
|
||||
assert "ExpectedMaintenanceSha256" in source
|
||||
assert '"Global\\WoooAgent99StaleQueueMaintenanceV1"' in source
|
||||
assert '"Global\\WoooAgent99RecoveryTransportV1"' in source
|
||||
assert 'Get-ScheduledTask `' in source
|
||||
assert '"Wooo-Agent99-Control-Loop"' in source
|
||||
assert 'throw "control_loop_not_frozen"' in source
|
||||
assert 'throw "active_control_or_recover_process"' in source
|
||||
assert 'throw "recovery_lease_present"' in source
|
||||
assert 'throw "recovery_mutex_active"' in source
|
||||
assert '[string]$command.mode -ne "Recover"' in source
|
||||
assert '[string]$command.source -ne "agent99-recovery-observer"' in source
|
||||
assert "$command.controlledApply -isnot [bool]" in source
|
||||
assert '"atomic_move_exact_queue_item_to_failed"' in source
|
||||
assert "[IO.File]::Move($sourcePath, $destinationPath)" in source
|
||||
assert "[IO.File]::Move($destinationPath, $sourcePath)" in source
|
||||
assert '"verified_quarantined"' in source
|
||||
assert '"failed_rolled_back"' in source
|
||||
assert '"rollback_unverified"' in source
|
||||
|
||||
|
||||
def test_stale_queue_quarantine_does_not_execute_or_delete_command() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "commandExecuted = $false" in source
|
||||
assert "deletePerformed = $false" in source
|
||||
assert "otherQueueMutationPerformed = $false" in source
|
||||
assert "rawInstructionStored = $false" in source
|
||||
assert "secretValueRead = $false" in source
|
||||
assert "Remove-Item -LiteralPath $sourcePath" not in source
|
||||
assert "Start-ScheduledTask" not in source
|
||||
assert "Invoke-Expression" not in source
|
||||
assert "& $command" not in source
|
||||
Reference in New Issue
Block a user