fix(agent99): reserve pre-promotion readiness
This commit is contained in:
ogt
@@ -2,7 +2,11 @@
|
||||
param(
|
||||
[ValidateSet("Verify")]
|
||||
[string]$Mode = "Verify",
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99"
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99",
|
||||
[ValidateSet("FullRuntime", "PromotionReserve")]
|
||||
[string]$ReadinessScope = "FullRuntime",
|
||||
[ValidateRange(0, 20)]
|
||||
[int]$MinimumEvidenceFreshnessReserveMinutes = 0
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@@ -11,10 +15,12 @@ function Get-AgentLivePreflightDecision {
|
||||
param(
|
||||
[bool]$AgentRootPresent,
|
||||
[object]$Manifest,
|
||||
[bool]$ManifestCheckRequired,
|
||||
[int]$ExpectedRuntimeFileCount,
|
||||
[int]$TaskFailureCount,
|
||||
[int]$RelayListenerCount,
|
||||
[int]$RequiredEvidenceStaleCount,
|
||||
[int]$RequiredEvidenceReserveFailureCount,
|
||||
[int]$RequiredEvidenceParseFailureCount,
|
||||
[int]$RequiredEvidenceContentFailureCount,
|
||||
[string]$SelfHealthSeverity,
|
||||
@@ -44,21 +50,24 @@ function Get-AgentLivePreflightDecision {
|
||||
)
|
||||
|
||||
if (-not $AgentRootPresent) { $blockingReasons += "agent_root_missing" }
|
||||
if (-not $manifestExists) {
|
||||
$blockingReasons += "runtime_manifest_missing"
|
||||
} elseif (-not $manifestParseReady) {
|
||||
$blockingReasons += "runtime_manifest_parse_failed"
|
||||
} else {
|
||||
if (-not $manifestIdentityReady) {
|
||||
$blockingReasons += "runtime_manifest_identity_invalid"
|
||||
}
|
||||
if (-not $manifestContentReady) {
|
||||
$blockingReasons += "runtime_manifest_mismatch"
|
||||
if ($ManifestCheckRequired) {
|
||||
if (-not $manifestExists) {
|
||||
$blockingReasons += "runtime_manifest_missing"
|
||||
} elseif (-not $manifestParseReady) {
|
||||
$blockingReasons += "runtime_manifest_parse_failed"
|
||||
} else {
|
||||
if (-not $manifestIdentityReady) {
|
||||
$blockingReasons += "runtime_manifest_identity_invalid"
|
||||
}
|
||||
if (-not $manifestContentReady) {
|
||||
$blockingReasons += "runtime_manifest_mismatch"
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($TaskFailureCount -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
|
||||
if ($RelayListenerCount -lt 1) { $blockingReasons += "sre_alert_relay_not_listening" }
|
||||
if ($RequiredEvidenceStaleCount -gt 0) { $blockingReasons += "required_evidence_stale" }
|
||||
if ($RequiredEvidenceReserveFailureCount -gt 0) { $blockingReasons += "required_evidence_freshness_reserve_insufficient" }
|
||||
if ($RequiredEvidenceParseFailureCount -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
|
||||
if ($RequiredEvidenceContentFailureCount -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" }
|
||||
|
||||
@@ -91,6 +100,7 @@ function Get-AgentLivePreflightDecision {
|
||||
blockingReasons = @($blockingReasons)
|
||||
warningReasons = @($warningReasons)
|
||||
warningCount = [int]$warningReasons.Count
|
||||
manifestCheckRequired = $ManifestCheckRequired
|
||||
manifestIdentityReady = $manifestIdentityReady
|
||||
manifestContentReady = $manifestContentReady
|
||||
selfHealthClassification = $normalizedSelfHealth
|
||||
@@ -98,11 +108,16 @@ function Get-AgentLivePreflightDecision {
|
||||
}
|
||||
|
||||
function Get-TaskReadback {
|
||||
param([string]$TaskName)
|
||||
param(
|
||||
[string]$TaskPath,
|
||||
[string]$TaskName
|
||||
)
|
||||
|
||||
try {
|
||||
$task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop
|
||||
$info = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction Stop
|
||||
$tasks = @(Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop)
|
||||
if ($tasks.Count -ne 1) { throw "exact_root_task_count_invalid" }
|
||||
$task = $tasks[0]
|
||||
$info = Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop
|
||||
$lastResult = [int64]$info.LastTaskResult
|
||||
$state = [string]$task.State
|
||||
$healthy = [bool](
|
||||
@@ -112,6 +127,7 @@ function Get-TaskReadback {
|
||||
|
||||
return [pscustomobject]@{
|
||||
name = $TaskName
|
||||
taskPath = $TaskPath
|
||||
exists = $true
|
||||
enabled = [bool]$task.Settings.Enabled
|
||||
state = $state
|
||||
@@ -123,6 +139,7 @@ function Get-TaskReadback {
|
||||
} catch {
|
||||
return [pscustomobject]@{
|
||||
name = $TaskName
|
||||
taskPath = $TaskPath
|
||||
exists = $false
|
||||
enabled = $false
|
||||
state = "Missing"
|
||||
@@ -165,7 +182,8 @@ function Get-EvidenceReadback {
|
||||
[string]$Name,
|
||||
[string]$Pattern,
|
||||
[int]$MaxAgeMinutes,
|
||||
[bool]$Required
|
||||
[bool]$Required,
|
||||
[int]$MinimumFreshnessReserveMinutes = 0
|
||||
)
|
||||
|
||||
$evidenceDir = Join-Path $AgentRoot "evidence"
|
||||
@@ -203,6 +221,12 @@ function Get-EvidenceReadback {
|
||||
}
|
||||
$ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null }
|
||||
$fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes)
|
||||
$freshnessHeadroomMinutes = if ($file) { [math]::Round($MaxAgeMinutes - $ageMinutes, 1) } else { $null }
|
||||
$freshnessReserveReady = [bool](
|
||||
$fresh -and
|
||||
$null -ne $freshnessHeadroomMinutes -and
|
||||
$freshnessHeadroomMinutes -ge $MinimumFreshnessReserveMinutes
|
||||
)
|
||||
$safeSummary = $null
|
||||
$parseError = ""
|
||||
$contentHealthy = $true
|
||||
@@ -345,7 +369,7 @@ function Get-EvidenceReadback {
|
||||
}
|
||||
|
||||
$parsed = [bool]($file -and -not $parseError)
|
||||
$usable = [bool]($fresh -and $parsed -and $contentHealthy)
|
||||
$usable = [bool]($fresh -and $freshnessReserveReady -and $parsed -and $contentHealthy)
|
||||
return [pscustomobject]@{
|
||||
name = $Name
|
||||
pattern = $Pattern
|
||||
@@ -355,6 +379,9 @@ function Get-EvidenceReadback {
|
||||
ageMinutes = $ageMinutes
|
||||
maxAgeMinutes = $MaxAgeMinutes
|
||||
fresh = $fresh
|
||||
minimumFreshnessReserveMinutes = $MinimumFreshnessReserveMinutes
|
||||
freshnessHeadroomMinutes = $freshnessHeadroomMinutes
|
||||
freshnessReserveReady = $freshnessReserveReady
|
||||
parsed = $parsed
|
||||
contentHealthy = $contentHealthy
|
||||
usable = $usable
|
||||
@@ -366,18 +393,19 @@ function Get-EvidenceReadback {
|
||||
}
|
||||
}
|
||||
|
||||
$rootTaskPath = "\"
|
||||
$requiredTasks = @(
|
||||
"Wooo-Agent99-Startup-Recovery",
|
||||
"Wooo-Agent99-Heartbeat",
|
||||
"Wooo-Agent99-Performance-Guard",
|
||||
"Wooo-Agent99-Control-Loop",
|
||||
"Wooo-Agent99-Telegram-Inbox",
|
||||
"Wooo-Agent99-SRE-Alert-Inbox",
|
||||
"Wooo-Agent99-SRE-Alert-Relay",
|
||||
"Wooo-Agent99-Self-Health",
|
||||
"Wooo-Agent99-Backup-Health"
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health" },
|
||||
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health" }
|
||||
)
|
||||
$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback $_ })
|
||||
$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback -TaskPath $_.taskPath -TaskName $_.name })
|
||||
|
||||
$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
||||
$manifest = [pscustomobject]@{
|
||||
@@ -430,20 +458,21 @@ $queues = @(
|
||||
(Get-DirectoryReadback "queue\failed")
|
||||
)
|
||||
$evidence = @(
|
||||
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true),
|
||||
(Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false)
|
||||
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true $MinimumEvidenceFreshnessReserveMinutes),
|
||||
(Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0)
|
||||
)
|
||||
|
||||
$operatingSystem = Get-CimInstance Win32_OperatingSystem
|
||||
$taskFailures = @($tasks | Where-Object { -not $_.healthy })
|
||||
$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy })
|
||||
$requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh })
|
||||
$requiredEvidenceReserveFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.freshnessReserveReady })
|
||||
$requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed })
|
||||
$requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy })
|
||||
$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
|
||||
@@ -456,10 +485,12 @@ $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-
|
||||
$decision = Get-AgentLivePreflightDecision `
|
||||
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
|
||||
-Manifest $manifest `
|
||||
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
|
||||
-ExpectedRuntimeFileCount 16 `
|
||||
-TaskFailureCount $taskFailures.Count `
|
||||
-RelayListenerCount $relayListeners.Count `
|
||||
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `
|
||||
-RequiredEvidenceReserveFailureCount $requiredEvidenceReserveFailures.Count `
|
||||
-RequiredEvidenceParseFailureCount $requiredEvidenceParseFailures.Count `
|
||||
-RequiredEvidenceContentFailureCount $requiredEvidenceContentFailures.Count `
|
||||
-SelfHealthSeverity $selfHealthSeverity `
|
||||
@@ -474,6 +505,8 @@ $warningReasons = @($decision.warningReasons)
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_live_preflight_v1"
|
||||
mode = $Mode
|
||||
readinessScope = $ReadinessScope
|
||||
minimumEvidenceFreshnessReserveMinutes = $MinimumEvidenceFreshnessReserveMinutes
|
||||
observedAt = (Get-Date).ToString("o")
|
||||
host = $env:COMPUTERNAME
|
||||
lastBootUpTime = $operatingSystem.LastBootUpTime.ToString("o")
|
||||
@@ -492,6 +525,8 @@ $result = [pscustomobject]@{
|
||||
freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count
|
||||
usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count
|
||||
requiredEvidenceFailureCount = $requiredEvidenceFailures.Count
|
||||
requiredEvidenceStaleCount = $requiredEvidenceStale.Count
|
||||
requiredEvidenceReserveFailureCount = $requiredEvidenceReserveFailures.Count
|
||||
requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count
|
||||
requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count
|
||||
selfHealthSeverity = $selfHealthSeverity
|
||||
@@ -504,6 +539,7 @@ $result = [pscustomobject]@{
|
||||
relayListenerCount = $relayListeners.Count
|
||||
preflightGreen = [bool]$decision.deploymentEligible
|
||||
deploymentEligible = [bool]$decision.deploymentEligible
|
||||
manifestCheckRequired = [bool]$decision.manifestCheckRequired
|
||||
manifestIdentityReady = [bool]$decision.manifestIdentityReady
|
||||
manifestContentReady = [bool]$decision.manifestContentReady
|
||||
blockingReasons = $blockingReasons
|
||||
@@ -515,6 +551,8 @@ $result = [pscustomobject]@{
|
||||
Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)"
|
||||
Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)"
|
||||
Write-Output "MODE=$Mode"
|
||||
Write-Output "READINESS_SCOPE=$ReadinessScope"
|
||||
Write-Output "MINIMUM_EVIDENCE_FRESHNESS_RESERVE_MINUTES=$MinimumEvidenceFreshnessReserveMinutes"
|
||||
Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)"
|
||||
Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
|
||||
Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)"
|
||||
|
||||
@@ -21,6 +21,12 @@ $RelayPort = 8787
|
||||
$RelayStopTimeoutSeconds = 60
|
||||
$RelayStartTimeoutSeconds = 360
|
||||
$RelayPollIntervalSeconds = 1
|
||||
$ValidateOnlyTimeoutSeconds = 300
|
||||
$LiveDeployTimeoutSeconds = 300
|
||||
$LivePreflightTimeoutSeconds = 120
|
||||
$PromotionReadinessReserveMinutes = 15
|
||||
$PromotionReadinessMaxAttempts = 2
|
||||
$PromotionReadinessRetryWaitSeconds = 30
|
||||
$FixedRuntimeFiles = @(
|
||||
"agent99-bootstrap.ps1",
|
||||
"agent99-contract-check.ps1",
|
||||
@@ -289,7 +295,7 @@ function Invoke-AgentDeployChild {
|
||||
"-SourceRevision", $SourceRevision
|
||||
)
|
||||
if ($ValidateOnly) { $arguments += "-ValidateOnly" }
|
||||
$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }
|
||||
$timeoutSeconds = if ($ValidateOnly) { $ValidateOnlyTimeoutSeconds } else { $LiveDeployTimeoutSeconds }
|
||||
$processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds $timeoutSeconds
|
||||
return [pscustomobject]@{
|
||||
ok = [bool]($processResult.exitCode -eq 0 -and -not $processResult.timedOut)
|
||||
@@ -301,10 +307,23 @@ function Invoke-AgentDeployChild {
|
||||
}
|
||||
|
||||
function Invoke-AgentLivePreflight {
|
||||
param([string]$PreflightPath)
|
||||
param(
|
||||
[string]$PreflightPath,
|
||||
[ValidateSet("FullRuntime", "PromotionReserve")]
|
||||
[string]$ReadinessScope = "FullRuntime",
|
||||
[ValidateRange(0, 20)]
|
||||
[int]$MinimumEvidenceFreshnessReserveMinutes = 0
|
||||
)
|
||||
|
||||
$arguments = @("-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", $PreflightPath, "-Mode", "Verify", "-AgentRoot", $AgentRoot)
|
||||
$processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds 120
|
||||
$arguments = @(
|
||||
"-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
|
||||
"-File", $PreflightPath,
|
||||
"-Mode", "Verify",
|
||||
"-AgentRoot", $AgentRoot,
|
||||
"-ReadinessScope", $ReadinessScope,
|
||||
"-MinimumEvidenceFreshnessReserveMinutes", ([string]$MinimumEvidenceFreshnessReserveMinutes)
|
||||
)
|
||||
$processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds $LivePreflightTimeoutSeconds
|
||||
$text = [string]$processResult.stdout
|
||||
$match = [regex]::Match($text, "(?s)JSON_BEGIN\r?\n(?<json>\{.*?\})\r?\nJSON_END")
|
||||
$payload = $null
|
||||
@@ -322,18 +341,75 @@ function Invoke-AgentLivePreflight {
|
||||
}
|
||||
[string[]]$externalBlockingReasons = @($blockingReasons | Where-Object { $_ -eq "self_health_not_ok" })
|
||||
[string[]]$runtimePlaneBlockingReasons = @($blockingReasons | Where-Object { $_ -ne "self_health_not_ok" })
|
||||
$safeTaskNames = @(
|
||||
"Wooo-Agent99-Startup-Recovery",
|
||||
"Wooo-Agent99-Heartbeat",
|
||||
"Wooo-Agent99-Performance-Guard",
|
||||
"Wooo-Agent99-Control-Loop",
|
||||
"Wooo-Agent99-Telegram-Inbox",
|
||||
"Wooo-Agent99-SRE-Alert-Inbox",
|
||||
"Wooo-Agent99-SRE-Alert-Relay",
|
||||
"Wooo-Agent99-Self-Health",
|
||||
"Wooo-Agent99-Backup-Health"
|
||||
)
|
||||
$safeEvidenceNames = @(
|
||||
"self_check", "status", "control_tick", "sre_alert_inbox",
|
||||
"telegram_inbox", "performance", "backup", "recovery"
|
||||
)
|
||||
$failedTasks = if ($payload) {
|
||||
@($payload.tasks | Where-Object { -not [bool]$_.healthy } | ForEach-Object {
|
||||
$name = [string]$_.name
|
||||
$state = [string]$_.state
|
||||
[pscustomobject]@{
|
||||
name = if ($name -in $safeTaskNames) { $name } else { "unknown" }
|
||||
taskPath = if ([string]$_.taskPath -eq "\") { "\" } else { "invalid" }
|
||||
exists = [bool]$_.exists
|
||||
enabled = [bool]$_.enabled
|
||||
state = if ($state -in @("Ready", "Running", "Disabled", "Queued", "Missing")) { $state } else { "Unknown" }
|
||||
lastTaskResult = if ($null -ne $_.lastTaskResult) { [long]$_.lastTaskResult } else { $null }
|
||||
}
|
||||
})
|
||||
} else { @() }
|
||||
$failedEvidence = if ($payload) {
|
||||
@($payload.evidence | Where-Object { [bool]$_.required -and -not [bool]$_.healthy } | ForEach-Object {
|
||||
$name = [string]$_.name
|
||||
[pscustomobject]@{
|
||||
name = if ($name -in $safeEvidenceNames) { $name } else { "unknown" }
|
||||
ageMinutes = if ($null -ne $_.ageMinutes) { [double]$_.ageMinutes } else { $null }
|
||||
maxAgeMinutes = if ($null -ne $_.maxAgeMinutes) { [double]$_.maxAgeMinutes } else { $null }
|
||||
minimumFreshnessReserveMinutes = if ($null -ne $_.minimumFreshnessReserveMinutes) { [double]$_.minimumFreshnessReserveMinutes } else { $null }
|
||||
freshnessHeadroomMinutes = if ($null -ne $_.freshnessHeadroomMinutes) { [double]$_.freshnessHeadroomMinutes } else { $null }
|
||||
fresh = [bool]$_.fresh
|
||||
freshnessReserveReady = [bool]$_.freshnessReserveReady
|
||||
parsed = [bool]$_.parsed
|
||||
contentHealthy = [bool]$_.contentHealthy
|
||||
}
|
||||
})
|
||||
} else { @() }
|
||||
$acceptedExitCode = [bool]($processResult.exitCode -in @(0, 2))
|
||||
$scopeMatched = [bool](
|
||||
$payload -and
|
||||
[string]$payload.readinessScope -eq $ReadinessScope -and
|
||||
[int]$payload.minimumEvidenceFreshnessReserveMinutes -eq $MinimumEvidenceFreshnessReserveMinutes -and
|
||||
$summary -and
|
||||
[bool]$summary.manifestCheckRequired -eq [bool]($ReadinessScope -eq "FullRuntime")
|
||||
)
|
||||
$manifestReadyForScope = [bool](
|
||||
$ReadinessScope -eq "PromotionReserve" -or
|
||||
($payload -and $payload.manifest -and [bool]$payload.manifest.runtimeMatched)
|
||||
)
|
||||
$runtimePlaneReady = [bool](
|
||||
$acceptedExitCode -and
|
||||
-not $processResult.timedOut -and
|
||||
$payload -and
|
||||
$payload.manifest -and
|
||||
[bool]$payload.manifest.runtimeMatched -and
|
||||
$scopeMatched -and
|
||||
$manifestReadyForScope -and
|
||||
$summary -and
|
||||
[int]$summary.requiredTaskCount -gt 0 -and
|
||||
[int]$summary.healthyTaskCount -eq [int]$summary.requiredTaskCount -and
|
||||
[int]$summary.taskFailureCount -eq 0 -and
|
||||
[int]$summary.requiredEvidenceFailureCount -eq 0 -and
|
||||
[int]$summary.requiredEvidenceStaleCount -eq 0 -and
|
||||
[int]$summary.requiredEvidenceReserveFailureCount -eq 0 -and
|
||||
[int]$summary.requiredEvidenceParseFailureCount -eq 0 -and
|
||||
[int]$summary.requiredEvidenceContentFailureCount -eq 0 -and
|
||||
[int]$summary.relayListenerCount -gt 0 -and
|
||||
@@ -344,6 +420,9 @@ function Invoke-AgentLivePreflight {
|
||||
)
|
||||
return [pscustomobject]@{
|
||||
ok = $runtimePlaneReady
|
||||
readinessScope = $ReadinessScope
|
||||
minimumEvidenceFreshnessReserveMinutes = $MinimumEvidenceFreshnessReserveMinutes
|
||||
scopeMatched = $scopeMatched
|
||||
exitCode = [int]$processResult.exitCode
|
||||
acceptedExitCode = $acceptedExitCode
|
||||
timedOut = [bool]$processResult.timedOut
|
||||
@@ -354,6 +433,8 @@ function Invoke-AgentLivePreflight {
|
||||
requiredTaskCount = if ($summary) { [int]$summary.requiredTaskCount } else { 0 }
|
||||
taskFailureCount = if ($summary) { [int]$summary.taskFailureCount } else { -1 }
|
||||
requiredEvidenceFailureCount = if ($summary) { [int]$summary.requiredEvidenceFailureCount } else { -1 }
|
||||
requiredEvidenceStaleCount = if ($summary) { [int]$summary.requiredEvidenceStaleCount } else { -1 }
|
||||
requiredEvidenceReserveFailureCount = if ($summary) { [int]$summary.requiredEvidenceReserveFailureCount } else { -1 }
|
||||
requiredEvidenceParseFailureCount = if ($summary) { [int]$summary.requiredEvidenceParseFailureCount } else { -1 }
|
||||
requiredEvidenceContentFailureCount = if ($summary) { [int]$summary.requiredEvidenceContentFailureCount } else { -1 }
|
||||
relayListenerCount = if ($summary) { [int]$summary.relayListenerCount } else { 0 }
|
||||
@@ -377,12 +458,60 @@ function Invoke-AgentLivePreflight {
|
||||
overallPreflightGreen = [bool]($summary -and $summary.preflightGreen)
|
||||
runtimePlaneReady = $runtimePlaneReady
|
||||
externalDegraded = [bool]($runtimePlaneReady -and $externalBlockingReasons.Count -gt 0)
|
||||
failedTaskNames = @($failedTasks | ForEach-Object { [string]$_.name })
|
||||
failedTasks = $failedTasks
|
||||
failedEvidenceNames = @($failedEvidence | ForEach-Object { [string]$_.name })
|
||||
failedEvidence = $failedEvidence
|
||||
blockingReasons = $blockingReasons
|
||||
runtimePlaneBlockingReasons = $runtimePlaneBlockingReasons
|
||||
externalBlockingReasons = $externalBlockingReasons
|
||||
}
|
||||
}
|
||||
|
||||
function Test-AgentPromotionReadinessRetryable {
|
||||
param([object]$Readiness)
|
||||
|
||||
if (
|
||||
-not $Readiness -or
|
||||
$Readiness.ok -or
|
||||
-not $Readiness.parsed -or
|
||||
-not $Readiness.scopeMatched -or
|
||||
[string]$Readiness.readinessScope -ne "PromotionReserve" -or
|
||||
[int]$Readiness.minimumEvidenceFreshnessReserveMinutes -ne $PromotionReadinessReserveMinutes
|
||||
) {
|
||||
return $false
|
||||
}
|
||||
$allowedReasons = @(
|
||||
"scheduled_task_unhealthy",
|
||||
"required_evidence_stale",
|
||||
"required_evidence_freshness_reserve_insufficient"
|
||||
)
|
||||
$reasons = @($Readiness.runtimePlaneBlockingReasons | ForEach-Object { [string]$_ })
|
||||
if ($reasons.Count -eq 0 -or @($reasons | Where-Object { $_ -notin $allowedReasons }).Count -gt 0) {
|
||||
return $false
|
||||
}
|
||||
if (
|
||||
"scheduled_task_unhealthy" -in $reasons -and
|
||||
@($Readiness.failedTasks).Count -eq 0
|
||||
) { return $false }
|
||||
if (
|
||||
@($reasons | Where-Object { $_ -in @("required_evidence_stale", "required_evidence_freshness_reserve_insufficient") }).Count -gt 0 -and
|
||||
@($Readiness.failedEvidence).Count -eq 0
|
||||
) { return $false }
|
||||
if (@($Readiness.failedTasks | Where-Object {
|
||||
-not $_.exists -or
|
||||
-not $_.enabled -or
|
||||
$_.taskPath -ne "\" -or
|
||||
$_.state -notin @("Ready", "Queued")
|
||||
}).Count -gt 0) {
|
||||
return $false
|
||||
}
|
||||
if (@($Readiness.failedEvidence | Where-Object { -not $_.parsed -or -not $_.contentHealthy }).Count -gt 0) {
|
||||
return $false
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Get-AgentRelayRuntime {
|
||||
$task = $null
|
||||
$taskError = ""
|
||||
@@ -1070,6 +1199,78 @@ try {
|
||||
Write-AgentResultAndExit $receipt 2
|
||||
}
|
||||
|
||||
$script:ReceiverOperationPhase = "pre_promotion_readiness"
|
||||
$promotionReadinessAttempts = @()
|
||||
$prePromotionReadiness = $null
|
||||
for ($readinessAttempt = 1; $readinessAttempt -le $PromotionReadinessMaxAttempts; $readinessAttempt += 1) {
|
||||
$prePromotionReadiness = Invoke-AgentLivePreflight `
|
||||
(Join-Path $stagePath "agent99-live-preflight.ps1") `
|
||||
-ReadinessScope "PromotionReserve" `
|
||||
-MinimumEvidenceFreshnessReserveMinutes $PromotionReadinessReserveMinutes
|
||||
$prePromotionReadiness | Add-Member -NotePropertyName attempt -NotePropertyValue $readinessAttempt -Force
|
||||
$promotionReadinessAttempts += $prePromotionReadiness
|
||||
if ($prePromotionReadiness.ok) { break }
|
||||
if (
|
||||
$readinessAttempt -ge $PromotionReadinessMaxAttempts -or
|
||||
-not (Test-AgentPromotionReadinessRetryable $prePromotionReadiness)
|
||||
) { break }
|
||||
Start-Sleep -Seconds $PromotionReadinessRetryWaitSeconds
|
||||
}
|
||||
$promotionReadinessRetryCount = [math]::Max(0, $promotionReadinessAttempts.Count - 1)
|
||||
$promotionReadinessStillRetryable = Test-AgentPromotionReadinessRetryable $prePromotionReadiness
|
||||
if (-not $prePromotionReadiness.ok) {
|
||||
$receipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = "blocked_pre_promotion_readiness_no_live_write"
|
||||
mode = "apply"
|
||||
traceId = $traceId
|
||||
runId = $runId
|
||||
workItemId = $workItemId
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
canonicalReceiptPath = $receiptPath
|
||||
attemptReceiptPath = $attemptReceiptPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
|
||||
prePromotionReadiness = $prePromotionReadiness
|
||||
promotionReadinessAttempts = $promotionReadinessAttempts
|
||||
retryPolicy = [pscustomobject]@{
|
||||
retryableTransientOnly = $true
|
||||
automaticTaskTriggerAllowed = $false
|
||||
maxAttempts = $PromotionReadinessMaxAttempts
|
||||
attemptCount = $promotionReadinessAttempts.Count
|
||||
retryCount = $promotionReadinessRetryCount
|
||||
retryWaitSeconds = $PromotionReadinessRetryWaitSeconds
|
||||
finalFailureRetryableTransient = $promotionReadinessStillRetryable
|
||||
unboundedRetryAllowed = $false
|
||||
}
|
||||
durableReceiptWritten = $true
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionAttempted = $false
|
||||
livePromotionPerformed = $false
|
||||
readinessCheckWritePerformed = $false
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
uiInteraction = $false
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $false
|
||||
scheduledTaskModification = $false
|
||||
nextSafeAction = if ($promotionReadinessStillRetryable) {
|
||||
"allow_natural_root_task_and_evidence_refresh_then_retry_same_identity_once"
|
||||
} else {
|
||||
"repair_exact_reported_readiness_blockers_before_retry"
|
||||
}
|
||||
}
|
||||
$script:ReceiverOperationPhase = "pre_promotion_readiness_failure_receipt_write"
|
||||
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
|
||||
Write-AgentResultAndExit $receipt 2
|
||||
}
|
||||
|
||||
$beforeManifest = Get-AgentSafeRuntimeManifest
|
||||
$beforeBundleDigest = Get-AgentLiveBundleDigest
|
||||
$beforeConfigDigest = Get-AgentSha256File $ConfigPath
|
||||
@@ -1215,6 +1416,9 @@ try {
|
||||
backupPath = $backupPath
|
||||
canonicalReceiptPath = $receiptPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status }
|
||||
prePromotionReadiness = $prePromotionReadiness
|
||||
promotionReadinessAttempts = $promotionReadinessAttempts
|
||||
promotionReadinessRetryCount = $promotionReadinessRetryCount
|
||||
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
|
||||
relayReload = $relayReload
|
||||
runtimeManifest = $runtimeManifest
|
||||
@@ -1331,6 +1535,9 @@ try {
|
||||
stagingPath = $stagePath
|
||||
backupPath = $backupPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
|
||||
prePromotionReadiness = $prePromotionReadiness
|
||||
promotionReadinessAttempts = $promotionReadinessAttempts
|
||||
promotionReadinessRetryCount = $promotionReadinessRetryCount
|
||||
deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } }
|
||||
failurePhase = $failurePhase
|
||||
failureType = $failureType
|
||||
|
||||
@@ -40,9 +40,11 @@ def _base_case() -> dict[str, Any]:
|
||||
"mismatchCount": 0,
|
||||
"parseError": "",
|
||||
},
|
||||
"manifestCheckRequired": True,
|
||||
"taskFailureCount": 0,
|
||||
"relayListenerCount": 1,
|
||||
"requiredEvidenceStaleCount": 0,
|
||||
"requiredEvidenceReserveFailureCount": 0,
|
||||
"requiredEvidenceParseFailureCount": 0,
|
||||
"requiredEvidenceContentFailureCount": 0,
|
||||
"selfHealthSeverity": "ok",
|
||||
@@ -81,10 +83,12 @@ $case = @'
|
||||
$parameters = @{{
|
||||
AgentRootPresent = [bool]$case.agentRootPresent
|
||||
Manifest = $case.manifest
|
||||
ManifestCheckRequired = [bool]$case.manifestCheckRequired
|
||||
ExpectedRuntimeFileCount = 16
|
||||
TaskFailureCount = [int]$case.taskFailureCount
|
||||
RelayListenerCount = [int]$case.relayListenerCount
|
||||
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
|
||||
RequiredEvidenceReserveFailureCount = [int]$case.requiredEvidenceReserveFailureCount
|
||||
RequiredEvidenceParseFailureCount = [int]$case.requiredEvidenceParseFailureCount
|
||||
RequiredEvidenceContentFailureCount = [int]$case.requiredEvidenceContentFailureCount
|
||||
SelfHealthSeverity = [string]$case.selfHealthSeverity
|
||||
@@ -123,6 +127,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
assert '$blockingReasons += "runtime_manifest_mismatch"' in decision
|
||||
assert '$blockingReasons += "scheduled_task_unhealthy"' in decision
|
||||
assert '$blockingReasons += "sre_alert_relay_not_listening"' in decision
|
||||
assert (
|
||||
'$blockingReasons += "required_evidence_freshness_reserve_insufficient"'
|
||||
in decision
|
||||
)
|
||||
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
|
||||
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
|
||||
assert "-ExpectedRuntimeFileCount 16" in source
|
||||
@@ -139,6 +147,9 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
assert '"agent99_background_mode_deferred_v1"' in source
|
||||
assert '"agent99_recovery_transport_deferred_v1"' in source
|
||||
assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source
|
||||
assert '$rootTaskPath = "\\"' in source
|
||||
assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source
|
||||
assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source
|
||||
|
||||
|
||||
def test_evidence_selection_skips_deferred_but_not_malformed_candidates(
|
||||
@@ -224,6 +235,45 @@ Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true |
|
||||
assert malformed_payload["healthy"] is False
|
||||
|
||||
|
||||
def test_evidence_reserve_rejects_fresh_but_expiring_evidence_without_extending_slo(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
||||
|
||||
evidence = tmp_path / "evidence"
|
||||
evidence.mkdir()
|
||||
candidate = evidence / "agent99-Status-20260716-230000.json"
|
||||
candidate.write_text(json.dumps({"mode": "Status"}), encoding="utf-8")
|
||||
six_minutes_ago = candidate.stat().st_mtime - (6 * 60)
|
||||
os.utime(candidate, (six_minutes_ago, six_minutes_ago))
|
||||
|
||||
agent_root = str(tmp_path).replace("'", "''")
|
||||
command = f"""
|
||||
{_evidence_function_source()}
|
||||
$AgentRoot = '{agent_root}'
|
||||
Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 |
|
||||
ConvertTo-Json -Depth 8 -Compress
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
payload = json.loads(
|
||||
next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{"))
|
||||
)
|
||||
assert payload["fresh"] is True
|
||||
assert payload["maxAgeMinutes"] == 20
|
||||
assert payload["minimumFreshnessReserveMinutes"] == 15
|
||||
assert payload["freshnessReserveReady"] is False
|
||||
assert payload["usable"] is False
|
||||
assert payload["healthy"] is False
|
||||
|
||||
|
||||
def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
@@ -252,6 +302,12 @@ def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
("scheduled_task_unhealthy",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(requiredEvidenceReserveFailureCount=1),
|
||||
False,
|
||||
("required_evidence_freshness_reserve_insufficient",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(manifest={"runtimeMatched": False, "mismatchCount": 1}),
|
||||
False,
|
||||
@@ -305,6 +361,29 @@ def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
assert set(_as_list(result.get("blockingReasons"))) == set(expected_blockers)
|
||||
assert set(_as_list(result.get("warningReasons"))) == set(expected_warnings)
|
||||
|
||||
promotion_reserve = _run_decision(
|
||||
powershell,
|
||||
_with_overrides(
|
||||
manifestCheckRequired=False,
|
||||
manifest={"sourceRevision": "old", "fileCount": 15, "runtimeMatched": False},
|
||||
),
|
||||
)
|
||||
assert promotion_reserve["deploymentEligible"] is True
|
||||
assert promotion_reserve["manifestCheckRequired"] is False
|
||||
|
||||
promotion_reserve_task_failure = _run_decision(
|
||||
powershell,
|
||||
_with_overrides(
|
||||
manifestCheckRequired=False,
|
||||
manifest={"sourceRevision": "old", "fileCount": 15, "runtimeMatched": False},
|
||||
taskFailureCount=1,
|
||||
),
|
||||
)
|
||||
assert promotion_reserve_task_failure["deploymentEligible"] is False
|
||||
assert promotion_reserve_task_failure["blockingReasons"] == [
|
||||
"scheduled_task_unhealthy"
|
||||
]
|
||||
|
||||
preflight_path = str(PREFLIGHT).replace("'", "''")
|
||||
parser_command = (
|
||||
"$tokens=$null; $errors=$null; "
|
||||
|
||||
@@ -179,7 +179,10 @@ def test_live_preflight_rejects_fresh_but_unparseable_required_evidence() -> Non
|
||||
preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1")
|
||||
|
||||
assert '$parsed = [bool]($file -and -not $parseError)' in preflight
|
||||
assert '$usable = [bool]($fresh -and $parsed -and $contentHealthy)' in preflight
|
||||
assert (
|
||||
"$usable = [bool]($fresh -and $freshnessReserveReady -and $parsed -and "
|
||||
"$contentHealthy)" in preflight
|
||||
)
|
||||
assert 'healthy = [bool](-not $Required -or $usable)' in preflight
|
||||
assert '$_.required -and -not $_.healthy' in preflight
|
||||
assert 'required_evidence_parse_failed' in preflight
|
||||
|
||||
@@ -209,8 +209,10 @@ def test_receiver_validates_manifest_before_staging_and_validate_only_before_pro
|
||||
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
|
||||
"-SourceRevision $sourceRevision"
|
||||
)
|
||||
assert manifest_check < stage_write < validate < promote
|
||||
readiness = source.index('$script:ReceiverOperationPhase = "pre_promotion_readiness"')
|
||||
assert manifest_check < stage_write < validate < readiness < promote
|
||||
assert 'status = "validate_only_failed_no_promotion"' in source
|
||||
assert 'status = "blocked_pre_promotion_readiness_no_live_write"' in source
|
||||
assert "liveRuntimeWritePerformed = $false" in source
|
||||
assert "livePromotionPerformed = $false" in source
|
||||
|
||||
@@ -236,7 +238,12 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
|
||||
assert "function Invoke-AgentBoundedPowerShell" in source
|
||||
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
|
||||
assert "$process.Kill()" in source
|
||||
assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source
|
||||
assert "$ValidateOnlyTimeoutSeconds = 300" in source
|
||||
assert "$LiveDeployTimeoutSeconds = 300" in source
|
||||
assert (
|
||||
"$timeoutSeconds = if ($ValidateOnly) { $ValidateOnlyTimeoutSeconds } "
|
||||
"else { $LiveDeployTimeoutSeconds }" in source
|
||||
)
|
||||
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
|
||||
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
|
||||
assert '$runtimeManifest.fileCount -eq 16' in source
|
||||
@@ -301,6 +308,60 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
|
||||
assert "[bool]$prior.relayReload.newGenerationVerified" in source
|
||||
|
||||
|
||||
def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
validate = source.index(
|
||||
"$validate = Invoke-AgentDeployChild -SourceRoot $stagePath "
|
||||
"-SourceRevision $sourceRevision -ValidateOnly"
|
||||
)
|
||||
readiness = source.index('$script:ReceiverOperationPhase = "pre_promotion_readiness"')
|
||||
live_write = source.index('$script:ReceiverOperationPhase = "live_deploy"')
|
||||
assert validate < readiness < live_write
|
||||
|
||||
assert '-ReadinessScope "PromotionReserve"' in source
|
||||
assert (
|
||||
"-MinimumEvidenceFreshnessReserveMinutes "
|
||||
"$PromotionReadinessReserveMinutes" in source
|
||||
)
|
||||
assert "$PromotionReadinessReserveMinutes = 15" in source
|
||||
assert "$PromotionReadinessMaxAttempts = 2" in source
|
||||
assert "$PromotionReadinessRetryWaitSeconds = 30" in source
|
||||
assert "function Test-AgentPromotionReadinessRetryable" in source
|
||||
assert '"scheduled_task_unhealthy"' in source
|
||||
assert '"required_evidence_stale"' in source
|
||||
assert '"required_evidence_freshness_reserve_insufficient"' in source
|
||||
assert '[string]$Readiness.readinessScope -ne "PromotionReserve"' in source
|
||||
assert (
|
||||
"[int]$Readiness.minimumEvidenceFreshnessReserveMinutes -ne "
|
||||
"$PromotionReadinessReserveMinutes" in source
|
||||
)
|
||||
assert '"scheduled_task_unhealthy" -in $reasons' in source
|
||||
assert "@($Readiness.failedTasks).Count -eq 0" in source
|
||||
assert "@($Readiness.failedEvidence).Count -eq 0" in source
|
||||
assert '$_.state -notin @("Ready", "Queued")' in source
|
||||
assert "automaticTaskTriggerAllowed = $false" in source
|
||||
assert "unboundedRetryAllowed = $false" in source
|
||||
assert "Start-ScheduledTask" not in source[readiness:live_write]
|
||||
|
||||
blocked = source[
|
||||
source.index('$status = "blocked_pre_promotion_readiness_no_live_write"')
|
||||
if '$status = "blocked_pre_promotion_readiness_no_live_write"' in source
|
||||
else source.index('status = "blocked_pre_promotion_readiness_no_live_write"') : live_write
|
||||
]
|
||||
assert "liveRuntimeWritePerformed = $false" in blocked
|
||||
assert "livePromotionAttempted = $false" in blocked
|
||||
assert "livePromotionPerformed = $false" in blocked
|
||||
assert "readinessCheckWritePerformed = $false" in blocked
|
||||
assert "failedTaskNames" in source
|
||||
assert "failedTasks" in source
|
||||
assert "failedEvidenceNames" in source
|
||||
assert "failedEvidence" in source
|
||||
assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source
|
||||
assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source
|
||||
assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source
|
||||
|
||||
|
||||
def test_check_and_apply_receipts_project_the_same_run_identity() -> None:
|
||||
sender = SENDER.read_text(encoding="utf-8")
|
||||
projector = sender[
|
||||
@@ -510,6 +571,11 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
assert "$RelayStopTimeoutSeconds = 60" in source
|
||||
assert "$RelayStartTimeoutSeconds = 360" in source
|
||||
assert "$RelayPollIntervalSeconds = 1" in source
|
||||
assert "$ValidateOnlyTimeoutSeconds = 300" in source
|
||||
assert "$LiveDeployTimeoutSeconds = 300" in source
|
||||
assert "$LivePreflightTimeoutSeconds = 120" in source
|
||||
assert "$PromotionReadinessMaxAttempts = 2" in source
|
||||
assert "$PromotionReadinessRetryWaitSeconds = 30" in source
|
||||
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
@@ -545,18 +611,21 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in wrapper
|
||||
wrapper_budget_seconds = 2400
|
||||
validate_seconds = 300
|
||||
deploy_seconds = 900
|
||||
promotion_readiness_attempts = 2
|
||||
promotion_readiness_retry_wait_seconds = 30
|
||||
deploy_seconds = 300
|
||||
live_preflight_seconds = 120
|
||||
worst_case_seconds = (
|
||||
validate_seconds
|
||||
+ (promotion_readiness_attempts * live_preflight_seconds)
|
||||
+ promotion_readiness_retry_wait_seconds
|
||||
+ deploy_seconds
|
||||
+ live_preflight_seconds
|
||||
+ (2 * (60 + 360))
|
||||
)
|
||||
assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source
|
||||
assert "-TimeoutSeconds 120" in source
|
||||
assert worst_case_seconds == 2160
|
||||
assert wrapper_budget_seconds - worst_case_seconds == 240
|
||||
assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source
|
||||
assert worst_case_seconds == 1830
|
||||
assert wrapper_budget_seconds - worst_case_seconds == 570
|
||||
|
||||
deploy = source.index(
|
||||
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
|
||||
|
||||
Reference in New Issue
Block a user