From d3d53a4f69f29b85fbf0073dc1318b2d2e6a67cf Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 16 Jul 2026 23:47:05 +0800 Subject: [PATCH] fix(agent99): reserve pre-promotion readiness --- .../agent99-live-preflight.ps1 | 106 ++++++--- .../agent99-remote-atomic-deploy-receiver.ps1 | 221 +++++++++++++++++- .../test_agent99_live_preflight_decision.py | 79 +++++++ ...t_agent99_recovery_coordinator_contract.py | 5 +- ...remote_atomic_deploy_transport_contract.py | 83 ++++++- 5 files changed, 445 insertions(+), 49 deletions(-) diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 32b770bd4..d0e2b79a0 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -2,7 +2,11 @@ param( [ValidateSet("Verify")] [string]$Mode = "Verify", - [string]$AgentRoot = "C:\Wooo\Agent99" + [string]$AgentRoot = "C:\Wooo\Agent99", + [ValidateSet("FullRuntime", "PromotionReserve")] + [string]$ReadinessScope = "FullRuntime", + [ValidateRange(0, 20)] + [int]$MinimumEvidenceFreshnessReserveMinutes = 0 ) $ErrorActionPreference = "Stop" @@ -11,10 +15,12 @@ function Get-AgentLivePreflightDecision { param( [bool]$AgentRootPresent, [object]$Manifest, + [bool]$ManifestCheckRequired, [int]$ExpectedRuntimeFileCount, [int]$TaskFailureCount, [int]$RelayListenerCount, [int]$RequiredEvidenceStaleCount, + [int]$RequiredEvidenceReserveFailureCount, [int]$RequiredEvidenceParseFailureCount, [int]$RequiredEvidenceContentFailureCount, [string]$SelfHealthSeverity, @@ -44,21 +50,24 @@ function Get-AgentLivePreflightDecision { ) if (-not $AgentRootPresent) { $blockingReasons += "agent_root_missing" } - if (-not $manifestExists) { - $blockingReasons += "runtime_manifest_missing" - } elseif (-not $manifestParseReady) { - $blockingReasons += "runtime_manifest_parse_failed" - } else { - if (-not $manifestIdentityReady) { - $blockingReasons += "runtime_manifest_identity_invalid" - } - if (-not $manifestContentReady) { - $blockingReasons += "runtime_manifest_mismatch" + if ($ManifestCheckRequired) { + if (-not $manifestExists) { + $blockingReasons += "runtime_manifest_missing" + } elseif (-not $manifestParseReady) { + $blockingReasons += "runtime_manifest_parse_failed" + } else { + if (-not $manifestIdentityReady) { + $blockingReasons += "runtime_manifest_identity_invalid" + } + if (-not $manifestContentReady) { + $blockingReasons += "runtime_manifest_mismatch" + } } } if ($TaskFailureCount -gt 0) { $blockingReasons += "scheduled_task_unhealthy" } if ($RelayListenerCount -lt 1) { $blockingReasons += "sre_alert_relay_not_listening" } if ($RequiredEvidenceStaleCount -gt 0) { $blockingReasons += "required_evidence_stale" } + if ($RequiredEvidenceReserveFailureCount -gt 0) { $blockingReasons += "required_evidence_freshness_reserve_insufficient" } if ($RequiredEvidenceParseFailureCount -gt 0) { $blockingReasons += "required_evidence_parse_failed" } if ($RequiredEvidenceContentFailureCount -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" } @@ -91,6 +100,7 @@ function Get-AgentLivePreflightDecision { blockingReasons = @($blockingReasons) warningReasons = @($warningReasons) warningCount = [int]$warningReasons.Count + manifestCheckRequired = $ManifestCheckRequired manifestIdentityReady = $manifestIdentityReady manifestContentReady = $manifestContentReady selfHealthClassification = $normalizedSelfHealth @@ -98,11 +108,16 @@ function Get-AgentLivePreflightDecision { } function Get-TaskReadback { - param([string]$TaskName) + param( + [string]$TaskPath, + [string]$TaskName + ) try { - $task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop - $info = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction Stop + $tasks = @(Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop) + if ($tasks.Count -ne 1) { throw "exact_root_task_count_invalid" } + $task = $tasks[0] + $info = Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop $lastResult = [int64]$info.LastTaskResult $state = [string]$task.State $healthy = [bool]( @@ -112,6 +127,7 @@ function Get-TaskReadback { return [pscustomobject]@{ name = $TaskName + taskPath = $TaskPath exists = $true enabled = [bool]$task.Settings.Enabled state = $state @@ -123,6 +139,7 @@ function Get-TaskReadback { } catch { return [pscustomobject]@{ name = $TaskName + taskPath = $TaskPath exists = $false enabled = $false state = "Missing" @@ -165,7 +182,8 @@ function Get-EvidenceReadback { [string]$Name, [string]$Pattern, [int]$MaxAgeMinutes, - [bool]$Required + [bool]$Required, + [int]$MinimumFreshnessReserveMinutes = 0 ) $evidenceDir = Join-Path $AgentRoot "evidence" @@ -203,6 +221,12 @@ function Get-EvidenceReadback { } $ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null } $fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes) + $freshnessHeadroomMinutes = if ($file) { [math]::Round($MaxAgeMinutes - $ageMinutes, 1) } else { $null } + $freshnessReserveReady = [bool]( + $fresh -and + $null -ne $freshnessHeadroomMinutes -and + $freshnessHeadroomMinutes -ge $MinimumFreshnessReserveMinutes + ) $safeSummary = $null $parseError = "" $contentHealthy = $true @@ -345,7 +369,7 @@ function Get-EvidenceReadback { } $parsed = [bool]($file -and -not $parseError) - $usable = [bool]($fresh -and $parsed -and $contentHealthy) + $usable = [bool]($fresh -and $freshnessReserveReady -and $parsed -and $contentHealthy) return [pscustomobject]@{ name = $Name pattern = $Pattern @@ -355,6 +379,9 @@ function Get-EvidenceReadback { ageMinutes = $ageMinutes maxAgeMinutes = $MaxAgeMinutes fresh = $fresh + minimumFreshnessReserveMinutes = $MinimumFreshnessReserveMinutes + freshnessHeadroomMinutes = $freshnessHeadroomMinutes + freshnessReserveReady = $freshnessReserveReady parsed = $parsed contentHealthy = $contentHealthy usable = $usable @@ -366,18 +393,19 @@ function Get-EvidenceReadback { } } +$rootTaskPath = "\" $requiredTasks = @( - "Wooo-Agent99-Startup-Recovery", - "Wooo-Agent99-Heartbeat", - "Wooo-Agent99-Performance-Guard", - "Wooo-Agent99-Control-Loop", - "Wooo-Agent99-Telegram-Inbox", - "Wooo-Agent99-SRE-Alert-Inbox", - "Wooo-Agent99-SRE-Alert-Relay", - "Wooo-Agent99-Self-Health", - "Wooo-Agent99-Backup-Health" + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery" }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat" }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard" }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop" }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox" }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox" }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay" }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health" }, + [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health" } ) -$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback $_ }) +$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback -TaskPath $_.taskPath -TaskName $_.name }) $manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json" $manifest = [pscustomobject]@{ @@ -430,20 +458,21 @@ $queues = @( (Get-DirectoryReadback "queue\failed") ) $evidence = @( - (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true), - (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true), - (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true), - (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true), - (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true), - (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true), - (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true), - (Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false) + (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0) ) $operatingSystem = Get-CimInstance Win32_OperatingSystem $taskFailures = @($tasks | Where-Object { -not $_.healthy }) $requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy }) $requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh }) +$requiredEvidenceReserveFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.freshnessReserveReady }) $requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed }) $requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy }) $selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1 @@ -456,10 +485,12 @@ $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select- $decision = Get-AgentLivePreflightDecision ` -AgentRootPresent ([bool](Test-Path $AgentRoot)) ` -Manifest $manifest ` + -ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) ` -ExpectedRuntimeFileCount 16 ` -TaskFailureCount $taskFailures.Count ` -RelayListenerCount $relayListeners.Count ` -RequiredEvidenceStaleCount $requiredEvidenceStale.Count ` + -RequiredEvidenceReserveFailureCount $requiredEvidenceReserveFailures.Count ` -RequiredEvidenceParseFailureCount $requiredEvidenceParseFailures.Count ` -RequiredEvidenceContentFailureCount $requiredEvidenceContentFailures.Count ` -SelfHealthSeverity $selfHealthSeverity ` @@ -474,6 +505,8 @@ $warningReasons = @($decision.warningReasons) $result = [pscustomobject]@{ schemaVersion = "agent99_live_preflight_v1" mode = $Mode + readinessScope = $ReadinessScope + minimumEvidenceFreshnessReserveMinutes = $MinimumEvidenceFreshnessReserveMinutes observedAt = (Get-Date).ToString("o") host = $env:COMPUTERNAME lastBootUpTime = $operatingSystem.LastBootUpTime.ToString("o") @@ -492,6 +525,8 @@ $result = [pscustomobject]@{ freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count requiredEvidenceFailureCount = $requiredEvidenceFailures.Count + requiredEvidenceStaleCount = $requiredEvidenceStale.Count + requiredEvidenceReserveFailureCount = $requiredEvidenceReserveFailures.Count requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count selfHealthSeverity = $selfHealthSeverity @@ -504,6 +539,7 @@ $result = [pscustomobject]@{ relayListenerCount = $relayListeners.Count preflightGreen = [bool]$decision.deploymentEligible deploymentEligible = [bool]$decision.deploymentEligible + manifestCheckRequired = [bool]$decision.manifestCheckRequired manifestIdentityReady = [bool]$decision.manifestIdentityReady manifestContentReady = [bool]$decision.manifestContentReady blockingReasons = $blockingReasons @@ -515,6 +551,8 @@ $result = [pscustomobject]@{ Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)" Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)" Write-Output "MODE=$Mode" +Write-Output "READINESS_SCOPE=$ReadinessScope" +Write-Output "MINIMUM_EVIDENCE_FRESHNESS_RESERVE_MINUTES=$MinimumEvidenceFreshnessReserveMinutes" Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)" Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)" Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)" diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index 633dd8bc5..dadc5b584 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -21,6 +21,12 @@ $RelayPort = 8787 $RelayStopTimeoutSeconds = 60 $RelayStartTimeoutSeconds = 360 $RelayPollIntervalSeconds = 1 +$ValidateOnlyTimeoutSeconds = 300 +$LiveDeployTimeoutSeconds = 300 +$LivePreflightTimeoutSeconds = 120 +$PromotionReadinessReserveMinutes = 15 +$PromotionReadinessMaxAttempts = 2 +$PromotionReadinessRetryWaitSeconds = 30 $FixedRuntimeFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", @@ -289,7 +295,7 @@ function Invoke-AgentDeployChild { "-SourceRevision", $SourceRevision ) if ($ValidateOnly) { $arguments += "-ValidateOnly" } - $timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 } + $timeoutSeconds = if ($ValidateOnly) { $ValidateOnlyTimeoutSeconds } else { $LiveDeployTimeoutSeconds } $processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds $timeoutSeconds return [pscustomobject]@{ ok = [bool]($processResult.exitCode -eq 0 -and -not $processResult.timedOut) @@ -301,10 +307,23 @@ function Invoke-AgentDeployChild { } function Invoke-AgentLivePreflight { - param([string]$PreflightPath) + param( + [string]$PreflightPath, + [ValidateSet("FullRuntime", "PromotionReserve")] + [string]$ReadinessScope = "FullRuntime", + [ValidateRange(0, 20)] + [int]$MinimumEvidenceFreshnessReserveMinutes = 0 + ) - $arguments = @("-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", $PreflightPath, "-Mode", "Verify", "-AgentRoot", $AgentRoot) - $processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds 120 + $arguments = @( + "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", + "-File", $PreflightPath, + "-Mode", "Verify", + "-AgentRoot", $AgentRoot, + "-ReadinessScope", $ReadinessScope, + "-MinimumEvidenceFreshnessReserveMinutes", ([string]$MinimumEvidenceFreshnessReserveMinutes) + ) + $processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds $LivePreflightTimeoutSeconds $text = [string]$processResult.stdout $match = [regex]::Match($text, "(?s)JSON_BEGIN\r?\n(?\{.*?\})\r?\nJSON_END") $payload = $null @@ -322,18 +341,75 @@ function Invoke-AgentLivePreflight { } [string[]]$externalBlockingReasons = @($blockingReasons | Where-Object { $_ -eq "self_health_not_ok" }) [string[]]$runtimePlaneBlockingReasons = @($blockingReasons | Where-Object { $_ -ne "self_health_not_ok" }) + $safeTaskNames = @( + "Wooo-Agent99-Startup-Recovery", + "Wooo-Agent99-Heartbeat", + "Wooo-Agent99-Performance-Guard", + "Wooo-Agent99-Control-Loop", + "Wooo-Agent99-Telegram-Inbox", + "Wooo-Agent99-SRE-Alert-Inbox", + "Wooo-Agent99-SRE-Alert-Relay", + "Wooo-Agent99-Self-Health", + "Wooo-Agent99-Backup-Health" + ) + $safeEvidenceNames = @( + "self_check", "status", "control_tick", "sre_alert_inbox", + "telegram_inbox", "performance", "backup", "recovery" + ) + $failedTasks = if ($payload) { + @($payload.tasks | Where-Object { -not [bool]$_.healthy } | ForEach-Object { + $name = [string]$_.name + $state = [string]$_.state + [pscustomobject]@{ + name = if ($name -in $safeTaskNames) { $name } else { "unknown" } + taskPath = if ([string]$_.taskPath -eq "\") { "\" } else { "invalid" } + exists = [bool]$_.exists + enabled = [bool]$_.enabled + state = if ($state -in @("Ready", "Running", "Disabled", "Queued", "Missing")) { $state } else { "Unknown" } + lastTaskResult = if ($null -ne $_.lastTaskResult) { [long]$_.lastTaskResult } else { $null } + } + }) + } else { @() } + $failedEvidence = if ($payload) { + @($payload.evidence | Where-Object { [bool]$_.required -and -not [bool]$_.healthy } | ForEach-Object { + $name = [string]$_.name + [pscustomobject]@{ + name = if ($name -in $safeEvidenceNames) { $name } else { "unknown" } + ageMinutes = if ($null -ne $_.ageMinutes) { [double]$_.ageMinutes } else { $null } + maxAgeMinutes = if ($null -ne $_.maxAgeMinutes) { [double]$_.maxAgeMinutes } else { $null } + minimumFreshnessReserveMinutes = if ($null -ne $_.minimumFreshnessReserveMinutes) { [double]$_.minimumFreshnessReserveMinutes } else { $null } + freshnessHeadroomMinutes = if ($null -ne $_.freshnessHeadroomMinutes) { [double]$_.freshnessHeadroomMinutes } else { $null } + fresh = [bool]$_.fresh + freshnessReserveReady = [bool]$_.freshnessReserveReady + parsed = [bool]$_.parsed + contentHealthy = [bool]$_.contentHealthy + } + }) + } else { @() } $acceptedExitCode = [bool]($processResult.exitCode -in @(0, 2)) + $scopeMatched = [bool]( + $payload -and + [string]$payload.readinessScope -eq $ReadinessScope -and + [int]$payload.minimumEvidenceFreshnessReserveMinutes -eq $MinimumEvidenceFreshnessReserveMinutes -and + $summary -and + [bool]$summary.manifestCheckRequired -eq [bool]($ReadinessScope -eq "FullRuntime") + ) + $manifestReadyForScope = [bool]( + $ReadinessScope -eq "PromotionReserve" -or + ($payload -and $payload.manifest -and [bool]$payload.manifest.runtimeMatched) + ) $runtimePlaneReady = [bool]( $acceptedExitCode -and -not $processResult.timedOut -and - $payload -and - $payload.manifest -and - [bool]$payload.manifest.runtimeMatched -and + $scopeMatched -and + $manifestReadyForScope -and $summary -and [int]$summary.requiredTaskCount -gt 0 -and [int]$summary.healthyTaskCount -eq [int]$summary.requiredTaskCount -and [int]$summary.taskFailureCount -eq 0 -and [int]$summary.requiredEvidenceFailureCount -eq 0 -and + [int]$summary.requiredEvidenceStaleCount -eq 0 -and + [int]$summary.requiredEvidenceReserveFailureCount -eq 0 -and [int]$summary.requiredEvidenceParseFailureCount -eq 0 -and [int]$summary.requiredEvidenceContentFailureCount -eq 0 -and [int]$summary.relayListenerCount -gt 0 -and @@ -344,6 +420,9 @@ function Invoke-AgentLivePreflight { ) return [pscustomobject]@{ ok = $runtimePlaneReady + readinessScope = $ReadinessScope + minimumEvidenceFreshnessReserveMinutes = $MinimumEvidenceFreshnessReserveMinutes + scopeMatched = $scopeMatched exitCode = [int]$processResult.exitCode acceptedExitCode = $acceptedExitCode timedOut = [bool]$processResult.timedOut @@ -354,6 +433,8 @@ function Invoke-AgentLivePreflight { requiredTaskCount = if ($summary) { [int]$summary.requiredTaskCount } else { 0 } taskFailureCount = if ($summary) { [int]$summary.taskFailureCount } else { -1 } requiredEvidenceFailureCount = if ($summary) { [int]$summary.requiredEvidenceFailureCount } else { -1 } + requiredEvidenceStaleCount = if ($summary) { [int]$summary.requiredEvidenceStaleCount } else { -1 } + requiredEvidenceReserveFailureCount = if ($summary) { [int]$summary.requiredEvidenceReserveFailureCount } else { -1 } requiredEvidenceParseFailureCount = if ($summary) { [int]$summary.requiredEvidenceParseFailureCount } else { -1 } requiredEvidenceContentFailureCount = if ($summary) { [int]$summary.requiredEvidenceContentFailureCount } else { -1 } relayListenerCount = if ($summary) { [int]$summary.relayListenerCount } else { 0 } @@ -377,12 +458,60 @@ function Invoke-AgentLivePreflight { overallPreflightGreen = [bool]($summary -and $summary.preflightGreen) runtimePlaneReady = $runtimePlaneReady externalDegraded = [bool]($runtimePlaneReady -and $externalBlockingReasons.Count -gt 0) + failedTaskNames = @($failedTasks | ForEach-Object { [string]$_.name }) + failedTasks = $failedTasks + failedEvidenceNames = @($failedEvidence | ForEach-Object { [string]$_.name }) + failedEvidence = $failedEvidence blockingReasons = $blockingReasons runtimePlaneBlockingReasons = $runtimePlaneBlockingReasons externalBlockingReasons = $externalBlockingReasons } } +function Test-AgentPromotionReadinessRetryable { + param([object]$Readiness) + + if ( + -not $Readiness -or + $Readiness.ok -or + -not $Readiness.parsed -or + -not $Readiness.scopeMatched -or + [string]$Readiness.readinessScope -ne "PromotionReserve" -or + [int]$Readiness.minimumEvidenceFreshnessReserveMinutes -ne $PromotionReadinessReserveMinutes + ) { + return $false + } + $allowedReasons = @( + "scheduled_task_unhealthy", + "required_evidence_stale", + "required_evidence_freshness_reserve_insufficient" + ) + $reasons = @($Readiness.runtimePlaneBlockingReasons | ForEach-Object { [string]$_ }) + if ($reasons.Count -eq 0 -or @($reasons | Where-Object { $_ -notin $allowedReasons }).Count -gt 0) { + return $false + } + if ( + "scheduled_task_unhealthy" -in $reasons -and + @($Readiness.failedTasks).Count -eq 0 + ) { return $false } + if ( + @($reasons | Where-Object { $_ -in @("required_evidence_stale", "required_evidence_freshness_reserve_insufficient") }).Count -gt 0 -and + @($Readiness.failedEvidence).Count -eq 0 + ) { return $false } + if (@($Readiness.failedTasks | Where-Object { + -not $_.exists -or + -not $_.enabled -or + $_.taskPath -ne "\" -or + $_.state -notin @("Ready", "Queued") + }).Count -gt 0) { + return $false + } + if (@($Readiness.failedEvidence | Where-Object { -not $_.parsed -or -not $_.contentHealthy }).Count -gt 0) { + return $false + } + return $true +} + function Get-AgentRelayRuntime { $task = $null $taskError = "" @@ -1070,6 +1199,78 @@ try { Write-AgentResultAndExit $receipt 2 } + $script:ReceiverOperationPhase = "pre_promotion_readiness" + $promotionReadinessAttempts = @() + $prePromotionReadiness = $null + for ($readinessAttempt = 1; $readinessAttempt -le $PromotionReadinessMaxAttempts; $readinessAttempt += 1) { + $prePromotionReadiness = Invoke-AgentLivePreflight ` + (Join-Path $stagePath "agent99-live-preflight.ps1") ` + -ReadinessScope "PromotionReserve" ` + -MinimumEvidenceFreshnessReserveMinutes $PromotionReadinessReserveMinutes + $prePromotionReadiness | Add-Member -NotePropertyName attempt -NotePropertyValue $readinessAttempt -Force + $promotionReadinessAttempts += $prePromotionReadiness + if ($prePromotionReadiness.ok) { break } + if ( + $readinessAttempt -ge $PromotionReadinessMaxAttempts -or + -not (Test-AgentPromotionReadinessRetryable $prePromotionReadiness) + ) { break } + Start-Sleep -Seconds $PromotionReadinessRetryWaitSeconds + } + $promotionReadinessRetryCount = [math]::Max(0, $promotionReadinessAttempts.Count - 1) + $promotionReadinessStillRetryable = Test-AgentPromotionReadinessRetryable $prePromotionReadiness + if (-not $prePromotionReadiness.ok) { + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "blocked_pre_promotion_readiness_no_live_write" + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + canonicalReceiptPath = $receiptPath + attemptReceiptPath = $attemptReceiptPath + validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } + prePromotionReadiness = $prePromotionReadiness + promotionReadinessAttempts = $promotionReadinessAttempts + retryPolicy = [pscustomobject]@{ + retryableTransientOnly = $true + automaticTaskTriggerAllowed = $false + maxAttempts = $PromotionReadinessMaxAttempts + attemptCount = $promotionReadinessAttempts.Count + retryCount = $promotionReadinessRetryCount + retryWaitSeconds = $PromotionReadinessRetryWaitSeconds + finalFailureRetryableTransient = $promotionReadinessStillRetryable + unboundedRetryAllowed = $false + } + durableReceiptWritten = $true + remoteWritePerformed = $script:ReceiverRemoteWritePerformed + liveRuntimeWritePerformed = $false + livePromotionAttempted = $false + livePromotionPerformed = $false + readinessCheckWritePerformed = $false + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskRestart = $false + scheduledTaskModification = $false + nextSafeAction = if ($promotionReadinessStillRetryable) { + "allow_natural_root_task_and_evidence_refresh_then_retry_same_identity_once" + } else { + "repair_exact_reported_readiness_blockers_before_retry" + } + } + $script:ReceiverOperationPhase = "pre_promotion_readiness_failure_receipt_write" + Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt + Write-AgentResultAndExit $receipt 2 + } + $beforeManifest = Get-AgentSafeRuntimeManifest $beforeBundleDigest = Get-AgentLiveBundleDigest $beforeConfigDigest = Get-AgentSha256File $ConfigPath @@ -1215,6 +1416,9 @@ try { backupPath = $backupPath canonicalReceiptPath = $receiptPath validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status } + prePromotionReadiness = $prePromotionReadiness + promotionReadinessAttempts = $promotionReadinessAttempts + promotionReadinessRetryCount = $promotionReadinessRetryCount deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status } relayReload = $relayReload runtimeManifest = $runtimeManifest @@ -1331,6 +1535,9 @@ try { stagingPath = $stagePath backupPath = $backupPath validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } + prePromotionReadiness = $prePromotionReadiness + promotionReadinessAttempts = $promotionReadinessAttempts + promotionReadinessRetryCount = $promotionReadinessRetryCount deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } } failurePhase = $failurePhase failureType = $failureType diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index 1637b0635..774839fd8 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -40,9 +40,11 @@ def _base_case() -> dict[str, Any]: "mismatchCount": 0, "parseError": "", }, + "manifestCheckRequired": True, "taskFailureCount": 0, "relayListenerCount": 1, "requiredEvidenceStaleCount": 0, + "requiredEvidenceReserveFailureCount": 0, "requiredEvidenceParseFailureCount": 0, "requiredEvidenceContentFailureCount": 0, "selfHealthSeverity": "ok", @@ -81,10 +83,12 @@ $case = @' $parameters = @{{ AgentRootPresent = [bool]$case.agentRootPresent Manifest = $case.manifest + ManifestCheckRequired = [bool]$case.manifestCheckRequired ExpectedRuntimeFileCount = 16 TaskFailureCount = [int]$case.taskFailureCount RelayListenerCount = [int]$case.relayListenerCount RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount + RequiredEvidenceReserveFailureCount = [int]$case.requiredEvidenceReserveFailureCount RequiredEvidenceParseFailureCount = [int]$case.requiredEvidenceParseFailureCount RequiredEvidenceContentFailureCount = [int]$case.requiredEvidenceContentFailureCount SelfHealthSeverity = [string]$case.selfHealthSeverity @@ -123,6 +127,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '$blockingReasons += "runtime_manifest_mismatch"' in decision assert '$blockingReasons += "scheduled_task_unhealthy"' in decision assert '$blockingReasons += "sre_alert_relay_not_listening"' in decision + assert ( + '$blockingReasons += "required_evidence_freshness_reserve_insufficient"' + in decision + ) assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision assert "preflightGreen = [bool]$decision.deploymentEligible" in source assert "-ExpectedRuntimeFileCount 16" in source @@ -139,6 +147,9 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '"agent99_background_mode_deferred_v1"' in source assert '"agent99_recovery_transport_deferred_v1"' in source assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source + assert '$rootTaskPath = "\\"' in source + assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source + assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source def test_evidence_selection_skips_deferred_but_not_malformed_candidates( @@ -224,6 +235,45 @@ Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true | assert malformed_payload["healthy"] is False +def test_evidence_reserve_rejects_fresh_but_expiring_evidence_without_extending_slo( + tmp_path: Path, +) -> None: + powershell = shutil.which("pwsh") or shutil.which("powershell") + if powershell is None: + pytest.skip("PowerShell runtime is not installed on this macOS verifier") + + evidence = tmp_path / "evidence" + evidence.mkdir() + candidate = evidence / "agent99-Status-20260716-230000.json" + candidate.write_text(json.dumps({"mode": "Status"}), encoding="utf-8") + six_minutes_ago = candidate.stat().st_mtime - (6 * 60) + os.utime(candidate, (six_minutes_ago, six_minutes_ago)) + + agent_root = str(tmp_path).replace("'", "''") + command = f""" +{_evidence_function_source()} +$AgentRoot = '{agent_root}' +Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 | + ConvertTo-Json -Depth 8 -Compress +""" + result = subprocess.run( + [powershell, "-NoProfile", "-NonInteractive", "-Command", command], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + payload = json.loads( + next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{")) + ) + assert payload["fresh"] is True + assert payload["maxAgeMinutes"] == 20 + assert payload["minimumFreshnessReserveMinutes"] == 15 + assert payload["freshnessReserveReady"] is False + assert payload["usable"] is False + assert payload["healthy"] is False + + def test_decision_behavior_when_powershell_is_available() -> None: powershell = shutil.which("pwsh") or shutil.which("powershell") if powershell is None: @@ -252,6 +302,12 @@ def test_decision_behavior_when_powershell_is_available() -> None: ("scheduled_task_unhealthy",), (), ), + ( + _with_overrides(requiredEvidenceReserveFailureCount=1), + False, + ("required_evidence_freshness_reserve_insufficient",), + (), + ), ( _with_overrides(manifest={"runtimeMatched": False, "mismatchCount": 1}), False, @@ -305,6 +361,29 @@ def test_decision_behavior_when_powershell_is_available() -> None: assert set(_as_list(result.get("blockingReasons"))) == set(expected_blockers) assert set(_as_list(result.get("warningReasons"))) == set(expected_warnings) + promotion_reserve = _run_decision( + powershell, + _with_overrides( + manifestCheckRequired=False, + manifest={"sourceRevision": "old", "fileCount": 15, "runtimeMatched": False}, + ), + ) + assert promotion_reserve["deploymentEligible"] is True + assert promotion_reserve["manifestCheckRequired"] is False + + promotion_reserve_task_failure = _run_decision( + powershell, + _with_overrides( + manifestCheckRequired=False, + manifest={"sourceRevision": "old", "fileCount": 15, "runtimeMatched": False}, + taskFailureCount=1, + ), + ) + assert promotion_reserve_task_failure["deploymentEligible"] is False + assert promotion_reserve_task_failure["blockingReasons"] == [ + "scheduled_task_unhealthy" + ] + preflight_path = str(PREFLIGHT).replace("'", "''") parser_command = ( "$tokens=$null; $errors=$null; " diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index f78b88507..56c38e40b 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -179,7 +179,10 @@ def test_live_preflight_rejects_fresh_but_unparseable_required_evidence() -> Non preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1") assert '$parsed = [bool]($file -and -not $parseError)' in preflight - assert '$usable = [bool]($fresh -and $parsed -and $contentHealthy)' in preflight + assert ( + "$usable = [bool]($fresh -and $freshnessReserveReady -and $parsed -and " + "$contentHealthy)" in preflight + ) assert 'healthy = [bool](-not $Required -or $usable)' in preflight assert '$_.required -and -not $_.healthy' in preflight assert 'required_evidence_parse_failed' in preflight diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 7d5e0ba16..966a4383a 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -209,8 +209,10 @@ def test_receiver_validates_manifest_before_staging_and_validate_only_before_pro "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath " "-SourceRevision $sourceRevision" ) - assert manifest_check < stage_write < validate < promote + readiness = source.index('$script:ReceiverOperationPhase = "pre_promotion_readiness"') + assert manifest_check < stage_write < validate < readiness < promote assert 'status = "validate_only_failed_no_promotion"' in source + assert 'status = "blocked_pre_promotion_readiness_no_live_write"' in source assert "liveRuntimeWritePerformed = $false" in source assert "livePromotionPerformed = $false" in source @@ -236,7 +238,12 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() assert "function Invoke-AgentBoundedPowerShell" in source assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source assert "$process.Kill()" in source - assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source + assert "$ValidateOnlyTimeoutSeconds = 300" in source + assert "$LiveDeployTimeoutSeconds = 300" in source + assert ( + "$timeoutSeconds = if ($ValidateOnly) { $ValidateOnlyTimeoutSeconds } " + "else { $LiveDeployTimeoutSeconds }" in source + ) assert "$livePreflight.sourceRevision -eq $sourceRevision" in source assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source assert '$runtimeManifest.fileCount -eq 16' in source @@ -301,6 +308,60 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary() assert "[bool]$prior.relayReload.newGenerationVerified" in source +def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + validate = source.index( + "$validate = Invoke-AgentDeployChild -SourceRoot $stagePath " + "-SourceRevision $sourceRevision -ValidateOnly" + ) + readiness = source.index('$script:ReceiverOperationPhase = "pre_promotion_readiness"') + live_write = source.index('$script:ReceiverOperationPhase = "live_deploy"') + assert validate < readiness < live_write + + assert '-ReadinessScope "PromotionReserve"' in source + assert ( + "-MinimumEvidenceFreshnessReserveMinutes " + "$PromotionReadinessReserveMinutes" in source + ) + assert "$PromotionReadinessReserveMinutes = 15" in source + assert "$PromotionReadinessMaxAttempts = 2" in source + assert "$PromotionReadinessRetryWaitSeconds = 30" in source + assert "function Test-AgentPromotionReadinessRetryable" in source + assert '"scheduled_task_unhealthy"' in source + assert '"required_evidence_stale"' in source + assert '"required_evidence_freshness_reserve_insufficient"' in source + assert '[string]$Readiness.readinessScope -ne "PromotionReserve"' in source + assert ( + "[int]$Readiness.minimumEvidenceFreshnessReserveMinutes -ne " + "$PromotionReadinessReserveMinutes" in source + ) + assert '"scheduled_task_unhealthy" -in $reasons' in source + assert "@($Readiness.failedTasks).Count -eq 0" in source + assert "@($Readiness.failedEvidence).Count -eq 0" in source + assert '$_.state -notin @("Ready", "Queued")' in source + assert "automaticTaskTriggerAllowed = $false" in source + assert "unboundedRetryAllowed = $false" in source + assert "Start-ScheduledTask" not in source[readiness:live_write] + + blocked = source[ + source.index('$status = "blocked_pre_promotion_readiness_no_live_write"') + if '$status = "blocked_pre_promotion_readiness_no_live_write"' in source + else source.index('status = "blocked_pre_promotion_readiness_no_live_write"') : live_write + ] + assert "liveRuntimeWritePerformed = $false" in blocked + assert "livePromotionAttempted = $false" in blocked + assert "livePromotionPerformed = $false" in blocked + assert "readinessCheckWritePerformed = $false" in blocked + assert "failedTaskNames" in source + assert "failedTasks" in source + assert "failedEvidenceNames" in source + assert "failedEvidence" in source + assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source + assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source + assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source + + def test_check_and_apply_receipts_project_the_same_run_identity() -> None: sender = SENDER.read_text(encoding="utf-8") projector = sender[ @@ -510,6 +571,11 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert "$RelayStopTimeoutSeconds = 60" in source assert "$RelayStartTimeoutSeconds = 360" in source assert "$RelayPollIntervalSeconds = 1" in source + assert "$ValidateOnlyTimeoutSeconds = 300" in source + assert "$LiveDeployTimeoutSeconds = 300" in source + assert "$LivePreflightTimeoutSeconds = 120" in source + assert "$PromotionReadinessMaxAttempts = 2" in source + assert "$PromotionReadinessRetryWaitSeconds = 30" in source assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source assert "Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source @@ -545,18 +611,21 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in wrapper wrapper_budget_seconds = 2400 validate_seconds = 300 - deploy_seconds = 900 + promotion_readiness_attempts = 2 + promotion_readiness_retry_wait_seconds = 30 + deploy_seconds = 300 live_preflight_seconds = 120 worst_case_seconds = ( validate_seconds + + (promotion_readiness_attempts * live_preflight_seconds) + + promotion_readiness_retry_wait_seconds + deploy_seconds + live_preflight_seconds + (2 * (60 + 360)) ) - assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source - assert "-TimeoutSeconds 120" in source - assert worst_case_seconds == 2160 - assert wrapper_budget_seconds - worst_case_seconds == 240 + assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source + assert worst_case_seconds == 1830 + assert wrapper_budget_seconds - worst_case_seconds == 570 deploy = source.index( "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "