From a2ed888e84ff35fa37b522c89447cdfad8584469 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 22 Jul 2026 16:39:58 +0800 Subject: [PATCH] fix(agent99): verify frozen post-promotion runtime --- .../agent99-live-preflight.ps1 | 69 ++++++++++++++++--- .../agent99-remote-atomic-deploy-receiver.ps1 | 48 ++++++++++--- .../test_agent99_live_preflight_decision.py | 16 ++++- ...remote_atomic_deploy_transport_contract.py | 41 +++++++++-- 4 files changed, 149 insertions(+), 25 deletions(-) diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 52009db19..78fc7cf0e 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -3,7 +3,7 @@ param( [ValidateSet("Verify")] [string]$Mode = "Verify", [string]$AgentRoot = "C:\Wooo\Agent99", - [ValidateSet("FullRuntime", "PromotionReserve")] + [ValidateSet("FullRuntime", "PromotionReserve", "PostPromotionMaintenance")] [string]$ReadinessScope = "FullRuntime", [ValidateRange(0, 20)] [int]$MinimumEvidenceFreshnessReserveMinutes = 0 @@ -451,7 +451,7 @@ function Get-AgentControlLoopMaintenanceReadback { function Get-AgentPendingCommandPromotionReadback { param( [object]$MaintenanceReadback, - [ValidateSet("FullRuntime", "PromotionReserve")] + [ValidateSet("FullRuntime", "PromotionReserve", "PostPromotionMaintenance")] [string]$Scope ) @@ -888,18 +888,49 @@ $requiredTasks = @( [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health"; markers = @("agent99-run.ps1", "-Mode SelfCheck") }, [pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health"; markers = @("agent99-run.ps1", "-Mode BackupCheck") } ) -$tasks = @($requiredTasks | ForEach-Object { - Get-TaskReadback ` - -TaskPath $_.taskPath ` - -TaskName $_.name ` - -RequiredArgumentMarkers $_.markers -}) -$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback +$maintenanceSnapshotMutex = [Threading.Mutex]::new( + $false, + "Global\WoooAgent99ControlLoopMaintenanceV1" +) +$maintenanceSnapshotMutexAcquired = $false +try { + try { + $maintenanceSnapshotMutexAcquired = [bool]$maintenanceSnapshotMutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $maintenanceSnapshotMutexAcquired = $true + } + if (-not $maintenanceSnapshotMutexAcquired) { + throw "maintenance_snapshot_mutex_unavailable" + } + + # The control task definition/state and its maintenance receipt are one + # consistency boundary. Freeze/restore holds this same mutex, so a verifier + # can never combine task state from one side of a transition with receipt + # state from the other side. + $tasks = @($requiredTasks | ForEach-Object { + Get-TaskReadback ` + -TaskPath $_.taskPath ` + -TaskName $_.name ` + -RequiredArgumentMarkers $_.markers + }) + $controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback + $controlLoopMaintenance | Add-Member ` + -NotePropertyName snapshotMutexAcquired ` + -NotePropertyValue $true ` + -Force +} finally { + if ($maintenanceSnapshotMutexAcquired -and $maintenanceSnapshotMutex) { + try { $maintenanceSnapshotMutex.ReleaseMutex() } catch {} + } + if ($maintenanceSnapshotMutex) { $maintenanceSnapshotMutex.Dispose() } +} if ($ReadinessScope -eq "PromotionReserve") { foreach ($task in $tasks) { $runtimeHealthy = [bool]$task.healthy $maintenanceFreezeAccepted = [bool]( $task.name -eq "Wooo-Agent99-Control-Loop" -and + $task.exists -and + $task.definitionReady -and -not $task.enabled -and $controlLoopMaintenance.valid ) @@ -913,6 +944,22 @@ if ($ReadinessScope -eq "PromotionReserve") { $task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force $task.healthy = $promotionReady } +} elseif ($ReadinessScope -eq "PostPromotionMaintenance") { + foreach ($task in $tasks) { + $runtimeHealthy = [bool]$task.healthy + $maintenanceFreezeAccepted = [bool]( + $task.name -eq "Wooo-Agent99-Control-Loop" -and + $task.exists -and + $task.definitionReady -and + -not $task.enabled -and + $controlLoopMaintenance.valid + ) + $postPromotionReady = [bool]($runtimeHealthy -or $maintenanceFreezeAccepted) + $task | Add-Member -NotePropertyName runtimeHealthy -NotePropertyValue $runtimeHealthy -Force + $task | Add-Member -NotePropertyName promotionReady -NotePropertyValue $postPromotionReady -Force + $task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force + $task.healthy = $postPromotionReady + } } $manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json" @@ -965,7 +1012,7 @@ $queues = @( (Get-DirectoryReadback "queue" -RootQueueFilesOnly), (Get-DirectoryReadback "queue\failed") ) -$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime") +$promotionEvidenceRequired = [bool]($ReadinessScope -ne "PromotionReserve") $evidence = @( (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), @@ -997,7 +1044,7 @@ $pendingCommandPromotion = Get-AgentPendingCommandPromotionReadback ` $decision = Get-AgentLivePreflightDecision ` -AgentRootPresent ([bool](Test-Path $AgentRoot)) ` -Manifest $manifest ` - -ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) ` + -ManifestCheckRequired ([bool]($ReadinessScope -ne "PromotionReserve")) ` -ExpectedRuntimeFileCount 20 ` -TaskFailureCount $taskFailures.Count ` -RelayListenerCount $relayListeners.Count ` diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index b8c9af3f4..4f0303f18 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -380,7 +380,7 @@ function Invoke-AgentWindowsControlBaseline { function Invoke-AgentLivePreflight { param( [string]$PreflightPath, - [ValidateSet("FullRuntime", "PromotionReserve")] + [ValidateSet("FullRuntime", "PromotionReserve", "PostPromotionMaintenance")] [string]$ReadinessScope = "FullRuntime", [ValidateRange(0, 20)] [int]$MinimumEvidenceFreshnessReserveMinutes = 0 @@ -466,7 +466,7 @@ function Invoke-AgentLivePreflight { [string]$payload.readinessScope -eq $ReadinessScope -and [int]$payload.minimumEvidenceFreshnessReserveMinutes -eq $MinimumEvidenceFreshnessReserveMinutes -and $summary -and - [bool]$summary.manifestCheckRequired -eq [bool]($ReadinessScope -eq "FullRuntime") + [bool]$summary.manifestCheckRequired -eq [bool]($ReadinessScope -ne "PromotionReserve") ) $manifestReadyForScope = [bool]( $ReadinessScope -eq "PromotionReserve" -or @@ -500,6 +500,14 @@ function Invoke-AgentLivePreflight { $pendingCommandsReadyForScope -and $runtimePlaneBlockingReasons.Count -eq 0 ) + $controlLoopMaintenanceRestorePending = [bool]( + $ReadinessScope -eq "PostPromotionMaintenance" -and + $payload -and + @($payload.tasks | Where-Object { + [string]$_.name -eq "Wooo-Agent99-Control-Loop" -and + [bool]$_.maintenanceFreezeAccepted + }).Count -eq 1 + ) return [pscustomobject]@{ ok = $runtimePlaneReady readinessScope = $ReadinessScope @@ -549,6 +557,7 @@ function Invoke-AgentLivePreflight { preflightGreen = [bool]($summary -and $summary.preflightGreen) overallPreflightGreen = [bool]($summary -and $summary.preflightGreen) runtimePlaneReady = $runtimePlaneReady + controlLoopMaintenanceRestorePending = $controlLoopMaintenanceRestorePending externalDegraded = [bool]($runtimePlaneReady -and $externalBlockingReasons.Count -gt 0) failedTaskNames = @($failedTasks | ForEach-Object { [string]$_.name }) failedTasks = $failedTasks @@ -632,7 +641,7 @@ function Test-AgentPostPromotionReadinessRetryable { $Readiness.ok -or -not $Readiness.parsed -or -not $Readiness.scopeMatched -or - [string]$Readiness.readinessScope -ne "FullRuntime" -or + [string]$Readiness.readinessScope -ne "PostPromotionMaintenance" -or $Readiness.timedOut -or -not $Readiness.acceptedExitCode -or [int]$Readiness.relayListenerCount -lt 1 -or @@ -1237,7 +1246,10 @@ try { $liveLauncherContract = Get-AgentLauncherContract if ( $prior -and - [string]$prior.status -eq "deployed_verified" -and + ([string]$prior.status) -in @( + "deployed_verified", + "deployed_verified_maintenance_restore_pending" + ) -and [string]$prior.traceId -eq $traceId -and [string]$prior.runId -eq $runId -and [string]$prior.workItemId -eq $workItemId -and @@ -1255,7 +1267,12 @@ try { [string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256 ) { $script:ReceiverOperationPhase = "idempotent_replay_post_verifier" - $replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") + $replayReadinessScope = if ( + [string]$prior.status -eq "deployed_verified_maintenance_restore_pending" + ) { "PostPromotionMaintenance" } else { "FullRuntime" } + $replayPreflight = Invoke-AgentLivePreflight ` + (Join-Path $stagePath "agent99-live-preflight.ps1") ` + -ReadinessScope $replayReadinessScope $replayWindowsControlBaseline = Invoke-AgentWindowsControlBaseline $traceId $runId $workItemId $sourceRevision if ($replayPreflight.ok -and $replayPreflight.sourceRevision -eq $sourceRevision -and $replayPreflight.runtimeMatched -and $replayWindowsControlBaseline.ok) { $originalOperationBoundaries = $prior.operationBoundaries @@ -1318,7 +1335,10 @@ try { if ($canonicalReceiptExists) { $canonicalStatus = if (-not $prior) { "blocked_canonical_receipt_invalid_no_promotion" - } elseif ([string]$prior.status -eq "deployed_verified") { + } elseif (([string]$prior.status) -in @( + "deployed_verified", + "deployed_verified_maintenance_restore_pending" + )) { "blocked_canonical_success_receipt_runtime_drift_no_promotion" } else { "blocked_canonical_receipt_reserved_no_promotion" @@ -1560,7 +1580,9 @@ try { $runtimeManifest = Get-AgentSafeRuntimeManifest $launcherContract = Get-AgentLauncherContract for ($postVerifyAttempt = 1; $postVerifyAttempt -le $PostVerifyMaxAttempts; $postVerifyAttempt += 1) { - $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") + $livePreflight = Invoke-AgentLivePreflight ` + (Join-Path $stagePath "agent99-live-preflight.ps1") ` + -ReadinessScope "PostPromotionMaintenance" $livePreflight | Add-Member -NotePropertyName attempt -NotePropertyValue $postVerifyAttempt -Force $postVerifyAttempts += $livePreflight $postVerified = [bool]( @@ -1615,9 +1637,10 @@ try { if (-not $failurePhase) { try { $script:ReceiverOperationPhase = "success_receipt_construction" + $maintenanceRestorePending = [bool]$livePreflight.controlLoopMaintenanceRestorePending $successReceipt = [pscustomobject]@{ schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" - status = "deployed_verified" + status = if ($maintenanceRestorePending) { "deployed_verified_maintenance_restore_pending" } else { "deployed_verified" } mode = "apply" traceId = $traceId runId = $runId @@ -1647,6 +1670,9 @@ try { windowsControlBaseline = $windowsControlBaseline durableReceiptWritten = $true idempotentReplay = $false + controlLoopMaintenanceRestorePending = $maintenanceRestorePending + fullRuntimeVerifierPending = $maintenanceRestorePending + completionClaim = [bool](-not $maintenanceRestorePending) operationBoundaries = [pscustomobject]@{ remoteWritePerformed = $true liveRuntimeWritePerformed = $true @@ -1664,7 +1690,11 @@ try { scheduledTaskModification = $false windowsControlSettingsWritePerformed = [bool]$windowsControlBaseline.settingsWritePerformed } - nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback" + nextSafeAction = if ($maintenanceRestorePending) { + "restore_control_loop_then_run_full_runtime_independent_verifier" + } else { + "run_existing_production_principal_verifier_and_same_trace_completion_readback" + } } $script:ReceiverOperationPhase = "success_receipt_write" Write-AgentRemoteDeployReceipt $receiptPath $successReceipt diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index e182cf040..251597a32 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -177,10 +177,24 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '$activeControlProcesses.Count -eq 0' in source assert '-not $leasePresent' in source assert '-not $recoveryMutexActive' in source - assert '$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")' in source + assert '$promotionEvidenceRequired = [bool]($ReadinessScope -ne "PromotionReserve")' in source + assert '"PostPromotionMaintenance"' in source + assert '"Global\\WoooAgent99ControlLoopMaintenanceV1"' in source + assert "$maintenanceSnapshotMutex.WaitOne(0)" in source + assert 'throw "maintenance_snapshot_mutex_unavailable"' in source + assert source.index("$maintenanceSnapshotMutex.WaitOne(0)") < source.index( + "$tasks = @($requiredTasks" + ) < source.index("$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback") + assert source.index("$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback") < source.index( + "$maintenanceSnapshotMutex.ReleaseMutex()" + ) + assert "$task.exists -and" in source + assert "$task.definitionReady -and" in source assert '$task | Add-Member -NotePropertyName runtimeHealthy' in source assert '$task | Add-Member -NotePropertyName promotionReady' in source assert '$controlLoopMaintenance.valid' in source + assert '$postPromotionReady = [bool]($runtimeHealthy -or $maintenanceFreezeAccepted)' in source + assert '$task.healthy = $postPromotionReady' in source assert "function Get-AgentPendingCommandPromotionReadback" in source assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index a00284d64..0c4005e07 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -284,7 +284,11 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() assert "$evidence.risk.userLogoff -eq $false" in source assert "windowsControlSettingsWritePerformed" in source assert '$runtimeManifest.mismatchCount -eq 0' in source - assert 'status = "deployed_verified"' in source + assert ( + 'status = if ($maintenanceRestorePending) { ' + '"deployed_verified_maintenance_restore_pending" } else { "deployed_verified" }' + in source + ) assert '$failurePhase = "deploy"' in source assert '$failurePhase = "relay_reload"' in source assert '$failurePhase = "post_verifier"' in source @@ -408,7 +412,7 @@ def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> assert "$PostVerifyMaxAttempts = 4" in source assert "$PostVerifyRetryWaitSeconds = 45" in source assert "function Test-AgentPostPromotionReadinessRetryable" in source - assert '[string]$Readiness.readinessScope -ne "FullRuntime"' in source + assert '[string]$Readiness.readinessScope -ne "PostPromotionMaintenance"' in source assert '"required_evidence_stale"' in source assert '"required_evidence_content_unhealthy"' in source assert '$_.name -notin @("telegram_inbox", "sre_alert_inbox")' in source @@ -424,6 +428,14 @@ def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> assert "automaticTaskTriggerAllowed = $false" in source assert "postVerifyAttempts = $postVerifyAttempts" in source assert "postVerifyRetryCount = $postVerifyRetryCount" in source + assert "controlLoopMaintenanceRestorePending = $controlLoopMaintenanceRestorePending" in source + assert '"deployed_verified_maintenance_restore_pending"' in source + assert "fullRuntimeVerifierPending = $maintenanceRestorePending" in source + assert "completionClaim = [bool](-not $maintenanceRestorePending)" in source + assert '"restore_control_loop_then_run_full_runtime_independent_verifier"' in source + assert '"deployed_verified_maintenance_restore_pending"' in source + assert '"PostPromotionMaintenance" } else { "FullRuntime" }' in source + assert "-ReadinessScope $replayReadinessScope" in source def test_check_and_apply_receipts_project_the_same_run_identity() -> None: @@ -705,11 +717,32 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati "@($beforeRelayRuntime.taskGeneration)" ) postverify = source.index( - '$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath ' - '"agent99-live-preflight.ps1")', + '$livePreflight = Invoke-AgentLivePreflight `', reload_task, ) assert deploy < reload_task < postverify + assert '-ReadinessScope "PostPromotionMaintenance"' in source[postverify:] + assert 'status = if ($maintenanceRestorePending) { "deployed_verified_maintenance_restore_pending" }' in source[postverify:] + assert '"restore_control_loop_then_run_full_runtime_independent_verifier"' in source[postverify:] + assert "completionClaim = [bool](-not $maintenanceRestorePending)" in source[postverify:] + + +def test_pending_maintenance_success_receipt_is_idempotently_replayable_without_promotion() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + replay_gate = source.index("$live = Get-AgentSafeRuntimeManifest") + replay_scope = source.index("$replayReadinessScope = if", replay_gate) + replay_verify = source.index("$replayPreflight = Invoke-AgentLivePreflight `", replay_scope) + canonical_guard = source.index("if ($canonicalReceiptExists)", replay_verify) + + gate = source[replay_gate:replay_scope] + replay = source[replay_scope:canonical_guard] + assert '"deployed_verified_maintenance_restore_pending"' in gate + assert '[string]$prior.status -eq "deployed_verified_maintenance_restore_pending"' in replay + assert '"PostPromotionMaintenance" } else { "FullRuntime" }' in replay + assert "-ReadinessScope $replayReadinessScope" in replay + assert "livePromotionAttempted = $false" in replay + assert "livePromotionPerformed = $false" in replay def test_relay_start_window_accepts_a_late_verified_listener_but_remains_fail_closed() -> None: