wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(agent99): verify frozen post-promotion runtime

Browse Source
This commit is contained in:
Your Name
2026-07-22 16:39:58 +08:00
parent 1cec67295b
commit a2ed888e84
4 changed files with 149 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
scripts/reboot-recovery/agent99-live-preflight.ps1
View File

@@ -3,7 +3,7 @@ param(
[ValidateSet("Verify")]
[string]$Mode = "Verify",
[string]$AgentRoot = "C:\Wooo\Agent99",
[ValidateSet("FullRuntime", "PromotionReserve")]
[ValidateSet("FullRuntime", "PromotionReserve", "PostPromotionMaintenance")]
[string]$ReadinessScope = "FullRuntime",
[ValidateRange(0, 20)]
[int]$MinimumEvidenceFreshnessReserveMinutes = 0
@@ -451,7 +451,7 @@ function Get-AgentControlLoopMaintenanceReadback {
function Get-AgentPendingCommandPromotionReadback {
param(
[object]$MaintenanceReadback,
[ValidateSet("FullRuntime", "PromotionReserve")]
[ValidateSet("FullRuntime", "PromotionReserve", "PostPromotionMaintenance")]
[string]$Scope
)
@@ -888,18 +888,49 @@ $requiredTasks = @(
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health"; markers = @("agent99-run.ps1", "-Mode SelfCheck") },
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health"; markers = @("agent99-run.ps1", "-Mode BackupCheck") }
)
$tasks = @($requiredTasks | ForEach-Object {
Get-TaskReadback `
-TaskPath $_.taskPath `
-TaskName $_.name `
-RequiredArgumentMarkers $_.markers
})
$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback
$maintenanceSnapshotMutex = [Threading.Mutex]::new(
$false,
"Global\WoooAgent99ControlLoopMaintenanceV1"
)
$maintenanceSnapshotMutexAcquired = $false
try {
try {
$maintenanceSnapshotMutexAcquired = [bool]$maintenanceSnapshotMutex.WaitOne(0)
} catch [Threading.AbandonedMutexException] {
$maintenanceSnapshotMutexAcquired = $true
}
if (-not $maintenanceSnapshotMutexAcquired) {
throw "maintenance_snapshot_mutex_unavailable"
}
# The control task definition/state and its maintenance receipt are one
# consistency boundary. Freeze/restore holds this same mutex, so a verifier
# can never combine task state from one side of a transition with receipt
# state from the other side.
$tasks = @($requiredTasks | ForEach-Object {
Get-TaskReadback `
-TaskPath $_.taskPath `
-TaskName $_.name `
-RequiredArgumentMarkers $_.markers
})
$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback
$controlLoopMaintenance | Add-Member `
-NotePropertyName snapshotMutexAcquired `
-NotePropertyValue $true `
-Force
} finally {
if ($maintenanceSnapshotMutexAcquired -and $maintenanceSnapshotMutex) {
try { $maintenanceSnapshotMutex.ReleaseMutex() } catch {}
}
if ($maintenanceSnapshotMutex) { $maintenanceSnapshotMutex.Dispose() }
}
if ($ReadinessScope -eq "PromotionReserve") {
foreach ($task in $tasks) {
$runtimeHealthy = [bool]$task.healthy
$maintenanceFreezeAccepted = [bool](
$task.name -eq "Wooo-Agent99-Control-Loop" -and
$task.exists -and
$task.definitionReady -and
-not $task.enabled -and
$controlLoopMaintenance.valid
)
@@ -913,6 +944,22 @@ if ($ReadinessScope -eq "PromotionReserve") {
$task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force
$task.healthy = $promotionReady
}
} elseif ($ReadinessScope -eq "PostPromotionMaintenance") {
foreach ($task in $tasks) {
$runtimeHealthy = [bool]$task.healthy
$maintenanceFreezeAccepted = [bool](
$task.name -eq "Wooo-Agent99-Control-Loop" -and
$task.exists -and
$task.definitionReady -and
-not $task.enabled -and
$controlLoopMaintenance.valid
)
$postPromotionReady = [bool]($runtimeHealthy -or $maintenanceFreezeAccepted)
$task | Add-Member -NotePropertyName runtimeHealthy -NotePropertyValue $runtimeHealthy -Force
$task | Add-Member -NotePropertyName promotionReady -NotePropertyValue $postPromotionReady -Force
$task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force
$task.healthy = $postPromotionReady
}
}
$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
@@ -965,7 +1012,7 @@ $queues = @(
(Get-DirectoryReadback "queue" -RootQueueFilesOnly),
(Get-DirectoryReadback "queue\failed")
)
$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")
$promotionEvidenceRequired = [bool]($ReadinessScope -ne "PromotionReserve")
$evidence = @(
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
@@ -997,7 +1044,7 @@ $pendingCommandPromotion = Get-AgentPendingCommandPromotionReadback `
$decision = Get-AgentLivePreflightDecision `
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
-Manifest $manifest `
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
-ManifestCheckRequired ([bool]($ReadinessScope -ne "PromotionReserve")) `
-ExpectedRuntimeFileCount 20 `
-TaskFailureCount $taskFailures.Count `
-RelayListenerCount $relayListeners.Count `

48
scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1
View File

@@ -380,7 +380,7 @@ function Invoke-AgentWindowsControlBaseline {
function Invoke-AgentLivePreflight {
param(
[string]$PreflightPath,
[ValidateSet("FullRuntime", "PromotionReserve")]
[ValidateSet("FullRuntime", "PromotionReserve", "PostPromotionMaintenance")]
[string]$ReadinessScope = "FullRuntime",
[ValidateRange(0, 20)]
[int]$MinimumEvidenceFreshnessReserveMinutes = 0
@@ -466,7 +466,7 @@ function Invoke-AgentLivePreflight {
[string]$payload.readinessScope -eq $ReadinessScope -and
[int]$payload.minimumEvidenceFreshnessReserveMinutes -eq $MinimumEvidenceFreshnessReserveMinutes -and
$summary -and
[bool]$summary.manifestCheckRequired -eq [bool]($ReadinessScope -eq "FullRuntime")
[bool]$summary.manifestCheckRequired -eq [bool]($ReadinessScope -ne "PromotionReserve")
)
$manifestReadyForScope = [bool](
$ReadinessScope -eq "PromotionReserve" -or
@@ -500,6 +500,14 @@ function Invoke-AgentLivePreflight {
$pendingCommandsReadyForScope -and
$runtimePlaneBlockingReasons.Count -eq 0
)
$controlLoopMaintenanceRestorePending = [bool](
$ReadinessScope -eq "PostPromotionMaintenance" -and
$payload -and
@($payload.tasks | Where-Object {
[string]$_.name -eq "Wooo-Agent99-Control-Loop" -and
[bool]$_.maintenanceFreezeAccepted
}).Count -eq 1
)
return [pscustomobject]@{
ok = $runtimePlaneReady
readinessScope = $ReadinessScope
@@ -549,6 +557,7 @@ function Invoke-AgentLivePreflight {
preflightGreen = [bool]($summary -and $summary.preflightGreen)
overallPreflightGreen = [bool]($summary -and $summary.preflightGreen)
runtimePlaneReady = $runtimePlaneReady
controlLoopMaintenanceRestorePending = $controlLoopMaintenanceRestorePending
externalDegraded = [bool]($runtimePlaneReady -and $externalBlockingReasons.Count -gt 0)
failedTaskNames = @($failedTasks | ForEach-Object { [string]$_.name })
failedTasks = $failedTasks
@@ -632,7 +641,7 @@ function Test-AgentPostPromotionReadinessRetryable {
$Readiness.ok -or
-not $Readiness.parsed -or
-not $Readiness.scopeMatched -or
[string]$Readiness.readinessScope -ne "FullRuntime" -or
[string]$Readiness.readinessScope -ne "PostPromotionMaintenance" -or
$Readiness.timedOut -or
-not $Readiness.acceptedExitCode -or
[int]$Readiness.relayListenerCount -lt 1 -or
@@ -1237,7 +1246,10 @@ try {
$liveLauncherContract = Get-AgentLauncherContract
if (
$prior -and
[string]$prior.status -eq "deployed_verified" -and
([string]$prior.status) -in @(
"deployed_verified",
"deployed_verified_maintenance_restore_pending"
) -and
[string]$prior.traceId -eq $traceId -and
[string]$prior.runId -eq $runId -and
[string]$prior.workItemId -eq $workItemId -and
@@ -1255,7 +1267,12 @@ try {
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
) {
$script:ReceiverOperationPhase = "idempotent_replay_post_verifier"
$replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$replayReadinessScope = if (
[string]$prior.status -eq "deployed_verified_maintenance_restore_pending"
) { "PostPromotionMaintenance" } else { "FullRuntime" }
$replayPreflight = Invoke-AgentLivePreflight `
(Join-Path $stagePath "agent99-live-preflight.ps1") `
-ReadinessScope $replayReadinessScope
$replayWindowsControlBaseline = Invoke-AgentWindowsControlBaseline $traceId $runId $workItemId $sourceRevision
if ($replayPreflight.ok -and $replayPreflight.sourceRevision -eq $sourceRevision -and $replayPreflight.runtimeMatched -and $replayWindowsControlBaseline.ok) {
$originalOperationBoundaries = $prior.operationBoundaries
@@ -1318,7 +1335,10 @@ try {
if ($canonicalReceiptExists) {
$canonicalStatus = if (-not $prior) {
"blocked_canonical_receipt_invalid_no_promotion"
} elseif ([string]$prior.status -eq "deployed_verified") {
} elseif (([string]$prior.status) -in @(
"deployed_verified",
"deployed_verified_maintenance_restore_pending"
)) {
"blocked_canonical_success_receipt_runtime_drift_no_promotion"
} else {
"blocked_canonical_receipt_reserved_no_promotion"
@@ -1560,7 +1580,9 @@ try {
$runtimeManifest = Get-AgentSafeRuntimeManifest
$launcherContract = Get-AgentLauncherContract
for ($postVerifyAttempt = 1; $postVerifyAttempt -le $PostVerifyMaxAttempts; $postVerifyAttempt += 1) {
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$livePreflight = Invoke-AgentLivePreflight `
(Join-Path $stagePath "agent99-live-preflight.ps1") `
-ReadinessScope "PostPromotionMaintenance"
$livePreflight | Add-Member -NotePropertyName attempt -NotePropertyValue $postVerifyAttempt -Force
$postVerifyAttempts += $livePreflight
$postVerified = [bool](
@@ -1615,9 +1637,10 @@ try {
if (-not $failurePhase) {
try {
$script:ReceiverOperationPhase = "success_receipt_construction"
$maintenanceRestorePending = [bool]$livePreflight.controlLoopMaintenanceRestorePending
$successReceipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "deployed_verified"
status = if ($maintenanceRestorePending) { "deployed_verified_maintenance_restore_pending" } else { "deployed_verified" }
mode = "apply"
traceId = $traceId
runId = $runId
@@ -1647,6 +1670,9 @@ try {
windowsControlBaseline = $windowsControlBaseline
durableReceiptWritten = $true
idempotentReplay = $false
controlLoopMaintenanceRestorePending = $maintenanceRestorePending
fullRuntimeVerifierPending = $maintenanceRestorePending
completionClaim = [bool](-not $maintenanceRestorePending)
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $true
liveRuntimeWritePerformed = $true
@@ -1664,7 +1690,11 @@ try {
scheduledTaskModification = $false
windowsControlSettingsWritePerformed = [bool]$windowsControlBaseline.settingsWritePerformed
}
nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback"
nextSafeAction = if ($maintenanceRestorePending) {
"restore_control_loop_then_run_full_runtime_independent_verifier"
} else {
"run_existing_production_principal_verifier_and_same_trace_completion_readback"
}
}
$script:ReceiverOperationPhase = "success_receipt_write"
Write-AgentRemoteDeployReceipt $receiptPath $successReceipt

16
scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py
View File

@@ -177,10 +177,24 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
assert '$activeControlProcesses.Count -eq 0' in source
assert '-not $leasePresent' in source
assert '-not $recoveryMutexActive' in source
assert '$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")' in source
assert '$promotionEvidenceRequired = [bool]($ReadinessScope -ne "PromotionReserve")' in source
assert '"PostPromotionMaintenance"' in source
assert '"Global\\WoooAgent99ControlLoopMaintenanceV1"' in source
assert "$maintenanceSnapshotMutex.WaitOne(0)" in source
assert 'throw "maintenance_snapshot_mutex_unavailable"' in source
assert source.index("$maintenanceSnapshotMutex.WaitOne(0)") < source.index(
"$tasks = @($requiredTasks"
) < source.index("$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback")
assert source.index("$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback") < source.index(
"$maintenanceSnapshotMutex.ReleaseMutex()"
)
assert "$task.exists -and" in source
assert "$task.definitionReady -and" in source
assert '$task | Add-Member -NotePropertyName runtimeHealthy' in source
assert '$task | Add-Member -NotePropertyName promotionReady' in source
assert '$controlLoopMaintenance.valid' in source
assert '$postPromotionReady = [bool]($runtimeHealthy -or $maintenanceFreezeAccepted)' in source
assert '$task.healthy = $postPromotionReady' in source
assert "function Get-AgentPendingCommandPromotionReadback" in source
assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source
assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source

41
scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py
View File

@@ -284,7 +284,11 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
assert "$evidence.risk.userLogoff -eq $false" in source
assert "windowsControlSettingsWritePerformed" in source
assert '$runtimeManifest.mismatchCount -eq 0' in source
assert 'status = "deployed_verified"' in source
assert (
'status = if ($maintenanceRestorePending) { '
'"deployed_verified_maintenance_restore_pending" } else { "deployed_verified" }'
in source
)
assert '$failurePhase = "deploy"' in source
assert '$failurePhase = "relay_reload"' in source
assert '$failurePhase = "post_verifier"' in source
@@ -408,7 +412,7 @@ def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() ->
assert "$PostVerifyMaxAttempts = 4" in source
assert "$PostVerifyRetryWaitSeconds = 45" in source
assert "function Test-AgentPostPromotionReadinessRetryable" in source
assert '[string]$Readiness.readinessScope -ne "FullRuntime"' in source
assert '[string]$Readiness.readinessScope -ne "PostPromotionMaintenance"' in source
assert '"required_evidence_stale"' in source
assert '"required_evidence_content_unhealthy"' in source
assert '$_.name -notin @("telegram_inbox", "sre_alert_inbox")' in source
@@ -424,6 +428,14 @@ def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() ->
assert "automaticTaskTriggerAllowed = $false" in source
assert "postVerifyAttempts = $postVerifyAttempts" in source
assert "postVerifyRetryCount = $postVerifyRetryCount" in source
assert "controlLoopMaintenanceRestorePending = $controlLoopMaintenanceRestorePending" in source
assert '"deployed_verified_maintenance_restore_pending"' in source
assert "fullRuntimeVerifierPending = $maintenanceRestorePending" in source
assert "completionClaim = [bool](-not $maintenanceRestorePending)" in source
assert '"restore_control_loop_then_run_full_runtime_independent_verifier"' in source
assert '"deployed_verified_maintenance_restore_pending"' in source
assert '"PostPromotionMaintenance" } else { "FullRuntime" }' in source
assert "-ReadinessScope $replayReadinessScope" in source
def test_check_and_apply_receipts_project_the_same_run_identity() -> None:
@@ -705,11 +717,32 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
"@($beforeRelayRuntime.taskGeneration)"
)
postverify = source.index(
'$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath '
'"agent99-live-preflight.ps1")',
'$livePreflight = Invoke-AgentLivePreflight `',
reload_task,
)
assert deploy < reload_task < postverify
assert '-ReadinessScope "PostPromotionMaintenance"' in source[postverify:]
assert 'status = if ($maintenanceRestorePending) { "deployed_verified_maintenance_restore_pending" }' in source[postverify:]
assert '"restore_control_loop_then_run_full_runtime_independent_verifier"' in source[postverify:]
assert "completionClaim = [bool](-not $maintenanceRestorePending)" in source[postverify:]
def test_pending_maintenance_success_receipt_is_idempotently_replayable_without_promotion() -> None:
source = RECEIVER.read_text(encoding="utf-8")
replay_gate = source.index("$live = Get-AgentSafeRuntimeManifest")
replay_scope = source.index("$replayReadinessScope = if", replay_gate)
replay_verify = source.index("$replayPreflight = Invoke-AgentLivePreflight `", replay_scope)
canonical_guard = source.index("if ($canonicalReceiptExists)", replay_verify)
gate = source[replay_gate:replay_scope]
replay = source[replay_scope:canonical_guard]
assert '"deployed_verified_maintenance_restore_pending"' in gate
assert '[string]$prior.status -eq "deployed_verified_maintenance_restore_pending"' in replay
assert '"PostPromotionMaintenance" } else { "FullRuntime" }' in replay
assert "-ReadinessScope $replayReadinessScope" in replay
assert "livePromotionAttempted = $false" in replay
assert "livePromotionPerformed = $false" in replay
def test_relay_start_window_accepts_a_late_verified_listener_but_remains_fail_closed() -> None: