From 3de0ba821e1414842fdfd3a3813c3591efc9684a Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 21:13:32 +0800 Subject: [PATCH] fix(agent99): tolerate fresh durable alerts during deploy --- .../agent99-live-preflight.ps1 | 8 ++++++- .../agent99-remote-atomic-deploy-receiver.ps1 | 3 ++- .../test_agent99_live_preflight_decision.py | 24 +++++++++++++++++++ ...remote_atomic_deploy_transport_contract.py | 5 ++-- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 20f8954e1..1860b99dd 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -21,6 +21,7 @@ function Get-AgentLivePreflightDecision { [int]$SelfHealthPerformanceIssueCount, [int]$StaleAlertExecutionCount, [int]$AlertIncomingCount, + [int]$StaleAlertIncomingCount, [int]$PendingCommandCount ) @@ -75,8 +76,10 @@ function Get-AgentLivePreflightDecision { } if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" } - if ($AlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) { + if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) { $blockingReasons += "pending_work_before_reboot" + } elseif ($AlertIncomingCount -gt 0) { + $warningReasons += "fresh_alert_pending_next_inbox_tick" } return [pscustomobject]@{ @@ -428,6 +431,7 @@ $decision = Get-AgentLivePreflightDecision ` -SelfHealthPerformanceIssueCount $selfHealthPerformanceIssueCount ` -StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes ` -AlertIncomingCount $alertIncoming.count ` + -StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes ` -PendingCommandCount $commandQueue.count $blockingReasons = @($decision.blockingReasons) $warningReasons = @($decision.warningReasons) @@ -459,6 +463,7 @@ $result = [pscustomobject]@{ selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity selfHealthPerformanceIssueCount = $selfHealthPerformanceIssueCount alertIncomingCount = [int]$alertIncoming.count + alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes alertRunningCount = [int]$alertRunning.count pendingCommandCount = [int]$commandQueue.count relayListenerCount = $relayListeners.Count @@ -480,6 +485,7 @@ Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)" Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)" Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)" Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)" +Write-Output "ALERT_INCOMING_STALE_COUNT=$($result.summary.alertIncomingStaleCount)" Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)" Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)" Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')" diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index e5aff93fc..92583eedc 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -284,7 +284,7 @@ function Invoke-AgentLivePreflight { [int]$summary.requiredEvidenceParseFailureCount -eq 0 -and [int]$summary.requiredEvidenceContentFailureCount -eq 0 -and [int]$summary.relayListenerCount -gt 0 -and - [int]$summary.alertIncomingCount -eq 0 -and + [int]$summary.alertIncomingStaleCount -eq 0 -and [int]$summary.alertRunningCount -eq 0 -and [int]$summary.pendingCommandCount -eq 0 -and $runtimePlaneBlockingReasons.Count -eq 0 @@ -317,6 +317,7 @@ function Invoke-AgentLivePreflight { } | Where-Object { $_ } | Sort-Object -Unique) } else { @() } alertIncomingCount = if ($summary) { [int]$summary.alertIncomingCount } else { -1 } + alertIncomingStaleCount = if ($summary) { [int]$summary.alertIncomingStaleCount } else { -1 } alertRunningCount = if ($summary) { [int]$summary.alertRunningCount } else { -1 } pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 } preflightGreen = [bool]($summary -and $summary.preflightGreen) diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index cd0ead9d1..ef96b054e 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -41,6 +41,7 @@ def _base_case() -> dict[str, Any]: "selfHealthPerformanceIssueCount": 0, "staleAlertExecutionCount": 0, "alertIncomingCount": 0, + "staleAlertIncomingCount": 0, "pendingCommandCount": 0, } @@ -82,6 +83,7 @@ $parameters = @{{ SelfHealthPerformanceIssueCount = [int]$case.selfHealthPerformanceIssueCount StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount AlertIncomingCount = [int]$case.alertIncomingCount + StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount PendingCommandCount = [int]$case.pendingCommandCount }} Get-AgentLivePreflightDecision @parameters | @@ -106,6 +108,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '$warningReasons += "self_health_warning"' in decision assert '$warningReasons += "performance_warning"' in decision + assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision assert '"critical" { $blockingReasons += "self_health_critical" }' in decision assert 'default { $blockingReasons += "self_health_unclassified" }' in decision assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision @@ -172,6 +175,27 @@ def test_decision_behavior_when_powershell_is_available() -> None: ("sre_alert_relay_not_listening",), (), ), + ( + _with_overrides(alertIncomingCount=1), + True, + (), + ("fresh_alert_pending_next_inbox_tick",), + ), + ( + _with_overrides( + alertIncomingCount=1, + staleAlertIncomingCount=1, + ), + False, + ("pending_work_before_reboot",), + (), + ), + ( + _with_overrides(pendingCommandCount=1), + False, + ("pending_work_before_reboot",), + (), + ), ) for case, expected_eligible, expected_blockers, expected_warnings in cases: diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index bb6975af5..b008d6d8f 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -513,7 +513,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() - assert '$_ -ne "self_health_not_ok"' in preflight assert '[int]$summary.healthyTaskCount -eq [int]$summary.requiredTaskCount' in preflight assert '[int]$summary.requiredEvidenceFailureCount -eq 0' in preflight - assert '[int]$summary.alertIncomingCount -eq 0' in preflight + assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight + assert '[int]$summary.alertIncomingCount -eq 0' not in preflight assert '[int]$summary.alertRunningCount -eq 0' in preflight assert '[int]$summary.pendingCommandCount -eq 0' in preflight assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight @@ -651,4 +652,4 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport( assert "Administrator@192.168.0.99" in args assert str(identity) in args payload_bytes = int(bytes_path.read_text(encoding="utf-8")) - assert 50_000 < payload_bytes < 500_000 + assert 50_000 < payload_bytes < 1_048_576