From d369b60aaff2fa08daa587f6e9b8af7343c6192c Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 16 Jul 2026 22:35:38 +0800 Subject: [PATCH] fix(agent99): bound late relay readiness --- .../agent99-remote-atomic-deploy-receiver.ps1 | 46 +++++++- ...remote_atomic_deploy_transport_contract.py | 109 ++++++++++++++++++ 2 files changed, 150 insertions(+), 5 deletions(-) diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index b8d172fc0..633dd8bc5 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -18,6 +18,9 @@ $WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" $RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay" $RelayTaskPath = "\" $RelayPort = 8787 +$RelayStopTimeoutSeconds = 60 +$RelayStartTimeoutSeconds = 360 +$RelayPollIntervalSeconds = 1 $FixedRuntimeFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", @@ -481,6 +484,9 @@ function Test-AgentGenerationIntersection { function Invoke-AgentRelayTaskRestart { param([string[]]$ForbiddenGenerations = @()) + $restartStartedAt = Get-Date + $stopAttemptCount = 0 + $startAttemptCount = 0 $before = Get-AgentRelayRuntime $generationCandidates = @($ForbiddenGenerations) + @($before.taskGeneration) $forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique) @@ -497,6 +503,15 @@ function Invoke-AgentRelayTaskRestart { newGenerationVerified = $false listenerStopObserved = $false taskGenerationChanged = $false + stopTimeoutSeconds = $RelayStopTimeoutSeconds + startTimeoutSeconds = $RelayStartTimeoutSeconds + pollIntervalSeconds = $RelayPollIntervalSeconds + elapsedSeconds = [Math]::Round(((Get-Date) - $restartStartedAt).TotalSeconds, 3) + stopAttemptCount = $stopAttemptCount + startAttemptCount = $startAttemptCount + lastObservedTaskState = [string]$before.taskState + lastObservedTaskResult = $before.taskLastTaskResult + lastObservedListenerCount = [int]$before.listenerCount } } @@ -507,9 +522,10 @@ function Invoke-AgentRelayTaskRestart { $taskGenerationChanged = $false try { Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop - $stopDeadline = (Get-Date).AddSeconds(60) + $stopDeadline = (Get-Date).AddSeconds($RelayStopTimeoutSeconds) do { - Start-Sleep -Seconds 1 + $stopAttemptCount += 1 + Start-Sleep -Seconds $RelayPollIntervalSeconds $afterStop = Get-AgentRelayRuntime $listenerStopObserved = [bool]($afterStop.listenerCount -eq 0) $stopVerified = [bool]($listenerStopObserved -and -not $afterStop.taskRunning) @@ -519,9 +535,10 @@ function Invoke-AgentRelayTaskRestart { $stage = "start" Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop - $startDeadline = (Get-Date).AddSeconds(120) + $startDeadline = (Get-Date).AddSeconds($RelayStartTimeoutSeconds) do { - Start-Sleep -Seconds 1 + $startAttemptCount += 1 + Start-Sleep -Seconds $RelayPollIntervalSeconds $after = Get-AgentRelayRuntime $newTaskGeneration = [string]$after.taskGeneration $taskGenerationChanged = [bool]($newTaskGeneration -and $newTaskGeneration -notin $forbidden) @@ -550,8 +567,18 @@ function Invoke-AgentRelayTaskRestart { newGenerationVerified = $newVerified listenerStopObserved = $listenerStopObserved taskGenerationChanged = $taskGenerationChanged + stopTimeoutSeconds = $RelayStopTimeoutSeconds + startTimeoutSeconds = $RelayStartTimeoutSeconds + pollIntervalSeconds = $RelayPollIntervalSeconds + elapsedSeconds = [Math]::Round(((Get-Date) - $restartStartedAt).TotalSeconds, 3) + stopAttemptCount = $stopAttemptCount + startAttemptCount = $startAttemptCount + lastObservedTaskState = [string]$after.taskState + lastObservedTaskResult = $after.taskLastTaskResult + lastObservedListenerCount = [int]$after.listenerCount } } catch { + $after = Get-AgentRelayRuntime return [pscustomobject]@{ attempted = $true ok = $false @@ -560,12 +587,21 @@ function Invoke-AgentRelayTaskRestart { errorType = $_.Exception.GetType().Name before = $before afterStop = $afterStop - after = (Get-AgentRelayRuntime) + after = $after forbiddenGenerations = $forbidden oldGenerationsGone = $listenerStopObserved newGenerationVerified = $false listenerStopObserved = $listenerStopObserved taskGenerationChanged = $taskGenerationChanged + stopTimeoutSeconds = $RelayStopTimeoutSeconds + startTimeoutSeconds = $RelayStartTimeoutSeconds + pollIntervalSeconds = $RelayPollIntervalSeconds + elapsedSeconds = [Math]::Round(((Get-Date) - $restartStartedAt).TotalSeconds, 3) + stopAttemptCount = $stopAttemptCount + startAttemptCount = $startAttemptCount + lastObservedTaskState = [string]$after.taskState + lastObservedTaskResult = $after.taskLastTaskResult + lastObservedListenerCount = [int]$after.listenerCount } } } diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 5c45814d1..7d5e0ba16 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -507,6 +507,9 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert '$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"' in source assert '$RelayTaskPath = "\\"' in source assert "$RelayPort = 8787" in source + assert "$RelayStopTimeoutSeconds = 60" in source + assert "$RelayStartTimeoutSeconds = 360" in source + assert "$RelayPollIntervalSeconds = 1" in source assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source assert "Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source @@ -519,6 +522,18 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert "$after.taskEnabled" in source assert "$listenerStopObserved" in source assert "$taskGenerationChanged" in source + assert "AddSeconds($RelayStopTimeoutSeconds)" in source + assert "AddSeconds($RelayStartTimeoutSeconds)" in source + assert "Start-Sleep -Seconds $RelayPollIntervalSeconds" in source + assert "stopTimeoutSeconds = $RelayStopTimeoutSeconds" in source + assert "startTimeoutSeconds = $RelayStartTimeoutSeconds" in source + assert "pollIntervalSeconds = $RelayPollIntervalSeconds" in source + assert "elapsedSeconds = [Math]::Round" in source + assert "stopAttemptCount = $stopAttemptCount" in source + assert "startAttemptCount = $startAttemptCount" in source + assert "lastObservedTaskState = [string]$after.taskState" in source + assert "lastObservedTaskResult = $after.taskLastTaskResult" in source + assert "lastObservedListenerCount = [int]$after.listenerCount" in source assert "$newTaskGeneration -notin $forbidden" in source assert "$relayReload.oldGenerationsGone" in source assert "$relayReload.newGenerationVerified" in source @@ -526,6 +541,23 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert "$livePreflight.relayListenerGenerations" in source assert "scheduledTaskRestart = $true" in source + wrapper = SENDER.read_text(encoding="utf-8") + assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in wrapper + wrapper_budget_seconds = 2400 + validate_seconds = 300 + deploy_seconds = 900 + live_preflight_seconds = 120 + worst_case_seconds = ( + validate_seconds + + deploy_seconds + + live_preflight_seconds + + (2 * (60 + 360)) + ) + assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source + assert "-TimeoutSeconds 120" in source + assert worst_case_seconds == 2160 + assert wrapper_budget_seconds - worst_case_seconds == 240 + deploy = source.index( "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath " "-SourceRevision $sourceRevision" @@ -542,6 +574,83 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert deploy < reload_task < postverify +def test_relay_start_window_accepts_a_late_verified_listener_but_remains_fail_closed() -> None: + source = RECEIVER.read_text(encoding="utf-8") + timeout_match = re.search(r"\$RelayStartTimeoutSeconds = (\d+)", source) + poll_match = re.search(r"\$RelayPollIntervalSeconds = (\d+)", source) + assert timeout_match is not None + assert poll_match is not None + timeout_seconds = int(timeout_match.group(1)) + poll_seconds = int(poll_match.group(1)) + + def verifier(observations: list[dict[str, object]]) -> bool: + for observation in observations: + if int(observation["elapsedSeconds"]) > timeout_seconds: + break + ready = bool( + observation["taskRunning"] + and observation["taskEnabled"] + and observation["generationReady"] + and int(observation["listenerCount"]) > 0 + and observation["listenerStopObserved"] + and observation["taskGenerationChanged"] + ) + if ready: + return True + return False + + assert poll_seconds == 1 + assert timeout_seconds == 360 + assert verifier( + [ + { + "elapsedSeconds": 120, + "taskRunning": True, + "taskEnabled": True, + "generationReady": False, + "listenerCount": 0, + "listenerStopObserved": True, + "taskGenerationChanged": True, + }, + { + "elapsedSeconds": 300, + "taskRunning": True, + "taskEnabled": True, + "generationReady": True, + "listenerCount": 1, + "listenerStopObserved": True, + "taskGenerationChanged": True, + }, + ] + ) + assert not verifier( + [ + { + "elapsedSeconds": timeout_seconds, + "taskRunning": True, + "taskEnabled": True, + "generationReady": False, + "listenerCount": 0, + "listenerStopObserved": True, + "taskGenerationChanged": True, + } + ] + ) + assert not verifier( + [ + { + "elapsedSeconds": timeout_seconds + poll_seconds, + "taskRunning": True, + "taskEnabled": True, + "generationReady": True, + "listenerCount": 1, + "listenerStopObserved": True, + "taskGenerationChanged": True, + } + ] + ) + + def test_receiver_does_not_treat_http_sys_pid4_as_the_userspace_generation() -> None: source = RECEIVER.read_text(encoding="utf-8") restart = source[source.index("function Invoke-AgentRelayTaskRestart") :]