diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index f73a18702..9902af325 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -301,6 +301,24 @@ function Get-AgentRelayRuntime { $taskError = $_.Exception.GetType().Name } + $taskInfoError = "" + $taskLastRunTime = "" + $taskLastTaskResult = $null + $taskGeneration = "" + if ($task) { + try { + $taskInfo = Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop + $lastRunTime = [datetime]$taskInfo.LastRunTime + if ($lastRunTime.Year -gt 1900) { + $taskLastRunTime = $lastRunTime.ToUniversalTime().ToString("o") + $taskGeneration = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$RelayTaskName|$taskLastRunTime")) + } + $taskLastTaskResult = [long]$taskInfo.LastTaskResult + } catch { + $taskInfoError = $_.Exception.GetType().Name + } + } + $listeners = @() $listenerError = "" try { @@ -343,11 +361,16 @@ function Get-AgentRelayRuntime { taskRunning = [bool]($task -and [string]$task.State -eq "Running") taskEnabled = $taskEnabled taskError = $taskError + taskInfoError = $taskInfoError + taskLastRunTime = $taskLastRunTime + taskLastTaskResult = $taskLastTaskResult + taskGeneration = $taskGeneration + taskGenerationReady = [bool]$taskGeneration listenerPort = $RelayPort listenerCount = $listeners.Count listenerProcessCount = $processIds.Count listenerGenerations = $listenerGenerations - generationReady = [bool]($listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count) + generationReady = [bool]($taskGeneration -and $listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count) listeners = $listeners listenerError = $listenerError } @@ -366,7 +389,7 @@ function Invoke-AgentRelayTaskRestart { param([string[]]$ForbiddenGenerations = @()) $before = Get-AgentRelayRuntime - $generationCandidates = @($ForbiddenGenerations) + @($before.listenerGenerations) + $generationCandidates = @($ForbiddenGenerations) + @($before.taskGeneration) $forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique) if (-not $before.taskExists -or -not $before.taskEnabled) { return [pscustomobject]@{ @@ -379,20 +402,24 @@ function Invoke-AgentRelayTaskRestart { forbiddenGenerations = $forbidden oldGenerationsGone = $false newGenerationVerified = $false + listenerStopObserved = $false + taskGenerationChanged = $false } } $stage = "stop" $afterStop = $before $after = $before + $listenerStopObserved = $false + $taskGenerationChanged = $false try { Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop $stopDeadline = (Get-Date).AddSeconds(60) do { Start-Sleep -Seconds 1 $afterStop = Get-AgentRelayRuntime - $oldGone = -not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden) - $stopVerified = [bool]($oldGone -and -not $afterStop.taskRunning) + $listenerStopObserved = [bool]($afterStop.listenerCount -eq 0) + $stopVerified = [bool]($listenerStopObserved -and -not $afterStop.taskRunning) if ($stopVerified) { break } } while ((Get-Date) -lt $stopDeadline) if (-not $stopVerified) { throw "relay_old_listener_generation_stop_timeout" } @@ -403,15 +430,15 @@ function Invoke-AgentRelayTaskRestart { do { Start-Sleep -Seconds 1 $after = Get-AgentRelayRuntime - $oldGone = -not (Test-AgentGenerationIntersection @($after.listenerGenerations) $forbidden) - $newGenerations = @($after.listenerGenerations | Where-Object { $_ -and $_ -notin $forbidden }) + $newTaskGeneration = [string]$after.taskGeneration + $taskGenerationChanged = [bool]($newTaskGeneration -and $newTaskGeneration -notin $forbidden) $newVerified = [bool]( $after.taskRunning -and $after.taskEnabled -and $after.generationReady -and $after.listenerCount -gt 0 -and - $oldGone -and - $newGenerations.Count -gt 0 + $listenerStopObserved -and + $taskGenerationChanged ) if ($newVerified) { break } } while ((Get-Date) -lt $startDeadline) @@ -426,8 +453,10 @@ function Invoke-AgentRelayTaskRestart { afterStop = $afterStop after = $after forbiddenGenerations = $forbidden - oldGenerationsGone = $oldGone + oldGenerationsGone = $listenerStopObserved newGenerationVerified = $newVerified + listenerStopObserved = $listenerStopObserved + taskGenerationChanged = $taskGenerationChanged } } catch { return [pscustomobject]@{ @@ -440,8 +469,10 @@ function Invoke-AgentRelayTaskRestart { afterStop = $afterStop after = (Get-AgentRelayRuntime) forbiddenGenerations = $forbidden - oldGenerationsGone = [bool](-not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden)) + oldGenerationsGone = $listenerStopObserved newGenerationVerified = $false + listenerStopObserved = $listenerStopObserved + taskGenerationChanged = $taskGenerationChanged } } } @@ -946,7 +977,7 @@ try { $script:ReceiverLivePromotionPerformed = $true $script:ReceiverOperationPhase = "relay_reload" $script:ReceiverScheduledTaskRestartPerformed = $true - $relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations) + $relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.taskGeneration) if (-not $relayReload.ok) { $failurePhase = "relay_reload" $failureType = "relay_reload_failed" @@ -1041,13 +1072,13 @@ try { $script:ReceiverOperationPhase = "runtime_rollback" $script:ReceiverRollbackAttempted = $true $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent - $rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations) + $rollbackForbiddenGenerations = @($beforeRelayRuntime.taskGeneration) if ($relayReload) { - $rollbackForbiddenGenerations += @($relayReload.before.listenerGenerations) - $rollbackForbiddenGenerations += @($relayReload.after.listenerGenerations) + $rollbackForbiddenGenerations += @($relayReload.before.taskGeneration) + $rollbackForbiddenGenerations += @($relayReload.after.taskGeneration) } $relayBeforeRollback = Get-AgentRelayRuntime - $rollbackForbiddenGenerations += @($relayBeforeRollback.listenerGenerations) + $rollbackForbiddenGenerations += @($relayBeforeRollback.taskGeneration) $rollbackForbiddenGenerations = @($rollbackForbiddenGenerations | Where-Object { $_ } | Sort-Object -Unique) $rollbackRelayRestart = if ($rollback.restored) { Invoke-AgentRelayTaskRestart -ForbiddenGenerations $rollbackForbiddenGenerations diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index a56407313..a25484ee7 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -440,13 +440,18 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert '$RelayTaskPath = "\\"' in source assert "$RelayPort = 8787" in source assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source + assert "Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source assert "Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source assert "Get-NetTCPConnection -LocalPort $RelayPort -State Listen" in source assert 'GetBytes("$processId|$startedAt")' in source + assert 'GetBytes("$RelayTaskName|$taskLastRunTime")' in source assert "$afterStop.taskRunning" in source assert "$beforeRelayRuntime.taskEnabled" in source assert "$after.taskEnabled" in source + assert "$listenerStopObserved" in source + assert "$taskGenerationChanged" in source + assert "$newTaskGeneration -notin $forbidden" in source assert "$relayReload.oldGenerationsGone" in source assert "$relayReload.newGenerationVerified" in source assert "$relayReload.after.taskRunning" in source @@ -459,7 +464,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati ) reload_task = source.index( "$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations " - "@($beforeRelayRuntime.listenerGenerations)" + "@($beforeRelayRuntime.taskGeneration)" ) postverify = source.index( '$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath ' @@ -469,6 +474,18 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati assert deploy < reload_task < postverify +def test_receiver_does_not_treat_http_sys_pid4_as_the_userspace_generation() -> None: + source = RECEIVER.read_text(encoding="utf-8") + restart = source[source.index("function Invoke-AgentRelayTaskRestart") :] + restart = restart[: restart.index("function Copy-AgentFileWithRetry")] + + assert "$afterStop.listenerCount -eq 0" in restart + assert "$newTaskGeneration = [string]$after.taskGeneration" in restart + assert "$taskGenerationChanged" in restart + assert "$after.listenerGenerations | Where-Object" not in restart + assert "$after.listenerGenerations" not in restart + + def test_transport_only_restarts_existing_task_without_mutating_its_definition() -> None: source = "\n".join( (