fix(agent99): verify relay task generation
This commit is contained in:
@@ -301,6 +301,24 @@ function Get-AgentRelayRuntime {
|
||||
$taskError = $_.Exception.GetType().Name
|
||||
}
|
||||
|
||||
$taskInfoError = ""
|
||||
$taskLastRunTime = ""
|
||||
$taskLastTaskResult = $null
|
||||
$taskGeneration = ""
|
||||
if ($task) {
|
||||
try {
|
||||
$taskInfo = Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
|
||||
$lastRunTime = [datetime]$taskInfo.LastRunTime
|
||||
if ($lastRunTime.Year -gt 1900) {
|
||||
$taskLastRunTime = $lastRunTime.ToUniversalTime().ToString("o")
|
||||
$taskGeneration = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$RelayTaskName|$taskLastRunTime"))
|
||||
}
|
||||
$taskLastTaskResult = [long]$taskInfo.LastTaskResult
|
||||
} catch {
|
||||
$taskInfoError = $_.Exception.GetType().Name
|
||||
}
|
||||
}
|
||||
|
||||
$listeners = @()
|
||||
$listenerError = ""
|
||||
try {
|
||||
@@ -343,11 +361,16 @@ function Get-AgentRelayRuntime {
|
||||
taskRunning = [bool]($task -and [string]$task.State -eq "Running")
|
||||
taskEnabled = $taskEnabled
|
||||
taskError = $taskError
|
||||
taskInfoError = $taskInfoError
|
||||
taskLastRunTime = $taskLastRunTime
|
||||
taskLastTaskResult = $taskLastTaskResult
|
||||
taskGeneration = $taskGeneration
|
||||
taskGenerationReady = [bool]$taskGeneration
|
||||
listenerPort = $RelayPort
|
||||
listenerCount = $listeners.Count
|
||||
listenerProcessCount = $processIds.Count
|
||||
listenerGenerations = $listenerGenerations
|
||||
generationReady = [bool]($listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count)
|
||||
generationReady = [bool]($taskGeneration -and $listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count)
|
||||
listeners = $listeners
|
||||
listenerError = $listenerError
|
||||
}
|
||||
@@ -366,7 +389,7 @@ function Invoke-AgentRelayTaskRestart {
|
||||
param([string[]]$ForbiddenGenerations = @())
|
||||
|
||||
$before = Get-AgentRelayRuntime
|
||||
$generationCandidates = @($ForbiddenGenerations) + @($before.listenerGenerations)
|
||||
$generationCandidates = @($ForbiddenGenerations) + @($before.taskGeneration)
|
||||
$forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique)
|
||||
if (-not $before.taskExists -or -not $before.taskEnabled) {
|
||||
return [pscustomobject]@{
|
||||
@@ -379,20 +402,24 @@ function Invoke-AgentRelayTaskRestart {
|
||||
forbiddenGenerations = $forbidden
|
||||
oldGenerationsGone = $false
|
||||
newGenerationVerified = $false
|
||||
listenerStopObserved = $false
|
||||
taskGenerationChanged = $false
|
||||
}
|
||||
}
|
||||
|
||||
$stage = "stop"
|
||||
$afterStop = $before
|
||||
$after = $before
|
||||
$listenerStopObserved = $false
|
||||
$taskGenerationChanged = $false
|
||||
try {
|
||||
Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
|
||||
$stopDeadline = (Get-Date).AddSeconds(60)
|
||||
do {
|
||||
Start-Sleep -Seconds 1
|
||||
$afterStop = Get-AgentRelayRuntime
|
||||
$oldGone = -not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden)
|
||||
$stopVerified = [bool]($oldGone -and -not $afterStop.taskRunning)
|
||||
$listenerStopObserved = [bool]($afterStop.listenerCount -eq 0)
|
||||
$stopVerified = [bool]($listenerStopObserved -and -not $afterStop.taskRunning)
|
||||
if ($stopVerified) { break }
|
||||
} while ((Get-Date) -lt $stopDeadline)
|
||||
if (-not $stopVerified) { throw "relay_old_listener_generation_stop_timeout" }
|
||||
@@ -403,15 +430,15 @@ function Invoke-AgentRelayTaskRestart {
|
||||
do {
|
||||
Start-Sleep -Seconds 1
|
||||
$after = Get-AgentRelayRuntime
|
||||
$oldGone = -not (Test-AgentGenerationIntersection @($after.listenerGenerations) $forbidden)
|
||||
$newGenerations = @($after.listenerGenerations | Where-Object { $_ -and $_ -notin $forbidden })
|
||||
$newTaskGeneration = [string]$after.taskGeneration
|
||||
$taskGenerationChanged = [bool]($newTaskGeneration -and $newTaskGeneration -notin $forbidden)
|
||||
$newVerified = [bool](
|
||||
$after.taskRunning -and
|
||||
$after.taskEnabled -and
|
||||
$after.generationReady -and
|
||||
$after.listenerCount -gt 0 -and
|
||||
$oldGone -and
|
||||
$newGenerations.Count -gt 0
|
||||
$listenerStopObserved -and
|
||||
$taskGenerationChanged
|
||||
)
|
||||
if ($newVerified) { break }
|
||||
} while ((Get-Date) -lt $startDeadline)
|
||||
@@ -426,8 +453,10 @@ function Invoke-AgentRelayTaskRestart {
|
||||
afterStop = $afterStop
|
||||
after = $after
|
||||
forbiddenGenerations = $forbidden
|
||||
oldGenerationsGone = $oldGone
|
||||
oldGenerationsGone = $listenerStopObserved
|
||||
newGenerationVerified = $newVerified
|
||||
listenerStopObserved = $listenerStopObserved
|
||||
taskGenerationChanged = $taskGenerationChanged
|
||||
}
|
||||
} catch {
|
||||
return [pscustomobject]@{
|
||||
@@ -440,8 +469,10 @@ function Invoke-AgentRelayTaskRestart {
|
||||
afterStop = $afterStop
|
||||
after = (Get-AgentRelayRuntime)
|
||||
forbiddenGenerations = $forbidden
|
||||
oldGenerationsGone = [bool](-not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden))
|
||||
oldGenerationsGone = $listenerStopObserved
|
||||
newGenerationVerified = $false
|
||||
listenerStopObserved = $listenerStopObserved
|
||||
taskGenerationChanged = $taskGenerationChanged
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -946,7 +977,7 @@ try {
|
||||
$script:ReceiverLivePromotionPerformed = $true
|
||||
$script:ReceiverOperationPhase = "relay_reload"
|
||||
$script:ReceiverScheduledTaskRestartPerformed = $true
|
||||
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations)
|
||||
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.taskGeneration)
|
||||
if (-not $relayReload.ok) {
|
||||
$failurePhase = "relay_reload"
|
||||
$failureType = "relay_reload_failed"
|
||||
@@ -1041,13 +1072,13 @@ try {
|
||||
$script:ReceiverOperationPhase = "runtime_rollback"
|
||||
$script:ReceiverRollbackAttempted = $true
|
||||
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
|
||||
$rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations)
|
||||
$rollbackForbiddenGenerations = @($beforeRelayRuntime.taskGeneration)
|
||||
if ($relayReload) {
|
||||
$rollbackForbiddenGenerations += @($relayReload.before.listenerGenerations)
|
||||
$rollbackForbiddenGenerations += @($relayReload.after.listenerGenerations)
|
||||
$rollbackForbiddenGenerations += @($relayReload.before.taskGeneration)
|
||||
$rollbackForbiddenGenerations += @($relayReload.after.taskGeneration)
|
||||
}
|
||||
$relayBeforeRollback = Get-AgentRelayRuntime
|
||||
$rollbackForbiddenGenerations += @($relayBeforeRollback.listenerGenerations)
|
||||
$rollbackForbiddenGenerations += @($relayBeforeRollback.taskGeneration)
|
||||
$rollbackForbiddenGenerations = @($rollbackForbiddenGenerations | Where-Object { $_ } | Sort-Object -Unique)
|
||||
$rollbackRelayRestart = if ($rollback.restored) {
|
||||
Invoke-AgentRelayTaskRestart -ForbiddenGenerations $rollbackForbiddenGenerations
|
||||
|
||||
@@ -440,13 +440,18 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
assert '$RelayTaskPath = "\\"' in source
|
||||
assert "$RelayPort = 8787" in source
|
||||
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Get-NetTCPConnection -LocalPort $RelayPort -State Listen" in source
|
||||
assert 'GetBytes("$processId|$startedAt")' in source
|
||||
assert 'GetBytes("$RelayTaskName|$taskLastRunTime")' in source
|
||||
assert "$afterStop.taskRunning" in source
|
||||
assert "$beforeRelayRuntime.taskEnabled" in source
|
||||
assert "$after.taskEnabled" in source
|
||||
assert "$listenerStopObserved" in source
|
||||
assert "$taskGenerationChanged" in source
|
||||
assert "$newTaskGeneration -notin $forbidden" in source
|
||||
assert "$relayReload.oldGenerationsGone" in source
|
||||
assert "$relayReload.newGenerationVerified" in source
|
||||
assert "$relayReload.after.taskRunning" in source
|
||||
@@ -459,7 +464,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
)
|
||||
reload_task = source.index(
|
||||
"$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
|
||||
"@($beforeRelayRuntime.listenerGenerations)"
|
||||
"@($beforeRelayRuntime.taskGeneration)"
|
||||
)
|
||||
postverify = source.index(
|
||||
'$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath '
|
||||
@@ -469,6 +474,18 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
|
||||
assert deploy < reload_task < postverify
|
||||
|
||||
|
||||
def test_receiver_does_not_treat_http_sys_pid4_as_the_userspace_generation() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
restart = source[source.index("function Invoke-AgentRelayTaskRestart") :]
|
||||
restart = restart[: restart.index("function Copy-AgentFileWithRetry")]
|
||||
|
||||
assert "$afterStop.listenerCount -eq 0" in restart
|
||||
assert "$newTaskGeneration = [string]$after.taskGeneration" in restart
|
||||
assert "$taskGenerationChanged" in restart
|
||||
assert "$after.listenerGenerations | Where-Object" not in restart
|
||||
assert "$after.listenerGenerations" not in restart
|
||||
|
||||
|
||||
def test_transport_only_restarts_existing_task_without_mutating_its_definition() -> None:
|
||||
source = "\n".join(
|
||||
(
|
||||
|
||||
Reference in New Issue
Block a user