fix(agent99): verify relay task generation

This commit is contained in:
ogt
2026-07-14 14:15:50 +08:00
parent 73d0d94c7f
commit 018b796ec3
2 changed files with 64 additions and 16 deletions

View File

@@ -301,6 +301,24 @@ function Get-AgentRelayRuntime {
$taskError = $_.Exception.GetType().Name
}
$taskInfoError = ""
$taskLastRunTime = ""
$taskLastTaskResult = $null
$taskGeneration = ""
if ($task) {
try {
$taskInfo = Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
$lastRunTime = [datetime]$taskInfo.LastRunTime
if ($lastRunTime.Year -gt 1900) {
$taskLastRunTime = $lastRunTime.ToUniversalTime().ToString("o")
$taskGeneration = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$RelayTaskName|$taskLastRunTime"))
}
$taskLastTaskResult = [long]$taskInfo.LastTaskResult
} catch {
$taskInfoError = $_.Exception.GetType().Name
}
}
$listeners = @()
$listenerError = ""
try {
@@ -343,11 +361,16 @@ function Get-AgentRelayRuntime {
taskRunning = [bool]($task -and [string]$task.State -eq "Running")
taskEnabled = $taskEnabled
taskError = $taskError
taskInfoError = $taskInfoError
taskLastRunTime = $taskLastRunTime
taskLastTaskResult = $taskLastTaskResult
taskGeneration = $taskGeneration
taskGenerationReady = [bool]$taskGeneration
listenerPort = $RelayPort
listenerCount = $listeners.Count
listenerProcessCount = $processIds.Count
listenerGenerations = $listenerGenerations
generationReady = [bool]($listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count)
generationReady = [bool]($taskGeneration -and $listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count)
listeners = $listeners
listenerError = $listenerError
}
@@ -366,7 +389,7 @@ function Invoke-AgentRelayTaskRestart {
param([string[]]$ForbiddenGenerations = @())
$before = Get-AgentRelayRuntime
$generationCandidates = @($ForbiddenGenerations) + @($before.listenerGenerations)
$generationCandidates = @($ForbiddenGenerations) + @($before.taskGeneration)
$forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique)
if (-not $before.taskExists -or -not $before.taskEnabled) {
return [pscustomobject]@{
@@ -379,20 +402,24 @@ function Invoke-AgentRelayTaskRestart {
forbiddenGenerations = $forbidden
oldGenerationsGone = $false
newGenerationVerified = $false
listenerStopObserved = $false
taskGenerationChanged = $false
}
}
$stage = "stop"
$afterStop = $before
$after = $before
$listenerStopObserved = $false
$taskGenerationChanged = $false
try {
Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
$stopDeadline = (Get-Date).AddSeconds(60)
do {
Start-Sleep -Seconds 1
$afterStop = Get-AgentRelayRuntime
$oldGone = -not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden)
$stopVerified = [bool]($oldGone -and -not $afterStop.taskRunning)
$listenerStopObserved = [bool]($afterStop.listenerCount -eq 0)
$stopVerified = [bool]($listenerStopObserved -and -not $afterStop.taskRunning)
if ($stopVerified) { break }
} while ((Get-Date) -lt $stopDeadline)
if (-not $stopVerified) { throw "relay_old_listener_generation_stop_timeout" }
@@ -403,15 +430,15 @@ function Invoke-AgentRelayTaskRestart {
do {
Start-Sleep -Seconds 1
$after = Get-AgentRelayRuntime
$oldGone = -not (Test-AgentGenerationIntersection @($after.listenerGenerations) $forbidden)
$newGenerations = @($after.listenerGenerations | Where-Object { $_ -and $_ -notin $forbidden })
$newTaskGeneration = [string]$after.taskGeneration
$taskGenerationChanged = [bool]($newTaskGeneration -and $newTaskGeneration -notin $forbidden)
$newVerified = [bool](
$after.taskRunning -and
$after.taskEnabled -and
$after.generationReady -and
$after.listenerCount -gt 0 -and
$oldGone -and
$newGenerations.Count -gt 0
$listenerStopObserved -and
$taskGenerationChanged
)
if ($newVerified) { break }
} while ((Get-Date) -lt $startDeadline)
@@ -426,8 +453,10 @@ function Invoke-AgentRelayTaskRestart {
afterStop = $afterStop
after = $after
forbiddenGenerations = $forbidden
oldGenerationsGone = $oldGone
oldGenerationsGone = $listenerStopObserved
newGenerationVerified = $newVerified
listenerStopObserved = $listenerStopObserved
taskGenerationChanged = $taskGenerationChanged
}
} catch {
return [pscustomobject]@{
@@ -440,8 +469,10 @@ function Invoke-AgentRelayTaskRestart {
afterStop = $afterStop
after = (Get-AgentRelayRuntime)
forbiddenGenerations = $forbidden
oldGenerationsGone = [bool](-not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden))
oldGenerationsGone = $listenerStopObserved
newGenerationVerified = $false
listenerStopObserved = $listenerStopObserved
taskGenerationChanged = $taskGenerationChanged
}
}
}
@@ -946,7 +977,7 @@ try {
$script:ReceiverLivePromotionPerformed = $true
$script:ReceiverOperationPhase = "relay_reload"
$script:ReceiverScheduledTaskRestartPerformed = $true
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations)
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.taskGeneration)
if (-not $relayReload.ok) {
$failurePhase = "relay_reload"
$failureType = "relay_reload_failed"
@@ -1041,13 +1072,13 @@ try {
$script:ReceiverOperationPhase = "runtime_rollback"
$script:ReceiverRollbackAttempted = $true
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
$rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations)
$rollbackForbiddenGenerations = @($beforeRelayRuntime.taskGeneration)
if ($relayReload) {
$rollbackForbiddenGenerations += @($relayReload.before.listenerGenerations)
$rollbackForbiddenGenerations += @($relayReload.after.listenerGenerations)
$rollbackForbiddenGenerations += @($relayReload.before.taskGeneration)
$rollbackForbiddenGenerations += @($relayReload.after.taskGeneration)
}
$relayBeforeRollback = Get-AgentRelayRuntime
$rollbackForbiddenGenerations += @($relayBeforeRollback.listenerGenerations)
$rollbackForbiddenGenerations += @($relayBeforeRollback.taskGeneration)
$rollbackForbiddenGenerations = @($rollbackForbiddenGenerations | Where-Object { $_ } | Sort-Object -Unique)
$rollbackRelayRestart = if ($rollback.restored) {
Invoke-AgentRelayTaskRestart -ForbiddenGenerations $rollbackForbiddenGenerations

View File

@@ -440,13 +440,18 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert '$RelayTaskPath = "\\"' in source
assert "$RelayPort = 8787" in source
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Get-NetTCPConnection -LocalPort $RelayPort -State Listen" in source
assert 'GetBytes("$processId|$startedAt")' in source
assert 'GetBytes("$RelayTaskName|$taskLastRunTime")' in source
assert "$afterStop.taskRunning" in source
assert "$beforeRelayRuntime.taskEnabled" in source
assert "$after.taskEnabled" in source
assert "$listenerStopObserved" in source
assert "$taskGenerationChanged" in source
assert "$newTaskGeneration -notin $forbidden" in source
assert "$relayReload.oldGenerationsGone" in source
assert "$relayReload.newGenerationVerified" in source
assert "$relayReload.after.taskRunning" in source
@@ -459,7 +464,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
)
reload_task = source.index(
"$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
"@($beforeRelayRuntime.listenerGenerations)"
"@($beforeRelayRuntime.taskGeneration)"
)
postverify = source.index(
'$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath '
@@ -469,6 +474,18 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert deploy < reload_task < postverify
def test_receiver_does_not_treat_http_sys_pid4_as_the_userspace_generation() -> None:
source = RECEIVER.read_text(encoding="utf-8")
restart = source[source.index("function Invoke-AgentRelayTaskRestart") :]
restart = restart[: restart.index("function Copy-AgentFileWithRetry")]
assert "$afterStop.listenerCount -eq 0" in restart
assert "$newTaskGeneration = [string]$after.taskGeneration" in restart
assert "$taskGenerationChanged" in restart
assert "$after.listenerGenerations | Where-Object" not in restart
assert "$after.listenerGenerations" not in restart
def test_transport_only_restarts_existing_task_without_mutating_its_definition() -> None:
source = "\n".join(
(