wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(agent99): bound late relay readiness

Browse Source
This commit is contained in:
ogt
2026-07-16 22:35:38 +08:00
parent 32923355be
commit d369b60aaf
2 changed files with 150 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1
View File

@@ -18,6 +18,9 @@ $WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"
$RelayTaskPath = "\"
$RelayPort = 8787
$RelayStopTimeoutSeconds = 60
$RelayStartTimeoutSeconds = 360
$RelayPollIntervalSeconds = 1
$FixedRuntimeFiles = @(
"agent99-bootstrap.ps1",
"agent99-contract-check.ps1",
@@ -481,6 +484,9 @@ function Test-AgentGenerationIntersection {
function Invoke-AgentRelayTaskRestart {
param([string[]]$ForbiddenGenerations = @())
$restartStartedAt = Get-Date
$stopAttemptCount = 0
$startAttemptCount = 0
$before = Get-AgentRelayRuntime
$generationCandidates = @($ForbiddenGenerations) + @($before.taskGeneration)
$forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique)
@@ -497,6 +503,15 @@ function Invoke-AgentRelayTaskRestart {
newGenerationVerified = $false
listenerStopObserved = $false
taskGenerationChanged = $false
stopTimeoutSeconds = $RelayStopTimeoutSeconds
startTimeoutSeconds = $RelayStartTimeoutSeconds
pollIntervalSeconds = $RelayPollIntervalSeconds
elapsedSeconds = [Math]::Round(((Get-Date) - $restartStartedAt).TotalSeconds, 3)
stopAttemptCount = $stopAttemptCount
startAttemptCount = $startAttemptCount
lastObservedTaskState = [string]$before.taskState
lastObservedTaskResult = $before.taskLastTaskResult
lastObservedListenerCount = [int]$before.listenerCount
}
}
@@ -507,9 +522,10 @@ function Invoke-AgentRelayTaskRestart {
$taskGenerationChanged = $false
try {
Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
$stopDeadline = (Get-Date).AddSeconds(60)
$stopDeadline = (Get-Date).AddSeconds($RelayStopTimeoutSeconds)
do {
Start-Sleep -Seconds 1
$stopAttemptCount += 1
Start-Sleep -Seconds $RelayPollIntervalSeconds
$afterStop = Get-AgentRelayRuntime
$listenerStopObserved = [bool]($afterStop.listenerCount -eq 0)
$stopVerified = [bool]($listenerStopObserved -and -not $afterStop.taskRunning)
@@ -519,9 +535,10 @@ function Invoke-AgentRelayTaskRestart {
$stage = "start"
Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
$startDeadline = (Get-Date).AddSeconds(120)
$startDeadline = (Get-Date).AddSeconds($RelayStartTimeoutSeconds)
do {
Start-Sleep -Seconds 1
$startAttemptCount += 1
Start-Sleep -Seconds $RelayPollIntervalSeconds
$after = Get-AgentRelayRuntime
$newTaskGeneration = [string]$after.taskGeneration
$taskGenerationChanged = [bool]($newTaskGeneration -and $newTaskGeneration -notin $forbidden)
@@ -550,8 +567,18 @@ function Invoke-AgentRelayTaskRestart {
newGenerationVerified = $newVerified
listenerStopObserved = $listenerStopObserved
taskGenerationChanged = $taskGenerationChanged
stopTimeoutSeconds = $RelayStopTimeoutSeconds
startTimeoutSeconds = $RelayStartTimeoutSeconds
pollIntervalSeconds = $RelayPollIntervalSeconds
elapsedSeconds = [Math]::Round(((Get-Date) - $restartStartedAt).TotalSeconds, 3)
stopAttemptCount = $stopAttemptCount
startAttemptCount = $startAttemptCount
lastObservedTaskState = [string]$after.taskState
lastObservedTaskResult = $after.taskLastTaskResult
lastObservedListenerCount = [int]$after.listenerCount
}
} catch {
$after = Get-AgentRelayRuntime
return [pscustomobject]@{
attempted = $true
ok = $false
@@ -560,12 +587,21 @@ function Invoke-AgentRelayTaskRestart {
errorType = $_.Exception.GetType().Name
before = $before
afterStop = $afterStop
after = (Get-AgentRelayRuntime)
after = $after
forbiddenGenerations = $forbidden
oldGenerationsGone = $listenerStopObserved
newGenerationVerified = $false
listenerStopObserved = $listenerStopObserved
taskGenerationChanged = $taskGenerationChanged
stopTimeoutSeconds = $RelayStopTimeoutSeconds
startTimeoutSeconds = $RelayStartTimeoutSeconds
pollIntervalSeconds = $RelayPollIntervalSeconds
elapsedSeconds = [Math]::Round(((Get-Date) - $restartStartedAt).TotalSeconds, 3)
stopAttemptCount = $stopAttemptCount
startAttemptCount = $startAttemptCount
lastObservedTaskState = [string]$after.taskState
lastObservedTaskResult = $after.taskLastTaskResult
lastObservedListenerCount = [int]$after.listenerCount
}
}
}

109
scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py
View File

@@ -507,6 +507,9 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert '$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"' in source
assert '$RelayTaskPath = "\\"' in source
assert "$RelayPort = 8787" in source
assert "$RelayStopTimeoutSeconds = 60" in source
assert "$RelayStartTimeoutSeconds = 360" in source
assert "$RelayPollIntervalSeconds = 1" in source
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
@@ -519,6 +522,18 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert "$after.taskEnabled" in source
assert "$listenerStopObserved" in source
assert "$taskGenerationChanged" in source
assert "AddSeconds($RelayStopTimeoutSeconds)" in source
assert "AddSeconds($RelayStartTimeoutSeconds)" in source
assert "Start-Sleep -Seconds $RelayPollIntervalSeconds" in source
assert "stopTimeoutSeconds = $RelayStopTimeoutSeconds" in source
assert "startTimeoutSeconds = $RelayStartTimeoutSeconds" in source
assert "pollIntervalSeconds = $RelayPollIntervalSeconds" in source
assert "elapsedSeconds = [Math]::Round" in source
assert "stopAttemptCount = $stopAttemptCount" in source
assert "startAttemptCount = $startAttemptCount" in source
assert "lastObservedTaskState = [string]$after.taskState" in source
assert "lastObservedTaskResult = $after.taskLastTaskResult" in source
assert "lastObservedListenerCount = [int]$after.listenerCount" in source
assert "$newTaskGeneration -notin $forbidden" in source
assert "$relayReload.oldGenerationsGone" in source
assert "$relayReload.newGenerationVerified" in source
@@ -526,6 +541,23 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert "$livePreflight.relayListenerGenerations" in source
assert "scheduledTaskRestart = $true" in source
wrapper = SENDER.read_text(encoding="utf-8")
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in wrapper
wrapper_budget_seconds = 2400
validate_seconds = 300
deploy_seconds = 900
live_preflight_seconds = 120
worst_case_seconds = (
validate_seconds
+ deploy_seconds
+ live_preflight_seconds
+ (2 * (60 + 360))
)
assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source
assert "-TimeoutSeconds 120" in source
assert worst_case_seconds == 2160
assert wrapper_budget_seconds - worst_case_seconds == 240
deploy = source.index(
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
"-SourceRevision $sourceRevision"
@@ -542,6 +574,83 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert deploy < reload_task < postverify
def test_relay_start_window_accepts_a_late_verified_listener_but_remains_fail_closed() -> None:
source = RECEIVER.read_text(encoding="utf-8")
timeout_match = re.search(r"\$RelayStartTimeoutSeconds = (\d+)", source)
poll_match = re.search(r"\$RelayPollIntervalSeconds = (\d+)", source)
assert timeout_match is not None
assert poll_match is not None
timeout_seconds = int(timeout_match.group(1))
poll_seconds = int(poll_match.group(1))
def verifier(observations: list[dict[str, object]]) -> bool:
for observation in observations:
if int(observation["elapsedSeconds"]) > timeout_seconds:
break
ready = bool(
observation["taskRunning"]
and observation["taskEnabled"]
and observation["generationReady"]
and int(observation["listenerCount"]) > 0
and observation["listenerStopObserved"]
and observation["taskGenerationChanged"]
)
if ready:
return True
return False
assert poll_seconds == 1
assert timeout_seconds == 360
assert verifier(
[
{
"elapsedSeconds": 120,
"taskRunning": True,
"taskEnabled": True,
"generationReady": False,
"listenerCount": 0,
"listenerStopObserved": True,
"taskGenerationChanged": True,
},
{
"elapsedSeconds": 300,
"taskRunning": True,
"taskEnabled": True,
"generationReady": True,
"listenerCount": 1,
"listenerStopObserved": True,
"taskGenerationChanged": True,
},
]
)
assert not verifier(
[
{
"elapsedSeconds": timeout_seconds,
"taskRunning": True,
"taskEnabled": True,
"generationReady": False,
"listenerCount": 0,
"listenerStopObserved": True,
"taskGenerationChanged": True,
}
]
)
assert not verifier(
[
{
"elapsedSeconds": timeout_seconds + poll_seconds,
"taskRunning": True,
"taskEnabled": True,
"generationReady": True,
"listenerCount": 1,
"listenerStopObserved": True,
"taskGenerationChanged": True,
}
]
)
def test_receiver_does_not_treat_http_sys_pid4_as_the_userspace_generation() -> None:
source = RECEIVER.read_text(encoding="utf-8")
restart = source[source.index("function Invoke-AgentRelayTaskRestart") :]