fix(agent99): verify relay generation on atomic deploy

This commit is contained in:
ogt
2026-07-14 11:11:05 +08:00
parent 745b86af01
commit 2180993e4a
3 changed files with 516 additions and 88 deletions

View File

@@ -13,6 +13,9 @@ $EvidenceDir = Join-Path $AgentRoot "evidence"
$DeployRoot = Join-Path $AgentRoot "deploy"
$ManifestPath = Join-Path $StateDir "runtime-manifest.json"
$WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"
$RelayTaskPath = "\"
$RelayPort = 8787
$FixedRuntimeFiles = @(
"agent99-bootstrap.ps1",
"agent99-contract-check.ps1",
@@ -251,12 +254,183 @@ function Invoke-AgentLivePreflight {
healthyTaskCount = if ($summary) { [int]$summary.healthyTaskCount } else { 0 }
requiredTaskCount = if ($summary) { [int]$summary.requiredTaskCount } else { 0 }
relayListenerCount = if ($summary) { [int]$summary.relayListenerCount } else { 0 }
relayListenerGenerations = if ($payload) {
@($payload.relayListeners | ForEach-Object {
$processId = [int]$_.processId
$process = Get-Process -Id $processId -ErrorAction SilentlyContinue
if ($process) {
try {
$startedAt = $process.StartTime.ToUniversalTime().ToString("o")
Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$processId|$startedAt"))
} catch {}
}
} | Where-Object { $_ } | Sort-Object -Unique)
} else { @() }
pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 }
preflightGreen = [bool]($summary -and $summary.preflightGreen)
blockingReasons = if ($processResult.timedOut) { @("preflight_process_timeout") } elseif ($summary) { @($summary.blockingReasons | ForEach-Object { [string]$_ }) } else { @("preflight_output_unparseable") }
}
}
function Get-AgentRelayRuntime {
$task = $null
$taskError = ""
try {
$tasks = @(Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop)
if ($tasks.Count -ne 1) {
$taskError = "exact_relay_task_count_$($tasks.Count)"
} else {
$task = $tasks[0]
}
} catch {
$taskError = $_.Exception.GetType().Name
}
$listeners = @()
$listenerError = ""
try {
$connections = @(Get-NetTCPConnection -LocalPort $RelayPort -State Listen -ErrorAction Stop)
foreach ($connection in $connections) {
$processId = [int]$connection.OwningProcess
$process = Get-Process -Id $processId -ErrorAction SilentlyContinue
$startedAt = ""
$generation = ""
if ($process) {
try {
$startedAt = $process.StartTime.ToUniversalTime().ToString("o")
$generation = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$processId|$startedAt"))
} catch {}
}
$listeners += [pscustomobject]@{
address = [string]$connection.LocalAddress
port = [int]$connection.LocalPort
processId = $processId
processName = if ($process) { [string]$process.ProcessName } else { "" }
startedAt = $startedAt
generation = $generation
}
}
} catch {
$listenerError = $_.Exception.GetType().Name
}
$processIds = @($listeners | ForEach-Object { [int]$_.processId } | Sort-Object -Unique)
$listenerGenerations = @($listeners | ForEach-Object { [string]$_.generation } | Where-Object { $_ } | Sort-Object -Unique)
$taskEnabled = $false
if ($task) {
try { $taskEnabled = [bool]$task.Settings.Enabled } catch { $taskEnabled = $false }
}
return [pscustomobject]@{
taskName = $RelayTaskName
taskPath = $RelayTaskPath
taskExists = [bool]($null -ne $task)
taskState = if ($task) { [string]$task.State } else { "" }
taskRunning = [bool]($task -and [string]$task.State -eq "Running")
taskEnabled = $taskEnabled
taskError = $taskError
listenerPort = $RelayPort
listenerCount = $listeners.Count
listenerProcessCount = $processIds.Count
listenerGenerations = $listenerGenerations
generationReady = [bool]($listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count)
listeners = $listeners
listenerError = $listenerError
}
}
function Test-AgentGenerationIntersection {
param([string[]]$Left, [string[]]$Right)
foreach ($value in @($Left)) {
if ($value -and $value -in @($Right)) { return $true }
}
return $false
}
function Invoke-AgentRelayTaskRestart {
param([string[]]$ForbiddenGenerations = @())
$before = Get-AgentRelayRuntime
$generationCandidates = @($ForbiddenGenerations) + @($before.listenerGenerations)
$forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique)
if (-not $before.taskExists -or -not $before.taskEnabled) {
return [pscustomobject]@{
attempted = $false
ok = $false
stage = "precondition"
reason = if ($before.taskExists) { "exact_existing_relay_task_disabled" } else { "exact_existing_relay_task_missing" }
before = $before
after = $before
forbiddenGenerations = $forbidden
oldGenerationsGone = $false
newGenerationVerified = $false
}
}
$stage = "stop"
$afterStop = $before
$after = $before
try {
Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
$stopDeadline = (Get-Date).AddSeconds(60)
do {
Start-Sleep -Seconds 1
$afterStop = Get-AgentRelayRuntime
$oldGone = -not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden)
$stopVerified = [bool]($oldGone -and -not $afterStop.taskRunning)
if ($stopVerified) { break }
} while ((Get-Date) -lt $stopDeadline)
if (-not $stopVerified) { throw "relay_old_listener_generation_stop_timeout" }
$stage = "start"
Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
$startDeadline = (Get-Date).AddSeconds(120)
do {
Start-Sleep -Seconds 1
$after = Get-AgentRelayRuntime
$oldGone = -not (Test-AgentGenerationIntersection @($after.listenerGenerations) $forbidden)
$newGenerations = @($after.listenerGenerations | Where-Object { $_ -and $_ -notin $forbidden })
$newVerified = [bool](
$after.taskRunning -and
$after.taskEnabled -and
$after.generationReady -and
$after.listenerCount -gt 0 -and
$oldGone -and
$newGenerations.Count -gt 0
)
if ($newVerified) { break }
} while ((Get-Date) -lt $startDeadline)
if (-not $newVerified) { throw "relay_new_listener_generation_start_timeout" }
return [pscustomobject]@{
attempted = $true
ok = $true
stage = "verified"
reason = "exact_existing_relay_task_restarted"
before = $before
afterStop = $afterStop
after = $after
forbiddenGenerations = $forbidden
oldGenerationsGone = $oldGone
newGenerationVerified = $newVerified
}
} catch {
return [pscustomobject]@{
attempted = $true
ok = $false
stage = $stage
reason = "relay_task_restart_$($stage)_failed"
errorType = $_.Exception.GetType().Name
before = $before
afterStop = $afterStop
after = (Get-AgentRelayRuntime)
forbiddenGenerations = $forbidden
oldGenerationsGone = [bool](-not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden))
newGenerationVerified = $false
}
}
}
function Copy-AgentFileWithRetry {
param([string]$Source, [string]$Destination)
@@ -275,9 +449,13 @@ function Test-AgentBackupPath {
param([string]$BackupPath)
if (-not $BackupPath) { return $false }
$expectedPrefix = [IO.Path]::GetFullPath((Join-Path $DeployRoot "backup-"))
$candidate = [IO.Path]::GetFullPath($BackupPath)
return [bool]($candidate.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase))
try {
$expectedPrefix = [IO.Path]::GetFullPath((Join-Path $DeployRoot "backup-"))
$candidate = [IO.Path]::GetFullPath($BackupPath)
return [bool]($candidate.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase))
} catch {
return $false
}
}
function Restore-AgentBundle {
@@ -394,7 +572,7 @@ try {
$preflightBytes = [Convert]::FromBase64String([string]$preflight.contentBase64)
if ((Get-AgentSha256Bytes $preflightBytes) -ne ([string]$preflight.sha256).ToLowerInvariant()) { throw "live_preflight_sha256_mismatch" }
$stageIdentity = if ($mode -eq "apply") { "$runId|$sourceRevision" } else { "check|$sourceRevision" }
$stageIdentity = "$mode|$traceId|$runId|$workItemId|$sourceRevision"
$stageToken = (Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($stageIdentity))).Substring(0, 20)
$stagePath = Join-Path $DeployRoot "remote-source-$stageToken"
$receiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json"
@@ -421,6 +599,7 @@ try {
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskModification = $false
}
nextSafeAction = "rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check"
@@ -492,6 +671,8 @@ try {
[string]$prior.runId -eq $runId -and
[string]$prior.workItemId -eq $workItemId -and
[string]$prior.sourceRevision -eq $sourceRevision -and
$prior.PSObject.Properties["relayReload"] -and
[bool]$prior.relayReload.newGenerationVerified -and
$live.runtimeMatched -and
$live.sourceRevision -eq $sourceRevision -and
$live.fileCount -eq 14 -and
@@ -516,6 +697,7 @@ try {
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskModification = $false
}) -Force
Write-AgentResultAndExit $prior 0
@@ -568,33 +750,22 @@ try {
foreach ($name in $FixedRuntimeFiles) { $beforeFilePresence[$name] = Test-Path -LiteralPath (Join-Path $BinDir $name) -PathType Leaf }
$beforeConfigPresent = Test-Path -LiteralPath $ConfigPath -PathType Leaf
$beforeManifestPresent = Test-Path -LiteralPath $ManifestPath -PathType Leaf
$beforeRelayRuntime = Get-AgentRelayRuntime
$backupDirsBefore = @{}
foreach ($directory in @(Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue)) {
$backupDirsBefore[$directory.FullName] = $true
}
$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision
$backupPath = if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) { [string]$deploy.payload.backupDir } else { "" }
if (-not $backupPath) {
$newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue |
Where-Object { -not $backupDirsBefore.ContainsKey($_.FullName) } |
Sort-Object LastWriteTime -Descending |
Select-Object -First 1
if ($newBackup) { $backupPath = $newBackup.FullName }
}
if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) {
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
$afterManifest = Get-AgentSafeRuntimeManifest
$rollbackVerified = [bool](
(Test-AgentManifestEqual $beforeManifest $afterManifest) -and
(Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and
(Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and
(Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest
)
if (
-not $beforeRelayRuntime.taskExists -or
-not $beforeRelayRuntime.taskEnabled -or
-not $beforeRelayRuntime.taskRunning -or
-not $beforeRelayRuntime.generationReady -or
$beforeRelayRuntime.listenerCount -lt 1
) {
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" }
status = "blocked_relay_runtime_precondition_no_promotion"
mode = "apply"
traceId = $traceId
runId = $runId
@@ -602,76 +773,208 @@ try {
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
backupPath = $backupPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut }
rollback = $rollback
rollbackVerified = $rollbackVerified
runtimeManifest = $afterManifest
relayRuntime = $beforeRelayRuntime
remoteWritePerformed = $true
liveRuntimeWritePerformed = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskModification = $false
nextSafeAction = "repair_exact_existing_relay_task_runtime_then_retry_same_identity"
}
Write-AgentRemoteDeployReceipt $receiptPath $receipt
Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 })
Write-AgentResultAndExit $receipt 2
}
$runtimeManifest = Get-AgentSafeRuntimeManifest
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$postVerified = [bool](
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 14 -and
$runtimeManifest.mismatchCount -eq 0 -and
$livePreflight.ok -and
$livePreflight.sourceRevision -eq $sourceRevision -and
$livePreflight.runtimeMatched
)
if (-not $postVerified) {
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
$afterManifest = Get-AgentSafeRuntimeManifest
$rollbackVerified = [bool](
$rollback.restored -and
(Test-AgentManifestEqual $beforeManifest $afterManifest) -and
(Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and
(Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and
(Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest
)
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" }
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
backupPath = $backupPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut }
failedRuntimeManifest = $runtimeManifest
failedLivePreflight = $livePreflight
rollback = $rollback
rollbackVerified = $rollbackVerified
runtimeManifest = $afterManifest
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskModification = $false
$deploy = $null
$relayReload = $null
$runtimeManifest = $null
$livePreflight = $null
$postVerified = $false
$failurePhase = ""
$failureType = ""
$backupPath = ""
try {
$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision
if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) {
$backupPath = [string]$deploy.payload.backupDir
}
if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) {
$failurePhase = "deploy"
$failureType = "deploy_child_failed_or_invalid"
} else {
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations)
if (-not $relayReload.ok) {
$failurePhase = "relay_reload"
$failureType = "relay_reload_failed"
} else {
$runtimeManifest = Get-AgentSafeRuntimeManifest
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$postVerified = [bool](
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 14 -and
$runtimeManifest.mismatchCount -eq 0 -and
$relayReload.oldGenerationsGone -and
$relayReload.newGenerationVerified -and
$relayReload.after.taskEnabled -and
$relayReload.after.taskRunning -and
$livePreflight.ok -and
$livePreflight.sourceRevision -eq $sourceRevision -and
$livePreflight.runtimeMatched -and
$livePreflight.relayListenerCount -gt 0 -and
(Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations))
)
if (-not $postVerified) {
$failurePhase = "post_verifier"
$failureType = "independent_post_verifier_failed"
}
}
}
} catch {
$failurePhase = if ($deploy -and $deploy.ok) { "reload_or_post_verifier_exception" } else { "deploy_exception" }
$failureType = $_.Exception.GetType().Name
}
if ($failurePhase) {
if (-not $backupPath) {
$newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue |
Where-Object { -not $backupDirsBefore.ContainsKey($_.FullName) } |
Sort-Object LastWriteTime -Descending |
Select-Object -First 1
if ($newBackup) { $backupPath = $newBackup.FullName }
}
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
$rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations)
if ($relayReload) {
$rollbackForbiddenGenerations += @($relayReload.before.listenerGenerations)
$rollbackForbiddenGenerations += @($relayReload.after.listenerGenerations)
}
$relayBeforeRollback = Get-AgentRelayRuntime
$rollbackForbiddenGenerations += @($relayBeforeRollback.listenerGenerations)
$rollbackForbiddenGenerations = @($rollbackForbiddenGenerations | Where-Object { $_ } | Sort-Object -Unique)
$rollbackRelayRestart = if ($rollback.restored) {
Invoke-AgentRelayTaskRestart -ForbiddenGenerations $rollbackForbiddenGenerations
} else {
[pscustomobject]@{
attempted = $false
ok = $false
stage = "restore"
reason = "bundle_restore_not_verified_relay_restart_withheld"
before = $relayBeforeRollback
after = $relayBeforeRollback
forbiddenGenerations = $rollbackForbiddenGenerations
oldGenerationsGone = $false
newGenerationVerified = $false
}
}
$afterManifest = [pscustomobject]@{
exists = $false
sourceRevision = ""
runtimeMatched = $false
fileCount = 0
mismatchCount = -1
parseError = "rollback_verifier_not_completed"
}
$rollbackVerified = $false
$rollbackVerifierErrorType = ""
try {
$afterManifest = Get-AgentSafeRuntimeManifest
$rollbackVerified = [bool](
$rollback.restored -and
(Test-AgentManifestEqual $beforeManifest $afterManifest) -and
(Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and
(Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and
(Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest -and
$rollbackRelayRestart.ok -and
$rollbackRelayRestart.oldGenerationsGone -and
$rollbackRelayRestart.newGenerationVerified -and
$rollbackRelayRestart.after.taskEnabled -and
$rollbackRelayRestart.after.taskRunning -and
$rollbackRelayRestart.after.listenerCount -gt 0
)
} catch {
$rollbackVerified = $false
$rollbackVerifierErrorType = $_.Exception.GetType().Name
}
try {
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = if ($failurePhase -eq "deploy") {
if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" }
} else {
if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" }
}
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
backupPath = $backupPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } }
failurePhase = $failurePhase
failureType = $failureType
relayReload = $relayReload
failedRuntimeManifest = $runtimeManifest
failedLivePreflight = $livePreflight
rollback = $rollback
rollbackRelayRestart = $rollbackRelayRestart
rollbackVerified = $rollbackVerified
rollbackVerifierErrorType = $rollbackVerifierErrorType
runtimeManifest = $afterManifest
durableReceiptWritten = $true
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = [bool](($relayReload -and $relayReload.attempted) -or $rollbackRelayRestart.attempted)
scheduledTaskModification = $false
}
} catch {
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "rollback_receipt_construction_failed"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
failurePhase = $failurePhase
failureType = $failureType
rollbackVerified = $false
runtimeRollbackWasVerifiedBeforeReceiptFailure = $rollbackVerified
receiptConstructionErrorType = $_.Exception.GetType().Name
durableReceiptWritten = $true
secretValueRead = $false
environmentSecretRead = $false
scheduledTaskModification = $false
}
$rollbackVerified = $false
}
try {
Write-AgentRemoteDeployReceipt $receiptPath $receipt
} catch {
$receipt.status = "rollback_receipt_write_failed"
$receipt.durableReceiptWritten = $false
$receipt | Add-Member -NotePropertyName receiptWriteErrorType -NotePropertyValue $_.Exception.GetType().Name -Force
Write-AgentResultAndExit $receipt 3
}
Write-AgentRemoteDeployReceipt $receiptPath $receipt
Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 })
}
@@ -688,6 +991,7 @@ try {
backupPath = $backupPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status }
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
relayReload = $relayReload
runtimeManifest = $runtimeManifest
livePreflight = $livePreflight
idempotentReplay = $false
@@ -701,6 +1005,7 @@ try {
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $true
scheduledTaskModification = $false
}
nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback"
@@ -721,6 +1026,8 @@ try {
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskModification = $false
}) 70
} finally {
if ($script:DeployLock) {

View File

@@ -5,6 +5,9 @@ readonly SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
readonly DEFAULT_SOURCE_ROOT="$(CDPATH= cd -- "${SCRIPT_DIR}/../.." && pwd)"
readonly TARGET="Administrator@192.168.0.99"
readonly SOURCE_HOST_IPV4="192.168.0.110"
readonly GITEA_ORIGIN_HTTPS="https://gitea.wooo.work/wooo/awoooi.git"
readonly GITEA_ORIGIN_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git"
readonly GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git"
readonly RECEIVER_RELATIVE="scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
readonly PREFLIGHT_RELATIVE="scripts/reboot-recovery/agent99-live-preflight.ps1"
readonly WRAPPER_RELATIVE="scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
@@ -136,7 +139,7 @@ done
(( CONNECT_TIMEOUT_SECONDS >= 1 && CONNECT_TIMEOUT_SECONDS <= 30 )) || fail "invalid_connect_timeout"
if [[ "${MODE}" == "apply" ]]; then
REMOTE_EXECUTION_TIMEOUT_SECONDS="1500"
REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"
readonly SAFE_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$'
[[ "${TRACE_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_trace_id_missing_or_invalid"
[[ "${RUN_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_run_id_missing_or_invalid"
@@ -159,17 +162,28 @@ done
[[ -d "${SOURCE_ROOT}" ]] || fail "source_root_missing"
SOURCE_ROOT="$(CDPATH= cd -- "${SOURCE_ROOT}" && pwd)"
ORIGIN_URL="$(git -C "${SOURCE_ROOT}" remote get-url origin 2>/dev/null)" || fail "origin_remote_missing"
case "${ORIGIN_URL}" in
"${GITEA_ORIGIN_HTTPS}"|"${GITEA_ORIGIN_HTTP_INTERNAL}"|"${GITEA_ORIGIN_SSH_INTERNAL}") ;;
*) fail "origin_is_not_fixed_internal_gitea" ;;
esac
git -C "${SOURCE_ROOT}" fetch --quiet --no-tags origin \
'+refs/heads/main:refs/remotes/origin/main' || fail "fresh_origin_main_fetch_failed"
ORIGIN_MAIN_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'refs/remotes/origin/main^{commit}' 2>/dev/null)" \
|| fail "fresh_origin_main_revision_unavailable"
ORIGIN_MAIN_REVISION="$(printf '%s' "${ORIGIN_MAIN_REVISION}" | tr '[:upper:]' '[:lower:]')"
[[ -f "${KNOWN_HOSTS_FILE}" ]] || fail "known_hosts_file_missing"
if [[ -n "${IDENTITY_FILE}" ]]; then
[[ -f "${IDENTITY_FILE}" ]] || fail "identity_file_missing"
fi
if [[ -z "${SOURCE_REVISION}" ]]; then
SOURCE_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'HEAD^{commit}')" || fail "source_revision_unavailable"
SOURCE_REVISION="${ORIGIN_MAIN_REVISION}"
fi
SOURCE_REVISION="$(printf '%s' "${SOURCE_REVISION}" | tr '[:upper:]' '[:lower:]')"
[[ "${SOURCE_REVISION}" =~ ^[0-9a-f]{40}$ ]] || fail "source_revision_must_be_full_commit_sha"
git -C "${SOURCE_ROOT}" cat-file -e "${SOURCE_REVISION}^{commit}" 2>/dev/null || fail "source_revision_not_found"
[[ "${SOURCE_REVISION}" == "${ORIGIN_MAIN_REVISION}" ]] || fail "source_revision_is_not_fresh_origin_main"
SOURCE_BOUND_FILES=(
"${RUNTIME_FILES[@]}"

View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import os
import re
import shlex
import shutil
import subprocess
from pathlib import Path
@@ -56,7 +57,7 @@ def test_sender_is_valid_shell_with_check_as_the_default() -> None:
assert "apply_work_item_id_missing_or_invalid" in source
assert 'MODE_SELECTION="default"' in source
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="45"' in source
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="1500"' in source
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in source
assert "timeout --signal=TERM --kill-after=10" in source
@@ -106,6 +107,23 @@ def test_sender_is_fixed_to_110_to_windows99_public_key_transport() -> None:
def test_sender_binds_every_transported_source_to_a_full_git_revision() -> None:
source = SENDER.read_text(encoding="utf-8")
assert 'GITEA_ORIGIN_HTTPS="https://gitea.wooo.work/wooo/awoooi.git"' in source
assert (
'GITEA_ORIGIN_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git"'
in source
)
assert (
'GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git"'
in source
)
assert 'remote get-url origin' in source
assert "origin_is_not_fixed_internal_gitea" in source
assert "fetch --quiet --no-tags origin" in source
assert "+refs/heads/main:refs/remotes/origin/main" in source
assert "fresh_origin_main_fetch_failed" in source
assert "refs/remotes/origin/main^{commit}" in source
assert '[[ "${SOURCE_REVISION}" == "${ORIGIN_MAIN_REVISION}" ]]' in source
assert "source_revision_is_not_fresh_origin_main" in source
assert "source_revision_must_be_full_commit_sha" in source
assert "source_revision_not_found" in source
assert "source_bound_file_untracked" in source
@@ -166,9 +184,11 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
assert "function Restore-AgentBundle" in source
assert "Test-AgentBackupPath" in source
assert 'Join-Path $DeployRoot "backup-"' in source
assert source.count("Restore-AgentBundle $backupPath") == 2
assert 'status = if ($rollbackVerified) { "deploy_failed_rollback_verified"' in source
assert 'status = if ($rollbackVerified) { "rolled_back_post_verifier_failed"' in source
assert source.count("Restore-AgentBundle $backupPath") == 1
assert '"deploy_failed_rollback_verified"' in source
assert '"deploy_failed_rollback_unverified"' in source
assert '"rolled_back_post_verifier_failed"' in source
assert '"rollback_failed_after_post_verifier"' in source
assert "Get-AgentLiveBundleDigest" in source
assert "Test-AgentManifestEqual $beforeManifest $afterManifest" in source
assert "Get-AgentSha256File $ConfigPath" in source
@@ -185,6 +205,29 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
assert '$runtimeManifest.fileCount -eq 14' in source
assert '$runtimeManifest.mismatchCount -eq 0' in source
assert 'status = "deployed_verified"' in source
assert '$failurePhase = "deploy"' in source
assert '$failurePhase = "relay_reload"' in source
assert '$failurePhase = "post_verifier"' in source
assert (
"Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
"$rollbackForbiddenGenerations"
in source
)
assert "$rollbackRelayRestart.newGenerationVerified" in source
assert "$rollbackVerifierErrorType = $_.Exception.GetType().Name" in source
assert 'status = "rollback_receipt_construction_failed"' in source
assert '$receipt.status = "rollback_receipt_write_failed"' in source
assert "durableReceiptWritten = $true" in source
assert "$receipt.durableReceiptWritten = $false" in source
rollback = source.index(
"$rollback = Restore-AgentBundle $backupPath $beforeFilePresence"
)
rollback_restart = source.index(
"Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
"$rollbackForbiddenGenerations"
)
rollback_verify = source.index("$rollbackVerified = [bool](")
assert rollback < rollback_restart < rollback_verify
def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary() -> None:
@@ -206,9 +249,53 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
assert "rawEnvelopeStored = $false" in source
assert "$env:" not in source
assert "GetEnvironmentVariable" not in source
assert (
'$stageIdentity = "$mode|$traceId|$runId|$workItemId|$sourceRevision"'
in source
)
assert 'Join-Path $DeployRoot "remote-source-$stageToken"' in source
assert 'Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json"' in source
assert '$prior.PSObject.Properties["relayReload"]' in source
assert "[bool]$prior.relayReload.newGenerationVerified" in source
def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> None:
def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generation() -> None:
source = RECEIVER.read_text(encoding="utf-8")
assert '$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"' in source
assert '$RelayTaskPath = "\\"' in source
assert "$RelayPort = 8787" in source
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Get-NetTCPConnection -LocalPort $RelayPort -State Listen" in source
assert 'GetBytes("$processId|$startedAt")' in source
assert "$afterStop.taskRunning" in source
assert "$beforeRelayRuntime.taskEnabled" in source
assert "$after.taskEnabled" in source
assert "$relayReload.oldGenerationsGone" in source
assert "$relayReload.newGenerationVerified" in source
assert "$relayReload.after.taskRunning" in source
assert "$livePreflight.relayListenerGenerations" in source
assert "scheduledTaskRestart = $true" in source
deploy = source.index(
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
"-SourceRevision $sourceRevision"
)
reload_task = source.index(
"$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
"@($beforeRelayRuntime.listenerGenerations)"
)
postverify = source.index(
'$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath '
'"agent99-live-preflight.ps1")',
reload_task,
)
assert deploy < reload_task < postverify
def test_transport_only_restarts_existing_task_without_mutating_its_definition() -> None:
source = "\n".join(
(
SENDER.read_text(encoding="utf-8"),
@@ -226,6 +313,8 @@ def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> No
"shutdown.exe",
"Register-ScheduledTask",
"Unregister-ScheduledTask",
"Set-ScheduledTask",
"New-ScheduledTask",
"-RegisterTasks",
)
for action in forbidden_actions:
@@ -235,6 +324,7 @@ def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> No
assert "vmPowerChange = $false" in source
assert "hostReboot = $false" in source
assert "serviceRestart = $false" in source
assert "scheduledTaskRestart = $true" in source
assert "scheduledTaskModification = $false" in source
assert "host112" not in source.lower()
assert "192.168.0.112" not in source
@@ -286,7 +376,24 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport(
"exec \"$@\"\n",
encoding="utf-8",
)
for executable in (fake_hostname, fake_ssh, fake_timeout):
real_git = shutil.which("git")
assert real_git is not None
fake_git = fake_bin / "git"
fake_git.write_text(
"#!/usr/bin/env bash\n"
"case \" $* \" in\n"
" *\" remote get-url origin \"*) "
"printf '%s\\n' 'https://gitea.wooo.work/wooo/awoooi.git'; exit 0 ;;\n"
" *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
" *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
"printf '%040d\\n' 0; exit 0 ;;\n"
" *\" cat-file -e \"*) exit 0 ;;\n"
" *\" diff --quiet \"*) exit 0 ;;\n"
"esac\n"
f"exec {shlex.quote(real_git)} \"$@\"\n",
encoding="utf-8",
)
for executable in (fake_hostname, fake_ssh, fake_timeout, fake_git):
executable.chmod(0o700)
environment = os.environ.copy()