fix(agent99): verify relay generation on atomic deploy
This commit is contained in:
@@ -13,6 +13,9 @@ $EvidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$DeployRoot = Join-Path $AgentRoot "deploy"
|
||||
$ManifestPath = Join-Path $StateDir "runtime-manifest.json"
|
||||
$WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"
|
||||
$RelayTaskPath = "\"
|
||||
$RelayPort = 8787
|
||||
$FixedRuntimeFiles = @(
|
||||
"agent99-bootstrap.ps1",
|
||||
"agent99-contract-check.ps1",
|
||||
@@ -251,12 +254,183 @@ function Invoke-AgentLivePreflight {
|
||||
healthyTaskCount = if ($summary) { [int]$summary.healthyTaskCount } else { 0 }
|
||||
requiredTaskCount = if ($summary) { [int]$summary.requiredTaskCount } else { 0 }
|
||||
relayListenerCount = if ($summary) { [int]$summary.relayListenerCount } else { 0 }
|
||||
relayListenerGenerations = if ($payload) {
|
||||
@($payload.relayListeners | ForEach-Object {
|
||||
$processId = [int]$_.processId
|
||||
$process = Get-Process -Id $processId -ErrorAction SilentlyContinue
|
||||
if ($process) {
|
||||
try {
|
||||
$startedAt = $process.StartTime.ToUniversalTime().ToString("o")
|
||||
Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$processId|$startedAt"))
|
||||
} catch {}
|
||||
}
|
||||
} | Where-Object { $_ } | Sort-Object -Unique)
|
||||
} else { @() }
|
||||
pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 }
|
||||
preflightGreen = [bool]($summary -and $summary.preflightGreen)
|
||||
blockingReasons = if ($processResult.timedOut) { @("preflight_process_timeout") } elseif ($summary) { @($summary.blockingReasons | ForEach-Object { [string]$_ }) } else { @("preflight_output_unparseable") }
|
||||
}
|
||||
}
|
||||
|
||||
function Get-AgentRelayRuntime {
|
||||
$task = $null
|
||||
$taskError = ""
|
||||
try {
|
||||
$tasks = @(Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop)
|
||||
if ($tasks.Count -ne 1) {
|
||||
$taskError = "exact_relay_task_count_$($tasks.Count)"
|
||||
} else {
|
||||
$task = $tasks[0]
|
||||
}
|
||||
} catch {
|
||||
$taskError = $_.Exception.GetType().Name
|
||||
}
|
||||
|
||||
$listeners = @()
|
||||
$listenerError = ""
|
||||
try {
|
||||
$connections = @(Get-NetTCPConnection -LocalPort $RelayPort -State Listen -ErrorAction Stop)
|
||||
foreach ($connection in $connections) {
|
||||
$processId = [int]$connection.OwningProcess
|
||||
$process = Get-Process -Id $processId -ErrorAction SilentlyContinue
|
||||
$startedAt = ""
|
||||
$generation = ""
|
||||
if ($process) {
|
||||
try {
|
||||
$startedAt = $process.StartTime.ToUniversalTime().ToString("o")
|
||||
$generation = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$processId|$startedAt"))
|
||||
} catch {}
|
||||
}
|
||||
$listeners += [pscustomobject]@{
|
||||
address = [string]$connection.LocalAddress
|
||||
port = [int]$connection.LocalPort
|
||||
processId = $processId
|
||||
processName = if ($process) { [string]$process.ProcessName } else { "" }
|
||||
startedAt = $startedAt
|
||||
generation = $generation
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
$listenerError = $_.Exception.GetType().Name
|
||||
}
|
||||
|
||||
$processIds = @($listeners | ForEach-Object { [int]$_.processId } | Sort-Object -Unique)
|
||||
$listenerGenerations = @($listeners | ForEach-Object { [string]$_.generation } | Where-Object { $_ } | Sort-Object -Unique)
|
||||
$taskEnabled = $false
|
||||
if ($task) {
|
||||
try { $taskEnabled = [bool]$task.Settings.Enabled } catch { $taskEnabled = $false }
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
taskName = $RelayTaskName
|
||||
taskPath = $RelayTaskPath
|
||||
taskExists = [bool]($null -ne $task)
|
||||
taskState = if ($task) { [string]$task.State } else { "" }
|
||||
taskRunning = [bool]($task -and [string]$task.State -eq "Running")
|
||||
taskEnabled = $taskEnabled
|
||||
taskError = $taskError
|
||||
listenerPort = $RelayPort
|
||||
listenerCount = $listeners.Count
|
||||
listenerProcessCount = $processIds.Count
|
||||
listenerGenerations = $listenerGenerations
|
||||
generationReady = [bool]($listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count)
|
||||
listeners = $listeners
|
||||
listenerError = $listenerError
|
||||
}
|
||||
}
|
||||
|
||||
function Test-AgentGenerationIntersection {
|
||||
param([string[]]$Left, [string[]]$Right)
|
||||
|
||||
foreach ($value in @($Left)) {
|
||||
if ($value -and $value -in @($Right)) { return $true }
|
||||
}
|
||||
return $false
|
||||
}
|
||||
|
||||
function Invoke-AgentRelayTaskRestart {
|
||||
param([string[]]$ForbiddenGenerations = @())
|
||||
|
||||
$before = Get-AgentRelayRuntime
|
||||
$generationCandidates = @($ForbiddenGenerations) + @($before.listenerGenerations)
|
||||
$forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique)
|
||||
if (-not $before.taskExists -or -not $before.taskEnabled) {
|
||||
return [pscustomobject]@{
|
||||
attempted = $false
|
||||
ok = $false
|
||||
stage = "precondition"
|
||||
reason = if ($before.taskExists) { "exact_existing_relay_task_disabled" } else { "exact_existing_relay_task_missing" }
|
||||
before = $before
|
||||
after = $before
|
||||
forbiddenGenerations = $forbidden
|
||||
oldGenerationsGone = $false
|
||||
newGenerationVerified = $false
|
||||
}
|
||||
}
|
||||
|
||||
$stage = "stop"
|
||||
$afterStop = $before
|
||||
$after = $before
|
||||
try {
|
||||
Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
|
||||
$stopDeadline = (Get-Date).AddSeconds(60)
|
||||
do {
|
||||
Start-Sleep -Seconds 1
|
||||
$afterStop = Get-AgentRelayRuntime
|
||||
$oldGone = -not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden)
|
||||
$stopVerified = [bool]($oldGone -and -not $afterStop.taskRunning)
|
||||
if ($stopVerified) { break }
|
||||
} while ((Get-Date) -lt $stopDeadline)
|
||||
if (-not $stopVerified) { throw "relay_old_listener_generation_stop_timeout" }
|
||||
|
||||
$stage = "start"
|
||||
Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
|
||||
$startDeadline = (Get-Date).AddSeconds(120)
|
||||
do {
|
||||
Start-Sleep -Seconds 1
|
||||
$after = Get-AgentRelayRuntime
|
||||
$oldGone = -not (Test-AgentGenerationIntersection @($after.listenerGenerations) $forbidden)
|
||||
$newGenerations = @($after.listenerGenerations | Where-Object { $_ -and $_ -notin $forbidden })
|
||||
$newVerified = [bool](
|
||||
$after.taskRunning -and
|
||||
$after.taskEnabled -and
|
||||
$after.generationReady -and
|
||||
$after.listenerCount -gt 0 -and
|
||||
$oldGone -and
|
||||
$newGenerations.Count -gt 0
|
||||
)
|
||||
if ($newVerified) { break }
|
||||
} while ((Get-Date) -lt $startDeadline)
|
||||
if (-not $newVerified) { throw "relay_new_listener_generation_start_timeout" }
|
||||
|
||||
return [pscustomobject]@{
|
||||
attempted = $true
|
||||
ok = $true
|
||||
stage = "verified"
|
||||
reason = "exact_existing_relay_task_restarted"
|
||||
before = $before
|
||||
afterStop = $afterStop
|
||||
after = $after
|
||||
forbiddenGenerations = $forbidden
|
||||
oldGenerationsGone = $oldGone
|
||||
newGenerationVerified = $newVerified
|
||||
}
|
||||
} catch {
|
||||
return [pscustomobject]@{
|
||||
attempted = $true
|
||||
ok = $false
|
||||
stage = $stage
|
||||
reason = "relay_task_restart_$($stage)_failed"
|
||||
errorType = $_.Exception.GetType().Name
|
||||
before = $before
|
||||
afterStop = $afterStop
|
||||
after = (Get-AgentRelayRuntime)
|
||||
forbiddenGenerations = $forbidden
|
||||
oldGenerationsGone = [bool](-not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden))
|
||||
newGenerationVerified = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Copy-AgentFileWithRetry {
|
||||
param([string]$Source, [string]$Destination)
|
||||
|
||||
@@ -275,9 +449,13 @@ function Test-AgentBackupPath {
|
||||
param([string]$BackupPath)
|
||||
|
||||
if (-not $BackupPath) { return $false }
|
||||
$expectedPrefix = [IO.Path]::GetFullPath((Join-Path $DeployRoot "backup-"))
|
||||
$candidate = [IO.Path]::GetFullPath($BackupPath)
|
||||
return [bool]($candidate.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase))
|
||||
try {
|
||||
$expectedPrefix = [IO.Path]::GetFullPath((Join-Path $DeployRoot "backup-"))
|
||||
$candidate = [IO.Path]::GetFullPath($BackupPath)
|
||||
return [bool]($candidate.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase))
|
||||
} catch {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
function Restore-AgentBundle {
|
||||
@@ -394,7 +572,7 @@ try {
|
||||
$preflightBytes = [Convert]::FromBase64String([string]$preflight.contentBase64)
|
||||
if ((Get-AgentSha256Bytes $preflightBytes) -ne ([string]$preflight.sha256).ToLowerInvariant()) { throw "live_preflight_sha256_mismatch" }
|
||||
|
||||
$stageIdentity = if ($mode -eq "apply") { "$runId|$sourceRevision" } else { "check|$sourceRevision" }
|
||||
$stageIdentity = "$mode|$traceId|$runId|$workItemId|$sourceRevision"
|
||||
$stageToken = (Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($stageIdentity))).Substring(0, 20)
|
||||
$stagePath = Join-Path $DeployRoot "remote-source-$stageToken"
|
||||
$receiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json"
|
||||
@@ -421,6 +599,7 @@ try {
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $false
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
nextSafeAction = "rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check"
|
||||
@@ -492,6 +671,8 @@ try {
|
||||
[string]$prior.runId -eq $runId -and
|
||||
[string]$prior.workItemId -eq $workItemId -and
|
||||
[string]$prior.sourceRevision -eq $sourceRevision -and
|
||||
$prior.PSObject.Properties["relayReload"] -and
|
||||
[bool]$prior.relayReload.newGenerationVerified -and
|
||||
$live.runtimeMatched -and
|
||||
$live.sourceRevision -eq $sourceRevision -and
|
||||
$live.fileCount -eq 14 -and
|
||||
@@ -516,6 +697,7 @@ try {
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $false
|
||||
scheduledTaskModification = $false
|
||||
}) -Force
|
||||
Write-AgentResultAndExit $prior 0
|
||||
@@ -568,33 +750,22 @@ try {
|
||||
foreach ($name in $FixedRuntimeFiles) { $beforeFilePresence[$name] = Test-Path -LiteralPath (Join-Path $BinDir $name) -PathType Leaf }
|
||||
$beforeConfigPresent = Test-Path -LiteralPath $ConfigPath -PathType Leaf
|
||||
$beforeManifestPresent = Test-Path -LiteralPath $ManifestPath -PathType Leaf
|
||||
$beforeRelayRuntime = Get-AgentRelayRuntime
|
||||
$backupDirsBefore = @{}
|
||||
foreach ($directory in @(Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue)) {
|
||||
$backupDirsBefore[$directory.FullName] = $true
|
||||
}
|
||||
|
||||
$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision
|
||||
$backupPath = if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) { [string]$deploy.payload.backupDir } else { "" }
|
||||
if (-not $backupPath) {
|
||||
$newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue |
|
||||
Where-Object { -not $backupDirsBefore.ContainsKey($_.FullName) } |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -First 1
|
||||
if ($newBackup) { $backupPath = $newBackup.FullName }
|
||||
}
|
||||
|
||||
if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) {
|
||||
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
|
||||
$afterManifest = Get-AgentSafeRuntimeManifest
|
||||
$rollbackVerified = [bool](
|
||||
(Test-AgentManifestEqual $beforeManifest $afterManifest) -and
|
||||
(Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and
|
||||
(Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and
|
||||
(Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest
|
||||
)
|
||||
if (
|
||||
-not $beforeRelayRuntime.taskExists -or
|
||||
-not $beforeRelayRuntime.taskEnabled -or
|
||||
-not $beforeRelayRuntime.taskRunning -or
|
||||
-not $beforeRelayRuntime.generationReady -or
|
||||
$beforeRelayRuntime.listenerCount -lt 1
|
||||
) {
|
||||
$receipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" }
|
||||
status = "blocked_relay_runtime_precondition_no_promotion"
|
||||
mode = "apply"
|
||||
traceId = $traceId
|
||||
runId = $runId
|
||||
@@ -602,76 +773,208 @@ try {
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
backupPath = $backupPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
|
||||
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut }
|
||||
rollback = $rollback
|
||||
rollbackVerified = $rollbackVerified
|
||||
runtimeManifest = $afterManifest
|
||||
relayRuntime = $beforeRelayRuntime
|
||||
remoteWritePerformed = $true
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionPerformed = $false
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
uiInteraction = $false
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $false
|
||||
scheduledTaskModification = $false
|
||||
nextSafeAction = "repair_exact_existing_relay_task_runtime_then_retry_same_identity"
|
||||
}
|
||||
Write-AgentRemoteDeployReceipt $receiptPath $receipt
|
||||
Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 })
|
||||
Write-AgentResultAndExit $receipt 2
|
||||
}
|
||||
|
||||
$runtimeManifest = Get-AgentSafeRuntimeManifest
|
||||
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
|
||||
$postVerified = [bool](
|
||||
$runtimeManifest.exists -and
|
||||
$runtimeManifest.runtimeMatched -and
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 14 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$livePreflight.ok -and
|
||||
$livePreflight.sourceRevision -eq $sourceRevision -and
|
||||
$livePreflight.runtimeMatched
|
||||
)
|
||||
|
||||
if (-not $postVerified) {
|
||||
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
|
||||
$afterManifest = Get-AgentSafeRuntimeManifest
|
||||
$rollbackVerified = [bool](
|
||||
$rollback.restored -and
|
||||
(Test-AgentManifestEqual $beforeManifest $afterManifest) -and
|
||||
(Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and
|
||||
(Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and
|
||||
(Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest
|
||||
)
|
||||
$receipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" }
|
||||
mode = "apply"
|
||||
traceId = $traceId
|
||||
runId = $runId
|
||||
workItemId = $workItemId
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
backupPath = $backupPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
|
||||
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut }
|
||||
failedRuntimeManifest = $runtimeManifest
|
||||
failedLivePreflight = $livePreflight
|
||||
rollback = $rollback
|
||||
rollbackVerified = $rollbackVerified
|
||||
runtimeManifest = $afterManifest
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
uiInteraction = $false
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskModification = $false
|
||||
$deploy = $null
|
||||
$relayReload = $null
|
||||
$runtimeManifest = $null
|
||||
$livePreflight = $null
|
||||
$postVerified = $false
|
||||
$failurePhase = ""
|
||||
$failureType = ""
|
||||
$backupPath = ""
|
||||
try {
|
||||
$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision
|
||||
if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) {
|
||||
$backupPath = [string]$deploy.payload.backupDir
|
||||
}
|
||||
if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) {
|
||||
$failurePhase = "deploy"
|
||||
$failureType = "deploy_child_failed_or_invalid"
|
||||
} else {
|
||||
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations)
|
||||
if (-not $relayReload.ok) {
|
||||
$failurePhase = "relay_reload"
|
||||
$failureType = "relay_reload_failed"
|
||||
} else {
|
||||
$runtimeManifest = Get-AgentSafeRuntimeManifest
|
||||
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
|
||||
$postVerified = [bool](
|
||||
$runtimeManifest.exists -and
|
||||
$runtimeManifest.runtimeMatched -and
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 14 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$relayReload.oldGenerationsGone -and
|
||||
$relayReload.newGenerationVerified -and
|
||||
$relayReload.after.taskEnabled -and
|
||||
$relayReload.after.taskRunning -and
|
||||
$livePreflight.ok -and
|
||||
$livePreflight.sourceRevision -eq $sourceRevision -and
|
||||
$livePreflight.runtimeMatched -and
|
||||
$livePreflight.relayListenerCount -gt 0 -and
|
||||
(Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations))
|
||||
)
|
||||
if (-not $postVerified) {
|
||||
$failurePhase = "post_verifier"
|
||||
$failureType = "independent_post_verifier_failed"
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
$failurePhase = if ($deploy -and $deploy.ok) { "reload_or_post_verifier_exception" } else { "deploy_exception" }
|
||||
$failureType = $_.Exception.GetType().Name
|
||||
}
|
||||
|
||||
if ($failurePhase) {
|
||||
if (-not $backupPath) {
|
||||
$newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue |
|
||||
Where-Object { -not $backupDirsBefore.ContainsKey($_.FullName) } |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -First 1
|
||||
if ($newBackup) { $backupPath = $newBackup.FullName }
|
||||
}
|
||||
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
|
||||
$rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations)
|
||||
if ($relayReload) {
|
||||
$rollbackForbiddenGenerations += @($relayReload.before.listenerGenerations)
|
||||
$rollbackForbiddenGenerations += @($relayReload.after.listenerGenerations)
|
||||
}
|
||||
$relayBeforeRollback = Get-AgentRelayRuntime
|
||||
$rollbackForbiddenGenerations += @($relayBeforeRollback.listenerGenerations)
|
||||
$rollbackForbiddenGenerations = @($rollbackForbiddenGenerations | Where-Object { $_ } | Sort-Object -Unique)
|
||||
$rollbackRelayRestart = if ($rollback.restored) {
|
||||
Invoke-AgentRelayTaskRestart -ForbiddenGenerations $rollbackForbiddenGenerations
|
||||
} else {
|
||||
[pscustomobject]@{
|
||||
attempted = $false
|
||||
ok = $false
|
||||
stage = "restore"
|
||||
reason = "bundle_restore_not_verified_relay_restart_withheld"
|
||||
before = $relayBeforeRollback
|
||||
after = $relayBeforeRollback
|
||||
forbiddenGenerations = $rollbackForbiddenGenerations
|
||||
oldGenerationsGone = $false
|
||||
newGenerationVerified = $false
|
||||
}
|
||||
}
|
||||
$afterManifest = [pscustomobject]@{
|
||||
exists = $false
|
||||
sourceRevision = ""
|
||||
runtimeMatched = $false
|
||||
fileCount = 0
|
||||
mismatchCount = -1
|
||||
parseError = "rollback_verifier_not_completed"
|
||||
}
|
||||
$rollbackVerified = $false
|
||||
$rollbackVerifierErrorType = ""
|
||||
try {
|
||||
$afterManifest = Get-AgentSafeRuntimeManifest
|
||||
$rollbackVerified = [bool](
|
||||
$rollback.restored -and
|
||||
(Test-AgentManifestEqual $beforeManifest $afterManifest) -and
|
||||
(Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and
|
||||
(Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and
|
||||
(Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest -and
|
||||
$rollbackRelayRestart.ok -and
|
||||
$rollbackRelayRestart.oldGenerationsGone -and
|
||||
$rollbackRelayRestart.newGenerationVerified -and
|
||||
$rollbackRelayRestart.after.taskEnabled -and
|
||||
$rollbackRelayRestart.after.taskRunning -and
|
||||
$rollbackRelayRestart.after.listenerCount -gt 0
|
||||
)
|
||||
} catch {
|
||||
$rollbackVerified = $false
|
||||
$rollbackVerifierErrorType = $_.Exception.GetType().Name
|
||||
}
|
||||
try {
|
||||
$receipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = if ($failurePhase -eq "deploy") {
|
||||
if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" }
|
||||
} else {
|
||||
if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" }
|
||||
}
|
||||
mode = "apply"
|
||||
traceId = $traceId
|
||||
runId = $runId
|
||||
workItemId = $workItemId
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
backupPath = $backupPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
|
||||
deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } }
|
||||
failurePhase = $failurePhase
|
||||
failureType = $failureType
|
||||
relayReload = $relayReload
|
||||
failedRuntimeManifest = $runtimeManifest
|
||||
failedLivePreflight = $livePreflight
|
||||
rollback = $rollback
|
||||
rollbackRelayRestart = $rollbackRelayRestart
|
||||
rollbackVerified = $rollbackVerified
|
||||
rollbackVerifierErrorType = $rollbackVerifierErrorType
|
||||
runtimeManifest = $afterManifest
|
||||
durableReceiptWritten = $true
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
uiInteraction = $false
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = [bool](($relayReload -and $relayReload.attempted) -or $rollbackRelayRestart.attempted)
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
} catch {
|
||||
$receipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = "rollback_receipt_construction_failed"
|
||||
mode = "apply"
|
||||
traceId = $traceId
|
||||
runId = $runId
|
||||
workItemId = $workItemId
|
||||
sourceRevision = $sourceRevision
|
||||
failurePhase = $failurePhase
|
||||
failureType = $failureType
|
||||
rollbackVerified = $false
|
||||
runtimeRollbackWasVerifiedBeforeReceiptFailure = $rollbackVerified
|
||||
receiptConstructionErrorType = $_.Exception.GetType().Name
|
||||
durableReceiptWritten = $true
|
||||
secretValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
$rollbackVerified = $false
|
||||
}
|
||||
try {
|
||||
Write-AgentRemoteDeployReceipt $receiptPath $receipt
|
||||
} catch {
|
||||
$receipt.status = "rollback_receipt_write_failed"
|
||||
$receipt.durableReceiptWritten = $false
|
||||
$receipt | Add-Member -NotePropertyName receiptWriteErrorType -NotePropertyValue $_.Exception.GetType().Name -Force
|
||||
Write-AgentResultAndExit $receipt 3
|
||||
}
|
||||
Write-AgentRemoteDeployReceipt $receiptPath $receipt
|
||||
Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 })
|
||||
}
|
||||
|
||||
@@ -688,6 +991,7 @@ try {
|
||||
backupPath = $backupPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status }
|
||||
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
|
||||
relayReload = $relayReload
|
||||
runtimeManifest = $runtimeManifest
|
||||
livePreflight = $livePreflight
|
||||
idempotentReplay = $false
|
||||
@@ -701,6 +1005,7 @@ try {
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $true
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback"
|
||||
@@ -721,6 +1026,8 @@ try {
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $false
|
||||
scheduledTaskModification = $false
|
||||
}) 70
|
||||
} finally {
|
||||
if ($script:DeployLock) {
|
||||
|
||||
@@ -5,6 +5,9 @@ readonly SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
|
||||
readonly DEFAULT_SOURCE_ROOT="$(CDPATH= cd -- "${SCRIPT_DIR}/../.." && pwd)"
|
||||
readonly TARGET="Administrator@192.168.0.99"
|
||||
readonly SOURCE_HOST_IPV4="192.168.0.110"
|
||||
readonly GITEA_ORIGIN_HTTPS="https://gitea.wooo.work/wooo/awoooi.git"
|
||||
readonly GITEA_ORIGIN_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git"
|
||||
readonly GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git"
|
||||
readonly RECEIVER_RELATIVE="scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
|
||||
readonly PREFLIGHT_RELATIVE="scripts/reboot-recovery/agent99-live-preflight.ps1"
|
||||
readonly WRAPPER_RELATIVE="scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
|
||||
@@ -136,7 +139,7 @@ done
|
||||
(( CONNECT_TIMEOUT_SECONDS >= 1 && CONNECT_TIMEOUT_SECONDS <= 30 )) || fail "invalid_connect_timeout"
|
||||
|
||||
if [[ "${MODE}" == "apply" ]]; then
|
||||
REMOTE_EXECUTION_TIMEOUT_SECONDS="1500"
|
||||
REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"
|
||||
readonly SAFE_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$'
|
||||
[[ "${TRACE_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_trace_id_missing_or_invalid"
|
||||
[[ "${RUN_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_run_id_missing_or_invalid"
|
||||
@@ -159,17 +162,28 @@ done
|
||||
|
||||
[[ -d "${SOURCE_ROOT}" ]] || fail "source_root_missing"
|
||||
SOURCE_ROOT="$(CDPATH= cd -- "${SOURCE_ROOT}" && pwd)"
|
||||
ORIGIN_URL="$(git -C "${SOURCE_ROOT}" remote get-url origin 2>/dev/null)" || fail "origin_remote_missing"
|
||||
case "${ORIGIN_URL}" in
|
||||
"${GITEA_ORIGIN_HTTPS}"|"${GITEA_ORIGIN_HTTP_INTERNAL}"|"${GITEA_ORIGIN_SSH_INTERNAL}") ;;
|
||||
*) fail "origin_is_not_fixed_internal_gitea" ;;
|
||||
esac
|
||||
git -C "${SOURCE_ROOT}" fetch --quiet --no-tags origin \
|
||||
'+refs/heads/main:refs/remotes/origin/main' || fail "fresh_origin_main_fetch_failed"
|
||||
ORIGIN_MAIN_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'refs/remotes/origin/main^{commit}' 2>/dev/null)" \
|
||||
|| fail "fresh_origin_main_revision_unavailable"
|
||||
ORIGIN_MAIN_REVISION="$(printf '%s' "${ORIGIN_MAIN_REVISION}" | tr '[:upper:]' '[:lower:]')"
|
||||
[[ -f "${KNOWN_HOSTS_FILE}" ]] || fail "known_hosts_file_missing"
|
||||
if [[ -n "${IDENTITY_FILE}" ]]; then
|
||||
[[ -f "${IDENTITY_FILE}" ]] || fail "identity_file_missing"
|
||||
fi
|
||||
|
||||
if [[ -z "${SOURCE_REVISION}" ]]; then
|
||||
SOURCE_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'HEAD^{commit}')" || fail "source_revision_unavailable"
|
||||
SOURCE_REVISION="${ORIGIN_MAIN_REVISION}"
|
||||
fi
|
||||
SOURCE_REVISION="$(printf '%s' "${SOURCE_REVISION}" | tr '[:upper:]' '[:lower:]')"
|
||||
[[ "${SOURCE_REVISION}" =~ ^[0-9a-f]{40}$ ]] || fail "source_revision_must_be_full_commit_sha"
|
||||
git -C "${SOURCE_ROOT}" cat-file -e "${SOURCE_REVISION}^{commit}" 2>/dev/null || fail "source_revision_not_found"
|
||||
[[ "${SOURCE_REVISION}" == "${ORIGIN_MAIN_REVISION}" ]] || fail "source_revision_is_not_fresh_origin_main"
|
||||
|
||||
SOURCE_BOUND_FILES=(
|
||||
"${RUNTIME_FILES[@]}"
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
@@ -56,7 +57,7 @@ def test_sender_is_valid_shell_with_check_as_the_default() -> None:
|
||||
assert "apply_work_item_id_missing_or_invalid" in source
|
||||
assert 'MODE_SELECTION="default"' in source
|
||||
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="45"' in source
|
||||
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="1500"' in source
|
||||
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in source
|
||||
assert "timeout --signal=TERM --kill-after=10" in source
|
||||
|
||||
|
||||
@@ -106,6 +107,23 @@ def test_sender_is_fixed_to_110_to_windows99_public_key_transport() -> None:
|
||||
def test_sender_binds_every_transported_source_to_a_full_git_revision() -> None:
|
||||
source = SENDER.read_text(encoding="utf-8")
|
||||
|
||||
assert 'GITEA_ORIGIN_HTTPS="https://gitea.wooo.work/wooo/awoooi.git"' in source
|
||||
assert (
|
||||
'GITEA_ORIGIN_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git"'
|
||||
in source
|
||||
)
|
||||
assert (
|
||||
'GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git"'
|
||||
in source
|
||||
)
|
||||
assert 'remote get-url origin' in source
|
||||
assert "origin_is_not_fixed_internal_gitea" in source
|
||||
assert "fetch --quiet --no-tags origin" in source
|
||||
assert "+refs/heads/main:refs/remotes/origin/main" in source
|
||||
assert "fresh_origin_main_fetch_failed" in source
|
||||
assert "refs/remotes/origin/main^{commit}" in source
|
||||
assert '[[ "${SOURCE_REVISION}" == "${ORIGIN_MAIN_REVISION}" ]]' in source
|
||||
assert "source_revision_is_not_fresh_origin_main" in source
|
||||
assert "source_revision_must_be_full_commit_sha" in source
|
||||
assert "source_revision_not_found" in source
|
||||
assert "source_bound_file_untracked" in source
|
||||
@@ -166,9 +184,11 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
|
||||
assert "function Restore-AgentBundle" in source
|
||||
assert "Test-AgentBackupPath" in source
|
||||
assert 'Join-Path $DeployRoot "backup-"' in source
|
||||
assert source.count("Restore-AgentBundle $backupPath") == 2
|
||||
assert 'status = if ($rollbackVerified) { "deploy_failed_rollback_verified"' in source
|
||||
assert 'status = if ($rollbackVerified) { "rolled_back_post_verifier_failed"' in source
|
||||
assert source.count("Restore-AgentBundle $backupPath") == 1
|
||||
assert '"deploy_failed_rollback_verified"' in source
|
||||
assert '"deploy_failed_rollback_unverified"' in source
|
||||
assert '"rolled_back_post_verifier_failed"' in source
|
||||
assert '"rollback_failed_after_post_verifier"' in source
|
||||
assert "Get-AgentLiveBundleDigest" in source
|
||||
assert "Test-AgentManifestEqual $beforeManifest $afterManifest" in source
|
||||
assert "Get-AgentSha256File $ConfigPath" in source
|
||||
@@ -185,6 +205,29 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
|
||||
assert '$runtimeManifest.fileCount -eq 14' in source
|
||||
assert '$runtimeManifest.mismatchCount -eq 0' in source
|
||||
assert 'status = "deployed_verified"' in source
|
||||
assert '$failurePhase = "deploy"' in source
|
||||
assert '$failurePhase = "relay_reload"' in source
|
||||
assert '$failurePhase = "post_verifier"' in source
|
||||
assert (
|
||||
"Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
|
||||
"$rollbackForbiddenGenerations"
|
||||
in source
|
||||
)
|
||||
assert "$rollbackRelayRestart.newGenerationVerified" in source
|
||||
assert "$rollbackVerifierErrorType = $_.Exception.GetType().Name" in source
|
||||
assert 'status = "rollback_receipt_construction_failed"' in source
|
||||
assert '$receipt.status = "rollback_receipt_write_failed"' in source
|
||||
assert "durableReceiptWritten = $true" in source
|
||||
assert "$receipt.durableReceiptWritten = $false" in source
|
||||
rollback = source.index(
|
||||
"$rollback = Restore-AgentBundle $backupPath $beforeFilePresence"
|
||||
)
|
||||
rollback_restart = source.index(
|
||||
"Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
|
||||
"$rollbackForbiddenGenerations"
|
||||
)
|
||||
rollback_verify = source.index("$rollbackVerified = [bool](")
|
||||
assert rollback < rollback_restart < rollback_verify
|
||||
|
||||
|
||||
def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary() -> None:
|
||||
@@ -206,9 +249,53 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
|
||||
assert "rawEnvelopeStored = $false" in source
|
||||
assert "$env:" not in source
|
||||
assert "GetEnvironmentVariable" not in source
|
||||
assert (
|
||||
'$stageIdentity = "$mode|$traceId|$runId|$workItemId|$sourceRevision"'
|
||||
in source
|
||||
)
|
||||
assert 'Join-Path $DeployRoot "remote-source-$stageToken"' in source
|
||||
assert 'Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json"' in source
|
||||
assert '$prior.PSObject.Properties["relayReload"]' in source
|
||||
assert "[bool]$prior.relayReload.newGenerationVerified" in source
|
||||
|
||||
|
||||
def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> None:
|
||||
def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generation() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
assert '$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"' in source
|
||||
assert '$RelayTaskPath = "\\"' in source
|
||||
assert "$RelayPort = 8787" in source
|
||||
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
|
||||
assert "Get-NetTCPConnection -LocalPort $RelayPort -State Listen" in source
|
||||
assert 'GetBytes("$processId|$startedAt")' in source
|
||||
assert "$afterStop.taskRunning" in source
|
||||
assert "$beforeRelayRuntime.taskEnabled" in source
|
||||
assert "$after.taskEnabled" in source
|
||||
assert "$relayReload.oldGenerationsGone" in source
|
||||
assert "$relayReload.newGenerationVerified" in source
|
||||
assert "$relayReload.after.taskRunning" in source
|
||||
assert "$livePreflight.relayListenerGenerations" in source
|
||||
assert "scheduledTaskRestart = $true" in source
|
||||
|
||||
deploy = source.index(
|
||||
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
|
||||
"-SourceRevision $sourceRevision"
|
||||
)
|
||||
reload_task = source.index(
|
||||
"$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
|
||||
"@($beforeRelayRuntime.listenerGenerations)"
|
||||
)
|
||||
postverify = source.index(
|
||||
'$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath '
|
||||
'"agent99-live-preflight.ps1")',
|
||||
reload_task,
|
||||
)
|
||||
assert deploy < reload_task < postverify
|
||||
|
||||
|
||||
def test_transport_only_restarts_existing_task_without_mutating_its_definition() -> None:
|
||||
source = "\n".join(
|
||||
(
|
||||
SENDER.read_text(encoding="utf-8"),
|
||||
@@ -226,6 +313,8 @@ def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> No
|
||||
"shutdown.exe",
|
||||
"Register-ScheduledTask",
|
||||
"Unregister-ScheduledTask",
|
||||
"Set-ScheduledTask",
|
||||
"New-ScheduledTask",
|
||||
"-RegisterTasks",
|
||||
)
|
||||
for action in forbidden_actions:
|
||||
@@ -235,6 +324,7 @@ def test_transport_does_not_expose_ui_vm_power_reboot_or_service_actions() -> No
|
||||
assert "vmPowerChange = $false" in source
|
||||
assert "hostReboot = $false" in source
|
||||
assert "serviceRestart = $false" in source
|
||||
assert "scheduledTaskRestart = $true" in source
|
||||
assert "scheduledTaskModification = $false" in source
|
||||
assert "host112" not in source.lower()
|
||||
assert "192.168.0.112" not in source
|
||||
@@ -286,7 +376,24 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport(
|
||||
"exec \"$@\"\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for executable in (fake_hostname, fake_ssh, fake_timeout):
|
||||
real_git = shutil.which("git")
|
||||
assert real_git is not None
|
||||
fake_git = fake_bin / "git"
|
||||
fake_git.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"case \" $* \" in\n"
|
||||
" *\" remote get-url origin \"*) "
|
||||
"printf '%s\\n' 'https://gitea.wooo.work/wooo/awoooi.git'; exit 0 ;;\n"
|
||||
" *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
|
||||
" *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
|
||||
"printf '%040d\\n' 0; exit 0 ;;\n"
|
||||
" *\" cat-file -e \"*) exit 0 ;;\n"
|
||||
" *\" diff --quiet \"*) exit 0 ;;\n"
|
||||
"esac\n"
|
||||
f"exec {shlex.quote(real_git)} \"$@\"\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for executable in (fake_hostname, fake_ssh, fake_timeout, fake_git):
|
||||
executable.chmod(0o700)
|
||||
|
||||
environment = os.environ.copy()
|
||||
|
||||
Reference in New Issue
Block a user