Files
awoooi/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1

1037 lines
40 KiB
PowerShell

[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$EnvelopeJson
)
$ErrorActionPreference = "Stop"
$AgentRoot = "C:\Wooo\Agent99"
$BinDir = Join-Path $AgentRoot "bin"
$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json"
$StateDir = Join-Path $AgentRoot "state"
$EvidenceDir = Join-Path $AgentRoot "evidence"
$DeployRoot = Join-Path $AgentRoot "deploy"
$ManifestPath = Join-Path $StateDir "runtime-manifest.json"
$WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"
$RelayTaskPath = "\"
$RelayPort = 8787
$FixedRuntimeFiles = @(
"agent99-bootstrap.ps1",
"agent99-contract-check.ps1",
"agent99-control-plane.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-sre-alert-inbox.ps1",
"agent99-sre-alert-relay.ps1",
"agent99-submit-request.ps1",
"agent99-synthetic-tests.ps1",
"agent99-telegram-inbox.ps1",
"agent99-windows-update-policy.ps1",
"host-reboot-scorecard.ps1",
"agent99.config.example.json",
"agent99.config.99.example.json"
)
$script:DeployLock = $null
function Get-AgentSha256Bytes {
param([byte[]]$Bytes)
$sha = [Security.Cryptography.SHA256]::Create()
try {
return (($sha.ComputeHash($Bytes) | ForEach-Object { $_.ToString("x2") }) -join "")
} finally {
$sha.Dispose()
}
}
function Get-AgentSha256File {
param([string]$Path)
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { return "missing" }
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
}
function Get-AgentSafeRuntimeManifest {
if (-not (Test-Path -LiteralPath $ManifestPath -PathType Leaf)) {
return [pscustomobject]@{
exists = $false
sourceRevision = ""
runtimeMatched = $false
recordedRuntimeMatched = $false
fileCount = 0
mismatchCount = -1
recordedMismatchCount = -1
parseError = ""
}
}
try {
$manifest = Get-Content -LiteralPath $ManifestPath -Raw | ConvertFrom-Json
$manifestRows = @($manifest.files)
$rowsByName = @{}
$shapeValid = [bool]($manifestRows.Count -eq $FixedRuntimeFiles.Count)
foreach ($row in $manifestRows) {
$rowName = [string]$row.name
if (-not $rowName -or $rowsByName.ContainsKey($rowName)) {
$shapeValid = $false
continue
}
$rowsByName[$rowName] = $row
}
$actualMismatchCount = 0
foreach ($name in $FixedRuntimeFiles) {
if (-not $rowsByName.ContainsKey($name)) {
$shapeValid = $false
$actualMismatchCount += 1
continue
}
$recordedRuntimeSha = ([string]($rowsByName[$name].runtimeSha256)).ToLowerInvariant()
$actualRuntimeSha = Get-AgentSha256File (Join-Path $BinDir $name)
if (-not $recordedRuntimeSha -or $recordedRuntimeSha -ne $actualRuntimeSha) {
$actualMismatchCount += 1
}
}
$currentMatched = [bool](
$shapeValid -and
[bool]$manifest.runtimeMatched -and
[int]$manifest.fileCount -eq $FixedRuntimeFiles.Count -and
[int]$manifest.mismatchCount -eq 0 -and
$actualMismatchCount -eq 0
)
return [pscustomobject]@{
exists = $true
sourceRevision = [string]$manifest.sourceRevision
runtimeMatched = $currentMatched
recordedRuntimeMatched = [bool]$manifest.runtimeMatched
fileCount = [int]$manifest.fileCount
mismatchCount = [int]$actualMismatchCount
recordedMismatchCount = [int]$manifest.mismatchCount
parseError = ""
}
} catch {
return [pscustomobject]@{
exists = $true
sourceRevision = ""
runtimeMatched = $false
recordedRuntimeMatched = $false
fileCount = 0
mismatchCount = -1
recordedMismatchCount = -1
parseError = $_.Exception.GetType().Name
}
}
}
function Get-AgentLiveBundleDigest {
$rows = @()
foreach ($name in $FixedRuntimeFiles) {
$rows += "$name`t$(Get-AgentSha256File (Join-Path $BinDir $name))`n"
}
return Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes(($rows -join "")))
}
function Test-AgentSafeIdentity {
param([string]$Value)
return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$")
}
function Test-AgentManifestEqual {
param([object]$Left, [object]$Right)
return [bool](
$Left.exists -eq $Right.exists -and
$Left.sourceRevision -eq $Right.sourceRevision -and
$Left.runtimeMatched -eq $Right.runtimeMatched -and
$Left.fileCount -eq $Right.fileCount -and
$Left.mismatchCount -eq $Right.mismatchCount
)
}
function Convert-AgentChildJson {
param([object[]]$Lines)
$text = (@($Lines | ForEach-Object { [string]$_ }) -join "`n").Trim()
$start = $text.IndexOf("{")
$end = $text.LastIndexOf("}")
if ($start -lt 0 -or $end -le $start) { return $null }
try {
return $text.Substring($start, $end - $start + 1) | ConvertFrom-Json
} catch {
return $null
}
}
function Invoke-AgentBoundedPowerShell {
param(
[string[]]$Arguments,
[int]$TimeoutSeconds
)
if ($TimeoutSeconds -lt 1 -or @($Arguments | Where-Object { $_ -match '[\s"]' }).Count -gt 0) {
throw "unsafe_or_invalid_child_process_contract"
}
$startInfo = New-Object System.Diagnostics.ProcessStartInfo
$startInfo.FileName = $WindowsPowerShell
$startInfo.Arguments = $Arguments -join " "
$startInfo.UseShellExecute = $false
$startInfo.CreateNoWindow = $true
$startInfo.RedirectStandardOutput = $true
$startInfo.RedirectStandardError = $true
$process = New-Object System.Diagnostics.Process
$process.StartInfo = $startInfo
try {
if (-not $process.Start()) { throw "child_process_start_failed" }
$stdoutTask = $process.StandardOutput.ReadToEndAsync()
$stderrTask = $process.StandardError.ReadToEndAsync()
$finished = $process.WaitForExit($TimeoutSeconds * 1000)
$terminated = $finished
if (-not $finished) {
try { $process.Kill() } catch {}
try { $terminated = $process.WaitForExit(10000) } catch { $terminated = $false }
}
$stdout = ""
if ($terminated) {
try { $stdout = [string]$stdoutTask.Result } catch { $stdout = "" }
try { $null = [string]$stderrTask.Result } catch {}
}
return [pscustomobject]@{
exitCode = if ($finished) { [int]$process.ExitCode } else { 124 }
timedOut = [bool](-not $finished)
stdout = $stdout
}
} finally {
$process.Dispose()
}
}
function Invoke-AgentDeployChild {
param(
[string]$SourceRoot,
[string]$SourceRevision,
[switch]$ValidateOnly
)
$arguments = @(
"-NoProfile",
"-NonInteractive",
"-ExecutionPolicy", "Bypass",
"-File", (Join-Path $SourceRoot "agent99-deploy.ps1"),
"-SourceRoot", $SourceRoot,
"-AgentRoot", $AgentRoot,
"-SourceRevision", $SourceRevision
)
if ($ValidateOnly) { $arguments += "-ValidateOnly" }
$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }
$processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds $timeoutSeconds
return [pscustomobject]@{
ok = [bool]($processResult.exitCode -eq 0 -and -not $processResult.timedOut)
exitCode = [int]$processResult.exitCode
timedOut = [bool]$processResult.timedOut
payload = Convert-AgentChildJson @($processResult.stdout)
validateOnly = [bool]$ValidateOnly
}
}
function Invoke-AgentLivePreflight {
param([string]$PreflightPath)
$arguments = @("-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File", $PreflightPath, "-Mode", "Verify", "-AgentRoot", $AgentRoot)
$processResult = Invoke-AgentBoundedPowerShell -Arguments $arguments -TimeoutSeconds 120
$text = [string]$processResult.stdout
$match = [regex]::Match($text, "(?s)JSON_BEGIN\r?\n(?<json>\{.*?\})\r?\nJSON_END")
$payload = $null
if ($match.Success) {
try { $payload = $match.Groups["json"].Value | ConvertFrom-Json } catch { $payload = $null }
}
$summary = if ($payload -and $payload.PSObject.Properties["summary"]) { $payload.summary } else { $null }
return [pscustomobject]@{
ok = [bool]($processResult.exitCode -eq 0 -and -not $processResult.timedOut -and $summary -and $summary.preflightGreen)
exitCode = [int]$processResult.exitCode
timedOut = [bool]$processResult.timedOut
parsed = [bool]($null -ne $payload)
sourceRevision = if ($payload -and $payload.manifest) { [string]$payload.manifest.sourceRevision } else { "" }
runtimeMatched = [bool]($payload -and $payload.manifest -and $payload.manifest.runtimeMatched)
healthyTaskCount = if ($summary) { [int]$summary.healthyTaskCount } else { 0 }
requiredTaskCount = if ($summary) { [int]$summary.requiredTaskCount } else { 0 }
relayListenerCount = if ($summary) { [int]$summary.relayListenerCount } else { 0 }
relayListenerGenerations = if ($payload) {
@($payload.relayListeners | ForEach-Object {
$processId = [int]$_.processId
$process = Get-Process -Id $processId -ErrorAction SilentlyContinue
if ($process) {
try {
$startedAt = $process.StartTime.ToUniversalTime().ToString("o")
Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$processId|$startedAt"))
} catch {}
}
} | Where-Object { $_ } | Sort-Object -Unique)
} else { @() }
pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 }
preflightGreen = [bool]($summary -and $summary.preflightGreen)
blockingReasons = if ($processResult.timedOut) { @("preflight_process_timeout") } elseif ($summary) { @($summary.blockingReasons | ForEach-Object { [string]$_ }) } else { @("preflight_output_unparseable") }
}
}
function Get-AgentRelayRuntime {
$task = $null
$taskError = ""
try {
$tasks = @(Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop)
if ($tasks.Count -ne 1) {
$taskError = "exact_relay_task_count_$($tasks.Count)"
} else {
$task = $tasks[0]
}
} catch {
$taskError = $_.Exception.GetType().Name
}
$listeners = @()
$listenerError = ""
try {
$connections = @(Get-NetTCPConnection -LocalPort $RelayPort -State Listen -ErrorAction Stop)
foreach ($connection in $connections) {
$processId = [int]$connection.OwningProcess
$process = Get-Process -Id $processId -ErrorAction SilentlyContinue
$startedAt = ""
$generation = ""
if ($process) {
try {
$startedAt = $process.StartTime.ToUniversalTime().ToString("o")
$generation = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes("$processId|$startedAt"))
} catch {}
}
$listeners += [pscustomobject]@{
address = [string]$connection.LocalAddress
port = [int]$connection.LocalPort
processId = $processId
processName = if ($process) { [string]$process.ProcessName } else { "" }
startedAt = $startedAt
generation = $generation
}
}
} catch {
$listenerError = $_.Exception.GetType().Name
}
$processIds = @($listeners | ForEach-Object { [int]$_.processId } | Sort-Object -Unique)
$listenerGenerations = @($listeners | ForEach-Object { [string]$_.generation } | Where-Object { $_ } | Sort-Object -Unique)
$taskEnabled = $false
if ($task) {
try { $taskEnabled = [bool]$task.Settings.Enabled } catch { $taskEnabled = $false }
}
return [pscustomobject]@{
taskName = $RelayTaskName
taskPath = $RelayTaskPath
taskExists = [bool]($null -ne $task)
taskState = if ($task) { [string]$task.State } else { "" }
taskRunning = [bool]($task -and [string]$task.State -eq "Running")
taskEnabled = $taskEnabled
taskError = $taskError
listenerPort = $RelayPort
listenerCount = $listeners.Count
listenerProcessCount = $processIds.Count
listenerGenerations = $listenerGenerations
generationReady = [bool]($listeners.Count -gt 0 -and $processIds.Count -gt 0 -and $listenerGenerations.Count -eq $processIds.Count)
listeners = $listeners
listenerError = $listenerError
}
}
function Test-AgentGenerationIntersection {
param([string[]]$Left, [string[]]$Right)
foreach ($value in @($Left)) {
if ($value -and $value -in @($Right)) { return $true }
}
return $false
}
function Invoke-AgentRelayTaskRestart {
param([string[]]$ForbiddenGenerations = @())
$before = Get-AgentRelayRuntime
$generationCandidates = @($ForbiddenGenerations) + @($before.listenerGenerations)
$forbidden = @($generationCandidates | Where-Object { $_ } | Sort-Object -Unique)
if (-not $before.taskExists -or -not $before.taskEnabled) {
return [pscustomobject]@{
attempted = $false
ok = $false
stage = "precondition"
reason = if ($before.taskExists) { "exact_existing_relay_task_disabled" } else { "exact_existing_relay_task_missing" }
before = $before
after = $before
forbiddenGenerations = $forbidden
oldGenerationsGone = $false
newGenerationVerified = $false
}
}
$stage = "stop"
$afterStop = $before
$after = $before
try {
Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
$stopDeadline = (Get-Date).AddSeconds(60)
do {
Start-Sleep -Seconds 1
$afterStop = Get-AgentRelayRuntime
$oldGone = -not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden)
$stopVerified = [bool]($oldGone -and -not $afterStop.taskRunning)
if ($stopVerified) { break }
} while ((Get-Date) -lt $stopDeadline)
if (-not $stopVerified) { throw "relay_old_listener_generation_stop_timeout" }
$stage = "start"
Start-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName -ErrorAction Stop
$startDeadline = (Get-Date).AddSeconds(120)
do {
Start-Sleep -Seconds 1
$after = Get-AgentRelayRuntime
$oldGone = -not (Test-AgentGenerationIntersection @($after.listenerGenerations) $forbidden)
$newGenerations = @($after.listenerGenerations | Where-Object { $_ -and $_ -notin $forbidden })
$newVerified = [bool](
$after.taskRunning -and
$after.taskEnabled -and
$after.generationReady -and
$after.listenerCount -gt 0 -and
$oldGone -and
$newGenerations.Count -gt 0
)
if ($newVerified) { break }
} while ((Get-Date) -lt $startDeadline)
if (-not $newVerified) { throw "relay_new_listener_generation_start_timeout" }
return [pscustomobject]@{
attempted = $true
ok = $true
stage = "verified"
reason = "exact_existing_relay_task_restarted"
before = $before
afterStop = $afterStop
after = $after
forbiddenGenerations = $forbidden
oldGenerationsGone = $oldGone
newGenerationVerified = $newVerified
}
} catch {
return [pscustomobject]@{
attempted = $true
ok = $false
stage = $stage
reason = "relay_task_restart_$($stage)_failed"
errorType = $_.Exception.GetType().Name
before = $before
afterStop = $afterStop
after = (Get-AgentRelayRuntime)
forbiddenGenerations = $forbidden
oldGenerationsGone = [bool](-not (Test-AgentGenerationIntersection @($afterStop.listenerGenerations) $forbidden))
newGenerationVerified = $false
}
}
}
function Copy-AgentFileWithRetry {
param([string]$Source, [string]$Destination)
for ($attempt = 1; $attempt -le 20; $attempt += 1) {
try {
Copy-Item -LiteralPath $Source -Destination $Destination -Force
return
} catch [System.IO.IOException] {
if ($attempt -ge 20) { throw }
Start-Sleep -Milliseconds 500
}
}
}
function Test-AgentBackupPath {
param([string]$BackupPath)
if (-not $BackupPath) { return $false }
try {
$expectedPrefix = [IO.Path]::GetFullPath((Join-Path $DeployRoot "backup-"))
$candidate = [IO.Path]::GetFullPath($BackupPath)
return [bool]($candidate.StartsWith($expectedPrefix, [StringComparison]::OrdinalIgnoreCase))
} catch {
return $false
}
}
function Restore-AgentBundle {
param(
[string]$BackupPath,
[hashtable]$BeforeFilePresence,
[bool]$BeforeConfigPresent,
[bool]$BeforeManifestPresent
)
if (-not (Test-AgentBackupPath $BackupPath) -or -not (Test-Path -LiteralPath $BackupPath -PathType Container)) {
return [pscustomobject]@{ attempted = $false; restored = $false; reason = "bounded_backup_path_unavailable" }
}
try {
foreach ($name in $FixedRuntimeFiles) {
$livePath = Join-Path $BinDir $name
$backupFile = Join-Path $BackupPath $name
if ($BeforeFilePresence[$name]) {
if (-not (Test-Path -LiteralPath $backupFile -PathType Leaf)) { throw "required_runtime_backup_missing" }
Copy-AgentFileWithRetry $backupFile $livePath
} elseif (Test-Path -LiteralPath $livePath -PathType Leaf) {
Remove-Item -LiteralPath $livePath -Force
}
}
$backupConfig = Join-Path $BackupPath "agent99.config.json"
if ($BeforeConfigPresent) {
if (-not (Test-Path -LiteralPath $backupConfig -PathType Leaf)) { throw "required_config_backup_missing" }
Copy-AgentFileWithRetry $backupConfig $ConfigPath
} elseif (Test-Path -LiteralPath $ConfigPath -PathType Leaf) {
Remove-Item -LiteralPath $ConfigPath -Force
}
$backupManifest = Join-Path $BackupPath "runtime-manifest.json"
if ($BeforeManifestPresent) {
if (-not (Test-Path -LiteralPath $backupManifest -PathType Leaf)) { throw "required_manifest_backup_missing" }
Copy-AgentFileWithRetry $backupManifest $ManifestPath
} elseif (Test-Path -LiteralPath $ManifestPath -PathType Leaf) {
Remove-Item -LiteralPath $ManifestPath -Force
}
return [pscustomobject]@{ attempted = $true; restored = $true; reason = "bounded_bundle_restored" }
} catch {
return [pscustomobject]@{ attempted = $true; restored = $false; reason = $_.Exception.GetType().Name }
}
}
function Write-AgentRemoteDeployReceipt {
param([string]$Path, [object]$Receipt)
$temporary = "$Path.tmp.$PID"
$Receipt | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $temporary -Encoding UTF8
Move-Item -LiteralPath $temporary -Destination $Path -Force
}
function Write-AgentResultAndExit {
param([object]$Result, [int]$ExitCode)
$Result | ConvertTo-Json -Compress -Depth 12
exit $ExitCode
}
try {
$envelope = $EnvelopeJson | ConvertFrom-Json
if ([string]$envelope.schemaVersion -ne "agent99_remote_atomic_deploy_envelope_v1") { throw "invalid_envelope_schema" }
if ([string]$envelope.sourceHostAlias -ne "110") { throw "invalid_source_host_alias" }
if ([string]$envelope.targetHostAlias -ne "99") { throw "invalid_target_host_alias" }
if ([string]$envelope.transport -ne "110_to_99_ssh_publickey") { throw "invalid_transport" }
$mode = [string]$envelope.mode
if ($mode -notin @("check", "apply")) { throw "invalid_mode" }
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" }
$traceId = [string]$envelope.traceId
$runId = [string]$envelope.runId
$workItemId = [string]$envelope.workItemId
if ($mode -eq "apply") {
foreach ($identity in @($traceId, $runId, $workItemId)) {
if (-not (Test-AgentSafeIdentity $identity)) { throw "apply_identity_missing_or_invalid" }
}
}
$filesByName = @{}
foreach ($file in @($envelope.files)) {
$name = [string]$file.name
if ($filesByName.ContainsKey($name)) { throw "duplicate_runtime_file" }
$filesByName[$name] = $file
}
if ($filesByName.Count -ne $FixedRuntimeFiles.Count) { throw "runtime_file_count_mismatch" }
$manifestText = ""
$decodedFiles = @{}
foreach ($name in $FixedRuntimeFiles) {
if (-not $filesByName.ContainsKey($name)) { throw "required_runtime_file_missing" }
$row = $filesByName[$name]
$bytes = [Convert]::FromBase64String([string]$row.contentBase64)
$actualSha = Get-AgentSha256Bytes $bytes
if ($actualSha -ne ([string]$row.sha256).ToLowerInvariant()) { throw "runtime_file_sha256_mismatch" }
$decodedFiles[$name] = $bytes
$manifestText += "$name`t$actualSha`n"
}
$manifestDigest = Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($manifestText))
if ($manifestDigest -ne ([string]$envelope.manifestSha256).ToLowerInvariant()) { throw "manifest_sha256_mismatch" }
$sourceManifest = [pscustomobject]@{
schemaVersion = "agent99_remote_source_manifest_v1"
sourceRevision = $sourceRevision
fileCount = 14
manifestSha256 = $manifestDigest
files = @($FixedRuntimeFiles | ForEach-Object {
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
})
}
$preflight = $envelope.livePreflight
if (-not $preflight -or [string]$preflight.name -ne "agent99-live-preflight.ps1") { throw "live_preflight_missing" }
$preflightBytes = [Convert]::FromBase64String([string]$preflight.contentBase64)
if ((Get-AgentSha256Bytes $preflightBytes) -ne ([string]$preflight.sha256).ToLowerInvariant()) { throw "live_preflight_sha256_mismatch" }
$stageIdentity = "$mode|$traceId|$runId|$workItemId|$sourceRevision"
$stageToken = (Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($stageIdentity))).Substring(0, 20)
$stagePath = Join-Path $DeployRoot "remote-source-$stageToken"
$receiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json"
$currentManifest = Get-AgentSafeRuntimeManifest
if ($mode -eq "check") {
Write-AgentResultAndExit ([pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "check_ready"
mode = "check"
sourceRevision = $sourceRevision
expectedRuntimeFileCount = 14
manifestSha256 = $manifestDigest
plannedStagingPath = $stagePath
runtimeManifest = $currentManifest
livePreflight = [pscustomobject]@{ executed = $false; reason = "apply_only_after_validate_only" }
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskModification = $false
}
nextSafeAction = "rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check"
}) 0
}
New-Item -ItemType Directory -Force -Path $StateDir, $EvidenceDir, $DeployRoot | Out-Null
$lockPath = Join-Path $StateDir "remote-atomic-deploy.lock"
try {
$script:DeployLock = [IO.File]::Open($lockPath, [IO.FileMode]::OpenOrCreate, [IO.FileAccess]::ReadWrite, [IO.FileShare]::None)
} catch {
Write-AgentResultAndExit ([pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "blocked_single_flight_lock_unavailable"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
remoteWritePerformed = $false
liveRuntimeWritePerformed = $false
livePromotionPerformed = $false
}) 75
}
$prior = $null
if (Test-Path -LiteralPath $receiptPath -PathType Leaf) {
try { $prior = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json } catch { $prior = $null }
}
$stageWrittenThisRun = $false
if (-not (Test-Path -LiteralPath $stagePath -PathType Container)) {
$stageBuildPath = "$stagePath.partial.$PID.$([Guid]::NewGuid().ToString('N'))"
New-Item -ItemType Directory -Path $stageBuildPath | Out-Null
foreach ($name in $FixedRuntimeFiles) {
[IO.File]::WriteAllBytes((Join-Path $stageBuildPath $name), [byte[]]$decodedFiles[$name])
}
[IO.File]::WriteAllBytes((Join-Path $stageBuildPath "agent99-live-preflight.ps1"), $preflightBytes)
$sourceManifest | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $stageBuildPath "source-manifest.json") -Encoding UTF8
Move-Item -LiteralPath $stageBuildPath -Destination $stagePath
$stageWrittenThisRun = $true
} else {
foreach ($name in $FixedRuntimeFiles) {
if ((Get-AgentSha256File (Join-Path $stagePath $name)) -ne ([string]($filesByName[$name].sha256)).ToLowerInvariant()) {
throw "existing_staging_file_mismatch"
}
}
if ((Get-AgentSha256File (Join-Path $stagePath "agent99-live-preflight.ps1")) -ne ([string]$preflight.sha256).ToLowerInvariant()) {
throw "existing_staging_preflight_mismatch"
}
try {
$existingSourceManifest = Get-Content -LiteralPath (Join-Path $stagePath "source-manifest.json") -Raw | ConvertFrom-Json
if (
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
[int]$existingSourceManifest.fileCount -ne 14 -or
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
) { throw "existing_staging_manifest_mismatch" }
} catch {
throw "existing_staging_manifest_invalid"
}
}
$live = Get-AgentSafeRuntimeManifest
if (
$prior -and
[string]$prior.status -eq "deployed_verified" -and
[string]$prior.traceId -eq $traceId -and
[string]$prior.runId -eq $runId -and
[string]$prior.workItemId -eq $workItemId -and
[string]$prior.sourceRevision -eq $sourceRevision -and
$prior.PSObject.Properties["relayReload"] -and
[bool]$prior.relayReload.newGenerationVerified -and
$live.runtimeMatched -and
$live.sourceRevision -eq $sourceRevision -and
$live.fileCount -eq 14 -and
$live.mismatchCount -eq 0
) {
$replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
if ($replayPreflight.ok -and $replayPreflight.sourceRevision -eq $sourceRevision -and $replayPreflight.runtimeMatched) {
$originalOperationBoundaries = $prior.operationBoundaries
$prior | Add-Member -NotePropertyName idempotentReplay -NotePropertyValue $true -Force
$prior | Add-Member -NotePropertyName runtimeManifest -NotePropertyValue $live -Force
$prior | Add-Member -NotePropertyName livePreflight -NotePropertyValue $replayPreflight -Force
$prior | Add-Member -NotePropertyName originalOperationBoundaries -NotePropertyValue $originalOperationBoundaries -Force
$prior | Add-Member -NotePropertyName operationBoundaries -NotePropertyValue ([pscustomobject]@{
remoteWritePerformed = $stageWrittenThisRun
liveRuntimeWritePerformed = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskModification = $false
}) -Force
Write-AgentResultAndExit $prior 0
}
Write-AgentResultAndExit ([pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "idempotent_replay_post_verifier_failed_no_promotion"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
runtimeManifest = $live
livePreflight = $replayPreflight
remoteWritePerformed = $stageWrittenThisRun
liveRuntimeWritePerformed = $false
livePromotionPerformed = $false
nextSafeAction = "repair_reported_live_preflight_blocker_then_retry_same_identity"
}) 2
}
$validate = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision -ValidateOnly
if (-not $validate.ok -or -not $validate.payload -or [string]$validate.payload.status -ne "validated") {
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "validate_only_failed_no_promotion"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
remoteWritePerformed = $true
liveRuntimeWritePerformed = $false
livePromotionPerformed = $false
}
Write-AgentRemoteDeployReceipt $receiptPath $receipt
Write-AgentResultAndExit $receipt 2
}
$beforeManifest = Get-AgentSafeRuntimeManifest
$beforeBundleDigest = Get-AgentLiveBundleDigest
$beforeConfigDigest = Get-AgentSha256File $ConfigPath
$beforeManifestDigest = Get-AgentSha256File $ManifestPath
$beforeFilePresence = @{}
foreach ($name in $FixedRuntimeFiles) { $beforeFilePresence[$name] = Test-Path -LiteralPath (Join-Path $BinDir $name) -PathType Leaf }
$beforeConfigPresent = Test-Path -LiteralPath $ConfigPath -PathType Leaf
$beforeManifestPresent = Test-Path -LiteralPath $ManifestPath -PathType Leaf
$beforeRelayRuntime = Get-AgentRelayRuntime
$backupDirsBefore = @{}
foreach ($directory in @(Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue)) {
$backupDirsBefore[$directory.FullName] = $true
}
if (
-not $beforeRelayRuntime.taskExists -or
-not $beforeRelayRuntime.taskEnabled -or
-not $beforeRelayRuntime.taskRunning -or
-not $beforeRelayRuntime.generationReady -or
$beforeRelayRuntime.listenerCount -lt 1
) {
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "blocked_relay_runtime_precondition_no_promotion"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
relayRuntime = $beforeRelayRuntime
remoteWritePerformed = $true
liveRuntimeWritePerformed = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskModification = $false
nextSafeAction = "repair_exact_existing_relay_task_runtime_then_retry_same_identity"
}
Write-AgentRemoteDeployReceipt $receiptPath $receipt
Write-AgentResultAndExit $receipt 2
}
$deploy = $null
$relayReload = $null
$runtimeManifest = $null
$livePreflight = $null
$postVerified = $false
$failurePhase = ""
$failureType = ""
$backupPath = ""
try {
$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision
if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) {
$backupPath = [string]$deploy.payload.backupDir
}
if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) {
$failurePhase = "deploy"
$failureType = "deploy_child_failed_or_invalid"
} else {
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations)
if (-not $relayReload.ok) {
$failurePhase = "relay_reload"
$failureType = "relay_reload_failed"
} else {
$runtimeManifest = Get-AgentSafeRuntimeManifest
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$postVerified = [bool](
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 14 -and
$runtimeManifest.mismatchCount -eq 0 -and
$relayReload.oldGenerationsGone -and
$relayReload.newGenerationVerified -and
$relayReload.after.taskEnabled -and
$relayReload.after.taskRunning -and
$livePreflight.ok -and
$livePreflight.sourceRevision -eq $sourceRevision -and
$livePreflight.runtimeMatched -and
$livePreflight.relayListenerCount -gt 0 -and
(Test-AgentGenerationIntersection @($livePreflight.relayListenerGenerations) @($relayReload.after.listenerGenerations))
)
if (-not $postVerified) {
$failurePhase = "post_verifier"
$failureType = "independent_post_verifier_failed"
}
}
}
} catch {
$failurePhase = if ($deploy -and $deploy.ok) { "reload_or_post_verifier_exception" } else { "deploy_exception" }
$failureType = $_.Exception.GetType().Name
}
if ($failurePhase) {
if (-not $backupPath) {
$newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue |
Where-Object { -not $backupDirsBefore.ContainsKey($_.FullName) } |
Sort-Object LastWriteTime -Descending |
Select-Object -First 1
if ($newBackup) { $backupPath = $newBackup.FullName }
}
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
$rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations)
if ($relayReload) {
$rollbackForbiddenGenerations += @($relayReload.before.listenerGenerations)
$rollbackForbiddenGenerations += @($relayReload.after.listenerGenerations)
}
$relayBeforeRollback = Get-AgentRelayRuntime
$rollbackForbiddenGenerations += @($relayBeforeRollback.listenerGenerations)
$rollbackForbiddenGenerations = @($rollbackForbiddenGenerations | Where-Object { $_ } | Sort-Object -Unique)
$rollbackRelayRestart = if ($rollback.restored) {
Invoke-AgentRelayTaskRestart -ForbiddenGenerations $rollbackForbiddenGenerations
} else {
[pscustomobject]@{
attempted = $false
ok = $false
stage = "restore"
reason = "bundle_restore_not_verified_relay_restart_withheld"
before = $relayBeforeRollback
after = $relayBeforeRollback
forbiddenGenerations = $rollbackForbiddenGenerations
oldGenerationsGone = $false
newGenerationVerified = $false
}
}
$afterManifest = [pscustomobject]@{
exists = $false
sourceRevision = ""
runtimeMatched = $false
fileCount = 0
mismatchCount = -1
parseError = "rollback_verifier_not_completed"
}
$rollbackVerified = $false
$rollbackVerifierErrorType = ""
try {
$afterManifest = Get-AgentSafeRuntimeManifest
$rollbackVerified = [bool](
$rollback.restored -and
(Test-AgentManifestEqual $beforeManifest $afterManifest) -and
(Get-AgentLiveBundleDigest) -eq $beforeBundleDigest -and
(Get-AgentSha256File $ConfigPath) -eq $beforeConfigDigest -and
(Get-AgentSha256File $ManifestPath) -eq $beforeManifestDigest -and
$rollbackRelayRestart.ok -and
$rollbackRelayRestart.oldGenerationsGone -and
$rollbackRelayRestart.newGenerationVerified -and
$rollbackRelayRestart.after.taskEnabled -and
$rollbackRelayRestart.after.taskRunning -and
$rollbackRelayRestart.after.listenerCount -gt 0
)
} catch {
$rollbackVerified = $false
$rollbackVerifierErrorType = $_.Exception.GetType().Name
}
try {
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = if ($failurePhase -eq "deploy") {
if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" }
} else {
if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" }
}
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
backupPath = $backupPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } }
failurePhase = $failurePhase
failureType = $failureType
relayReload = $relayReload
failedRuntimeManifest = $runtimeManifest
failedLivePreflight = $livePreflight
rollback = $rollback
rollbackRelayRestart = $rollbackRelayRestart
rollbackVerified = $rollbackVerified
rollbackVerifierErrorType = $rollbackVerifierErrorType
runtimeManifest = $afterManifest
durableReceiptWritten = $true
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = [bool](($relayReload -and $relayReload.attempted) -or $rollbackRelayRestart.attempted)
scheduledTaskModification = $false
}
} catch {
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "rollback_receipt_construction_failed"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
failurePhase = $failurePhase
failureType = $failureType
rollbackVerified = $false
runtimeRollbackWasVerifiedBeforeReceiptFailure = $rollbackVerified
receiptConstructionErrorType = $_.Exception.GetType().Name
durableReceiptWritten = $true
secretValueRead = $false
environmentSecretRead = $false
scheduledTaskModification = $false
}
$rollbackVerified = $false
}
try {
Write-AgentRemoteDeployReceipt $receiptPath $receipt
} catch {
$receipt.status = "rollback_receipt_write_failed"
$receipt.durableReceiptWritten = $false
$receipt | Add-Member -NotePropertyName receiptWriteErrorType -NotePropertyValue $_.Exception.GetType().Name -Force
Write-AgentResultAndExit $receipt 3
}
Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 })
}
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "deployed_verified"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
backupPath = $backupPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status }
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
relayReload = $relayReload
runtimeManifest = $runtimeManifest
livePreflight = $livePreflight
idempotentReplay = $false
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $true
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $true
scheduledTaskModification = $false
}
nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback"
}
Write-AgentRemoteDeployReceipt $receiptPath $receipt
Write-AgentResultAndExit $receipt 0
} catch {
Write-AgentResultAndExit ([pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "transport_receiver_failed"
errorType = $_.Exception.GetType().Name
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
rawEnvelopeStored = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskModification = $false
}) 70
} finally {
if ($script:DeployLock) {
try { $script:DeployLock.Dispose() } catch {}
}
}