fix(agent99): complete same-run status reconcile
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m46s
CD Pipeline / build-and-deploy (push) Successful in 19m6s
CD Pipeline / post-deploy-checks (push) Successful in 2m4s

This commit is contained in:
ogt
2026-07-14 23:28:19 +08:00
parent 17bdcb194e
commit b233ac5c74
6 changed files with 617 additions and 32 deletions

View File

@@ -1,7 +1,8 @@
param(
[string]$AgentRoot = "C:\Wooo\Agent99",
[string]$SourceRevision = "",
[switch]$InstallScheduledTasks
[switch]$InstallScheduledTasks,
[switch]$WriteLauncherOnly
)
$ErrorActionPreference = "Stop"
@@ -9,6 +10,7 @@ $ErrorActionPreference = "Stop"
$SourceRoot = Split-Path -Parent $MyInvocation.MyCommand.Path
$BinDir = Join-Path $AgentRoot "bin"
$ConfigDir = Join-Path $AgentRoot "config"
$ConfigPath = Join-Path $ConfigDir "agent99.config.json"
$EvidenceDir = Join-Path $AgentRoot "evidence"
$LogDir = Join-Path $AgentRoot "logs"
$QueueDir = Join-Path $AgentRoot "queue"
@@ -18,6 +20,83 @@ $PlaybookDir = Join-Path $AgentRoot "playbooks"
$DashboardDir = Join-Path $AgentRoot "dashboard"
$StateDir = Join-Path $AgentRoot "state"
function Write-AgentLauncher {
param([string]$Path)
$launcherContent = @"
param(
[ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
[string]`$Mode = "Status",
[switch]`$ControlledApply,
[switch]`$ReconcileOnly,
[string]`$ReconcileEvidenceId = "",
[switch]`$ReconcileQueueOnly,
[string]`$ReconcileQueueId = "",
[string]`$AutomationRunId = "",
[string]`$TraceId = "",
[string]`$WorkItemId = "",
[switch]`$SuppressAlerts,
[string]`$SuppressReason = ""
)
& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ReconcileOnly:`$ReconcileOnly -ReconcileEvidenceId `$ReconcileEvidenceId -ReconcileQueueOnly:`$ReconcileQueueOnly -ReconcileQueueId `$ReconcileQueueId -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$ConfigPath" -EvidenceDir "$EvidenceDir"
"@
$parent = Split-Path -Parent $Path
if (-not (Test-Path -LiteralPath $parent -PathType Container)) {
throw "Agent99 launcher parent is unavailable: $parent"
}
$temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))"
try {
[IO.File]::WriteAllText($temporary, $launcherContent, (New-Object Text.UTF8Encoding($true)))
$tokens = $null
$parseErrors = $null
[Management.Automation.Language.Parser]::ParseFile($temporary, [ref]$tokens, [ref]$parseErrors) | Out-Null
if (@($parseErrors).Count -ne 0) {
throw "Agent99 launcher PowerShell parser check failed."
}
Move-Item -LiteralPath $temporary -Destination $Path -Force
$written = [IO.File]::ReadAllText($Path)
$requiredMarkers = @(
'[switch]$ReconcileOnly',
'[string]$ReconcileEvidenceId = ""',
'[switch]$ReconcileQueueOnly',
'[string]$ReconcileQueueId = ""',
'-ReconcileOnly:$ReconcileOnly',
'-ReconcileEvidenceId $ReconcileEvidenceId',
'-ReconcileQueueOnly:$ReconcileQueueOnly',
'-ReconcileQueueId $ReconcileQueueId'
)
$missingMarkers = @($requiredMarkers | Where-Object { -not $written.Contains($_) })
if ($missingMarkers.Count -ne 0) {
throw "Agent99 launcher reconcile contract verification failed."
}
return [pscustomobject]@{
schemaVersion = "agent99_launcher_contract_v1"
ok = $true
path = $Path
sha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
parseErrorCount = 0
requiredMarkerCount = $requiredMarkers.Count
missingMarkerCount = 0
reconcileOnlyForwarded = $true
reconcileQueueOnlyForwarded = $true
}
} finally {
if (Test-Path -LiteralPath $temporary) {
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue
}
}
}
if ($WriteLauncherOnly) {
$controlPlanePath = Join-Path $BinDir "agent99-control-plane.ps1"
if (-not (Test-Path -LiteralPath $controlPlanePath -PathType Leaf)) {
throw "Agent99 control plane is unavailable for launcher generation: $controlPlanePath"
}
Write-AgentLauncher (Join-Path $AgentRoot "agent99-run.ps1")
return
}
function Copy-AgentFile {
param([string]$Name)
$src = Join-Path $SourceRoot $Name
@@ -210,7 +289,6 @@ if (-not $runtimeMatched) {
}
$exampleConfig = Join-Path $SourceRoot "agent99.config.example.json"
$configPath = Join-Path $ConfigDir "agent99.config.json"
if (-not (Test-Path $configPath)) {
if (Test-Path $exampleConfig) {
Copy-Item $exampleConfig $configPath
@@ -290,24 +368,7 @@ if (-not (Test-Path $configPath)) {
Ensure-AgentHost112CanonicalSshConfig $configPath $host112BootstrapPolicyPath
$launcher = Join-Path $AgentRoot "agent99-run.ps1"
@"
param(
[ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
[string]`$Mode = "Status",
[switch]`$ControlledApply,
[switch]`$ReconcileOnly,
[string]`$ReconcileEvidenceId = "",
[switch]`$ReconcileQueueOnly,
[string]`$ReconcileQueueId = "",
[string]`$AutomationRunId = "",
[string]`$TraceId = "",
[string]`$WorkItemId = "",
[switch]`$SuppressAlerts,
[string]`$SuppressReason = ""
)
& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ReconcileOnly:`$ReconcileOnly -ReconcileEvidenceId `$ReconcileEvidenceId -ReconcileQueueOnly:`$ReconcileQueueOnly -ReconcileQueueId `$ReconcileQueueId -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir"
"@ | Set-Content -Encoding UTF8 $launcher
$null = Write-AgentLauncher $launcher
$submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd"
@"

View File

@@ -16,9 +16,18 @@ $DeployRoot = Join-Path $AgentRoot "deploy"
$StageDir = Join-Path $DeployRoot "staging-$stamp"
$BackupDir = Join-Path $DeployRoot "backup-$stamp"
$ManifestPath = Join-Path $StateDir "runtime-manifest.json"
$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1"
$EvidencePath = Join-Path $EvidenceDir "agent99-deploy-$stamp.json"
$script:ConfigMigrations = @()
$script:RuntimeCopyRetries = @()
$script:LauncherContract = [pscustomobject]@{
schemaVersion = "agent99_launcher_contract_v1"
ok = $false
scope = "not_checked"
exists = $false
parseErrorCount = -1
missingMarkerCount = -1
}
$runtimeFiles = @(
"agent99-bootstrap.ps1",
@@ -112,6 +121,55 @@ function Copy-AgentRuntimeFileWithRetry {
}
}
function Get-AgentLauncherContract {
param([string]$Path, [string]$Scope)
$requiredMarkers = @(
'[switch]$ReconcileOnly',
'[string]$ReconcileEvidenceId = ""',
'[switch]$ReconcileQueueOnly',
'[string]$ReconcileQueueId = ""',
'-ReconcileOnly:$ReconcileOnly',
'-ReconcileEvidenceId $ReconcileEvidenceId',
'-ReconcileQueueOnly:$ReconcileQueueOnly',
'-ReconcileQueueId $ReconcileQueueId'
)
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
return [pscustomobject]@{
schemaVersion = "agent99_launcher_contract_v1"
ok = $false
scope = $Scope
exists = $false
path = $Path
sha256 = ""
parseErrorCount = -1
requiredMarkerCount = $requiredMarkers.Count
missingMarkerCount = $requiredMarkers.Count
reconcileOnlyForwarded = $false
reconcileQueueOnlyForwarded = $false
}
}
$tokens = $null
$parseErrors = $null
[Management.Automation.Language.Parser]::ParseFile($Path, [ref]$tokens, [ref]$parseErrors) | Out-Null
$content = [IO.File]::ReadAllText($Path)
$missingMarkers = @($requiredMarkers | Where-Object { -not $content.Contains($_) })
$ok = [bool](@($parseErrors).Count -eq 0 -and $missingMarkers.Count -eq 0)
return [pscustomobject]@{
schemaVersion = "agent99_launcher_contract_v1"
ok = $ok
scope = $Scope
exists = $true
path = $Path
sha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
parseErrorCount = @($parseErrors).Count
requiredMarkerCount = $requiredMarkers.Count
missingMarkerCount = $missingMarkers.Count
reconcileOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileOnly:$ReconcileOnly' -and $missingMarkers -notcontains '-ReconcileEvidenceId $ReconcileEvidenceId')
reconcileQueueOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileQueueOnly:$ReconcileQueueOnly' -and $missingMarkers -notcontains '-ReconcileQueueId $ReconcileQueueId')
}
}
function Write-DeployEvidence {
param([bool]$Ok, [string]$Status, [string]$ErrorText, [object[]]$Files, [object]$TaskResult)
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null
@@ -132,6 +190,7 @@ function Write-DeployEvidence {
configMigrations = @($script:ConfigMigrations)
runtimeCopyRetryCount = @($script:RuntimeCopyRetries).Count
runtimeCopyRetries = @($script:RuntimeCopyRetries)
launcherContract = $script:LauncherContract
files = $Files
taskRegistration = $TaskResult
} | ConvertTo-Json -Depth 10 | Set-Content -Path $EvidencePath -Encoding UTF8
@@ -185,6 +244,17 @@ if ($LASTEXITCODE -ne 0) {
throw "Agent99 staged synthetic tests failed."
}
$launcherValidationRoot = Join-Path $StageDir "launcher-contract-work"
$launcherValidationBin = Join-Path $launcherValidationRoot "bin"
New-Item -ItemType Directory -Force -Path $launcherValidationBin | Out-Null
Copy-Item -Force (Join-Path $StageDir "agent99-control-plane.ps1") (Join-Path $launcherValidationBin "agent99-control-plane.ps1")
$null = & (Join-Path $StageDir "agent99-bootstrap.ps1") -AgentRoot $launcherValidationRoot -WriteLauncherOnly
$script:LauncherContract = Get-AgentLauncherContract (Join-Path $launcherValidationRoot "agent99-run.ps1") "staged_no_live_write"
if (-not $script:LauncherContract.ok) {
Write-DeployEvidence $false "preflight_failed" "launcher_contract_failed" @() $null
throw "Agent99 staged launcher contract check failed."
}
$stagedFiles = @($runtimeFiles | ForEach-Object {
$path = Join-Path $StageDir $_
[pscustomobject]@{ name = $_; sha256 = (Get-FileHash -Algorithm SHA256 -Path $path).Hash.ToLowerInvariant() }
@@ -196,7 +266,14 @@ $canonicalHost112Vmx = [string]$canonicalHost112Vm.vmx
if ($ValidateOnly) {
Write-DeployEvidence $true "validated" $null $stagedFiles $null
[pscustomobject]@{ ok = $true; status = "validated"; sourceRevision = $SourceRevision; evidence = $EvidencePath; files = $stagedFiles } | ConvertTo-Json -Depth 8
[pscustomobject]@{
ok = $true
status = "validated"
sourceRevision = $SourceRevision
evidence = $EvidencePath
launcherContract = $script:LauncherContract
files = $stagedFiles
} | ConvertTo-Json -Depth 8
exit 0
}
@@ -217,6 +294,16 @@ try {
$manifestBackedUp = $true
}
$launcherExisted = Test-Path -LiteralPath $LauncherPath -PathType Leaf
if ($launcherExisted) {
Copy-Item -Force $LauncherPath (Join-Path $BackupDir "agent99-run.ps1")
}
$promotionRows += [pscustomobject]@{
name = "agent99-run.ps1"
livePath = $LauncherPath
existed = [bool]$launcherExisted
}
foreach ($name in $runtimeFiles) {
$livePath = Join-Path $BinDir $name
$existed = Test-Path $livePath
@@ -225,6 +312,12 @@ try {
Copy-AgentRuntimeFileWithRetry (Join-Path $StageDir $name) $livePath
}
$null = & (Join-Path $BinDir "agent99-bootstrap.ps1") -AgentRoot $AgentRoot -WriteLauncherOnly
$script:LauncherContract = Get-AgentLauncherContract $LauncherPath "live_post_promotion"
if (-not $script:LauncherContract.ok) {
throw "Promoted Agent99 launcher reconcile contract verification failed."
}
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519")
Add-DefaultProperty $config "sshUsers" ([pscustomobject]@{})
@@ -499,6 +592,7 @@ Write-DeployEvidence $deployOk $deployStatus $(if ($taskResult -and -not $taskRe
backupDir = $BackupDir
runtimeManifest = $ManifestPath
taskRegistration = $taskResult
launcherContract = $script:LauncherContract
files = $stagedFiles
} | ConvertTo-Json -Depth 10
if (-not $deployOk) { exit 2 }

View File

@@ -17,10 +17,11 @@ $BinDir = Join-Path $AgentRoot "bin"
$QueueDir = Join-Path $AgentRoot "queue"
$ProcessedDir = Join-Path $QueueDir "processed"
$SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile"
$SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit"
$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored"
$SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1"
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir | Out-Null
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir, $SameRunRetryAuditDir | Out-Null
function Convert-AgentSafeFileToken {
param([string]$Value)
@@ -36,6 +37,9 @@ function Get-AgentField {
[string]$Name,
[object]$Default = $null
)
if ($Object -is [Collections.IDictionary] -and $Object.Contains($Name)) {
return $Object[$Name]
}
if ($Object -and $Object.PSObject.Properties[$Name]) {
return $Object.PSObject.Properties[$Name].Value
}
@@ -510,6 +514,180 @@ function Test-AgentSameRunOutcomeEligible {
)
}
function Test-AgentSameRunFailedNoWriteOutcomeRetryEligible {
param(
[object]$Result,
[object]$Identity,
[string]$SourceEvidence,
[string]$EvidenceReceiptId,
[string]$QueueId
)
if (-not $Result) { return $false }
$outcome = Get-AgentField $Result "outcome" $null
$canonicalIdentity = Get-AgentCanonicalOutcomeIdentity $Result $outcome
if (-not $outcome -or -not $canonicalIdentity) { return $false }
foreach ($name in @(
"id", "source", "reason", "mode", "reconcileEvidenceId", "evidenceReceiptId",
"automationRunId", "traceId", "workItemId", "idempotencyKey",
"executionGeneration", "incidentId", "approvalId", "projectId", "routeId",
"sourceEventResolutionPolicy"
)) {
if (-not $Result.PSObject.Properties[$name] -or (Get-AgentField $Result $name $null) -isnot [string]) {
return $false
}
}
foreach ($name in @("schemaVersion", "state", "verifierName", "sourceEventEvidence")) {
if (-not $outcome.PSObject.Properties[$name] -or (Get-AgentField $outcome $name $null) -isnot [string]) {
return $false
}
}
if (-not [string](Get-AgentField $outcome "verifierName" "")) { return $false }
$previousSourceEvidence = [string](Get-AgentField $outcome "sourceEventEvidence" "")
if (
-not (Test-AgentPublicReceiptRef $SourceEvidence) -or
-not $SourceEvidence.StartsWith("prometheus:cold-start:") -or
-not (Test-AgentPublicReceiptRef $previousSourceEvidence) -or
-not $previousSourceEvidence.StartsWith("prometheus:cold-start:")
) { return $false }
return [bool](
[string](Get-AgentField $Result "id" "") -eq $QueueId -and
[string](Get-AgentField $Result "source" "") -eq "agent99-same-run-reconcile" -and
[string](Get-AgentField $Result "reason" "") -eq "source_event_resolved_status_reconcile" -and
[string](Get-AgentField $Result "mode" "") -eq "Status" -and
(Get-AgentField $Result "controlledApply" $null) -is [bool] -and
-not [bool](Get-AgentField $Result "controlledApply" $true) -and
(Get-AgentField $Result "reconcileOnly" $null) -is [bool] -and
[bool](Get-AgentField $Result "reconcileOnly" $false) -and
(Get-AgentField $Result "runtimeWritePerformed" $null) -is [bool] -and
-not [bool](Get-AgentField $Result "runtimeWritePerformed" $true) -and
(Get-AgentField $Result "suppressTelegram" $null) -is [bool] -and
[bool](Get-AgentField $Result "suppressTelegram" $false) -and
[string](Get-AgentField $Result "reconcileEvidenceId" "") -eq $EvidenceReceiptId -and
[string](Get-AgentField $Result "evidenceReceiptId" "") -eq $EvidenceReceiptId -and
[string](Get-AgentField $Result "automationRunId" "") -eq [string](Get-AgentField $Identity "run_id" "") -and
[string](Get-AgentField $Result "traceId" "") -eq [string](Get-AgentField $Identity "trace_id" "") -and
[string](Get-AgentField $Result "workItemId" "") -eq [string](Get-AgentField $Identity "work_item_id" "") -and
[string](Get-AgentField $Result "idempotencyKey" "") -eq [string](Get-AgentField $Identity "idempotency_key" "") -and
[string](Get-AgentField $Result "executionGeneration" "") -eq [string](Get-AgentField $Identity "execution_generation" "") -and
[string](Get-AgentField $Result "incidentId" "") -eq [string](Get-AgentField $Identity "incident_id" "") -and
[string](Get-AgentField $Result "approvalId" "") -eq [string](Get-AgentField $Identity "approval_id" "") -and
[string](Get-AgentField $Result "projectId" "") -eq [string](Get-AgentField $Identity "project_id" "") -and
[string](Get-AgentField $Result "routeId" "") -eq [string](Get-AgentField $Identity "route_id" "") -and
(Test-AgentSameRunIdentityMatchesExisting $canonicalIdentity $Identity) -and
[string](Get-AgentField $Result "sourceEventResolutionPolicy" "") -eq "external_source_receipt_required" -and
[string](Get-AgentField $outcome "schemaVersion" "") -eq "agent99_outcome_contract_v1" -and
[string](Get-AgentField $outcome "state" "") -eq "failed" -and
(Get-AgentField $outcome "resolved" $null) -is [bool] -and
-not [bool](Get-AgentField $outcome "resolved" $true) -and
(Get-AgentField $outcome "transportOk" $null) -is [bool] -and
-not [bool](Get-AgentField $outcome "transportOk" $true) -and
(Get-AgentField $outcome "verifierPassed" $null) -is [bool] -and
-not [bool](Get-AgentField $outcome "verifierPassed" $true) -and
(Get-AgentField $outcome "sourceEventResolved" $null) -is [bool] -and
[bool](Get-AgentField $outcome "sourceEventResolved" $false)
)
}
function Move-AgentSameRunFailedNoWriteOutcomeForRetry {
param(
[string]$ProcessedPath,
[string]$LockPath,
[string]$RetryMarkerPath,
[string]$QueueId,
[object]$Identity,
[string]$SourceEvidence,
[string]$EvidenceReceiptId
)
if (Test-Path -LiteralPath $RetryMarkerPath) {
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_already_consumed_fail_closed" }
}
if (-not (Test-Path -LiteralPath $ProcessedPath) -or -not (Test-Path -LiteralPath $LockPath)) {
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_missing_fail_closed" }
}
try {
$failedResult = Get-Content -LiteralPath $ProcessedPath -Raw | ConvertFrom-Json
$lockReceipt = Get-Content -LiteralPath $LockPath -Raw | ConvertFrom-Json
$failedOutcomeSha256 = (Get-FileHash -LiteralPath $ProcessedPath -Algorithm SHA256).Hash.ToLowerInvariant()
$previousLockSha256 = (Get-FileHash -LiteralPath $LockPath -Algorithm SHA256).Hash.ToLowerInvariant()
} catch {
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_invalid_fail_closed" }
}
if ($failedOutcomeSha256 -notmatch "^[0-9a-f]{64}$" -or $previousLockSha256 -notmatch "^[0-9a-f]{64}$") {
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_digest_invalid_fail_closed" }
}
if (-not (Test-AgentSameRunFailedNoWriteOutcomeRetryEligible $failedResult $Identity $SourceEvidence $EvidenceReceiptId $QueueId)) {
return [pscustomobject]@{ ok = $false; reason = "existing_same_run_status_outcome_invalid_fail_closed" }
}
$runId = [string](Get-AgentField $Identity "run_id" "")
$canonicalDigest = [string](Get-AgentField $Identity "canonical_digest" "")
$lockValid = [bool](
[string](Get-AgentField $lockReceipt "schemaVersion" "") -eq "agent99_same_run_single_flight_lock_v1" -and
[string](Get-AgentField $lockReceipt "automationRunId" "") -eq $runId -and
[string](Get-AgentField $lockReceipt "evidenceReceiptId" "") -eq $EvidenceReceiptId -and
[string](Get-AgentField $lockReceipt "canonicalDigest" "") -eq $canonicalDigest -and
(Get-AgentField $lockReceipt "storesRawEvidence" $null) -is [bool] -and
-not [bool](Get-AgentField $lockReceipt "storesRawEvidence" $true)
)
if (-not $lockValid) {
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_lock_invalid_fail_closed" }
}
$outcomeAuditPath = Join-Path $SameRunRetryAuditDir "$QueueId.attempt-1.processed.json"
$lockAuditPath = Join-Path $SameRunRetryAuditDir "$QueueId.attempt-1.lock.json"
if ((Test-Path -LiteralPath $outcomeAuditPath) -or (Test-Path -LiteralPath $lockAuditPath)) {
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_audit_collision_fail_closed" }
}
try {
$markerStream = [IO.File]::Open($RetryMarkerPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
try {
$markerReceipt = @{
schemaVersion = "agent99_same_run_status_retry_v1"
status = "retry_reserved"
retryOrdinal = 1
automationRunId = $runId
evidenceReceiptId = $EvidenceReceiptId
canonicalDigest = $canonicalDigest
failedOutcomeAuditFile = (Split-Path -Leaf $outcomeAuditPath)
failedOutcomeSha256 = $failedOutcomeSha256
previousLockAuditFile = (Split-Path -Leaf $lockAuditPath)
previousLockSha256 = $previousLockSha256
preservesFailedOutcome = $true
controlledApply = $false
reconcileOnly = $true
runtimeWriteAuthorized = $false
storesRawEvidence = $false
createdAt = (Get-Date -Format o)
} | ConvertTo-Json -Compress
$markerBytes = [Text.Encoding]::UTF8.GetBytes($markerReceipt)
$markerStream.Write($markerBytes, 0, $markerBytes.Length)
$markerStream.Flush($true)
} finally {
$markerStream.Dispose()
}
} catch [System.IO.IOException] {
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_already_consumed_fail_closed" }
}
try {
Move-Item -LiteralPath $ProcessedPath -Destination $outcomeAuditPath
Move-Item -LiteralPath $LockPath -Destination $lockAuditPath
} catch {
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_archive_failed_fail_closed" }
}
return [pscustomobject]@{
ok = $true
reason = "same_run_status_failed_no_write_outcome_archived_for_single_retry"
retryOrdinal = 1
}
}
function Start-AgentSameRunStatusReconcile {
param([object]$Request)
@@ -558,6 +736,15 @@ function Start-AgentSameRunStatusReconcile {
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_dispatch_identity_mismatch"; httpStatus = 409 }
}
$safeRunId = Convert-AgentSafeFileToken $runId
$queueId = "same-run-status-$safeRunId"
$queuePath = Join-Path $QueueDir "$queueId.json"
$runningPath = Join-Path $QueueDir "running-$queueId.json"
$processedPath = Join-Path $ProcessedDir "processed-$queueId.json"
$lockPath = Join-Path $SameRunStateDir "$queueId.lock.json"
$retryMarkerPath = Join-Path $SameRunStateDir "$queueId.retry-1.json"
$singleRetryPrepared = $false
$existing = Get-AgentOutcomeReadback $runId
if (Test-AgentSameRunOutcomeEligible $existing $identity $sourceEvidence $evidenceReceiptId) {
return [pscustomobject]@{
@@ -572,15 +759,33 @@ function Start-AgentSameRunStatusReconcile {
}
}
if ($existing.found -and [bool](Get-AgentField $existing "reconcileOnly" $false)) {
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_same_run_status_outcome_invalid_fail_closed"; httpStatus = 409 }
$retryPreparation = Move-AgentSameRunFailedNoWriteOutcomeForRetry `
$processedPath $lockPath $retryMarkerPath $queueId $identity $sourceEvidence $evidenceReceiptId
if (-not $retryPreparation.ok) {
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = [string]$retryPreparation.reason; httpStatus = 409 }
}
$singleRetryPrepared = $true
}
$queuePending = Test-Path -LiteralPath $queuePath -PathType Leaf
$runningPending = Test-Path -LiteralPath $runningPath -PathType Leaf
$lockPending = Test-Path -LiteralPath $lockPath -PathType Leaf
if (-not $singleRetryPrepared -and (Test-Path -LiteralPath $retryMarkerPath -PathType Leaf)) {
if (-not (($queuePending -or $runningPending) -and $lockPending)) {
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "same_run_status_retry_already_consumed_fail_closed"; httpStatus = 409 }
}
return [pscustomobject]@{
ok = $true
status = "same_run_status_reconcile_pending"
httpStatus = 202
automationRunId = $runId
reconcileOnly = $true
controlledApply = $false
evidenceReceiptId = $evidenceReceiptId
queued = $false
}
}
$safeRunId = Convert-AgentSafeFileToken $runId
$queueId = "same-run-status-$safeRunId"
$queuePath = Join-Path $QueueDir "$queueId.json"
$runningPath = Join-Path $QueueDir "running-$queueId.json"
$lockPath = Join-Path $SameRunStateDir "$queueId.lock.json"
if ((Test-Path -LiteralPath $queuePath) -or (Test-Path -LiteralPath $runningPath) -or (Test-Path -LiteralPath $lockPath)) {
if ($queuePending -or $runningPending -or $lockPending) {
return [pscustomobject]@{
ok = $true
status = "same_run_status_reconcile_pending"

View File

@@ -829,3 +829,73 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() ->
assert "Get-AgentEvidenceAtPath $exactEvidencePath" in queue_branch
assert "Send-AgentTelegram" not in queue_branch
assert "Invoke-AgentCompletionCallback" not in queue_branch
def test_windows_relay_allows_only_one_audited_failed_no_write_status_retry() -> None:
root = Path(__file__).resolve().parents[3]
relay = (root / "agent99-sre-alert-relay.ps1").read_text(encoding="utf-8")
field_helper = relay[relay.index("function Get-AgentField") :]
field_helper = field_helper[: field_helper.index("function Get-AgentEnvValue")]
assert "$Object -is [Collections.IDictionary]" in field_helper
assert "$Object.Contains($Name)" in field_helper
assert "return $Object[$Name]" in field_helper
retry_start = relay.index(
"function Test-AgentSameRunFailedNoWriteOutcomeRetryEligible"
)
retry_end = relay.index("function Start-AgentSameRunStatusReconcile")
retry_contract = relay[retry_start:retry_end]
assert '$SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit"' in relay
assert '"agent99_same_run_status_retry_v1"' in retry_contract
assert 'retryOrdinal = 1' in retry_contract
assert 'status = "retry_reserved"' in retry_contract
assert 'preservesFailedOutcome = $true' in retry_contract
assert 'runtimeWriteAuthorized = $false' in retry_contract
assert '[string](Get-AgentField $Result "mode" "") -eq "Status"' in retry_contract
assert '[string](Get-AgentField $outcome "state" "") -eq "failed"' in retry_contract
assert '-not [bool](Get-AgentField $Result "controlledApply" $true)' in retry_contract
assert '[bool](Get-AgentField $Result "reconcileOnly" $false)' in retry_contract
assert '-not [bool](Get-AgentField $Result "runtimeWritePerformed" $true)' in retry_contract
assert '[bool](Get-AgentField $Result "suppressTelegram" $false)' in retry_contract
assert '-not [bool](Get-AgentField $outcome "resolved" $true)' in retry_contract
assert '-not [bool](Get-AgentField $outcome "transportOk" $true)' in retry_contract
assert '-not [bool](Get-AgentField $outcome "verifierPassed" $true)' in retry_contract
assert '[bool](Get-AgentField $outcome "sourceEventResolved" $false)' in retry_contract
assert "Test-AgentPublicReceiptRef $SourceEvidence" in retry_contract
assert "Test-AgentPublicReceiptRef $previousSourceEvidence" in retry_contract
assert '$previousSourceEvidence.StartsWith("prometheus:cold-start:")' in retry_contract
assert "Test-AgentSameRunIdentityMatchesExisting $canonicalIdentity $Identity" in retry_contract
assert '"agent99_same_run_single_flight_lock_v1"' in retry_contract
assert '[IO.FileMode]::CreateNew' in retry_contract
assert 'failedOutcomeSha256 = $failedOutcomeSha256' in retry_contract
assert 'previousLockSha256 = $previousLockSha256' in retry_contract
assert 'Move-Item -LiteralPath $ProcessedPath -Destination $outcomeAuditPath' in retry_contract
assert 'Move-Item -LiteralPath $LockPath -Destination $lockAuditPath' in retry_contract
assert 'same_run_status_retry_already_consumed_fail_closed' in retry_contract
assert 'same_run_status_retry_archive_failed_fail_closed' in retry_contract
assert 'mode = "Recover"' not in retry_contract
assert 'controlledApply = $true' not in retry_contract
start = relay[relay.index("function Start-AgentSameRunStatusReconcile") :]
marker_branch = start.index(
"if (-not $singleRetryPrepared -and "
"(Test-Path -LiteralPath $retryMarkerPath -PathType Leaf))"
)
marker_rejected = start.index(
'reason = "same_run_status_retry_already_consumed_fail_closed"',
marker_branch,
)
marker_pending = start.index(
'status = "same_run_status_reconcile_pending"',
marker_rejected,
)
general_pending = start.index(
"if ($queuePending -or $runningPending -or $lockPending)",
marker_pending,
)
assert marker_branch < marker_rejected < marker_pending < general_pending
assert "(($queuePending -or $runningPending) -and $lockPending)" in start[
marker_branch:marker_pending
]

View File

@@ -8,6 +8,9 @@ CONTROL = ROOT / "agent99-control-plane.ps1"
DEPLOY = ROOT / "agent99-deploy.ps1"
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
REMOTE_ATOMIC_RECEIVER = (
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
)
def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None:
@@ -312,6 +315,84 @@ def test_agent99_deployer_backs_up_manifest_before_runtime_promotion() -> None:
assert "Copy-AgentRuntimeFileWithRetry (Join-Path $StageDir $name) $livePath" in deploy
def test_agent99_atomic_deploy_regenerates_and_verifies_root_launcher() -> None:
bootstrap = BOOTSTRAP.read_text(encoding="utf-8")
deploy = DEPLOY.read_text(encoding="utf-8")
assert "[switch]$WriteLauncherOnly" in bootstrap
assert "function Write-AgentLauncher" in bootstrap
launcher_only = bootstrap[bootstrap.index("if ($WriteLauncherOnly)") :]
launcher_only = launcher_only[: launcher_only.index("function Copy-AgentFile")]
assert 'Join-Path $BinDir "agent99-control-plane.ps1"' in launcher_only
assert 'Write-AgentLauncher (Join-Path $AgentRoot "agent99-run.ps1")' in launcher_only
assert "Ensure-AgentHost112CanonicalSshConfig" not in launcher_only
assert "Set-AgentBootstrapProperty" not in launcher_only
staged_check = deploy[deploy.index('$launcherValidationRoot = Join-Path $StageDir') :]
staged_check = staged_check[: staged_check.index("$stagedFiles = @(")]
assert "-WriteLauncherOnly" in staged_check
assert '"staged_no_live_write"' in staged_check
assert 'Write-DeployEvidence $false "preflight_failed" "launcher_contract_failed"' in staged_check
promotion = deploy[deploy.index("$configBackedUp = $false") :]
promotion = promotion[: promotion.index("$config = Get-Content $ConfigPath")]
assert promotion.index('Copy-Item -Force $LauncherPath') < promotion.index(
"foreach ($name in $runtimeFiles)"
)
assert 'name = "agent99-run.ps1"' in promotion
assert 'Join-Path $BinDir "agent99-bootstrap.ps1"' in promotion
assert 'Get-AgentLauncherContract $LauncherPath "live_post_promotion"' in promotion
assert "Promoted Agent99 launcher reconcile contract verification failed." in promotion
assert "launcherContract = $script:LauncherContract" in deploy
def test_agent99_launcher_contract_covers_same_run_flags_and_forwarding() -> None:
bootstrap = BOOTSTRAP.read_text(encoding="utf-8")
deploy = DEPLOY.read_text(encoding="utf-8")
for marker in (
'[switch]`$ReconcileOnly',
'[string]`$ReconcileEvidenceId = ""',
'[switch]`$ReconcileQueueOnly',
'[string]`$ReconcileQueueId = ""',
"-ReconcileOnly:`$ReconcileOnly",
"-ReconcileEvidenceId `$ReconcileEvidenceId",
"-ReconcileQueueOnly:`$ReconcileQueueOnly",
"-ReconcileQueueId `$ReconcileQueueId",
):
assert marker in bootstrap
assert "Management.Automation.Language.Parser" in bootstrap
assert "Management.Automation.Language.Parser" in deploy
assert "reconcileOnlyForwarded = $true" in bootstrap
assert "reconcileQueueOnlyForwarded = $true" in bootstrap
def test_agent99_remote_rollback_and_post_verifier_include_root_launcher() -> None:
receiver = REMOTE_ATOMIC_RECEIVER.read_text(encoding="utf-8")
assert '$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1"' in receiver
assert "function Get-AgentLauncherContract" in receiver
assert '$rows += "agent99-run.ps1`t$(Get-AgentSha256File $LauncherPath)`n"' in receiver
restore = receiver[receiver.index("function Restore-AgentBundle") :]
restore = restore[: restore.index("function Write-AgentRemoteDeployReceipt")]
assert "[bool]$BeforeLauncherPresent" in restore
assert 'Join-Path $BackupPath "agent99-run.ps1"' in restore
assert "Copy-AgentFileWithRetry $backupLauncher $LauncherPath" in restore
assert "Remove-Item -LiteralPath $LauncherPath -Force" in restore
assert "$liveLauncherContract.ok" in receiver
assert (
'[string]$liveLauncherContract.sha256 -eq '
'[string]$prior.launcherContract.sha256'
) in receiver
assert "$launcherContract.ok" in receiver
assert (
'[string]$launcherContract.sha256 -eq '
'[string]$deploy.payload.launcherContract.sha256'
) in receiver
assert "launcherContract = $launcherContract" in receiver
assert "failedLauncherContract = $launcherContract" in receiver
def test_agent99_completion_callback_waits_for_durable_same_trace_readback() -> None:
source = CONTROL.read_text(encoding="utf-8")
deploy = DEPLOY.read_text(encoding="utf-8")

View File

@@ -13,6 +13,7 @@ $StateDir = Join-Path $AgentRoot "state"
$EvidenceDir = Join-Path $AgentRoot "evidence"
$DeployRoot = Join-Path $AgentRoot "deploy"
$ManifestPath = Join-Path $StateDir "runtime-manifest.json"
$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1"
$WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"
$RelayTaskPath = "\"
@@ -71,6 +72,48 @@ function Get-AgentSha256File {
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
}
function Get-AgentLauncherContract {
$requiredMarkers = @(
'[switch]$ReconcileOnly',
'[string]$ReconcileEvidenceId = ""',
'[switch]$ReconcileQueueOnly',
'[string]$ReconcileQueueId = ""',
'-ReconcileOnly:$ReconcileOnly',
'-ReconcileEvidenceId $ReconcileEvidenceId',
'-ReconcileQueueOnly:$ReconcileQueueOnly',
'-ReconcileQueueId $ReconcileQueueId'
)
if (-not (Test-Path -LiteralPath $LauncherPath -PathType Leaf)) {
return [pscustomobject]@{
schemaVersion = "agent99_launcher_contract_v1"
ok = $false
exists = $false
sha256 = "missing"
parseErrorCount = -1
requiredMarkerCount = $requiredMarkers.Count
missingMarkerCount = $requiredMarkers.Count
reconcileOnlyForwarded = $false
reconcileQueueOnlyForwarded = $false
}
}
$tokens = $null
$parseErrors = $null
[Management.Automation.Language.Parser]::ParseFile($LauncherPath, [ref]$tokens, [ref]$parseErrors) | Out-Null
$content = [IO.File]::ReadAllText($LauncherPath)
$missingMarkers = @($requiredMarkers | Where-Object { -not $content.Contains($_) })
return [pscustomobject]@{
schemaVersion = "agent99_launcher_contract_v1"
ok = [bool](@($parseErrors).Count -eq 0 -and $missingMarkers.Count -eq 0)
exists = $true
sha256 = (Get-AgentSha256File $LauncherPath)
parseErrorCount = @($parseErrors).Count
requiredMarkerCount = $requiredMarkers.Count
missingMarkerCount = $missingMarkers.Count
reconcileOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileOnly:$ReconcileOnly' -and $missingMarkers -notcontains '-ReconcileEvidenceId $ReconcileEvidenceId')
reconcileQueueOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileQueueOnly:$ReconcileQueueOnly' -and $missingMarkers -notcontains '-ReconcileQueueId $ReconcileQueueId')
}
}
function Get-AgentSafeRuntimeManifest {
if (-not (Test-Path -LiteralPath $ManifestPath -PathType Leaf)) {
return [pscustomobject]@{
@@ -146,6 +189,7 @@ function Get-AgentLiveBundleDigest {
foreach ($name in $FixedRuntimeFiles) {
$rows += "$name`t$(Get-AgentSha256File (Join-Path $BinDir $name))`n"
}
$rows += "agent99-run.ps1`t$(Get-AgentSha256File $LauncherPath)`n"
return Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes(($rows -join "")))
}
@@ -555,6 +599,7 @@ function Restore-AgentBundle {
param(
[string]$BackupPath,
[hashtable]$BeforeFilePresence,
[bool]$BeforeLauncherPresent,
[bool]$BeforeConfigPresent,
[bool]$BeforeManifestPresent
)
@@ -573,6 +618,13 @@ function Restore-AgentBundle {
Remove-Item -LiteralPath $livePath -Force
}
}
$backupLauncher = Join-Path $BackupPath "agent99-run.ps1"
if ($BeforeLauncherPresent) {
if (-not (Test-Path -LiteralPath $backupLauncher -PathType Leaf)) { throw "required_launcher_backup_missing" }
Copy-AgentFileWithRetry $backupLauncher $LauncherPath
} elseif (Test-Path -LiteralPath $LauncherPath -PathType Leaf) {
Remove-Item -LiteralPath $LauncherPath -Force
}
$backupConfig = Join-Path $BackupPath "agent99.config.json"
if ($BeforeConfigPresent) {
if (-not (Test-Path -LiteralPath $backupConfig -PathType Leaf)) { throw "required_config_backup_missing" }
@@ -832,6 +884,7 @@ try {
}
$live = Get-AgentSafeRuntimeManifest
$liveLauncherContract = Get-AgentLauncherContract
if (
$prior -and
[string]$prior.status -eq "deployed_verified" -and
@@ -841,10 +894,15 @@ try {
[string]$prior.sourceRevision -eq $sourceRevision -and
$prior.PSObject.Properties["relayReload"] -and
[bool]$prior.relayReload.newGenerationVerified -and
$prior.PSObject.Properties["launcherContract"] -and
[bool]$prior.launcherContract.ok -and
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
$live.runtimeMatched -and
$live.sourceRevision -eq $sourceRevision -and
$live.fileCount -eq 14 -and
$live.mismatchCount -eq 0
$live.mismatchCount -eq 0 -and
$liveLauncherContract.ok -and
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
) {
$script:ReceiverOperationPhase = "idempotent_replay_post_verifier"
$replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
@@ -852,6 +910,7 @@ try {
$originalOperationBoundaries = $prior.operationBoundaries
$prior | Add-Member -NotePropertyName idempotentReplay -NotePropertyValue $true -Force
$prior | Add-Member -NotePropertyName runtimeManifest -NotePropertyValue $live -Force
$prior | Add-Member -NotePropertyName launcherContract -NotePropertyValue $liveLauncherContract -Force
$prior | Add-Member -NotePropertyName livePreflight -NotePropertyValue $replayPreflight -Force
$prior | Add-Member -NotePropertyName originalOperationBoundaries -NotePropertyValue $originalOperationBoundaries -Force
$prior | Add-Member -NotePropertyName operationBoundaries -NotePropertyValue ([pscustomobject]@{
@@ -979,6 +1038,7 @@ try {
$beforeManifestDigest = Get-AgentSha256File $ManifestPath
$beforeFilePresence = @{}
foreach ($name in $FixedRuntimeFiles) { $beforeFilePresence[$name] = Test-Path -LiteralPath (Join-Path $BinDir $name) -PathType Leaf }
$beforeLauncherPresent = Test-Path -LiteralPath $LauncherPath -PathType Leaf
$beforeConfigPresent = Test-Path -LiteralPath $ConfigPath -PathType Leaf
$beforeManifestPresent = Test-Path -LiteralPath $ManifestPath -PathType Leaf
$beforeRelayRuntime = Get-AgentRelayRuntime
@@ -1034,6 +1094,7 @@ try {
$relayReload = $null
$runtimeManifest = $null
$livePreflight = $null
$launcherContract = $null
$postVerified = $false
$failurePhase = ""
$failureType = ""
@@ -1046,7 +1107,15 @@ try {
if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) {
$backupPath = [string]$deploy.payload.backupDir
}
if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) {
if (
-not $deploy.ok -or
-not $deploy.payload -or
[string]$deploy.payload.status -ne "deployed" -or
$deploy.payload.taskRegistration -or
-not $deploy.payload.PSObject.Properties["launcherContract"] -or
-not [bool]$deploy.payload.launcherContract.ok -or
[string]$deploy.payload.launcherContract.sha256 -notmatch "^[0-9a-f]{64}$"
) {
$failurePhase = "deploy"
$failureType = "deploy_child_failed_or_invalid"
} else {
@@ -1060,6 +1129,7 @@ try {
} else {
$script:ReceiverOperationPhase = "independent_post_verifier"
$runtimeManifest = Get-AgentSafeRuntimeManifest
$launcherContract = Get-AgentLauncherContract
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$postVerified = [bool](
$runtimeManifest.exists -and
@@ -1067,6 +1137,8 @@ try {
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 14 -and
$runtimeManifest.mismatchCount -eq 0 -and
$launcherContract.ok -and
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
$relayReload.oldGenerationsGone -and
$relayReload.newGenerationVerified -and
$relayReload.after.taskEnabled -and
@@ -1108,6 +1180,7 @@ try {
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
relayReload = $relayReload
runtimeManifest = $runtimeManifest
launcherContract = $launcherContract
livePreflight = $livePreflight
durableReceiptWritten = $true
idempotentReplay = $false
@@ -1147,7 +1220,7 @@ try {
}
$script:ReceiverOperationPhase = "runtime_rollback"
$script:ReceiverRollbackAttempted = $true
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeLauncherPresent $beforeConfigPresent $beforeManifestPresent
$rollbackForbiddenGenerations = @($beforeRelayRuntime.taskGeneration)
if ($relayReload) {
$rollbackForbiddenGenerations += @($relayReload.before.taskGeneration)
@@ -1227,6 +1300,7 @@ try {
attemptReceiptPath = $attemptReceiptPath
relayReload = $relayReload
failedRuntimeManifest = $runtimeManifest
failedLauncherContract = $launcherContract
failedLivePreflight = $livePreflight
rollback = $rollback
rollbackRelayRestart = $rollbackRelayRestart