From b233ac5c74eca465b256c25517a13e76c986f3b8 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 23:28:19 +0800 Subject: [PATCH] fix(agent99): complete same-run status reconcile --- agent99-bootstrap.ps1 | 101 ++++++-- agent99-deploy.ps1 | 96 +++++++- agent99-sre-alert-relay.ps1 | 221 +++++++++++++++++- .../tests/test_agent99_same_run_reconcile.py | 70 ++++++ ...nt99_transport_recovery_deploy_contract.py | 81 +++++++ .../agent99-remote-atomic-deploy-receiver.ps1 | 80 ++++++- 6 files changed, 617 insertions(+), 32 deletions(-) diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 2f3674a3c..4325a64ba 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -1,7 +1,8 @@ param( [string]$AgentRoot = "C:\Wooo\Agent99", [string]$SourceRevision = "", - [switch]$InstallScheduledTasks + [switch]$InstallScheduledTasks, + [switch]$WriteLauncherOnly ) $ErrorActionPreference = "Stop" @@ -9,6 +10,7 @@ $ErrorActionPreference = "Stop" $SourceRoot = Split-Path -Parent $MyInvocation.MyCommand.Path $BinDir = Join-Path $AgentRoot "bin" $ConfigDir = Join-Path $AgentRoot "config" +$ConfigPath = Join-Path $ConfigDir "agent99.config.json" $EvidenceDir = Join-Path $AgentRoot "evidence" $LogDir = Join-Path $AgentRoot "logs" $QueueDir = Join-Path $AgentRoot "queue" @@ -18,6 +20,83 @@ $PlaybookDir = Join-Path $AgentRoot "playbooks" $DashboardDir = Join-Path $AgentRoot "dashboard" $StateDir = Join-Path $AgentRoot "state" +function Write-AgentLauncher { + param([string]$Path) + + $launcherContent = @" +param( + [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] + [string]`$Mode = "Status", + [switch]`$ControlledApply, + [switch]`$ReconcileOnly, + [string]`$ReconcileEvidenceId = "", + [switch]`$ReconcileQueueOnly, + [string]`$ReconcileQueueId = "", + [string]`$AutomationRunId = "", + [string]`$TraceId = "", + [string]`$WorkItemId = "", + [switch]`$SuppressAlerts, + [string]`$SuppressReason = "" +) + +& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ReconcileOnly:`$ReconcileOnly -ReconcileEvidenceId `$ReconcileEvidenceId -ReconcileQueueOnly:`$ReconcileQueueOnly -ReconcileQueueId `$ReconcileQueueId -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$ConfigPath" -EvidenceDir "$EvidenceDir" +"@ + $parent = Split-Path -Parent $Path + if (-not (Test-Path -LiteralPath $parent -PathType Container)) { + throw "Agent99 launcher parent is unavailable: $parent" + } + $temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))" + try { + [IO.File]::WriteAllText($temporary, $launcherContent, (New-Object Text.UTF8Encoding($true))) + $tokens = $null + $parseErrors = $null + [Management.Automation.Language.Parser]::ParseFile($temporary, [ref]$tokens, [ref]$parseErrors) | Out-Null + if (@($parseErrors).Count -ne 0) { + throw "Agent99 launcher PowerShell parser check failed." + } + Move-Item -LiteralPath $temporary -Destination $Path -Force + $written = [IO.File]::ReadAllText($Path) + $requiredMarkers = @( + '[switch]$ReconcileOnly', + '[string]$ReconcileEvidenceId = ""', + '[switch]$ReconcileQueueOnly', + '[string]$ReconcileQueueId = ""', + '-ReconcileOnly:$ReconcileOnly', + '-ReconcileEvidenceId $ReconcileEvidenceId', + '-ReconcileQueueOnly:$ReconcileQueueOnly', + '-ReconcileQueueId $ReconcileQueueId' + ) + $missingMarkers = @($requiredMarkers | Where-Object { -not $written.Contains($_) }) + if ($missingMarkers.Count -ne 0) { + throw "Agent99 launcher reconcile contract verification failed." + } + return [pscustomobject]@{ + schemaVersion = "agent99_launcher_contract_v1" + ok = $true + path = $Path + sha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + parseErrorCount = 0 + requiredMarkerCount = $requiredMarkers.Count + missingMarkerCount = 0 + reconcileOnlyForwarded = $true + reconcileQueueOnlyForwarded = $true + } + } finally { + if (Test-Path -LiteralPath $temporary) { + Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue + } + } +} + +if ($WriteLauncherOnly) { + $controlPlanePath = Join-Path $BinDir "agent99-control-plane.ps1" + if (-not (Test-Path -LiteralPath $controlPlanePath -PathType Leaf)) { + throw "Agent99 control plane is unavailable for launcher generation: $controlPlanePath" + } + Write-AgentLauncher (Join-Path $AgentRoot "agent99-run.ps1") + return +} + function Copy-AgentFile { param([string]$Name) $src = Join-Path $SourceRoot $Name @@ -210,7 +289,6 @@ if (-not $runtimeMatched) { } $exampleConfig = Join-Path $SourceRoot "agent99.config.example.json" -$configPath = Join-Path $ConfigDir "agent99.config.json" if (-not (Test-Path $configPath)) { if (Test-Path $exampleConfig) { Copy-Item $exampleConfig $configPath @@ -290,24 +368,7 @@ if (-not (Test-Path $configPath)) { Ensure-AgentHost112CanonicalSshConfig $configPath $host112BootstrapPolicyPath $launcher = Join-Path $AgentRoot "agent99-run.ps1" -@" -param( - [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] - [string]`$Mode = "Status", - [switch]`$ControlledApply, - [switch]`$ReconcileOnly, - [string]`$ReconcileEvidenceId = "", - [switch]`$ReconcileQueueOnly, - [string]`$ReconcileQueueId = "", - [string]`$AutomationRunId = "", - [string]`$TraceId = "", - [string]`$WorkItemId = "", - [switch]`$SuppressAlerts, - [string]`$SuppressReason = "" -) - -& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ReconcileOnly:`$ReconcileOnly -ReconcileEvidenceId `$ReconcileEvidenceId -ReconcileQueueOnly:`$ReconcileQueueOnly -ReconcileQueueId `$ReconcileQueueId -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir" -"@ | Set-Content -Encoding UTF8 $launcher +$null = Write-AgentLauncher $launcher $submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd" @" diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 99a5a0d7b..c286130fb 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -16,9 +16,18 @@ $DeployRoot = Join-Path $AgentRoot "deploy" $StageDir = Join-Path $DeployRoot "staging-$stamp" $BackupDir = Join-Path $DeployRoot "backup-$stamp" $ManifestPath = Join-Path $StateDir "runtime-manifest.json" +$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1" $EvidencePath = Join-Path $EvidenceDir "agent99-deploy-$stamp.json" $script:ConfigMigrations = @() $script:RuntimeCopyRetries = @() +$script:LauncherContract = [pscustomobject]@{ + schemaVersion = "agent99_launcher_contract_v1" + ok = $false + scope = "not_checked" + exists = $false + parseErrorCount = -1 + missingMarkerCount = -1 +} $runtimeFiles = @( "agent99-bootstrap.ps1", @@ -112,6 +121,55 @@ function Copy-AgentRuntimeFileWithRetry { } } +function Get-AgentLauncherContract { + param([string]$Path, [string]$Scope) + + $requiredMarkers = @( + '[switch]$ReconcileOnly', + '[string]$ReconcileEvidenceId = ""', + '[switch]$ReconcileQueueOnly', + '[string]$ReconcileQueueId = ""', + '-ReconcileOnly:$ReconcileOnly', + '-ReconcileEvidenceId $ReconcileEvidenceId', + '-ReconcileQueueOnly:$ReconcileQueueOnly', + '-ReconcileQueueId $ReconcileQueueId' + ) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return [pscustomobject]@{ + schemaVersion = "agent99_launcher_contract_v1" + ok = $false + scope = $Scope + exists = $false + path = $Path + sha256 = "" + parseErrorCount = -1 + requiredMarkerCount = $requiredMarkers.Count + missingMarkerCount = $requiredMarkers.Count + reconcileOnlyForwarded = $false + reconcileQueueOnlyForwarded = $false + } + } + $tokens = $null + $parseErrors = $null + [Management.Automation.Language.Parser]::ParseFile($Path, [ref]$tokens, [ref]$parseErrors) | Out-Null + $content = [IO.File]::ReadAllText($Path) + $missingMarkers = @($requiredMarkers | Where-Object { -not $content.Contains($_) }) + $ok = [bool](@($parseErrors).Count -eq 0 -and $missingMarkers.Count -eq 0) + return [pscustomobject]@{ + schemaVersion = "agent99_launcher_contract_v1" + ok = $ok + scope = $Scope + exists = $true + path = $Path + sha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + parseErrorCount = @($parseErrors).Count + requiredMarkerCount = $requiredMarkers.Count + missingMarkerCount = $missingMarkers.Count + reconcileOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileOnly:$ReconcileOnly' -and $missingMarkers -notcontains '-ReconcileEvidenceId $ReconcileEvidenceId') + reconcileQueueOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileQueueOnly:$ReconcileQueueOnly' -and $missingMarkers -notcontains '-ReconcileQueueId $ReconcileQueueId') + } +} + function Write-DeployEvidence { param([bool]$Ok, [string]$Status, [string]$ErrorText, [object[]]$Files, [object]$TaskResult) New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null @@ -132,6 +190,7 @@ function Write-DeployEvidence { configMigrations = @($script:ConfigMigrations) runtimeCopyRetryCount = @($script:RuntimeCopyRetries).Count runtimeCopyRetries = @($script:RuntimeCopyRetries) + launcherContract = $script:LauncherContract files = $Files taskRegistration = $TaskResult } | ConvertTo-Json -Depth 10 | Set-Content -Path $EvidencePath -Encoding UTF8 @@ -185,6 +244,17 @@ if ($LASTEXITCODE -ne 0) { throw "Agent99 staged synthetic tests failed." } +$launcherValidationRoot = Join-Path $StageDir "launcher-contract-work" +$launcherValidationBin = Join-Path $launcherValidationRoot "bin" +New-Item -ItemType Directory -Force -Path $launcherValidationBin | Out-Null +Copy-Item -Force (Join-Path $StageDir "agent99-control-plane.ps1") (Join-Path $launcherValidationBin "agent99-control-plane.ps1") +$null = & (Join-Path $StageDir "agent99-bootstrap.ps1") -AgentRoot $launcherValidationRoot -WriteLauncherOnly +$script:LauncherContract = Get-AgentLauncherContract (Join-Path $launcherValidationRoot "agent99-run.ps1") "staged_no_live_write" +if (-not $script:LauncherContract.ok) { + Write-DeployEvidence $false "preflight_failed" "launcher_contract_failed" @() $null + throw "Agent99 staged launcher contract check failed." +} + $stagedFiles = @($runtimeFiles | ForEach-Object { $path = Join-Path $StageDir $_ [pscustomobject]@{ name = $_; sha256 = (Get-FileHash -Algorithm SHA256 -Path $path).Hash.ToLowerInvariant() } @@ -196,7 +266,14 @@ $canonicalHost112Vmx = [string]$canonicalHost112Vm.vmx if ($ValidateOnly) { Write-DeployEvidence $true "validated" $null $stagedFiles $null - [pscustomobject]@{ ok = $true; status = "validated"; sourceRevision = $SourceRevision; evidence = $EvidencePath; files = $stagedFiles } | ConvertTo-Json -Depth 8 + [pscustomobject]@{ + ok = $true + status = "validated" + sourceRevision = $SourceRevision + evidence = $EvidencePath + launcherContract = $script:LauncherContract + files = $stagedFiles + } | ConvertTo-Json -Depth 8 exit 0 } @@ -217,6 +294,16 @@ try { $manifestBackedUp = $true } + $launcherExisted = Test-Path -LiteralPath $LauncherPath -PathType Leaf + if ($launcherExisted) { + Copy-Item -Force $LauncherPath (Join-Path $BackupDir "agent99-run.ps1") + } + $promotionRows += [pscustomobject]@{ + name = "agent99-run.ps1" + livePath = $LauncherPath + existed = [bool]$launcherExisted + } + foreach ($name in $runtimeFiles) { $livePath = Join-Path $BinDir $name $existed = Test-Path $livePath @@ -225,6 +312,12 @@ try { Copy-AgentRuntimeFileWithRetry (Join-Path $StageDir $name) $livePath } + $null = & (Join-Path $BinDir "agent99-bootstrap.ps1") -AgentRoot $AgentRoot -WriteLauncherOnly + $script:LauncherContract = Get-AgentLauncherContract $LauncherPath "live_post_promotion" + if (-not $script:LauncherContract.ok) { + throw "Promoted Agent99 launcher reconcile contract verification failed." + } + $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519") Add-DefaultProperty $config "sshUsers" ([pscustomobject]@{}) @@ -499,6 +592,7 @@ Write-DeployEvidence $deployOk $deployStatus $(if ($taskResult -and -not $taskRe backupDir = $BackupDir runtimeManifest = $ManifestPath taskRegistration = $taskResult + launcherContract = $script:LauncherContract files = $stagedFiles } | ConvertTo-Json -Depth 10 if (-not $deployOk) { exit 2 } diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 index f158ea7f2..ed9ba508d 100644 --- a/agent99-sre-alert-relay.ps1 +++ b/agent99-sre-alert-relay.ps1 @@ -17,10 +17,11 @@ $BinDir = Join-Path $AgentRoot "bin" $QueueDir = Join-Path $AgentRoot "queue" $ProcessedDir = Join-Path $QueueDir "processed" $SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile" +$SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit" $AcceptedIgnoredDir = Join-Path $AlertRoot "ignored" $SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1" -New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir | Out-Null +New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir, $SameRunRetryAuditDir | Out-Null function Convert-AgentSafeFileToken { param([string]$Value) @@ -36,6 +37,9 @@ function Get-AgentField { [string]$Name, [object]$Default = $null ) + if ($Object -is [Collections.IDictionary] -and $Object.Contains($Name)) { + return $Object[$Name] + } if ($Object -and $Object.PSObject.Properties[$Name]) { return $Object.PSObject.Properties[$Name].Value } @@ -510,6 +514,180 @@ function Test-AgentSameRunOutcomeEligible { ) } +function Test-AgentSameRunFailedNoWriteOutcomeRetryEligible { + param( + [object]$Result, + [object]$Identity, + [string]$SourceEvidence, + [string]$EvidenceReceiptId, + [string]$QueueId + ) + + if (-not $Result) { return $false } + $outcome = Get-AgentField $Result "outcome" $null + $canonicalIdentity = Get-AgentCanonicalOutcomeIdentity $Result $outcome + if (-not $outcome -or -not $canonicalIdentity) { return $false } + foreach ($name in @( + "id", "source", "reason", "mode", "reconcileEvidenceId", "evidenceReceiptId", + "automationRunId", "traceId", "workItemId", "idempotencyKey", + "executionGeneration", "incidentId", "approvalId", "projectId", "routeId", + "sourceEventResolutionPolicy" + )) { + if (-not $Result.PSObject.Properties[$name] -or (Get-AgentField $Result $name $null) -isnot [string]) { + return $false + } + } + foreach ($name in @("schemaVersion", "state", "verifierName", "sourceEventEvidence")) { + if (-not $outcome.PSObject.Properties[$name] -or (Get-AgentField $outcome $name $null) -isnot [string]) { + return $false + } + } + if (-not [string](Get-AgentField $outcome "verifierName" "")) { return $false } + $previousSourceEvidence = [string](Get-AgentField $outcome "sourceEventEvidence" "") + if ( + -not (Test-AgentPublicReceiptRef $SourceEvidence) -or + -not $SourceEvidence.StartsWith("prometheus:cold-start:") -or + -not (Test-AgentPublicReceiptRef $previousSourceEvidence) -or + -not $previousSourceEvidence.StartsWith("prometheus:cold-start:") + ) { return $false } + + return [bool]( + [string](Get-AgentField $Result "id" "") -eq $QueueId -and + [string](Get-AgentField $Result "source" "") -eq "agent99-same-run-reconcile" -and + [string](Get-AgentField $Result "reason" "") -eq "source_event_resolved_status_reconcile" -and + [string](Get-AgentField $Result "mode" "") -eq "Status" -and + (Get-AgentField $Result "controlledApply" $null) -is [bool] -and + -not [bool](Get-AgentField $Result "controlledApply" $true) -and + (Get-AgentField $Result "reconcileOnly" $null) -is [bool] -and + [bool](Get-AgentField $Result "reconcileOnly" $false) -and + (Get-AgentField $Result "runtimeWritePerformed" $null) -is [bool] -and + -not [bool](Get-AgentField $Result "runtimeWritePerformed" $true) -and + (Get-AgentField $Result "suppressTelegram" $null) -is [bool] -and + [bool](Get-AgentField $Result "suppressTelegram" $false) -and + [string](Get-AgentField $Result "reconcileEvidenceId" "") -eq $EvidenceReceiptId -and + [string](Get-AgentField $Result "evidenceReceiptId" "") -eq $EvidenceReceiptId -and + [string](Get-AgentField $Result "automationRunId" "") -eq [string](Get-AgentField $Identity "run_id" "") -and + [string](Get-AgentField $Result "traceId" "") -eq [string](Get-AgentField $Identity "trace_id" "") -and + [string](Get-AgentField $Result "workItemId" "") -eq [string](Get-AgentField $Identity "work_item_id" "") -and + [string](Get-AgentField $Result "idempotencyKey" "") -eq [string](Get-AgentField $Identity "idempotency_key" "") -and + [string](Get-AgentField $Result "executionGeneration" "") -eq [string](Get-AgentField $Identity "execution_generation" "") -and + [string](Get-AgentField $Result "incidentId" "") -eq [string](Get-AgentField $Identity "incident_id" "") -and + [string](Get-AgentField $Result "approvalId" "") -eq [string](Get-AgentField $Identity "approval_id" "") -and + [string](Get-AgentField $Result "projectId" "") -eq [string](Get-AgentField $Identity "project_id" "") -and + [string](Get-AgentField $Result "routeId" "") -eq [string](Get-AgentField $Identity "route_id" "") -and + (Test-AgentSameRunIdentityMatchesExisting $canonicalIdentity $Identity) -and + [string](Get-AgentField $Result "sourceEventResolutionPolicy" "") -eq "external_source_receipt_required" -and + [string](Get-AgentField $outcome "schemaVersion" "") -eq "agent99_outcome_contract_v1" -and + [string](Get-AgentField $outcome "state" "") -eq "failed" -and + (Get-AgentField $outcome "resolved" $null) -is [bool] -and + -not [bool](Get-AgentField $outcome "resolved" $true) -and + (Get-AgentField $outcome "transportOk" $null) -is [bool] -and + -not [bool](Get-AgentField $outcome "transportOk" $true) -and + (Get-AgentField $outcome "verifierPassed" $null) -is [bool] -and + -not [bool](Get-AgentField $outcome "verifierPassed" $true) -and + (Get-AgentField $outcome "sourceEventResolved" $null) -is [bool] -and + [bool](Get-AgentField $outcome "sourceEventResolved" $false) + ) +} + +function Move-AgentSameRunFailedNoWriteOutcomeForRetry { + param( + [string]$ProcessedPath, + [string]$LockPath, + [string]$RetryMarkerPath, + [string]$QueueId, + [object]$Identity, + [string]$SourceEvidence, + [string]$EvidenceReceiptId + ) + + if (Test-Path -LiteralPath $RetryMarkerPath) { + return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_already_consumed_fail_closed" } + } + if (-not (Test-Path -LiteralPath $ProcessedPath) -or -not (Test-Path -LiteralPath $LockPath)) { + return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_missing_fail_closed" } + } + + try { + $failedResult = Get-Content -LiteralPath $ProcessedPath -Raw | ConvertFrom-Json + $lockReceipt = Get-Content -LiteralPath $LockPath -Raw | ConvertFrom-Json + $failedOutcomeSha256 = (Get-FileHash -LiteralPath $ProcessedPath -Algorithm SHA256).Hash.ToLowerInvariant() + $previousLockSha256 = (Get-FileHash -LiteralPath $LockPath -Algorithm SHA256).Hash.ToLowerInvariant() + } catch { + return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_invalid_fail_closed" } + } + if ($failedOutcomeSha256 -notmatch "^[0-9a-f]{64}$" -or $previousLockSha256 -notmatch "^[0-9a-f]{64}$") { + return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_digest_invalid_fail_closed" } + } + if (-not (Test-AgentSameRunFailedNoWriteOutcomeRetryEligible $failedResult $Identity $SourceEvidence $EvidenceReceiptId $QueueId)) { + return [pscustomobject]@{ ok = $false; reason = "existing_same_run_status_outcome_invalid_fail_closed" } + } + + $runId = [string](Get-AgentField $Identity "run_id" "") + $canonicalDigest = [string](Get-AgentField $Identity "canonical_digest" "") + $lockValid = [bool]( + [string](Get-AgentField $lockReceipt "schemaVersion" "") -eq "agent99_same_run_single_flight_lock_v1" -and + [string](Get-AgentField $lockReceipt "automationRunId" "") -eq $runId -and + [string](Get-AgentField $lockReceipt "evidenceReceiptId" "") -eq $EvidenceReceiptId -and + [string](Get-AgentField $lockReceipt "canonicalDigest" "") -eq $canonicalDigest -and + (Get-AgentField $lockReceipt "storesRawEvidence" $null) -is [bool] -and + -not [bool](Get-AgentField $lockReceipt "storesRawEvidence" $true) + ) + if (-not $lockValid) { + return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_lock_invalid_fail_closed" } + } + + $outcomeAuditPath = Join-Path $SameRunRetryAuditDir "$QueueId.attempt-1.processed.json" + $lockAuditPath = Join-Path $SameRunRetryAuditDir "$QueueId.attempt-1.lock.json" + if ((Test-Path -LiteralPath $outcomeAuditPath) -or (Test-Path -LiteralPath $lockAuditPath)) { + return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_audit_collision_fail_closed" } + } + + try { + $markerStream = [IO.File]::Open($RetryMarkerPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None) + try { + $markerReceipt = @{ + schemaVersion = "agent99_same_run_status_retry_v1" + status = "retry_reserved" + retryOrdinal = 1 + automationRunId = $runId + evidenceReceiptId = $EvidenceReceiptId + canonicalDigest = $canonicalDigest + failedOutcomeAuditFile = (Split-Path -Leaf $outcomeAuditPath) + failedOutcomeSha256 = $failedOutcomeSha256 + previousLockAuditFile = (Split-Path -Leaf $lockAuditPath) + previousLockSha256 = $previousLockSha256 + preservesFailedOutcome = $true + controlledApply = $false + reconcileOnly = $true + runtimeWriteAuthorized = $false + storesRawEvidence = $false + createdAt = (Get-Date -Format o) + } | ConvertTo-Json -Compress + $markerBytes = [Text.Encoding]::UTF8.GetBytes($markerReceipt) + $markerStream.Write($markerBytes, 0, $markerBytes.Length) + $markerStream.Flush($true) + } finally { + $markerStream.Dispose() + } + } catch [System.IO.IOException] { + return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_already_consumed_fail_closed" } + } + + try { + Move-Item -LiteralPath $ProcessedPath -Destination $outcomeAuditPath + Move-Item -LiteralPath $LockPath -Destination $lockAuditPath + } catch { + return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_archive_failed_fail_closed" } + } + + return [pscustomobject]@{ + ok = $true + reason = "same_run_status_failed_no_write_outcome_archived_for_single_retry" + retryOrdinal = 1 + } +} + function Start-AgentSameRunStatusReconcile { param([object]$Request) @@ -558,6 +736,15 @@ function Start-AgentSameRunStatusReconcile { return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_dispatch_identity_mismatch"; httpStatus = 409 } } + $safeRunId = Convert-AgentSafeFileToken $runId + $queueId = "same-run-status-$safeRunId" + $queuePath = Join-Path $QueueDir "$queueId.json" + $runningPath = Join-Path $QueueDir "running-$queueId.json" + $processedPath = Join-Path $ProcessedDir "processed-$queueId.json" + $lockPath = Join-Path $SameRunStateDir "$queueId.lock.json" + $retryMarkerPath = Join-Path $SameRunStateDir "$queueId.retry-1.json" + $singleRetryPrepared = $false + $existing = Get-AgentOutcomeReadback $runId if (Test-AgentSameRunOutcomeEligible $existing $identity $sourceEvidence $evidenceReceiptId) { return [pscustomobject]@{ @@ -572,15 +759,33 @@ function Start-AgentSameRunStatusReconcile { } } if ($existing.found -and [bool](Get-AgentField $existing "reconcileOnly" $false)) { - return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_same_run_status_outcome_invalid_fail_closed"; httpStatus = 409 } + $retryPreparation = Move-AgentSameRunFailedNoWriteOutcomeForRetry ` + $processedPath $lockPath $retryMarkerPath $queueId $identity $sourceEvidence $evidenceReceiptId + if (-not $retryPreparation.ok) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = [string]$retryPreparation.reason; httpStatus = 409 } + } + $singleRetryPrepared = $true + } + $queuePending = Test-Path -LiteralPath $queuePath -PathType Leaf + $runningPending = Test-Path -LiteralPath $runningPath -PathType Leaf + $lockPending = Test-Path -LiteralPath $lockPath -PathType Leaf + if (-not $singleRetryPrepared -and (Test-Path -LiteralPath $retryMarkerPath -PathType Leaf)) { + if (-not (($queuePending -or $runningPending) -and $lockPending)) { + return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "same_run_status_retry_already_consumed_fail_closed"; httpStatus = 409 } + } + return [pscustomobject]@{ + ok = $true + status = "same_run_status_reconcile_pending" + httpStatus = 202 + automationRunId = $runId + reconcileOnly = $true + controlledApply = $false + evidenceReceiptId = $evidenceReceiptId + queued = $false + } } - $safeRunId = Convert-AgentSafeFileToken $runId - $queueId = "same-run-status-$safeRunId" - $queuePath = Join-Path $QueueDir "$queueId.json" - $runningPath = Join-Path $QueueDir "running-$queueId.json" - $lockPath = Join-Path $SameRunStateDir "$queueId.lock.json" - if ((Test-Path -LiteralPath $queuePath) -or (Test-Path -LiteralPath $runningPath) -or (Test-Path -LiteralPath $lockPath)) { + if ($queuePending -or $runningPending -or $lockPending) { return [pscustomobject]@{ ok = $true status = "same_run_status_reconcile_pending" diff --git a/apps/api/tests/test_agent99_same_run_reconcile.py b/apps/api/tests/test_agent99_same_run_reconcile.py index 05063f0d3..78f92f100 100644 --- a/apps/api/tests/test_agent99_same_run_reconcile.py +++ b/apps/api/tests/test_agent99_same_run_reconcile.py @@ -829,3 +829,73 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> assert "Get-AgentEvidenceAtPath $exactEvidencePath" in queue_branch assert "Send-AgentTelegram" not in queue_branch assert "Invoke-AgentCompletionCallback" not in queue_branch + + +def test_windows_relay_allows_only_one_audited_failed_no_write_status_retry() -> None: + root = Path(__file__).resolve().parents[3] + relay = (root / "agent99-sre-alert-relay.ps1").read_text(encoding="utf-8") + + field_helper = relay[relay.index("function Get-AgentField") :] + field_helper = field_helper[: field_helper.index("function Get-AgentEnvValue")] + assert "$Object -is [Collections.IDictionary]" in field_helper + assert "$Object.Contains($Name)" in field_helper + assert "return $Object[$Name]" in field_helper + + retry_start = relay.index( + "function Test-AgentSameRunFailedNoWriteOutcomeRetryEligible" + ) + retry_end = relay.index("function Start-AgentSameRunStatusReconcile") + retry_contract = relay[retry_start:retry_end] + + assert '$SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit"' in relay + assert '"agent99_same_run_status_retry_v1"' in retry_contract + assert 'retryOrdinal = 1' in retry_contract + assert 'status = "retry_reserved"' in retry_contract + assert 'preservesFailedOutcome = $true' in retry_contract + assert 'runtimeWriteAuthorized = $false' in retry_contract + assert '[string](Get-AgentField $Result "mode" "") -eq "Status"' in retry_contract + assert '[string](Get-AgentField $outcome "state" "") -eq "failed"' in retry_contract + assert '-not [bool](Get-AgentField $Result "controlledApply" $true)' in retry_contract + assert '[bool](Get-AgentField $Result "reconcileOnly" $false)' in retry_contract + assert '-not [bool](Get-AgentField $Result "runtimeWritePerformed" $true)' in retry_contract + assert '[bool](Get-AgentField $Result "suppressTelegram" $false)' in retry_contract + assert '-not [bool](Get-AgentField $outcome "resolved" $true)' in retry_contract + assert '-not [bool](Get-AgentField $outcome "transportOk" $true)' in retry_contract + assert '-not [bool](Get-AgentField $outcome "verifierPassed" $true)' in retry_contract + assert '[bool](Get-AgentField $outcome "sourceEventResolved" $false)' in retry_contract + assert "Test-AgentPublicReceiptRef $SourceEvidence" in retry_contract + assert "Test-AgentPublicReceiptRef $previousSourceEvidence" in retry_contract + assert '$previousSourceEvidence.StartsWith("prometheus:cold-start:")' in retry_contract + assert "Test-AgentSameRunIdentityMatchesExisting $canonicalIdentity $Identity" in retry_contract + assert '"agent99_same_run_single_flight_lock_v1"' in retry_contract + assert '[IO.FileMode]::CreateNew' in retry_contract + assert 'failedOutcomeSha256 = $failedOutcomeSha256' in retry_contract + assert 'previousLockSha256 = $previousLockSha256' in retry_contract + assert 'Move-Item -LiteralPath $ProcessedPath -Destination $outcomeAuditPath' in retry_contract + assert 'Move-Item -LiteralPath $LockPath -Destination $lockAuditPath' in retry_contract + assert 'same_run_status_retry_already_consumed_fail_closed' in retry_contract + assert 'same_run_status_retry_archive_failed_fail_closed' in retry_contract + assert 'mode = "Recover"' not in retry_contract + assert 'controlledApply = $true' not in retry_contract + + start = relay[relay.index("function Start-AgentSameRunStatusReconcile") :] + marker_branch = start.index( + "if (-not $singleRetryPrepared -and " + "(Test-Path -LiteralPath $retryMarkerPath -PathType Leaf))" + ) + marker_rejected = start.index( + 'reason = "same_run_status_retry_already_consumed_fail_closed"', + marker_branch, + ) + marker_pending = start.index( + 'status = "same_run_status_reconcile_pending"', + marker_rejected, + ) + general_pending = start.index( + "if ($queuePending -or $runningPending -or $lockPending)", + marker_pending, + ) + assert marker_branch < marker_rejected < marker_pending < general_pending + assert "(($queuePending -or $runningPending) -and $lockPending)" in start[ + marker_branch:marker_pending + ] diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index edea82da1..112cb8c9e 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -8,6 +8,9 @@ CONTROL = ROOT / "agent99-control-plane.ps1" DEPLOY = ROOT / "agent99-deploy.ps1" BOOTSTRAP = ROOT / "agent99-bootstrap.ps1" SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1" +REMOTE_ATOMIC_RECEIVER = ( + ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" +) def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None: @@ -312,6 +315,84 @@ def test_agent99_deployer_backs_up_manifest_before_runtime_promotion() -> None: assert "Copy-AgentRuntimeFileWithRetry (Join-Path $StageDir $name) $livePath" in deploy +def test_agent99_atomic_deploy_regenerates_and_verifies_root_launcher() -> None: + bootstrap = BOOTSTRAP.read_text(encoding="utf-8") + deploy = DEPLOY.read_text(encoding="utf-8") + + assert "[switch]$WriteLauncherOnly" in bootstrap + assert "function Write-AgentLauncher" in bootstrap + launcher_only = bootstrap[bootstrap.index("if ($WriteLauncherOnly)") :] + launcher_only = launcher_only[: launcher_only.index("function Copy-AgentFile")] + assert 'Join-Path $BinDir "agent99-control-plane.ps1"' in launcher_only + assert 'Write-AgentLauncher (Join-Path $AgentRoot "agent99-run.ps1")' in launcher_only + assert "Ensure-AgentHost112CanonicalSshConfig" not in launcher_only + assert "Set-AgentBootstrapProperty" not in launcher_only + + staged_check = deploy[deploy.index('$launcherValidationRoot = Join-Path $StageDir') :] + staged_check = staged_check[: staged_check.index("$stagedFiles = @(")] + assert "-WriteLauncherOnly" in staged_check + assert '"staged_no_live_write"' in staged_check + assert 'Write-DeployEvidence $false "preflight_failed" "launcher_contract_failed"' in staged_check + + promotion = deploy[deploy.index("$configBackedUp = $false") :] + promotion = promotion[: promotion.index("$config = Get-Content $ConfigPath")] + assert promotion.index('Copy-Item -Force $LauncherPath') < promotion.index( + "foreach ($name in $runtimeFiles)" + ) + assert 'name = "agent99-run.ps1"' in promotion + assert 'Join-Path $BinDir "agent99-bootstrap.ps1"' in promotion + assert 'Get-AgentLauncherContract $LauncherPath "live_post_promotion"' in promotion + assert "Promoted Agent99 launcher reconcile contract verification failed." in promotion + assert "launcherContract = $script:LauncherContract" in deploy + + +def test_agent99_launcher_contract_covers_same_run_flags_and_forwarding() -> None: + bootstrap = BOOTSTRAP.read_text(encoding="utf-8") + deploy = DEPLOY.read_text(encoding="utf-8") + + for marker in ( + '[switch]`$ReconcileOnly', + '[string]`$ReconcileEvidenceId = ""', + '[switch]`$ReconcileQueueOnly', + '[string]`$ReconcileQueueId = ""', + "-ReconcileOnly:`$ReconcileOnly", + "-ReconcileEvidenceId `$ReconcileEvidenceId", + "-ReconcileQueueOnly:`$ReconcileQueueOnly", + "-ReconcileQueueId `$ReconcileQueueId", + ): + assert marker in bootstrap + assert "Management.Automation.Language.Parser" in bootstrap + assert "Management.Automation.Language.Parser" in deploy + assert "reconcileOnlyForwarded = $true" in bootstrap + assert "reconcileQueueOnlyForwarded = $true" in bootstrap + + +def test_agent99_remote_rollback_and_post_verifier_include_root_launcher() -> None: + receiver = REMOTE_ATOMIC_RECEIVER.read_text(encoding="utf-8") + + assert '$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1"' in receiver + assert "function Get-AgentLauncherContract" in receiver + assert '$rows += "agent99-run.ps1`t$(Get-AgentSha256File $LauncherPath)`n"' in receiver + restore = receiver[receiver.index("function Restore-AgentBundle") :] + restore = restore[: restore.index("function Write-AgentRemoteDeployReceipt")] + assert "[bool]$BeforeLauncherPresent" in restore + assert 'Join-Path $BackupPath "agent99-run.ps1"' in restore + assert "Copy-AgentFileWithRetry $backupLauncher $LauncherPath" in restore + assert "Remove-Item -LiteralPath $LauncherPath -Force" in restore + assert "$liveLauncherContract.ok" in receiver + assert ( + '[string]$liveLauncherContract.sha256 -eq ' + '[string]$prior.launcherContract.sha256' + ) in receiver + assert "$launcherContract.ok" in receiver + assert ( + '[string]$launcherContract.sha256 -eq ' + '[string]$deploy.payload.launcherContract.sha256' + ) in receiver + assert "launcherContract = $launcherContract" in receiver + assert "failedLauncherContract = $launcherContract" in receiver + + def test_agent99_completion_callback_waits_for_durable_same_trace_readback() -> None: source = CONTROL.read_text(encoding="utf-8") deploy = DEPLOY.read_text(encoding="utf-8") diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index a239de3cd..d67aa952a 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -13,6 +13,7 @@ $StateDir = Join-Path $AgentRoot "state" $EvidenceDir = Join-Path $AgentRoot "evidence" $DeployRoot = Join-Path $AgentRoot "deploy" $ManifestPath = Join-Path $StateDir "runtime-manifest.json" +$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1" $WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe" $RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay" $RelayTaskPath = "\" @@ -71,6 +72,48 @@ function Get-AgentSha256File { return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() } +function Get-AgentLauncherContract { + $requiredMarkers = @( + '[switch]$ReconcileOnly', + '[string]$ReconcileEvidenceId = ""', + '[switch]$ReconcileQueueOnly', + '[string]$ReconcileQueueId = ""', + '-ReconcileOnly:$ReconcileOnly', + '-ReconcileEvidenceId $ReconcileEvidenceId', + '-ReconcileQueueOnly:$ReconcileQueueOnly', + '-ReconcileQueueId $ReconcileQueueId' + ) + if (-not (Test-Path -LiteralPath $LauncherPath -PathType Leaf)) { + return [pscustomobject]@{ + schemaVersion = "agent99_launcher_contract_v1" + ok = $false + exists = $false + sha256 = "missing" + parseErrorCount = -1 + requiredMarkerCount = $requiredMarkers.Count + missingMarkerCount = $requiredMarkers.Count + reconcileOnlyForwarded = $false + reconcileQueueOnlyForwarded = $false + } + } + $tokens = $null + $parseErrors = $null + [Management.Automation.Language.Parser]::ParseFile($LauncherPath, [ref]$tokens, [ref]$parseErrors) | Out-Null + $content = [IO.File]::ReadAllText($LauncherPath) + $missingMarkers = @($requiredMarkers | Where-Object { -not $content.Contains($_) }) + return [pscustomobject]@{ + schemaVersion = "agent99_launcher_contract_v1" + ok = [bool](@($parseErrors).Count -eq 0 -and $missingMarkers.Count -eq 0) + exists = $true + sha256 = (Get-AgentSha256File $LauncherPath) + parseErrorCount = @($parseErrors).Count + requiredMarkerCount = $requiredMarkers.Count + missingMarkerCount = $missingMarkers.Count + reconcileOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileOnly:$ReconcileOnly' -and $missingMarkers -notcontains '-ReconcileEvidenceId $ReconcileEvidenceId') + reconcileQueueOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileQueueOnly:$ReconcileQueueOnly' -and $missingMarkers -notcontains '-ReconcileQueueId $ReconcileQueueId') + } +} + function Get-AgentSafeRuntimeManifest { if (-not (Test-Path -LiteralPath $ManifestPath -PathType Leaf)) { return [pscustomobject]@{ @@ -146,6 +189,7 @@ function Get-AgentLiveBundleDigest { foreach ($name in $FixedRuntimeFiles) { $rows += "$name`t$(Get-AgentSha256File (Join-Path $BinDir $name))`n" } + $rows += "agent99-run.ps1`t$(Get-AgentSha256File $LauncherPath)`n" return Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes(($rows -join ""))) } @@ -555,6 +599,7 @@ function Restore-AgentBundle { param( [string]$BackupPath, [hashtable]$BeforeFilePresence, + [bool]$BeforeLauncherPresent, [bool]$BeforeConfigPresent, [bool]$BeforeManifestPresent ) @@ -573,6 +618,13 @@ function Restore-AgentBundle { Remove-Item -LiteralPath $livePath -Force } } + $backupLauncher = Join-Path $BackupPath "agent99-run.ps1" + if ($BeforeLauncherPresent) { + if (-not (Test-Path -LiteralPath $backupLauncher -PathType Leaf)) { throw "required_launcher_backup_missing" } + Copy-AgentFileWithRetry $backupLauncher $LauncherPath + } elseif (Test-Path -LiteralPath $LauncherPath -PathType Leaf) { + Remove-Item -LiteralPath $LauncherPath -Force + } $backupConfig = Join-Path $BackupPath "agent99.config.json" if ($BeforeConfigPresent) { if (-not (Test-Path -LiteralPath $backupConfig -PathType Leaf)) { throw "required_config_backup_missing" } @@ -832,6 +884,7 @@ try { } $live = Get-AgentSafeRuntimeManifest + $liveLauncherContract = Get-AgentLauncherContract if ( $prior -and [string]$prior.status -eq "deployed_verified" -and @@ -841,10 +894,15 @@ try { [string]$prior.sourceRevision -eq $sourceRevision -and $prior.PSObject.Properties["relayReload"] -and [bool]$prior.relayReload.newGenerationVerified -and + $prior.PSObject.Properties["launcherContract"] -and + [bool]$prior.launcherContract.ok -and + [string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and $live.runtimeMatched -and $live.sourceRevision -eq $sourceRevision -and $live.fileCount -eq 14 -and - $live.mismatchCount -eq 0 + $live.mismatchCount -eq 0 -and + $liveLauncherContract.ok -and + [string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256 ) { $script:ReceiverOperationPhase = "idempotent_replay_post_verifier" $replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") @@ -852,6 +910,7 @@ try { $originalOperationBoundaries = $prior.operationBoundaries $prior | Add-Member -NotePropertyName idempotentReplay -NotePropertyValue $true -Force $prior | Add-Member -NotePropertyName runtimeManifest -NotePropertyValue $live -Force + $prior | Add-Member -NotePropertyName launcherContract -NotePropertyValue $liveLauncherContract -Force $prior | Add-Member -NotePropertyName livePreflight -NotePropertyValue $replayPreflight -Force $prior | Add-Member -NotePropertyName originalOperationBoundaries -NotePropertyValue $originalOperationBoundaries -Force $prior | Add-Member -NotePropertyName operationBoundaries -NotePropertyValue ([pscustomobject]@{ @@ -979,6 +1038,7 @@ try { $beforeManifestDigest = Get-AgentSha256File $ManifestPath $beforeFilePresence = @{} foreach ($name in $FixedRuntimeFiles) { $beforeFilePresence[$name] = Test-Path -LiteralPath (Join-Path $BinDir $name) -PathType Leaf } + $beforeLauncherPresent = Test-Path -LiteralPath $LauncherPath -PathType Leaf $beforeConfigPresent = Test-Path -LiteralPath $ConfigPath -PathType Leaf $beforeManifestPresent = Test-Path -LiteralPath $ManifestPath -PathType Leaf $beforeRelayRuntime = Get-AgentRelayRuntime @@ -1034,6 +1094,7 @@ try { $relayReload = $null $runtimeManifest = $null $livePreflight = $null + $launcherContract = $null $postVerified = $false $failurePhase = "" $failureType = "" @@ -1046,7 +1107,15 @@ try { if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) { $backupPath = [string]$deploy.payload.backupDir } - if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) { + if ( + -not $deploy.ok -or + -not $deploy.payload -or + [string]$deploy.payload.status -ne "deployed" -or + $deploy.payload.taskRegistration -or + -not $deploy.payload.PSObject.Properties["launcherContract"] -or + -not [bool]$deploy.payload.launcherContract.ok -or + [string]$deploy.payload.launcherContract.sha256 -notmatch "^[0-9a-f]{64}$" + ) { $failurePhase = "deploy" $failureType = "deploy_child_failed_or_invalid" } else { @@ -1060,6 +1129,7 @@ try { } else { $script:ReceiverOperationPhase = "independent_post_verifier" $runtimeManifest = Get-AgentSafeRuntimeManifest + $launcherContract = Get-AgentLauncherContract $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") $postVerified = [bool]( $runtimeManifest.exists -and @@ -1067,6 +1137,8 @@ try { $runtimeManifest.sourceRevision -eq $sourceRevision -and $runtimeManifest.fileCount -eq 14 -and $runtimeManifest.mismatchCount -eq 0 -and + $launcherContract.ok -and + [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and $relayReload.oldGenerationsGone -and $relayReload.newGenerationVerified -and $relayReload.after.taskEnabled -and @@ -1108,6 +1180,7 @@ try { deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status } relayReload = $relayReload runtimeManifest = $runtimeManifest + launcherContract = $launcherContract livePreflight = $livePreflight durableReceiptWritten = $true idempotentReplay = $false @@ -1147,7 +1220,7 @@ try { } $script:ReceiverOperationPhase = "runtime_rollback" $script:ReceiverRollbackAttempted = $true - $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent + $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeLauncherPresent $beforeConfigPresent $beforeManifestPresent $rollbackForbiddenGenerations = @($beforeRelayRuntime.taskGeneration) if ($relayReload) { $rollbackForbiddenGenerations += @($relayReload.before.taskGeneration) @@ -1227,6 +1300,7 @@ try { attemptReceiptPath = $attemptReceiptPath relayReload = $relayReload failedRuntimeManifest = $runtimeManifest + failedLauncherContract = $launcherContract failedLivePreflight = $livePreflight rollback = $rollback rollbackRelayRestart = $rollbackRelayRestart