fix(agent99): complete same-run status reconcile
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m46s
CD Pipeline / build-and-deploy (push) Successful in 19m6s
CD Pipeline / post-deploy-checks (push) Successful in 2m4s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m46s
CD Pipeline / build-and-deploy (push) Successful in 19m6s
CD Pipeline / post-deploy-checks (push) Successful in 2m4s
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
param(
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99",
|
||||
[string]$SourceRevision = "",
|
||||
[switch]$InstallScheduledTasks
|
||||
[switch]$InstallScheduledTasks,
|
||||
[switch]$WriteLauncherOnly
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@@ -9,6 +10,7 @@ $ErrorActionPreference = "Stop"
|
||||
$SourceRoot = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$BinDir = Join-Path $AgentRoot "bin"
|
||||
$ConfigDir = Join-Path $AgentRoot "config"
|
||||
$ConfigPath = Join-Path $ConfigDir "agent99.config.json"
|
||||
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$LogDir = Join-Path $AgentRoot "logs"
|
||||
$QueueDir = Join-Path $AgentRoot "queue"
|
||||
@@ -18,6 +20,83 @@ $PlaybookDir = Join-Path $AgentRoot "playbooks"
|
||||
$DashboardDir = Join-Path $AgentRoot "dashboard"
|
||||
$StateDir = Join-Path $AgentRoot "state"
|
||||
|
||||
function Write-AgentLauncher {
|
||||
param([string]$Path)
|
||||
|
||||
$launcherContent = @"
|
||||
param(
|
||||
[ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
|
||||
[string]`$Mode = "Status",
|
||||
[switch]`$ControlledApply,
|
||||
[switch]`$ReconcileOnly,
|
||||
[string]`$ReconcileEvidenceId = "",
|
||||
[switch]`$ReconcileQueueOnly,
|
||||
[string]`$ReconcileQueueId = "",
|
||||
[string]`$AutomationRunId = "",
|
||||
[string]`$TraceId = "",
|
||||
[string]`$WorkItemId = "",
|
||||
[switch]`$SuppressAlerts,
|
||||
[string]`$SuppressReason = ""
|
||||
)
|
||||
|
||||
& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ReconcileOnly:`$ReconcileOnly -ReconcileEvidenceId `$ReconcileEvidenceId -ReconcileQueueOnly:`$ReconcileQueueOnly -ReconcileQueueId `$ReconcileQueueId -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$ConfigPath" -EvidenceDir "$EvidenceDir"
|
||||
"@
|
||||
$parent = Split-Path -Parent $Path
|
||||
if (-not (Test-Path -LiteralPath $parent -PathType Container)) {
|
||||
throw "Agent99 launcher parent is unavailable: $parent"
|
||||
}
|
||||
$temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))"
|
||||
try {
|
||||
[IO.File]::WriteAllText($temporary, $launcherContent, (New-Object Text.UTF8Encoding($true)))
|
||||
$tokens = $null
|
||||
$parseErrors = $null
|
||||
[Management.Automation.Language.Parser]::ParseFile($temporary, [ref]$tokens, [ref]$parseErrors) | Out-Null
|
||||
if (@($parseErrors).Count -ne 0) {
|
||||
throw "Agent99 launcher PowerShell parser check failed."
|
||||
}
|
||||
Move-Item -LiteralPath $temporary -Destination $Path -Force
|
||||
$written = [IO.File]::ReadAllText($Path)
|
||||
$requiredMarkers = @(
|
||||
'[switch]$ReconcileOnly',
|
||||
'[string]$ReconcileEvidenceId = ""',
|
||||
'[switch]$ReconcileQueueOnly',
|
||||
'[string]$ReconcileQueueId = ""',
|
||||
'-ReconcileOnly:$ReconcileOnly',
|
||||
'-ReconcileEvidenceId $ReconcileEvidenceId',
|
||||
'-ReconcileQueueOnly:$ReconcileQueueOnly',
|
||||
'-ReconcileQueueId $ReconcileQueueId'
|
||||
)
|
||||
$missingMarkers = @($requiredMarkers | Where-Object { -not $written.Contains($_) })
|
||||
if ($missingMarkers.Count -ne 0) {
|
||||
throw "Agent99 launcher reconcile contract verification failed."
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_launcher_contract_v1"
|
||||
ok = $true
|
||||
path = $Path
|
||||
sha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
parseErrorCount = 0
|
||||
requiredMarkerCount = $requiredMarkers.Count
|
||||
missingMarkerCount = 0
|
||||
reconcileOnlyForwarded = $true
|
||||
reconcileQueueOnlyForwarded = $true
|
||||
}
|
||||
} finally {
|
||||
if (Test-Path -LiteralPath $temporary) {
|
||||
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ($WriteLauncherOnly) {
|
||||
$controlPlanePath = Join-Path $BinDir "agent99-control-plane.ps1"
|
||||
if (-not (Test-Path -LiteralPath $controlPlanePath -PathType Leaf)) {
|
||||
throw "Agent99 control plane is unavailable for launcher generation: $controlPlanePath"
|
||||
}
|
||||
Write-AgentLauncher (Join-Path $AgentRoot "agent99-run.ps1")
|
||||
return
|
||||
}
|
||||
|
||||
function Copy-AgentFile {
|
||||
param([string]$Name)
|
||||
$src = Join-Path $SourceRoot $Name
|
||||
@@ -210,7 +289,6 @@ if (-not $runtimeMatched) {
|
||||
}
|
||||
|
||||
$exampleConfig = Join-Path $SourceRoot "agent99.config.example.json"
|
||||
$configPath = Join-Path $ConfigDir "agent99.config.json"
|
||||
if (-not (Test-Path $configPath)) {
|
||||
if (Test-Path $exampleConfig) {
|
||||
Copy-Item $exampleConfig $configPath
|
||||
@@ -290,24 +368,7 @@ if (-not (Test-Path $configPath)) {
|
||||
Ensure-AgentHost112CanonicalSshConfig $configPath $host112BootstrapPolicyPath
|
||||
|
||||
$launcher = Join-Path $AgentRoot "agent99-run.ps1"
|
||||
@"
|
||||
param(
|
||||
[ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
|
||||
[string]`$Mode = "Status",
|
||||
[switch]`$ControlledApply,
|
||||
[switch]`$ReconcileOnly,
|
||||
[string]`$ReconcileEvidenceId = "",
|
||||
[switch]`$ReconcileQueueOnly,
|
||||
[string]`$ReconcileQueueId = "",
|
||||
[string]`$AutomationRunId = "",
|
||||
[string]`$TraceId = "",
|
||||
[string]`$WorkItemId = "",
|
||||
[switch]`$SuppressAlerts,
|
||||
[string]`$SuppressReason = ""
|
||||
)
|
||||
|
||||
& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ReconcileOnly:`$ReconcileOnly -ReconcileEvidenceId `$ReconcileEvidenceId -ReconcileQueueOnly:`$ReconcileQueueOnly -ReconcileQueueId `$ReconcileQueueId -AutomationRunId `$AutomationRunId -TraceId `$TraceId -WorkItemId `$WorkItemId -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir"
|
||||
"@ | Set-Content -Encoding UTF8 $launcher
|
||||
$null = Write-AgentLauncher $launcher
|
||||
|
||||
$submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd"
|
||||
@"
|
||||
|
||||
@@ -16,9 +16,18 @@ $DeployRoot = Join-Path $AgentRoot "deploy"
|
||||
$StageDir = Join-Path $DeployRoot "staging-$stamp"
|
||||
$BackupDir = Join-Path $DeployRoot "backup-$stamp"
|
||||
$ManifestPath = Join-Path $StateDir "runtime-manifest.json"
|
||||
$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1"
|
||||
$EvidencePath = Join-Path $EvidenceDir "agent99-deploy-$stamp.json"
|
||||
$script:ConfigMigrations = @()
|
||||
$script:RuntimeCopyRetries = @()
|
||||
$script:LauncherContract = [pscustomobject]@{
|
||||
schemaVersion = "agent99_launcher_contract_v1"
|
||||
ok = $false
|
||||
scope = "not_checked"
|
||||
exists = $false
|
||||
parseErrorCount = -1
|
||||
missingMarkerCount = -1
|
||||
}
|
||||
|
||||
$runtimeFiles = @(
|
||||
"agent99-bootstrap.ps1",
|
||||
@@ -112,6 +121,55 @@ function Copy-AgentRuntimeFileWithRetry {
|
||||
}
|
||||
}
|
||||
|
||||
function Get-AgentLauncherContract {
|
||||
param([string]$Path, [string]$Scope)
|
||||
|
||||
$requiredMarkers = @(
|
||||
'[switch]$ReconcileOnly',
|
||||
'[string]$ReconcileEvidenceId = ""',
|
||||
'[switch]$ReconcileQueueOnly',
|
||||
'[string]$ReconcileQueueId = ""',
|
||||
'-ReconcileOnly:$ReconcileOnly',
|
||||
'-ReconcileEvidenceId $ReconcileEvidenceId',
|
||||
'-ReconcileQueueOnly:$ReconcileQueueOnly',
|
||||
'-ReconcileQueueId $ReconcileQueueId'
|
||||
)
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_launcher_contract_v1"
|
||||
ok = $false
|
||||
scope = $Scope
|
||||
exists = $false
|
||||
path = $Path
|
||||
sha256 = ""
|
||||
parseErrorCount = -1
|
||||
requiredMarkerCount = $requiredMarkers.Count
|
||||
missingMarkerCount = $requiredMarkers.Count
|
||||
reconcileOnlyForwarded = $false
|
||||
reconcileQueueOnlyForwarded = $false
|
||||
}
|
||||
}
|
||||
$tokens = $null
|
||||
$parseErrors = $null
|
||||
[Management.Automation.Language.Parser]::ParseFile($Path, [ref]$tokens, [ref]$parseErrors) | Out-Null
|
||||
$content = [IO.File]::ReadAllText($Path)
|
||||
$missingMarkers = @($requiredMarkers | Where-Object { -not $content.Contains($_) })
|
||||
$ok = [bool](@($parseErrors).Count -eq 0 -and $missingMarkers.Count -eq 0)
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_launcher_contract_v1"
|
||||
ok = $ok
|
||||
scope = $Scope
|
||||
exists = $true
|
||||
path = $Path
|
||||
sha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
parseErrorCount = @($parseErrors).Count
|
||||
requiredMarkerCount = $requiredMarkers.Count
|
||||
missingMarkerCount = $missingMarkers.Count
|
||||
reconcileOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileOnly:$ReconcileOnly' -and $missingMarkers -notcontains '-ReconcileEvidenceId $ReconcileEvidenceId')
|
||||
reconcileQueueOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileQueueOnly:$ReconcileQueueOnly' -and $missingMarkers -notcontains '-ReconcileQueueId $ReconcileQueueId')
|
||||
}
|
||||
}
|
||||
|
||||
function Write-DeployEvidence {
|
||||
param([bool]$Ok, [string]$Status, [string]$ErrorText, [object[]]$Files, [object]$TaskResult)
|
||||
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null
|
||||
@@ -132,6 +190,7 @@ function Write-DeployEvidence {
|
||||
configMigrations = @($script:ConfigMigrations)
|
||||
runtimeCopyRetryCount = @($script:RuntimeCopyRetries).Count
|
||||
runtimeCopyRetries = @($script:RuntimeCopyRetries)
|
||||
launcherContract = $script:LauncherContract
|
||||
files = $Files
|
||||
taskRegistration = $TaskResult
|
||||
} | ConvertTo-Json -Depth 10 | Set-Content -Path $EvidencePath -Encoding UTF8
|
||||
@@ -185,6 +244,17 @@ if ($LASTEXITCODE -ne 0) {
|
||||
throw "Agent99 staged synthetic tests failed."
|
||||
}
|
||||
|
||||
$launcherValidationRoot = Join-Path $StageDir "launcher-contract-work"
|
||||
$launcherValidationBin = Join-Path $launcherValidationRoot "bin"
|
||||
New-Item -ItemType Directory -Force -Path $launcherValidationBin | Out-Null
|
||||
Copy-Item -Force (Join-Path $StageDir "agent99-control-plane.ps1") (Join-Path $launcherValidationBin "agent99-control-plane.ps1")
|
||||
$null = & (Join-Path $StageDir "agent99-bootstrap.ps1") -AgentRoot $launcherValidationRoot -WriteLauncherOnly
|
||||
$script:LauncherContract = Get-AgentLauncherContract (Join-Path $launcherValidationRoot "agent99-run.ps1") "staged_no_live_write"
|
||||
if (-not $script:LauncherContract.ok) {
|
||||
Write-DeployEvidence $false "preflight_failed" "launcher_contract_failed" @() $null
|
||||
throw "Agent99 staged launcher contract check failed."
|
||||
}
|
||||
|
||||
$stagedFiles = @($runtimeFiles | ForEach-Object {
|
||||
$path = Join-Path $StageDir $_
|
||||
[pscustomobject]@{ name = $_; sha256 = (Get-FileHash -Algorithm SHA256 -Path $path).Hash.ToLowerInvariant() }
|
||||
@@ -196,7 +266,14 @@ $canonicalHost112Vmx = [string]$canonicalHost112Vm.vmx
|
||||
|
||||
if ($ValidateOnly) {
|
||||
Write-DeployEvidence $true "validated" $null $stagedFiles $null
|
||||
[pscustomobject]@{ ok = $true; status = "validated"; sourceRevision = $SourceRevision; evidence = $EvidencePath; files = $stagedFiles } | ConvertTo-Json -Depth 8
|
||||
[pscustomobject]@{
|
||||
ok = $true
|
||||
status = "validated"
|
||||
sourceRevision = $SourceRevision
|
||||
evidence = $EvidencePath
|
||||
launcherContract = $script:LauncherContract
|
||||
files = $stagedFiles
|
||||
} | ConvertTo-Json -Depth 8
|
||||
exit 0
|
||||
}
|
||||
|
||||
@@ -217,6 +294,16 @@ try {
|
||||
$manifestBackedUp = $true
|
||||
}
|
||||
|
||||
$launcherExisted = Test-Path -LiteralPath $LauncherPath -PathType Leaf
|
||||
if ($launcherExisted) {
|
||||
Copy-Item -Force $LauncherPath (Join-Path $BackupDir "agent99-run.ps1")
|
||||
}
|
||||
$promotionRows += [pscustomobject]@{
|
||||
name = "agent99-run.ps1"
|
||||
livePath = $LauncherPath
|
||||
existed = [bool]$launcherExisted
|
||||
}
|
||||
|
||||
foreach ($name in $runtimeFiles) {
|
||||
$livePath = Join-Path $BinDir $name
|
||||
$existed = Test-Path $livePath
|
||||
@@ -225,6 +312,12 @@ try {
|
||||
Copy-AgentRuntimeFileWithRetry (Join-Path $StageDir $name) $livePath
|
||||
}
|
||||
|
||||
$null = & (Join-Path $BinDir "agent99-bootstrap.ps1") -AgentRoot $AgentRoot -WriteLauncherOnly
|
||||
$script:LauncherContract = Get-AgentLauncherContract $LauncherPath "live_post_promotion"
|
||||
if (-not $script:LauncherContract.ok) {
|
||||
throw "Promoted Agent99 launcher reconcile contract verification failed."
|
||||
}
|
||||
|
||||
$config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
|
||||
Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519")
|
||||
Add-DefaultProperty $config "sshUsers" ([pscustomobject]@{})
|
||||
@@ -499,6 +592,7 @@ Write-DeployEvidence $deployOk $deployStatus $(if ($taskResult -and -not $taskRe
|
||||
backupDir = $BackupDir
|
||||
runtimeManifest = $ManifestPath
|
||||
taskRegistration = $taskResult
|
||||
launcherContract = $script:LauncherContract
|
||||
files = $stagedFiles
|
||||
} | ConvertTo-Json -Depth 10
|
||||
if (-not $deployOk) { exit 2 }
|
||||
|
||||
@@ -17,10 +17,11 @@ $BinDir = Join-Path $AgentRoot "bin"
|
||||
$QueueDir = Join-Path $AgentRoot "queue"
|
||||
$ProcessedDir = Join-Path $QueueDir "processed"
|
||||
$SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile"
|
||||
$SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit"
|
||||
$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored"
|
||||
$SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir, $SameRunRetryAuditDir | Out-Null
|
||||
|
||||
function Convert-AgentSafeFileToken {
|
||||
param([string]$Value)
|
||||
@@ -36,6 +37,9 @@ function Get-AgentField {
|
||||
[string]$Name,
|
||||
[object]$Default = $null
|
||||
)
|
||||
if ($Object -is [Collections.IDictionary] -and $Object.Contains($Name)) {
|
||||
return $Object[$Name]
|
||||
}
|
||||
if ($Object -and $Object.PSObject.Properties[$Name]) {
|
||||
return $Object.PSObject.Properties[$Name].Value
|
||||
}
|
||||
@@ -510,6 +514,180 @@ function Test-AgentSameRunOutcomeEligible {
|
||||
)
|
||||
}
|
||||
|
||||
function Test-AgentSameRunFailedNoWriteOutcomeRetryEligible {
|
||||
param(
|
||||
[object]$Result,
|
||||
[object]$Identity,
|
||||
[string]$SourceEvidence,
|
||||
[string]$EvidenceReceiptId,
|
||||
[string]$QueueId
|
||||
)
|
||||
|
||||
if (-not $Result) { return $false }
|
||||
$outcome = Get-AgentField $Result "outcome" $null
|
||||
$canonicalIdentity = Get-AgentCanonicalOutcomeIdentity $Result $outcome
|
||||
if (-not $outcome -or -not $canonicalIdentity) { return $false }
|
||||
foreach ($name in @(
|
||||
"id", "source", "reason", "mode", "reconcileEvidenceId", "evidenceReceiptId",
|
||||
"automationRunId", "traceId", "workItemId", "idempotencyKey",
|
||||
"executionGeneration", "incidentId", "approvalId", "projectId", "routeId",
|
||||
"sourceEventResolutionPolicy"
|
||||
)) {
|
||||
if (-not $Result.PSObject.Properties[$name] -or (Get-AgentField $Result $name $null) -isnot [string]) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
foreach ($name in @("schemaVersion", "state", "verifierName", "sourceEventEvidence")) {
|
||||
if (-not $outcome.PSObject.Properties[$name] -or (Get-AgentField $outcome $name $null) -isnot [string]) {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
if (-not [string](Get-AgentField $outcome "verifierName" "")) { return $false }
|
||||
$previousSourceEvidence = [string](Get-AgentField $outcome "sourceEventEvidence" "")
|
||||
if (
|
||||
-not (Test-AgentPublicReceiptRef $SourceEvidence) -or
|
||||
-not $SourceEvidence.StartsWith("prometheus:cold-start:") -or
|
||||
-not (Test-AgentPublicReceiptRef $previousSourceEvidence) -or
|
||||
-not $previousSourceEvidence.StartsWith("prometheus:cold-start:")
|
||||
) { return $false }
|
||||
|
||||
return [bool](
|
||||
[string](Get-AgentField $Result "id" "") -eq $QueueId -and
|
||||
[string](Get-AgentField $Result "source" "") -eq "agent99-same-run-reconcile" -and
|
||||
[string](Get-AgentField $Result "reason" "") -eq "source_event_resolved_status_reconcile" -and
|
||||
[string](Get-AgentField $Result "mode" "") -eq "Status" -and
|
||||
(Get-AgentField $Result "controlledApply" $null) -is [bool] -and
|
||||
-not [bool](Get-AgentField $Result "controlledApply" $true) -and
|
||||
(Get-AgentField $Result "reconcileOnly" $null) -is [bool] -and
|
||||
[bool](Get-AgentField $Result "reconcileOnly" $false) -and
|
||||
(Get-AgentField $Result "runtimeWritePerformed" $null) -is [bool] -and
|
||||
-not [bool](Get-AgentField $Result "runtimeWritePerformed" $true) -and
|
||||
(Get-AgentField $Result "suppressTelegram" $null) -is [bool] -and
|
||||
[bool](Get-AgentField $Result "suppressTelegram" $false) -and
|
||||
[string](Get-AgentField $Result "reconcileEvidenceId" "") -eq $EvidenceReceiptId -and
|
||||
[string](Get-AgentField $Result "evidenceReceiptId" "") -eq $EvidenceReceiptId -and
|
||||
[string](Get-AgentField $Result "automationRunId" "") -eq [string](Get-AgentField $Identity "run_id" "") -and
|
||||
[string](Get-AgentField $Result "traceId" "") -eq [string](Get-AgentField $Identity "trace_id" "") -and
|
||||
[string](Get-AgentField $Result "workItemId" "") -eq [string](Get-AgentField $Identity "work_item_id" "") -and
|
||||
[string](Get-AgentField $Result "idempotencyKey" "") -eq [string](Get-AgentField $Identity "idempotency_key" "") -and
|
||||
[string](Get-AgentField $Result "executionGeneration" "") -eq [string](Get-AgentField $Identity "execution_generation" "") -and
|
||||
[string](Get-AgentField $Result "incidentId" "") -eq [string](Get-AgentField $Identity "incident_id" "") -and
|
||||
[string](Get-AgentField $Result "approvalId" "") -eq [string](Get-AgentField $Identity "approval_id" "") -and
|
||||
[string](Get-AgentField $Result "projectId" "") -eq [string](Get-AgentField $Identity "project_id" "") -and
|
||||
[string](Get-AgentField $Result "routeId" "") -eq [string](Get-AgentField $Identity "route_id" "") -and
|
||||
(Test-AgentSameRunIdentityMatchesExisting $canonicalIdentity $Identity) -and
|
||||
[string](Get-AgentField $Result "sourceEventResolutionPolicy" "") -eq "external_source_receipt_required" -and
|
||||
[string](Get-AgentField $outcome "schemaVersion" "") -eq "agent99_outcome_contract_v1" -and
|
||||
[string](Get-AgentField $outcome "state" "") -eq "failed" -and
|
||||
(Get-AgentField $outcome "resolved" $null) -is [bool] -and
|
||||
-not [bool](Get-AgentField $outcome "resolved" $true) -and
|
||||
(Get-AgentField $outcome "transportOk" $null) -is [bool] -and
|
||||
-not [bool](Get-AgentField $outcome "transportOk" $true) -and
|
||||
(Get-AgentField $outcome "verifierPassed" $null) -is [bool] -and
|
||||
-not [bool](Get-AgentField $outcome "verifierPassed" $true) -and
|
||||
(Get-AgentField $outcome "sourceEventResolved" $null) -is [bool] -and
|
||||
[bool](Get-AgentField $outcome "sourceEventResolved" $false)
|
||||
)
|
||||
}
|
||||
|
||||
function Move-AgentSameRunFailedNoWriteOutcomeForRetry {
|
||||
param(
|
||||
[string]$ProcessedPath,
|
||||
[string]$LockPath,
|
||||
[string]$RetryMarkerPath,
|
||||
[string]$QueueId,
|
||||
[object]$Identity,
|
||||
[string]$SourceEvidence,
|
||||
[string]$EvidenceReceiptId
|
||||
)
|
||||
|
||||
if (Test-Path -LiteralPath $RetryMarkerPath) {
|
||||
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_already_consumed_fail_closed" }
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $ProcessedPath) -or -not (Test-Path -LiteralPath $LockPath)) {
|
||||
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_missing_fail_closed" }
|
||||
}
|
||||
|
||||
try {
|
||||
$failedResult = Get-Content -LiteralPath $ProcessedPath -Raw | ConvertFrom-Json
|
||||
$lockReceipt = Get-Content -LiteralPath $LockPath -Raw | ConvertFrom-Json
|
||||
$failedOutcomeSha256 = (Get-FileHash -LiteralPath $ProcessedPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$previousLockSha256 = (Get-FileHash -LiteralPath $LockPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
} catch {
|
||||
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_invalid_fail_closed" }
|
||||
}
|
||||
if ($failedOutcomeSha256 -notmatch "^[0-9a-f]{64}$" -or $previousLockSha256 -notmatch "^[0-9a-f]{64}$") {
|
||||
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_digest_invalid_fail_closed" }
|
||||
}
|
||||
if (-not (Test-AgentSameRunFailedNoWriteOutcomeRetryEligible $failedResult $Identity $SourceEvidence $EvidenceReceiptId $QueueId)) {
|
||||
return [pscustomobject]@{ ok = $false; reason = "existing_same_run_status_outcome_invalid_fail_closed" }
|
||||
}
|
||||
|
||||
$runId = [string](Get-AgentField $Identity "run_id" "")
|
||||
$canonicalDigest = [string](Get-AgentField $Identity "canonical_digest" "")
|
||||
$lockValid = [bool](
|
||||
[string](Get-AgentField $lockReceipt "schemaVersion" "") -eq "agent99_same_run_single_flight_lock_v1" -and
|
||||
[string](Get-AgentField $lockReceipt "automationRunId" "") -eq $runId -and
|
||||
[string](Get-AgentField $lockReceipt "evidenceReceiptId" "") -eq $EvidenceReceiptId -and
|
||||
[string](Get-AgentField $lockReceipt "canonicalDigest" "") -eq $canonicalDigest -and
|
||||
(Get-AgentField $lockReceipt "storesRawEvidence" $null) -is [bool] -and
|
||||
-not [bool](Get-AgentField $lockReceipt "storesRawEvidence" $true)
|
||||
)
|
||||
if (-not $lockValid) {
|
||||
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_lock_invalid_fail_closed" }
|
||||
}
|
||||
|
||||
$outcomeAuditPath = Join-Path $SameRunRetryAuditDir "$QueueId.attempt-1.processed.json"
|
||||
$lockAuditPath = Join-Path $SameRunRetryAuditDir "$QueueId.attempt-1.lock.json"
|
||||
if ((Test-Path -LiteralPath $outcomeAuditPath) -or (Test-Path -LiteralPath $lockAuditPath)) {
|
||||
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_audit_collision_fail_closed" }
|
||||
}
|
||||
|
||||
try {
|
||||
$markerStream = [IO.File]::Open($RetryMarkerPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
||||
try {
|
||||
$markerReceipt = @{
|
||||
schemaVersion = "agent99_same_run_status_retry_v1"
|
||||
status = "retry_reserved"
|
||||
retryOrdinal = 1
|
||||
automationRunId = $runId
|
||||
evidenceReceiptId = $EvidenceReceiptId
|
||||
canonicalDigest = $canonicalDigest
|
||||
failedOutcomeAuditFile = (Split-Path -Leaf $outcomeAuditPath)
|
||||
failedOutcomeSha256 = $failedOutcomeSha256
|
||||
previousLockAuditFile = (Split-Path -Leaf $lockAuditPath)
|
||||
previousLockSha256 = $previousLockSha256
|
||||
preservesFailedOutcome = $true
|
||||
controlledApply = $false
|
||||
reconcileOnly = $true
|
||||
runtimeWriteAuthorized = $false
|
||||
storesRawEvidence = $false
|
||||
createdAt = (Get-Date -Format o)
|
||||
} | ConvertTo-Json -Compress
|
||||
$markerBytes = [Text.Encoding]::UTF8.GetBytes($markerReceipt)
|
||||
$markerStream.Write($markerBytes, 0, $markerBytes.Length)
|
||||
$markerStream.Flush($true)
|
||||
} finally {
|
||||
$markerStream.Dispose()
|
||||
}
|
||||
} catch [System.IO.IOException] {
|
||||
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_already_consumed_fail_closed" }
|
||||
}
|
||||
|
||||
try {
|
||||
Move-Item -LiteralPath $ProcessedPath -Destination $outcomeAuditPath
|
||||
Move-Item -LiteralPath $LockPath -Destination $lockAuditPath
|
||||
} catch {
|
||||
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_archive_failed_fail_closed" }
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
ok = $true
|
||||
reason = "same_run_status_failed_no_write_outcome_archived_for_single_retry"
|
||||
retryOrdinal = 1
|
||||
}
|
||||
}
|
||||
|
||||
function Start-AgentSameRunStatusReconcile {
|
||||
param([object]$Request)
|
||||
|
||||
@@ -558,6 +736,15 @@ function Start-AgentSameRunStatusReconcile {
|
||||
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_dispatch_identity_mismatch"; httpStatus = 409 }
|
||||
}
|
||||
|
||||
$safeRunId = Convert-AgentSafeFileToken $runId
|
||||
$queueId = "same-run-status-$safeRunId"
|
||||
$queuePath = Join-Path $QueueDir "$queueId.json"
|
||||
$runningPath = Join-Path $QueueDir "running-$queueId.json"
|
||||
$processedPath = Join-Path $ProcessedDir "processed-$queueId.json"
|
||||
$lockPath = Join-Path $SameRunStateDir "$queueId.lock.json"
|
||||
$retryMarkerPath = Join-Path $SameRunStateDir "$queueId.retry-1.json"
|
||||
$singleRetryPrepared = $false
|
||||
|
||||
$existing = Get-AgentOutcomeReadback $runId
|
||||
if (Test-AgentSameRunOutcomeEligible $existing $identity $sourceEvidence $evidenceReceiptId) {
|
||||
return [pscustomobject]@{
|
||||
@@ -572,15 +759,33 @@ function Start-AgentSameRunStatusReconcile {
|
||||
}
|
||||
}
|
||||
if ($existing.found -and [bool](Get-AgentField $existing "reconcileOnly" $false)) {
|
||||
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_same_run_status_outcome_invalid_fail_closed"; httpStatus = 409 }
|
||||
$retryPreparation = Move-AgentSameRunFailedNoWriteOutcomeForRetry `
|
||||
$processedPath $lockPath $retryMarkerPath $queueId $identity $sourceEvidence $evidenceReceiptId
|
||||
if (-not $retryPreparation.ok) {
|
||||
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = [string]$retryPreparation.reason; httpStatus = 409 }
|
||||
}
|
||||
$singleRetryPrepared = $true
|
||||
}
|
||||
$queuePending = Test-Path -LiteralPath $queuePath -PathType Leaf
|
||||
$runningPending = Test-Path -LiteralPath $runningPath -PathType Leaf
|
||||
$lockPending = Test-Path -LiteralPath $lockPath -PathType Leaf
|
||||
if (-not $singleRetryPrepared -and (Test-Path -LiteralPath $retryMarkerPath -PathType Leaf)) {
|
||||
if (-not (($queuePending -or $runningPending) -and $lockPending)) {
|
||||
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "same_run_status_retry_already_consumed_fail_closed"; httpStatus = 409 }
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
ok = $true
|
||||
status = "same_run_status_reconcile_pending"
|
||||
httpStatus = 202
|
||||
automationRunId = $runId
|
||||
reconcileOnly = $true
|
||||
controlledApply = $false
|
||||
evidenceReceiptId = $evidenceReceiptId
|
||||
queued = $false
|
||||
}
|
||||
}
|
||||
|
||||
$safeRunId = Convert-AgentSafeFileToken $runId
|
||||
$queueId = "same-run-status-$safeRunId"
|
||||
$queuePath = Join-Path $QueueDir "$queueId.json"
|
||||
$runningPath = Join-Path $QueueDir "running-$queueId.json"
|
||||
$lockPath = Join-Path $SameRunStateDir "$queueId.lock.json"
|
||||
if ((Test-Path -LiteralPath $queuePath) -or (Test-Path -LiteralPath $runningPath) -or (Test-Path -LiteralPath $lockPath)) {
|
||||
if ($queuePending -or $runningPending -or $lockPending) {
|
||||
return [pscustomobject]@{
|
||||
ok = $true
|
||||
status = "same_run_status_reconcile_pending"
|
||||
|
||||
@@ -829,3 +829,73 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() ->
|
||||
assert "Get-AgentEvidenceAtPath $exactEvidencePath" in queue_branch
|
||||
assert "Send-AgentTelegram" not in queue_branch
|
||||
assert "Invoke-AgentCompletionCallback" not in queue_branch
|
||||
|
||||
|
||||
def test_windows_relay_allows_only_one_audited_failed_no_write_status_retry() -> None:
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
relay = (root / "agent99-sre-alert-relay.ps1").read_text(encoding="utf-8")
|
||||
|
||||
field_helper = relay[relay.index("function Get-AgentField") :]
|
||||
field_helper = field_helper[: field_helper.index("function Get-AgentEnvValue")]
|
||||
assert "$Object -is [Collections.IDictionary]" in field_helper
|
||||
assert "$Object.Contains($Name)" in field_helper
|
||||
assert "return $Object[$Name]" in field_helper
|
||||
|
||||
retry_start = relay.index(
|
||||
"function Test-AgentSameRunFailedNoWriteOutcomeRetryEligible"
|
||||
)
|
||||
retry_end = relay.index("function Start-AgentSameRunStatusReconcile")
|
||||
retry_contract = relay[retry_start:retry_end]
|
||||
|
||||
assert '$SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit"' in relay
|
||||
assert '"agent99_same_run_status_retry_v1"' in retry_contract
|
||||
assert 'retryOrdinal = 1' in retry_contract
|
||||
assert 'status = "retry_reserved"' in retry_contract
|
||||
assert 'preservesFailedOutcome = $true' in retry_contract
|
||||
assert 'runtimeWriteAuthorized = $false' in retry_contract
|
||||
assert '[string](Get-AgentField $Result "mode" "") -eq "Status"' in retry_contract
|
||||
assert '[string](Get-AgentField $outcome "state" "") -eq "failed"' in retry_contract
|
||||
assert '-not [bool](Get-AgentField $Result "controlledApply" $true)' in retry_contract
|
||||
assert '[bool](Get-AgentField $Result "reconcileOnly" $false)' in retry_contract
|
||||
assert '-not [bool](Get-AgentField $Result "runtimeWritePerformed" $true)' in retry_contract
|
||||
assert '[bool](Get-AgentField $Result "suppressTelegram" $false)' in retry_contract
|
||||
assert '-not [bool](Get-AgentField $outcome "resolved" $true)' in retry_contract
|
||||
assert '-not [bool](Get-AgentField $outcome "transportOk" $true)' in retry_contract
|
||||
assert '-not [bool](Get-AgentField $outcome "verifierPassed" $true)' in retry_contract
|
||||
assert '[bool](Get-AgentField $outcome "sourceEventResolved" $false)' in retry_contract
|
||||
assert "Test-AgentPublicReceiptRef $SourceEvidence" in retry_contract
|
||||
assert "Test-AgentPublicReceiptRef $previousSourceEvidence" in retry_contract
|
||||
assert '$previousSourceEvidence.StartsWith("prometheus:cold-start:")' in retry_contract
|
||||
assert "Test-AgentSameRunIdentityMatchesExisting $canonicalIdentity $Identity" in retry_contract
|
||||
assert '"agent99_same_run_single_flight_lock_v1"' in retry_contract
|
||||
assert '[IO.FileMode]::CreateNew' in retry_contract
|
||||
assert 'failedOutcomeSha256 = $failedOutcomeSha256' in retry_contract
|
||||
assert 'previousLockSha256 = $previousLockSha256' in retry_contract
|
||||
assert 'Move-Item -LiteralPath $ProcessedPath -Destination $outcomeAuditPath' in retry_contract
|
||||
assert 'Move-Item -LiteralPath $LockPath -Destination $lockAuditPath' in retry_contract
|
||||
assert 'same_run_status_retry_already_consumed_fail_closed' in retry_contract
|
||||
assert 'same_run_status_retry_archive_failed_fail_closed' in retry_contract
|
||||
assert 'mode = "Recover"' not in retry_contract
|
||||
assert 'controlledApply = $true' not in retry_contract
|
||||
|
||||
start = relay[relay.index("function Start-AgentSameRunStatusReconcile") :]
|
||||
marker_branch = start.index(
|
||||
"if (-not $singleRetryPrepared -and "
|
||||
"(Test-Path -LiteralPath $retryMarkerPath -PathType Leaf))"
|
||||
)
|
||||
marker_rejected = start.index(
|
||||
'reason = "same_run_status_retry_already_consumed_fail_closed"',
|
||||
marker_branch,
|
||||
)
|
||||
marker_pending = start.index(
|
||||
'status = "same_run_status_reconcile_pending"',
|
||||
marker_rejected,
|
||||
)
|
||||
general_pending = start.index(
|
||||
"if ($queuePending -or $runningPending -or $lockPending)",
|
||||
marker_pending,
|
||||
)
|
||||
assert marker_branch < marker_rejected < marker_pending < general_pending
|
||||
assert "(($queuePending -or $runningPending) -and $lockPending)" in start[
|
||||
marker_branch:marker_pending
|
||||
]
|
||||
|
||||
@@ -8,6 +8,9 @@ CONTROL = ROOT / "agent99-control-plane.ps1"
|
||||
DEPLOY = ROOT / "agent99-deploy.ps1"
|
||||
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
|
||||
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
|
||||
REMOTE_ATOMIC_RECEIVER = (
|
||||
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
|
||||
)
|
||||
|
||||
|
||||
def test_agent99_uses_dedicated_identity_and_preferred_jump_route() -> None:
|
||||
@@ -312,6 +315,84 @@ def test_agent99_deployer_backs_up_manifest_before_runtime_promotion() -> None:
|
||||
assert "Copy-AgentRuntimeFileWithRetry (Join-Path $StageDir $name) $livePath" in deploy
|
||||
|
||||
|
||||
def test_agent99_atomic_deploy_regenerates_and_verifies_root_launcher() -> None:
|
||||
bootstrap = BOOTSTRAP.read_text(encoding="utf-8")
|
||||
deploy = DEPLOY.read_text(encoding="utf-8")
|
||||
|
||||
assert "[switch]$WriteLauncherOnly" in bootstrap
|
||||
assert "function Write-AgentLauncher" in bootstrap
|
||||
launcher_only = bootstrap[bootstrap.index("if ($WriteLauncherOnly)") :]
|
||||
launcher_only = launcher_only[: launcher_only.index("function Copy-AgentFile")]
|
||||
assert 'Join-Path $BinDir "agent99-control-plane.ps1"' in launcher_only
|
||||
assert 'Write-AgentLauncher (Join-Path $AgentRoot "agent99-run.ps1")' in launcher_only
|
||||
assert "Ensure-AgentHost112CanonicalSshConfig" not in launcher_only
|
||||
assert "Set-AgentBootstrapProperty" not in launcher_only
|
||||
|
||||
staged_check = deploy[deploy.index('$launcherValidationRoot = Join-Path $StageDir') :]
|
||||
staged_check = staged_check[: staged_check.index("$stagedFiles = @(")]
|
||||
assert "-WriteLauncherOnly" in staged_check
|
||||
assert '"staged_no_live_write"' in staged_check
|
||||
assert 'Write-DeployEvidence $false "preflight_failed" "launcher_contract_failed"' in staged_check
|
||||
|
||||
promotion = deploy[deploy.index("$configBackedUp = $false") :]
|
||||
promotion = promotion[: promotion.index("$config = Get-Content $ConfigPath")]
|
||||
assert promotion.index('Copy-Item -Force $LauncherPath') < promotion.index(
|
||||
"foreach ($name in $runtimeFiles)"
|
||||
)
|
||||
assert 'name = "agent99-run.ps1"' in promotion
|
||||
assert 'Join-Path $BinDir "agent99-bootstrap.ps1"' in promotion
|
||||
assert 'Get-AgentLauncherContract $LauncherPath "live_post_promotion"' in promotion
|
||||
assert "Promoted Agent99 launcher reconcile contract verification failed." in promotion
|
||||
assert "launcherContract = $script:LauncherContract" in deploy
|
||||
|
||||
|
||||
def test_agent99_launcher_contract_covers_same_run_flags_and_forwarding() -> None:
|
||||
bootstrap = BOOTSTRAP.read_text(encoding="utf-8")
|
||||
deploy = DEPLOY.read_text(encoding="utf-8")
|
||||
|
||||
for marker in (
|
||||
'[switch]`$ReconcileOnly',
|
||||
'[string]`$ReconcileEvidenceId = ""',
|
||||
'[switch]`$ReconcileQueueOnly',
|
||||
'[string]`$ReconcileQueueId = ""',
|
||||
"-ReconcileOnly:`$ReconcileOnly",
|
||||
"-ReconcileEvidenceId `$ReconcileEvidenceId",
|
||||
"-ReconcileQueueOnly:`$ReconcileQueueOnly",
|
||||
"-ReconcileQueueId `$ReconcileQueueId",
|
||||
):
|
||||
assert marker in bootstrap
|
||||
assert "Management.Automation.Language.Parser" in bootstrap
|
||||
assert "Management.Automation.Language.Parser" in deploy
|
||||
assert "reconcileOnlyForwarded = $true" in bootstrap
|
||||
assert "reconcileQueueOnlyForwarded = $true" in bootstrap
|
||||
|
||||
|
||||
def test_agent99_remote_rollback_and_post_verifier_include_root_launcher() -> None:
|
||||
receiver = REMOTE_ATOMIC_RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
assert '$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1"' in receiver
|
||||
assert "function Get-AgentLauncherContract" in receiver
|
||||
assert '$rows += "agent99-run.ps1`t$(Get-AgentSha256File $LauncherPath)`n"' in receiver
|
||||
restore = receiver[receiver.index("function Restore-AgentBundle") :]
|
||||
restore = restore[: restore.index("function Write-AgentRemoteDeployReceipt")]
|
||||
assert "[bool]$BeforeLauncherPresent" in restore
|
||||
assert 'Join-Path $BackupPath "agent99-run.ps1"' in restore
|
||||
assert "Copy-AgentFileWithRetry $backupLauncher $LauncherPath" in restore
|
||||
assert "Remove-Item -LiteralPath $LauncherPath -Force" in restore
|
||||
assert "$liveLauncherContract.ok" in receiver
|
||||
assert (
|
||||
'[string]$liveLauncherContract.sha256 -eq '
|
||||
'[string]$prior.launcherContract.sha256'
|
||||
) in receiver
|
||||
assert "$launcherContract.ok" in receiver
|
||||
assert (
|
||||
'[string]$launcherContract.sha256 -eq '
|
||||
'[string]$deploy.payload.launcherContract.sha256'
|
||||
) in receiver
|
||||
assert "launcherContract = $launcherContract" in receiver
|
||||
assert "failedLauncherContract = $launcherContract" in receiver
|
||||
|
||||
|
||||
def test_agent99_completion_callback_waits_for_durable_same_trace_readback() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
deploy = DEPLOY.read_text(encoding="utf-8")
|
||||
|
||||
@@ -13,6 +13,7 @@ $StateDir = Join-Path $AgentRoot "state"
|
||||
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$DeployRoot = Join-Path $AgentRoot "deploy"
|
||||
$ManifestPath = Join-Path $StateDir "runtime-manifest.json"
|
||||
$LauncherPath = Join-Path $AgentRoot "agent99-run.ps1"
|
||||
$WindowsPowerShell = "C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe"
|
||||
$RelayTaskName = "Wooo-Agent99-SRE-Alert-Relay"
|
||||
$RelayTaskPath = "\"
|
||||
@@ -71,6 +72,48 @@ function Get-AgentSha256File {
|
||||
return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Get-AgentLauncherContract {
|
||||
$requiredMarkers = @(
|
||||
'[switch]$ReconcileOnly',
|
||||
'[string]$ReconcileEvidenceId = ""',
|
||||
'[switch]$ReconcileQueueOnly',
|
||||
'[string]$ReconcileQueueId = ""',
|
||||
'-ReconcileOnly:$ReconcileOnly',
|
||||
'-ReconcileEvidenceId $ReconcileEvidenceId',
|
||||
'-ReconcileQueueOnly:$ReconcileQueueOnly',
|
||||
'-ReconcileQueueId $ReconcileQueueId'
|
||||
)
|
||||
if (-not (Test-Path -LiteralPath $LauncherPath -PathType Leaf)) {
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_launcher_contract_v1"
|
||||
ok = $false
|
||||
exists = $false
|
||||
sha256 = "missing"
|
||||
parseErrorCount = -1
|
||||
requiredMarkerCount = $requiredMarkers.Count
|
||||
missingMarkerCount = $requiredMarkers.Count
|
||||
reconcileOnlyForwarded = $false
|
||||
reconcileQueueOnlyForwarded = $false
|
||||
}
|
||||
}
|
||||
$tokens = $null
|
||||
$parseErrors = $null
|
||||
[Management.Automation.Language.Parser]::ParseFile($LauncherPath, [ref]$tokens, [ref]$parseErrors) | Out-Null
|
||||
$content = [IO.File]::ReadAllText($LauncherPath)
|
||||
$missingMarkers = @($requiredMarkers | Where-Object { -not $content.Contains($_) })
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_launcher_contract_v1"
|
||||
ok = [bool](@($parseErrors).Count -eq 0 -and $missingMarkers.Count -eq 0)
|
||||
exists = $true
|
||||
sha256 = (Get-AgentSha256File $LauncherPath)
|
||||
parseErrorCount = @($parseErrors).Count
|
||||
requiredMarkerCount = $requiredMarkers.Count
|
||||
missingMarkerCount = $missingMarkers.Count
|
||||
reconcileOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileOnly:$ReconcileOnly' -and $missingMarkers -notcontains '-ReconcileEvidenceId $ReconcileEvidenceId')
|
||||
reconcileQueueOnlyForwarded = [bool]($missingMarkers -notcontains '-ReconcileQueueOnly:$ReconcileQueueOnly' -and $missingMarkers -notcontains '-ReconcileQueueId $ReconcileQueueId')
|
||||
}
|
||||
}
|
||||
|
||||
function Get-AgentSafeRuntimeManifest {
|
||||
if (-not (Test-Path -LiteralPath $ManifestPath -PathType Leaf)) {
|
||||
return [pscustomobject]@{
|
||||
@@ -146,6 +189,7 @@ function Get-AgentLiveBundleDigest {
|
||||
foreach ($name in $FixedRuntimeFiles) {
|
||||
$rows += "$name`t$(Get-AgentSha256File (Join-Path $BinDir $name))`n"
|
||||
}
|
||||
$rows += "agent99-run.ps1`t$(Get-AgentSha256File $LauncherPath)`n"
|
||||
return Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes(($rows -join "")))
|
||||
}
|
||||
|
||||
@@ -555,6 +599,7 @@ function Restore-AgentBundle {
|
||||
param(
|
||||
[string]$BackupPath,
|
||||
[hashtable]$BeforeFilePresence,
|
||||
[bool]$BeforeLauncherPresent,
|
||||
[bool]$BeforeConfigPresent,
|
||||
[bool]$BeforeManifestPresent
|
||||
)
|
||||
@@ -573,6 +618,13 @@ function Restore-AgentBundle {
|
||||
Remove-Item -LiteralPath $livePath -Force
|
||||
}
|
||||
}
|
||||
$backupLauncher = Join-Path $BackupPath "agent99-run.ps1"
|
||||
if ($BeforeLauncherPresent) {
|
||||
if (-not (Test-Path -LiteralPath $backupLauncher -PathType Leaf)) { throw "required_launcher_backup_missing" }
|
||||
Copy-AgentFileWithRetry $backupLauncher $LauncherPath
|
||||
} elseif (Test-Path -LiteralPath $LauncherPath -PathType Leaf) {
|
||||
Remove-Item -LiteralPath $LauncherPath -Force
|
||||
}
|
||||
$backupConfig = Join-Path $BackupPath "agent99.config.json"
|
||||
if ($BeforeConfigPresent) {
|
||||
if (-not (Test-Path -LiteralPath $backupConfig -PathType Leaf)) { throw "required_config_backup_missing" }
|
||||
@@ -832,6 +884,7 @@ try {
|
||||
}
|
||||
|
||||
$live = Get-AgentSafeRuntimeManifest
|
||||
$liveLauncherContract = Get-AgentLauncherContract
|
||||
if (
|
||||
$prior -and
|
||||
[string]$prior.status -eq "deployed_verified" -and
|
||||
@@ -841,10 +894,15 @@ try {
|
||||
[string]$prior.sourceRevision -eq $sourceRevision -and
|
||||
$prior.PSObject.Properties["relayReload"] -and
|
||||
[bool]$prior.relayReload.newGenerationVerified -and
|
||||
$prior.PSObject.Properties["launcherContract"] -and
|
||||
[bool]$prior.launcherContract.ok -and
|
||||
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
|
||||
$live.runtimeMatched -and
|
||||
$live.sourceRevision -eq $sourceRevision -and
|
||||
$live.fileCount -eq 14 -and
|
||||
$live.mismatchCount -eq 0
|
||||
$live.mismatchCount -eq 0 -and
|
||||
$liveLauncherContract.ok -and
|
||||
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
|
||||
) {
|
||||
$script:ReceiverOperationPhase = "idempotent_replay_post_verifier"
|
||||
$replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
|
||||
@@ -852,6 +910,7 @@ try {
|
||||
$originalOperationBoundaries = $prior.operationBoundaries
|
||||
$prior | Add-Member -NotePropertyName idempotentReplay -NotePropertyValue $true -Force
|
||||
$prior | Add-Member -NotePropertyName runtimeManifest -NotePropertyValue $live -Force
|
||||
$prior | Add-Member -NotePropertyName launcherContract -NotePropertyValue $liveLauncherContract -Force
|
||||
$prior | Add-Member -NotePropertyName livePreflight -NotePropertyValue $replayPreflight -Force
|
||||
$prior | Add-Member -NotePropertyName originalOperationBoundaries -NotePropertyValue $originalOperationBoundaries -Force
|
||||
$prior | Add-Member -NotePropertyName operationBoundaries -NotePropertyValue ([pscustomobject]@{
|
||||
@@ -979,6 +1038,7 @@ try {
|
||||
$beforeManifestDigest = Get-AgentSha256File $ManifestPath
|
||||
$beforeFilePresence = @{}
|
||||
foreach ($name in $FixedRuntimeFiles) { $beforeFilePresence[$name] = Test-Path -LiteralPath (Join-Path $BinDir $name) -PathType Leaf }
|
||||
$beforeLauncherPresent = Test-Path -LiteralPath $LauncherPath -PathType Leaf
|
||||
$beforeConfigPresent = Test-Path -LiteralPath $ConfigPath -PathType Leaf
|
||||
$beforeManifestPresent = Test-Path -LiteralPath $ManifestPath -PathType Leaf
|
||||
$beforeRelayRuntime = Get-AgentRelayRuntime
|
||||
@@ -1034,6 +1094,7 @@ try {
|
||||
$relayReload = $null
|
||||
$runtimeManifest = $null
|
||||
$livePreflight = $null
|
||||
$launcherContract = $null
|
||||
$postVerified = $false
|
||||
$failurePhase = ""
|
||||
$failureType = ""
|
||||
@@ -1046,7 +1107,15 @@ try {
|
||||
if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) {
|
||||
$backupPath = [string]$deploy.payload.backupDir
|
||||
}
|
||||
if (-not $deploy.ok -or -not $deploy.payload -or [string]$deploy.payload.status -ne "deployed" -or $deploy.payload.taskRegistration) {
|
||||
if (
|
||||
-not $deploy.ok -or
|
||||
-not $deploy.payload -or
|
||||
[string]$deploy.payload.status -ne "deployed" -or
|
||||
$deploy.payload.taskRegistration -or
|
||||
-not $deploy.payload.PSObject.Properties["launcherContract"] -or
|
||||
-not [bool]$deploy.payload.launcherContract.ok -or
|
||||
[string]$deploy.payload.launcherContract.sha256 -notmatch "^[0-9a-f]{64}$"
|
||||
) {
|
||||
$failurePhase = "deploy"
|
||||
$failureType = "deploy_child_failed_or_invalid"
|
||||
} else {
|
||||
@@ -1060,6 +1129,7 @@ try {
|
||||
} else {
|
||||
$script:ReceiverOperationPhase = "independent_post_verifier"
|
||||
$runtimeManifest = Get-AgentSafeRuntimeManifest
|
||||
$launcherContract = Get-AgentLauncherContract
|
||||
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
|
||||
$postVerified = [bool](
|
||||
$runtimeManifest.exists -and
|
||||
@@ -1067,6 +1137,8 @@ try {
|
||||
$runtimeManifest.sourceRevision -eq $sourceRevision -and
|
||||
$runtimeManifest.fileCount -eq 14 -and
|
||||
$runtimeManifest.mismatchCount -eq 0 -and
|
||||
$launcherContract.ok -and
|
||||
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and
|
||||
$relayReload.oldGenerationsGone -and
|
||||
$relayReload.newGenerationVerified -and
|
||||
$relayReload.after.taskEnabled -and
|
||||
@@ -1108,6 +1180,7 @@ try {
|
||||
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
|
||||
relayReload = $relayReload
|
||||
runtimeManifest = $runtimeManifest
|
||||
launcherContract = $launcherContract
|
||||
livePreflight = $livePreflight
|
||||
durableReceiptWritten = $true
|
||||
idempotentReplay = $false
|
||||
@@ -1147,7 +1220,7 @@ try {
|
||||
}
|
||||
$script:ReceiverOperationPhase = "runtime_rollback"
|
||||
$script:ReceiverRollbackAttempted = $true
|
||||
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
|
||||
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeLauncherPresent $beforeConfigPresent $beforeManifestPresent
|
||||
$rollbackForbiddenGenerations = @($beforeRelayRuntime.taskGeneration)
|
||||
if ($relayReload) {
|
||||
$rollbackForbiddenGenerations += @($relayReload.before.taskGeneration)
|
||||
@@ -1227,6 +1300,7 @@ try {
|
||||
attemptReceiptPath = $attemptReceiptPath
|
||||
relayReload = $relayReload
|
||||
failedRuntimeManifest = $runtimeManifest
|
||||
failedLauncherContract = $launcherContract
|
||||
failedLivePreflight = $livePreflight
|
||||
rollback = $rollback
|
||||
rollbackRelayRestart = $rollbackRelayRestart
|
||||
|
||||
Reference in New Issue
Block a user