1702 lines
80 KiB
PowerShell
1702 lines
80 KiB
PowerShell
param(
|
|
[ValidateSet("Check", "Apply", "Verify", "RestoreCheck", "RestoreApply", "RestoreVerifyCleanup", "ToolchainCheck", "ToolchainApply", "ToolchainVerify", "ToolchainReconcile")]
|
|
[string]$Mode = "Check",
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$TraceId,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$RunId,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$WorkItemId,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$SourceRevision,
|
|
[string]$AttemptId = "",
|
|
[string]$ToolchainEnvelopeJson = "",
|
|
[switch]$ToolchainEnvelopeDeletedBeforeExecutor,
|
|
[string]$AgentRoot = "C:\Wooo\Agent99"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$CanonicalAsset = "signoz-110-control-plane-metadata"
|
|
$TargetHost = "192.168.0.110"
|
|
$RemoteUser = "wooo"
|
|
$IdentityFile = Join-Path $AgentRoot "keys\agent99_ed25519"
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
|
$RuntimeManifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
|
$EntrypointPath = [string]$MyInvocation.MyCommand.Path
|
|
$ToolchainTargetPath = Join-Path (Split-Path -Parent $EntrypointPath) "agent99-signoz-toolchain-target.py"
|
|
$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
|
$BundleId = "b83bfc898cd43cf8ec5688d75ec20231c2faa32de5be3fa886db33038d620dff"
|
|
$ToolchainRoot = "/backup/toolchains/signoz-metadata/$BundleId"
|
|
$PolicyPath = "$ToolchainRoot/metadata-export-policy.json"
|
|
$ExporterPath = "$ToolchainRoot/signoz-metadata-export.py"
|
|
$VerifierPath = "$ToolchainRoot/verify-signoz-metadata-export.py"
|
|
$RestoreDriverPath = "$ToolchainRoot/signoz-metadata-restore-drill.py"
|
|
$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"
|
|
$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"
|
|
$SharedContractPath = "$ToolchainRoot/signoz_metadata_contract.py"
|
|
$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"
|
|
$BaseUrl = "http://127.0.0.1:8080"
|
|
$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"
|
|
$RemoteReceipt = "/backup/deploy-receipts/signoz-metadata-toolchain/export-$RunId.json"
|
|
$RestoreRemoteReceipt = "/backup/deploy-receipts/signoz-metadata-toolchain/restore-$RunId.json"
|
|
$SignozImage = "signoz/signoz:v0.113.0"
|
|
$SignozImageId = "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b"
|
|
$SignozContainer = "signoz"
|
|
$OperationLock = "/tmp/awoooi-signoz-backup-operation.lock"
|
|
$DefaultTargetKillAfterSeconds = 15
|
|
$RestoreCleanupKillAfterSeconds = 600
|
|
$TargetGuardWorstCaseSeconds = 95
|
|
$CheckVerifyTargetTimeoutSeconds = 90
|
|
$ApplyTargetTimeoutSeconds = 720
|
|
$RestoreCheckTargetTimeoutSeconds = 180
|
|
$RestoreApplyTargetTimeoutSeconds = 1800
|
|
$RestoreCleanupTargetTimeoutSeconds = 300
|
|
$ToolchainCheckTargetTimeoutSeconds = 180
|
|
$ToolchainApplyTargetTimeoutSeconds = 600
|
|
$CheckVerifyControllerWaitSeconds = 300
|
|
$ApplyControllerWaitSeconds = 900
|
|
$RestoreCheckControllerWaitSeconds = 420
|
|
$RestoreApplyControllerWaitSeconds = 2550
|
|
$RestoreCleanupControllerWaitSeconds = 1050
|
|
$ToolchainCheckControllerWaitSeconds = 420
|
|
$ToolchainApplyControllerWaitSeconds = 900
|
|
$ControllerTransportReserveSeconds = 30
|
|
$TargetProgramTimeoutSeconds = switch ($Mode) {
|
|
"Apply" { $ApplyTargetTimeoutSeconds }
|
|
"RestoreCheck" { $RestoreCheckTargetTimeoutSeconds }
|
|
"RestoreApply" { $RestoreApplyTargetTimeoutSeconds }
|
|
"RestoreVerifyCleanup" { $RestoreCleanupTargetTimeoutSeconds }
|
|
"ToolchainApply" { $ToolchainApplyTargetTimeoutSeconds }
|
|
"ToolchainReconcile" { $ToolchainApplyTargetTimeoutSeconds }
|
|
"ToolchainCheck" { $ToolchainCheckTargetTimeoutSeconds }
|
|
"ToolchainVerify" { $ToolchainCheckTargetTimeoutSeconds }
|
|
default { $CheckVerifyTargetTimeoutSeconds }
|
|
}
|
|
$ControllerWaitSeconds = switch ($Mode) {
|
|
"Apply" { $ApplyControllerWaitSeconds }
|
|
"RestoreCheck" { $RestoreCheckControllerWaitSeconds }
|
|
"RestoreApply" { $RestoreApplyControllerWaitSeconds }
|
|
"RestoreVerifyCleanup" { $RestoreCleanupControllerWaitSeconds }
|
|
"ToolchainApply" { $ToolchainApplyControllerWaitSeconds }
|
|
"ToolchainReconcile" { $ToolchainApplyControllerWaitSeconds }
|
|
"ToolchainCheck" { $ToolchainCheckControllerWaitSeconds }
|
|
"ToolchainVerify" { $ToolchainCheckControllerWaitSeconds }
|
|
default { $CheckVerifyControllerWaitSeconds }
|
|
}
|
|
$TargetKillAfterSeconds = if ($Mode -in @("RestoreApply", "RestoreVerifyCleanup")) {
|
|
$RestoreCleanupKillAfterSeconds
|
|
} else {
|
|
$DefaultTargetKillAfterSeconds
|
|
}
|
|
$TargetWorstCaseSeconds = $TargetGuardWorstCaseSeconds + $TargetProgramTimeoutSeconds + $TargetKillAfterSeconds
|
|
$EvidenceRunId = if ($RunId -match $SafeIdPattern) { $RunId } else { "invalid-$PID" }
|
|
$EvidenceAttemptId = if ($AttemptId -match $SafeIdPattern) { $AttemptId } else { "invalid-$PID" }
|
|
$EvidenceIdentity = if ($Mode -in @("RestoreVerifyCleanup", "ToolchainReconcile")) {
|
|
"$EvidenceRunId-attempt-$EvidenceAttemptId"
|
|
} else {
|
|
$EvidenceRunId
|
|
}
|
|
$EvidencePath = Join-Path $EvidenceDir "agent99-signoz-metadata-$($Mode.ToLowerInvariant())-$EvidenceIdentity.json"
|
|
$EvidenceReservationPath = "$EvidencePath.reserve"
|
|
$script:TransientTransportOutputUsed = $false
|
|
$script:TransportCleanupSucceeded = $true
|
|
$script:EvidenceReserved = $false
|
|
$script:ExportCredentialDispatchAttempted = $false
|
|
$script:IsolatedApplyDispatchAttempted = $false
|
|
$script:ToolchainSourceWritePerformed = $false
|
|
$script:ToolchainSourceWriteAttempted = $false
|
|
|
|
$ToolchainFiles = [ordered]@{
|
|
$PolicyPath = "a050943d50631d34d6eb2580db484197d6c7101b3b9cc2b47bb7ee3d87dd3822"
|
|
$SharedContractPath = "d2b2281da9ec89afc44d5153711b2d61e972545ad107611b74a5e6dfb01defe8"
|
|
$ExporterPath = "8cd5cabf245e3814da32e33920c431ec28ed7180edc0495a2d0b9379fb4025fb"
|
|
$VerifierPath = "0bac6d8ef8063f3584fc744cf205de08fc50d1085b292ffbd061f6090c344c86"
|
|
$RestoreDriverPath = "eb766f1209af08c3b339756d58653165d55b3c23bc8dbd66438dfb46d8098156"
|
|
$IsolatedRestorePath = "dc57604919e220454f25955c1c652b7bd9dea632210019b0b4de527969e03c52"
|
|
$IsolatedClusterConfigPath = "e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b"
|
|
}
|
|
$ToolchainPayloadSpec = @(
|
|
[pscustomobject]@{ name = "metadata-export-policy.json"; path = $PolicyPath; sha256 = $ToolchainFiles[$PolicyPath]; mode = "0644" },
|
|
[pscustomobject]@{ name = "signoz_metadata_contract.py"; path = $SharedContractPath; sha256 = $ToolchainFiles[$SharedContractPath]; mode = "0644" },
|
|
[pscustomobject]@{ name = "signoz-metadata-export.py"; path = $ExporterPath; sha256 = $ToolchainFiles[$ExporterPath]; mode = "0755" },
|
|
[pscustomobject]@{ name = "verify-signoz-metadata-export.py"; path = $VerifierPath; sha256 = $ToolchainFiles[$VerifierPath]; mode = "0755" },
|
|
[pscustomobject]@{ name = "signoz-metadata-restore-drill.py"; path = $RestoreDriverPath; sha256 = $ToolchainFiles[$RestoreDriverPath]; mode = "0755" },
|
|
[pscustomobject]@{ name = "signoz-metadata-isolated-restore.py"; path = $IsolatedRestorePath; sha256 = $ToolchainFiles[$IsolatedRestorePath]; mode = "0755" },
|
|
[pscustomobject]@{ name = "isolated-cluster.xml"; path = $IsolatedClusterConfigPath; sha256 = $ToolchainFiles[$IsolatedClusterConfigPath]; mode = "0644" }
|
|
)
|
|
|
|
function Invoke-Agent99SignozFixedRemote {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$RemoteCommand,
|
|
[int]$TimeoutSeconds = 120
|
|
)
|
|
|
|
$arguments = @(
|
|
"-i", $IdentityFile,
|
|
"-o", "BatchMode=yes",
|
|
"-o", "IdentitiesOnly=yes",
|
|
"-o", "StrictHostKeyChecking=yes",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "ConnectTimeout=8",
|
|
"-o", "ServerAliveInterval=20",
|
|
"-o", "ServerAliveCountMax=3",
|
|
"-l", $RemoteUser,
|
|
$TargetHost,
|
|
$RemoteCommand
|
|
)
|
|
$token = [Guid]::NewGuid().ToString("N")
|
|
$stdoutPath = Join-Path $env:TEMP "agent99-signoz-metadata-$token.out"
|
|
$stderrPath = Join-Path $env:TEMP "agent99-signoz-metadata-$token.err"
|
|
$script:TransientTransportOutputUsed = $true
|
|
try {
|
|
$process = Start-Process -FilePath "ssh.exe" -ArgumentList $arguments `
|
|
-NoNewWindow -RedirectStandardOutput $stdoutPath `
|
|
-RedirectStandardError $stderrPath -PassThru
|
|
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
|
|
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
|
|
return [pscustomobject]@{
|
|
completed = $false
|
|
timedOut = $true
|
|
errorCode = "remote_timeout"
|
|
stdout = ""
|
|
}
|
|
}
|
|
$process.WaitForExit() | Out-Null
|
|
$stdout = if (Test-Path -LiteralPath $stdoutPath) {
|
|
([IO.File]::ReadAllText($stdoutPath, [Text.Encoding]::UTF8)).Trim()
|
|
} else { "" }
|
|
return [pscustomobject]@{
|
|
completed = $true
|
|
timedOut = $false
|
|
errorCode = ""
|
|
stdout = $stdout
|
|
}
|
|
} finally {
|
|
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue
|
|
if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) {
|
|
$script:TransportCleanupSucceeded = $false
|
|
}
|
|
}
|
|
}
|
|
|
|
function Test-Agent99SignozTransportOutcome {
|
|
param([object]$Result)
|
|
|
|
return [bool](
|
|
$Result -and
|
|
$Result.completed -eq $true -and
|
|
-not $Result.timedOut
|
|
)
|
|
}
|
|
|
|
function New-Agent99SignozPathStateCommand {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Path,
|
|
[ValidateSet("directory", "file")][string]$ExpectedKind
|
|
)
|
|
|
|
$kindTest = if ($ExpectedKind -eq "directory") { "-d" } else { "-f" }
|
|
return (
|
|
"sudo -n /bin/bash -c 'if [ -L `"$Path`" ]; then " +
|
|
"/usr/bin/printf `"agent99_path_state:symlink`"; elif [ $kindTest `"$Path`" ]; then " +
|
|
"/usr/bin/printf `"agent99_path_state:$ExpectedKind`"; elif [ -e `"$Path`" ]; then " +
|
|
"/usr/bin/printf `"agent99_path_state:other`"; else " +
|
|
"/usr/bin/printf `"agent99_path_state:absent`"; fi'"
|
|
)
|
|
}
|
|
|
|
function Get-Agent99SignozRemotePathState {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$Path,
|
|
[ValidateSet("directory", "file")][string]$ExpectedKind
|
|
)
|
|
$remote = Invoke-Agent99SignozFixedRemote (
|
|
New-Agent99SignozPathStateCommand $Path $ExpectedKind
|
|
)
|
|
if (
|
|
(Test-Agent99SignozTransportOutcome $remote) -and
|
|
[string]$remote.stdout -match "^agent99_path_state:(absent|directory|file|other|symlink)$"
|
|
) {
|
|
return [string]$Matches[1]
|
|
}
|
|
return "unverified"
|
|
}
|
|
|
|
function Convert-Agent99JsonReceipt {
|
|
param([object]$RemoteResult)
|
|
if (
|
|
-not $RemoteResult -or
|
|
$RemoteResult.completed -ne $true -or
|
|
$RemoteResult.timedOut -or
|
|
-not [string]$RemoteResult.stdout
|
|
) { return $null }
|
|
try {
|
|
return ([string]$RemoteResult.stdout | ConvertFrom-Json)
|
|
} catch {
|
|
return $null
|
|
}
|
|
}
|
|
|
|
function Get-Agent99SignozSha256Bytes {
|
|
param([byte[]]$Bytes)
|
|
$sha = [Security.Cryptography.SHA256]::Create()
|
|
try {
|
|
return ([BitConverter]::ToString($sha.ComputeHash($Bytes))).Replace("-", "").ToLowerInvariant()
|
|
} finally {
|
|
$sha.Dispose()
|
|
}
|
|
}
|
|
|
|
function Get-Agent99SignozToolchainManifestBase64 {
|
|
$rows = @($ToolchainPayloadSpec | ForEach-Object {
|
|
[ordered]@{
|
|
name = [string]$_.name
|
|
sha256 = [string]$_.sha256
|
|
mode = [string]$_.mode
|
|
}
|
|
})
|
|
$json = $rows | ConvertTo-Json -Compress
|
|
return [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($json))
|
|
}
|
|
|
|
function Expand-Agent99SignozToolchainEnvelope {
|
|
param([Parameter(Mandatory = $true)][string]$EnvelopeJson)
|
|
|
|
if (-not $EnvelopeJson -or $EnvelopeJson.Length -gt 4194304) {
|
|
throw "toolchain_envelope_size_invalid"
|
|
}
|
|
try { $envelope = $EnvelopeJson | ConvertFrom-Json } catch {
|
|
throw "toolchain_envelope_json_invalid"
|
|
}
|
|
if (
|
|
[string]$envelope.schemaVersion -ne "awoooi_signoz_metadata_toolchain_envelope_v1" -or
|
|
[string]$envelope.traceId -ne $TraceId -or
|
|
[string]$envelope.runId -ne $RunId -or
|
|
[string]$envelope.workItemId -ne $WorkItemId -or
|
|
[string]$envelope.sourceRevision -ne $SourceRevision -or
|
|
[string]$envelope.bundleId -ne $BundleId
|
|
) { throw "toolchain_envelope_identity_invalid" }
|
|
$files = @($envelope.files)
|
|
if ($files.Count -ne $ToolchainPayloadSpec.Count) {
|
|
throw "toolchain_envelope_file_count_invalid"
|
|
}
|
|
$manifestText = ""
|
|
$decodedRows = @()
|
|
$tempRoot = Join-Path $env:TEMP "agent99-signoz-toolchain-$RunId-$PID-$([Guid]::NewGuid().ToString('N'))"
|
|
if (Test-Path -LiteralPath $tempRoot) { throw "toolchain_temp_identity_collision" }
|
|
New-Item -ItemType Directory -Path $tempRoot | Out-Null
|
|
try {
|
|
for ($index = 0; $index -lt $ToolchainPayloadSpec.Count; $index += 1) {
|
|
$expected = $ToolchainPayloadSpec[$index]
|
|
$row = $files[$index]
|
|
if (
|
|
[string]$row.name -ne [string]$expected.name -or
|
|
[string]$row.sha256 -ne [string]$expected.sha256 -or
|
|
[string]$row.mode -ne [string]$expected.mode
|
|
) { throw "toolchain_envelope_file_identity_invalid" }
|
|
try { $bytes = [Convert]::FromBase64String([string]$row.contentBase64) } catch {
|
|
throw "toolchain_envelope_content_base64_invalid"
|
|
}
|
|
if ($bytes.Length -le 0 -or $bytes.Length -gt 2097152) {
|
|
throw "toolchain_envelope_content_size_invalid"
|
|
}
|
|
if ((Get-Agent99SignozSha256Bytes $bytes) -ne [string]$expected.sha256) {
|
|
throw "toolchain_envelope_content_hash_mismatch"
|
|
}
|
|
$destination = Join-Path $tempRoot ([string]$expected.name)
|
|
$stream = [IO.File]::Open(
|
|
$destination,
|
|
[IO.FileMode]::CreateNew,
|
|
[IO.FileAccess]::Write,
|
|
[IO.FileShare]::None
|
|
)
|
|
try {
|
|
$stream.Write($bytes, 0, $bytes.Length)
|
|
$stream.Flush($true)
|
|
} finally {
|
|
$stream.Dispose()
|
|
}
|
|
$decodedRows += [pscustomobject]@{
|
|
name = [string]$expected.name
|
|
path = $destination
|
|
}
|
|
$manifestText += "$([string]$expected.name)`t$([string]$expected.sha256)`t$([string]$expected.mode)`n"
|
|
}
|
|
$bundleMaterial = (($ToolchainPayloadSpec | ForEach-Object { "$([string]$_.sha256)`n" }) -join "")
|
|
$computedBundle = Get-Agent99SignozSha256Bytes ([Text.Encoding]::ASCII.GetBytes($bundleMaterial))
|
|
$computedManifest = Get-Agent99SignozSha256Bytes ([Text.Encoding]::UTF8.GetBytes($manifestText))
|
|
if ($computedBundle -ne $BundleId) { throw "toolchain_envelope_bundle_digest_invalid" }
|
|
if ([string]$envelope.manifestSha256 -ne $computedManifest) {
|
|
throw "toolchain_envelope_manifest_digest_invalid"
|
|
}
|
|
return [pscustomobject]@{
|
|
root = $tempRoot
|
|
files = $decodedRows
|
|
manifestSha256 = $computedManifest
|
|
}
|
|
} catch {
|
|
Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue
|
|
throw
|
|
}
|
|
}
|
|
|
|
function Invoke-Agent99SignozToolchainTarget {
|
|
param(
|
|
[ValidateSet("check", "prepare", "apply", "verify", "reconcile")][string]$TargetMode,
|
|
[string]$StageDir = "",
|
|
[int]$TimeoutSeconds = 420
|
|
)
|
|
if (-not (Test-Path -LiteralPath $ToolchainTargetPath -PathType Leaf)) {
|
|
return [pscustomobject]@{ completed = $false; timedOut = $false; exitCode = -1; receipt = $null; errorCode = "toolchain_target_missing" }
|
|
}
|
|
$manifestBase64 = Get-Agent99SignozToolchainManifestBase64
|
|
$targetCommand = (
|
|
"sudo -n /usr/bin/python3 - --mode $TargetMode " +
|
|
"--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId " +
|
|
"--source-revision $SourceRevision --bundle-id $BundleId " +
|
|
"--manifest-b64 $manifestBase64"
|
|
)
|
|
if ($StageDir) { $targetCommand += " --stage-dir $StageDir" }
|
|
if ($TargetMode -eq "reconcile") { $targetCommand += " --attempt-id $AttemptId" }
|
|
$arguments = @(
|
|
"-i", $IdentityFile,
|
|
"-o", "BatchMode=yes",
|
|
"-o", "IdentitiesOnly=yes",
|
|
"-o", "StrictHostKeyChecking=yes",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "ConnectTimeout=8",
|
|
"-o", "ServerAliveInterval=20",
|
|
"-o", "ServerAliveCountMax=3",
|
|
"-l", $RemoteUser,
|
|
$TargetHost,
|
|
$targetCommand
|
|
)
|
|
$token = [Guid]::NewGuid().ToString("N")
|
|
$stdoutPath = Join-Path $env:TEMP "agent99-signoz-toolchain-target-$token.out"
|
|
$stderrPath = Join-Path $env:TEMP "agent99-signoz-toolchain-target-$token.err"
|
|
$script:TransientTransportOutputUsed = $true
|
|
try {
|
|
$process = Start-Process -FilePath "ssh.exe" -ArgumentList $arguments `
|
|
-NoNewWindow -RedirectStandardInput $ToolchainTargetPath `
|
|
-RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
|
|
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
|
|
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
|
|
return [pscustomobject]@{ completed = $false; timedOut = $true; receipt = $null; errorCode = "toolchain_target_timeout" }
|
|
}
|
|
$process.WaitForExit() | Out-Null
|
|
$stdout = if (Test-Path -LiteralPath $stdoutPath) {
|
|
([IO.File]::ReadAllText($stdoutPath, [Text.Encoding]::UTF8)).Trim()
|
|
} else { "" }
|
|
if ($stdout.Length -gt 16384) {
|
|
return [pscustomobject]@{ completed = $false; timedOut = $false; receipt = $null; errorCode = "toolchain_target_output_too_large" }
|
|
}
|
|
try { $receipt = $stdout | ConvertFrom-Json } catch { $receipt = $null }
|
|
return [pscustomobject]@{
|
|
completed = $true
|
|
timedOut = $false
|
|
receipt = $receipt
|
|
errorCode = if ($receipt) { "" } else { "toolchain_target_receipt_invalid" }
|
|
}
|
|
} finally {
|
|
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue
|
|
if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) {
|
|
$script:TransportCleanupSucceeded = $false
|
|
}
|
|
}
|
|
}
|
|
|
|
function Test-Agent99SignozToolchainReceipt {
|
|
param(
|
|
[object]$Result,
|
|
[string[]]$AllowedTerminals
|
|
)
|
|
$value = if ($Result) { $Result.receipt } else { $null }
|
|
$mutationContract = [bool](
|
|
$value -and (
|
|
(
|
|
[string]$value.terminal -in @("check_pass_no_write", "verify_pass", "stage_not_required_exact") -and
|
|
$value.source_write_attempted -eq $false -and
|
|
$value.source_write_performed -eq $false
|
|
) -or (
|
|
[string]$value.terminal -eq "stage_prepared" -and
|
|
$value.source_write_attempted -eq $true -and
|
|
$value.source_write_performed -eq $true
|
|
) -or (
|
|
[string]$value.terminal -in @("apply_pass_deployed_inactive", "apply_pass_idempotent_exact") -and
|
|
$value.source_write_attempted -eq $true -and
|
|
$value.source_write_performed -eq $true -and
|
|
$value.durable_deploy_receipt_valid -eq $true -and
|
|
$value.quarantine_performed -eq $false
|
|
) -or (
|
|
[string]$value.terminal -eq "reconcile_exact_deployment_verified" -and
|
|
[string]$value.attempt_id -eq $AttemptId -and
|
|
$value.durable_deploy_receipt_valid -eq $true -and
|
|
$value.owned_live_residue_verified_absent -eq $true
|
|
) -or (
|
|
[string]$value.terminal -eq "reconcile_zero_owned_residue" -and
|
|
[string]$value.attempt_id -eq $AttemptId -and
|
|
$value.durable_deploy_receipt_valid -eq $false -and
|
|
$value.owned_live_residue_verified_absent -eq $true -and
|
|
$value.shared_target_preserved -eq $false
|
|
) -or (
|
|
[string]$value.terminal -in @(
|
|
"reconcile_exact_target_preserved_unverified",
|
|
"reconcile_shared_target_drift_preserved"
|
|
) -and
|
|
[string]$value.attempt_id -eq $AttemptId -and
|
|
$value.durable_deploy_receipt_valid -eq $false -and
|
|
$value.owned_live_residue_verified_absent -eq $true -and
|
|
$value.shared_target_preserved -eq $true
|
|
)
|
|
)
|
|
)
|
|
return [bool](
|
|
$Result -and $Result.completed -eq $true -and -not $Result.timedOut -and
|
|
$value -and
|
|
[string]$value.schema -eq "awoooi_agent99_signoz_toolchain_target_v1" -and
|
|
[string]$value.trace_id -eq $TraceId -and
|
|
[string]$value.run_id -eq $RunId -and
|
|
[string]$value.work_item_id -eq $WorkItemId -and
|
|
[string]$value.source_revision -eq $SourceRevision -and
|
|
[string]$value.bundle_id -eq $BundleId -and
|
|
[string]$value.terminal -in $AllowedTerminals -and
|
|
$value.production_service_mutation_performed -eq $false -and
|
|
$value.active_pointer_created -eq $false -and
|
|
$value.credential_value_read -eq $false -and
|
|
$value.completion_claim -eq $false -and
|
|
$mutationContract
|
|
)
|
|
}
|
|
|
|
function Update-Agent99SignozToolchainMutationState {
|
|
param([object]$Receipt)
|
|
if (-not $Receipt) { return }
|
|
if ($Receipt.source_write_attempted -eq $true) {
|
|
$script:ToolchainSourceWriteAttempted = $true
|
|
}
|
|
if ($Receipt.source_write_performed -eq $true) {
|
|
$script:ToolchainSourceWritePerformed = $true
|
|
}
|
|
}
|
|
|
|
function Invoke-Agent99SignozToolchainScp {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$LocalPath,
|
|
[Parameter(Mandatory = $true)][string]$RemotePath
|
|
)
|
|
$arguments = @(
|
|
"-q",
|
|
"-i", $IdentityFile,
|
|
"-o", "BatchMode=yes",
|
|
"-o", "IdentitiesOnly=yes",
|
|
"-o", "StrictHostKeyChecking=yes",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "ConnectTimeout=8",
|
|
$LocalPath,
|
|
"$RemoteUser@$TargetHost`:$RemotePath"
|
|
)
|
|
$token = [Guid]::NewGuid().ToString("N")
|
|
$stdoutPath = Join-Path $env:TEMP "agent99-signoz-toolchain-scp-$token.out"
|
|
$stderrPath = Join-Path $env:TEMP "agent99-signoz-toolchain-scp-$token.err"
|
|
$script:TransientTransportOutputUsed = $true
|
|
try {
|
|
$scpProcess = Start-Process -FilePath "scp.exe" -ArgumentList $arguments `
|
|
-NoNewWindow -RedirectStandardOutput $stdoutPath `
|
|
-RedirectStandardError $stderrPath -PassThru
|
|
if (-not $scpProcess.WaitForExit(180000)) {
|
|
try { & taskkill.exe /PID $scpProcess.Id /T /F 2>&1 | Out-Null } catch {}
|
|
return $false
|
|
}
|
|
$scpProcess.WaitForExit() | Out-Null
|
|
$scpProcess.Refresh()
|
|
return [bool]($scpProcess.HasExited -and [int]$scpProcess.ExitCode -eq 0)
|
|
} finally {
|
|
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue
|
|
if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) {
|
|
$script:TransportCleanupSucceeded = $false
|
|
}
|
|
}
|
|
}
|
|
|
|
function New-Agent99SignozLockedRemoteCommand {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$FixedCommand,
|
|
[Parameter(Mandatory = $true)][int]$TargetTimeoutSeconds,
|
|
[ValidateSet("absent", "present")][string]$ExpectedArtifactState
|
|
)
|
|
$artifactGuard = if ($ExpectedArtifactState -eq "present") {
|
|
"[ -d `"$OutputDir`" ] && [ ! -L `"$OutputDir`" ] && " +
|
|
"[ -f `"$RemoteReceipt`" ] && [ ! -L `"$RemoteReceipt`" ]; "
|
|
} else {
|
|
"[ ! -e `"$OutputDir`" ] && [ ! -L `"$OutputDir`" ] && " +
|
|
"[ ! -e `"$RemoteReceipt`" ] && [ ! -L `"$RemoteReceipt`" ]; "
|
|
}
|
|
$bashProgram = (
|
|
"set -euo pipefail; " +
|
|
"[ -f `"$OperationLock`" ] && [ ! -L `"$OperationLock`" ]; " +
|
|
"exec 8<`"$OperationLock`"; /usr/bin/flock -n 8; " +
|
|
$artifactGuard +
|
|
"[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker inspect --format={{.Image}} $SignozContainer)`" = `"$SignozImageId`" ]; " +
|
|
"[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker inspect --format={{.Config.Image}} $SignozContainer)`" = `"$SignozImage`" ]; " +
|
|
"[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker inspect --format={{.State.Running}} $SignozContainer)`" = `"true`" ]; " +
|
|
"[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/curl -sS -o /dev/null -w `"%{http_code}`" $BaseUrl/api/v1/health)`" = `"200`" ]; " +
|
|
"set +e; active_output=`$(/usr/bin/timeout --kill-after=5s 10s /usr/bin/pgrep -af `"([s]ignoz-metadata-export[.]py|[s]ignoz-metadata-restore-drill[.]py|[s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])([b]ackup-signoz|[c]lickhouse-native-backup|[c]lickhouse-native-restore-drill|[r]un-signoz-backup-canary)[.]sh([[:space:]]|$))`"); active_rc=`$?; set -e; " +
|
|
"[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]; " +
|
|
"exec /usr/bin/timeout --signal=TERM --kill-after=$($TargetKillAfterSeconds)s " +
|
|
"$($TargetTimeoutSeconds)s $FixedCommand"
|
|
)
|
|
$encodedProgram = [Convert]::ToBase64String(
|
|
[Text.Encoding]::UTF8.GetBytes($bashProgram)
|
|
)
|
|
return (
|
|
"/usr/bin/printf %s $encodedProgram | /usr/bin/base64 -d | " +
|
|
"sudo -n /bin/bash"
|
|
)
|
|
}
|
|
|
|
function New-Agent99SignozCleanupLockedRemoteCommand {
|
|
param(
|
|
[Parameter(Mandatory = $true)][string]$FixedCommand,
|
|
[Parameter(Mandatory = $true)][int]$TargetTimeoutSeconds
|
|
)
|
|
$bashProgram = (
|
|
"set -euo pipefail; " +
|
|
"[ -f `"$OperationLock`" ] && [ ! -L `"$OperationLock`" ]; " +
|
|
"exec 8<`"$OperationLock`"; /usr/bin/flock -n 8; " +
|
|
"/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker info >/dev/null; " +
|
|
"set +e; active_output=`$(/usr/bin/timeout --kill-after=5s 10s /usr/bin/pgrep -af `"([s]ignoz-metadata-export[.]py|[s]ignoz-metadata-restore-drill[.]py|[s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])([b]ackup-signoz|[c]lickhouse-native-backup|[c]lickhouse-native-restore-drill|[r]un-signoz-backup-canary)[.]sh([[:space:]]|$))`"); active_rc=`$?; set -e; " +
|
|
"[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]; " +
|
|
"exec /usr/bin/timeout --signal=TERM --kill-after=$($TargetKillAfterSeconds)s " +
|
|
"$($TargetTimeoutSeconds)s $FixedCommand"
|
|
)
|
|
$encodedProgram = [Convert]::ToBase64String(
|
|
[Text.Encoding]::UTF8.GetBytes($bashProgram)
|
|
)
|
|
return (
|
|
"/usr/bin/printf %s $encodedProgram | /usr/bin/base64 -d | " +
|
|
"sudo -n /bin/bash"
|
|
)
|
|
}
|
|
|
|
function Get-Agent99RuntimeSourceBinding {
|
|
$result = [ordered]@{
|
|
ok = $false
|
|
manifestPresent = $false
|
|
runtimeMatched = $false
|
|
fileCount = $null
|
|
mismatchCount = $null
|
|
sourceRevision = ""
|
|
entrypointTracked = $false
|
|
entrypointHashMatched = $false
|
|
toolchainTargetTracked = $false
|
|
toolchainTargetHashMatched = $false
|
|
}
|
|
if (-not (Test-Path -LiteralPath $RuntimeManifestPath -PathType Leaf)) {
|
|
return [pscustomobject]$result
|
|
}
|
|
$result.manifestPresent = $true
|
|
try {
|
|
$manifest = Get-Content -LiteralPath $RuntimeManifestPath -Raw | ConvertFrom-Json
|
|
$result.runtimeMatched = [bool]$manifest.runtimeMatched
|
|
$result.fileCount = [int]$manifest.fileCount
|
|
$result.mismatchCount = [int]$manifest.mismatchCount
|
|
$result.sourceRevision = [string]$manifest.sourceRevision
|
|
$rows = @($manifest.files | Where-Object {
|
|
[string]$_.name -eq "agent99-signoz-metadata-executor.ps1"
|
|
})
|
|
$result.entrypointTracked = [bool]($rows.Count -eq 1)
|
|
if ($rows.Count -eq 1 -and $EntrypointPath -and (Test-Path -LiteralPath $EntrypointPath -PathType Leaf)) {
|
|
$actualHash = (Get-FileHash -LiteralPath $EntrypointPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$result.entrypointHashMatched = [bool](
|
|
[string]$rows[0].runtimeSha256 -eq $actualHash
|
|
)
|
|
}
|
|
$targetRows = @($manifest.files | Where-Object {
|
|
[string]$_.name -eq "agent99-signoz-toolchain-target.py"
|
|
})
|
|
$result.toolchainTargetTracked = [bool]($targetRows.Count -eq 1)
|
|
if ($targetRows.Count -eq 1 -and (Test-Path -LiteralPath $ToolchainTargetPath -PathType Leaf)) {
|
|
$targetHash = (Get-FileHash -LiteralPath $ToolchainTargetPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$result.toolchainTargetHashMatched = [bool](
|
|
[string]$targetRows[0].runtimeSha256 -eq $targetHash
|
|
)
|
|
}
|
|
$result.ok = [bool](
|
|
$result.runtimeMatched -and
|
|
$result.fileCount -eq 21 -and
|
|
$result.mismatchCount -eq 0 -and
|
|
$result.sourceRevision -eq $SourceRevision -and
|
|
$result.entrypointTracked -and
|
|
$result.entrypointHashMatched -and
|
|
$result.toolchainTargetTracked -and
|
|
$result.toolchainTargetHashMatched
|
|
)
|
|
} catch {
|
|
$result.ok = $false
|
|
}
|
|
return [pscustomobject]$result
|
|
}
|
|
|
|
function Test-Agent99SignozReceiptIdentity {
|
|
param(
|
|
[object]$Receipt,
|
|
[string]$ExpectedPhase,
|
|
[string[]]$AllowedTerminals
|
|
)
|
|
return [bool](
|
|
$Receipt -and
|
|
[string]$Receipt.schema -eq "awoooi_signoz_metadata_receipt_v1" -and
|
|
[string]$Receipt.trace_id -eq $TraceId -and
|
|
[string]$Receipt.run_id -eq $RunId -and
|
|
[string]$Receipt.work_item_id -eq $WorkItemId -and
|
|
[string]$Receipt.phase -eq $ExpectedPhase -and
|
|
[string]$Receipt.terminal -in $AllowedTerminals
|
|
)
|
|
}
|
|
|
|
function Get-Agent99SignozCleanupPreflight {
|
|
$program = (
|
|
"set -euo pipefail; " +
|
|
"/usr/bin/echo agent99_hashes_begin; " +
|
|
"/usr/bin/sha256sum " + (($ToolchainFiles.Keys) -join " ") + "; " +
|
|
"/usr/bin/echo agent99_hashes_end; " +
|
|
"/usr/bin/printf agent99_docker:; " +
|
|
"/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker info --format={{.ServerVersion}}"
|
|
)
|
|
$encoded = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($program))
|
|
$remote = Invoke-Agent99SignozFixedRemote (
|
|
"/usr/bin/printf %s $encoded | /usr/bin/base64 -d | sudo -n /bin/bash"
|
|
)
|
|
$lines = if (Test-Agent99SignozTransportOutcome $remote) {
|
|
@(([string]$remote.stdout -split "`r?`n") | Where-Object { [string]$_ -ne "" })
|
|
} else { @() }
|
|
$shapeValid = [bool](
|
|
$lines.Count -eq $ToolchainFiles.Count + 3 -and
|
|
[string]$lines[0] -eq "agent99_hashes_begin" -and
|
|
[string]$lines[$lines.Count - 2] -eq "agent99_hashes_end" -and
|
|
[string]$lines[$lines.Count - 1] -match "^agent99_docker:[A-Za-z0-9._+-]+$"
|
|
)
|
|
$actualHashes = @{}
|
|
if ($shapeValid) {
|
|
foreach ($line in @($lines[1..($lines.Count - 3)])) {
|
|
if ($line -match "^([0-9a-f]{64})\s+(.+)$") {
|
|
$path = [string]$Matches[2]
|
|
if ($actualHashes.ContainsKey($path) -or -not $ToolchainFiles.Contains($path)) {
|
|
$shapeValid = $false
|
|
break
|
|
}
|
|
$actualHashes[$path] = [string]$Matches[1]
|
|
} else {
|
|
$shapeValid = $false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
$toolchainExact = [bool]($shapeValid -and $actualHashes.Count -eq $ToolchainFiles.Count)
|
|
foreach ($entry in $ToolchainFiles.GetEnumerator()) {
|
|
if (-not $actualHashes.ContainsKey([string]$entry.Key) -or $actualHashes[[string]$entry.Key] -ne [string]$entry.Value) {
|
|
$toolchainExact = $false
|
|
}
|
|
}
|
|
return [pscustomobject]@{
|
|
ready = [bool]($shapeValid -and $toolchainExact)
|
|
transportReadbackVerified = $shapeValid
|
|
toolchainBundleId = $BundleId
|
|
toolchainFileCount = $ToolchainFiles.Count
|
|
toolchainReadbackVerified = $shapeValid
|
|
toolchainExact = $toolchainExact
|
|
dockerDaemonReadbackVerified = $shapeValid
|
|
credentialValueRead = $false
|
|
credentialValueOutput = $false
|
|
productionHealthRequired = $false
|
|
exportArtifactRequired = $false
|
|
deterministicOwnershipEnforcedByController = $true
|
|
}
|
|
}
|
|
|
|
function Get-Agent99SignozPreflight {
|
|
param(
|
|
[ValidateSet("absent", "present")][string]$ExpectedArtifactState,
|
|
[bool]$RequireCredential
|
|
)
|
|
|
|
$credentialCommand = (
|
|
"sudo -n /bin/bash -c 'if [ -L `"$CredentialFile`" ]; then " +
|
|
"/usr/bin/printf `"agent99_credential:symlink`"; elif [ -f `"$CredentialFile`" ]; then " +
|
|
"/usr/bin/stat -c `"agent99_credential:present:%u:%a:%s`" `"$CredentialFile`"; " +
|
|
"elif [ -e `"$CredentialFile`" ]; then /usr/bin/printf `"agent99_credential:other`"; " +
|
|
"else /usr/bin/printf `"agent99_credential:absent`"; fi'"
|
|
)
|
|
$credential = Invoke-Agent99SignozFixedRemote $credentialCommand
|
|
$credentialOutcomeAccepted = Test-Agent99SignozTransportOutcome $credential
|
|
$credentialTypedOutput = if ($credentialOutcomeAccepted) { [string]$credential.stdout } else { "" }
|
|
$credentialState = if ($credentialTypedOutput -match "^agent99_credential:(absent|symlink|other)$") {
|
|
[string]$Matches[1]
|
|
} elseif ($credentialTypedOutput -match "^agent99_credential:(present:[0-9]+:[0-9]+:[0-9]+)$") {
|
|
[string]$Matches[1]
|
|
} else { "unverified" }
|
|
$credentialReadbackVerified = [bool](
|
|
(
|
|
$credentialState -in @("absent", "symlink", "other") -or
|
|
$credentialState -match "^present:[0-9]+:[0-9]+:[0-9]+$"
|
|
)
|
|
)
|
|
$credentialReferencePresent = if ($credentialReadbackVerified) {
|
|
[bool]($credentialState -ne "absent")
|
|
} else { $null }
|
|
$credentialOwner = $null
|
|
$credentialMode = $null
|
|
$credentialSize = $null
|
|
$credentialMetadataValid = $false
|
|
if ($credentialReadbackVerified -and $credentialState -match "^present:([0-9]+):([0-9]+):([0-9]+)$") {
|
|
$credentialOwner = [int]$Matches[1]
|
|
$credentialMode = [string]$Matches[2]
|
|
$credentialSize = [int64]$Matches[3]
|
|
$credentialMetadataValid = [bool](
|
|
$credentialOwner -eq 0 -and
|
|
$credentialMode -in @("400", "600") -and
|
|
$credentialSize -gt 0 -and
|
|
$credentialSize -le 4096
|
|
)
|
|
}
|
|
|
|
$hashCommand = (
|
|
"sudo -n /bin/bash -c 'set -euo pipefail; " +
|
|
"/usr/bin/echo agent99_hashes_begin; " +
|
|
"/usr/bin/sha256sum " + (($ToolchainFiles.Keys) -join " ") + "; " +
|
|
"/usr/bin/echo agent99_hashes_end'"
|
|
)
|
|
$hashReadback = Invoke-Agent99SignozFixedRemote $hashCommand
|
|
$actualHashes = @{}
|
|
$hashOutcomeAccepted = Test-Agent99SignozTransportOutcome $hashReadback
|
|
$hashLines = if ($hashOutcomeAccepted) {
|
|
@(([string]$hashReadback.stdout -split "`r?`n") | Where-Object { [string]$_ -ne "" })
|
|
} else { @() }
|
|
$hashShapeValid = [bool](
|
|
$hashLines.Count -eq $ToolchainFiles.Count + 2 -and
|
|
[string]$hashLines[0] -eq "agent99_hashes_begin" -and
|
|
[string]$hashLines[$hashLines.Count - 1] -eq "agent99_hashes_end"
|
|
)
|
|
if ($hashShapeValid) {
|
|
foreach ($line in @($hashLines[1..($hashLines.Count - 2)])) {
|
|
if ($line -match "^([0-9a-f]{64})\s+(.+)$") {
|
|
$path = [string]$Matches[2]
|
|
if ($actualHashes.ContainsKey($path) -or -not $ToolchainFiles.Contains($path)) {
|
|
$hashShapeValid = $false
|
|
break
|
|
}
|
|
$actualHashes[$path] = [string]$Matches[1]
|
|
} else {
|
|
$hashShapeValid = $false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
$toolchainReadbackVerified = [bool](
|
|
$hashShapeValid -and
|
|
$actualHashes.Count -eq $ToolchainFiles.Count
|
|
)
|
|
$toolchainExact = $toolchainReadbackVerified
|
|
foreach ($entry in $ToolchainFiles.GetEnumerator()) {
|
|
if (-not $actualHashes.ContainsKey([string]$entry.Key) -or $actualHashes[[string]$entry.Key] -ne [string]$entry.Value) {
|
|
$toolchainExact = $false
|
|
}
|
|
}
|
|
|
|
$image = Invoke-Agent99SignozFixedRemote (
|
|
"sudo -n /bin/bash -c 'set -euo pipefail; " +
|
|
"/usr/bin/printf `"agent99_image:`"; " +
|
|
"/usr/bin/docker image inspect --format={{.Id}} $SignozImage'"
|
|
)
|
|
$imageOutcomeAccepted = Test-Agent99SignozTransportOutcome $image
|
|
$imageId = if ($imageOutcomeAccepted -and [string]$image.stdout -match "^agent99_image:(sha256:[0-9a-f]{64})$") {
|
|
[string]$Matches[1]
|
|
} else { "" }
|
|
$imageReadbackVerified = [bool]($imageId -match "^sha256:[0-9a-f]{64}$")
|
|
$imageExact = [bool]($imageReadbackVerified -and $imageId -eq $SignozImageId)
|
|
$runtime = Invoke-Agent99SignozFixedRemote (
|
|
"sudo -n /bin/bash -c 'set -euo pipefail; " +
|
|
"/usr/bin/printf agent99_runtime:; " +
|
|
"/usr/bin/docker inspect --format={{.Id}},{{.Image}},{{.Config.Image}},{{.State.Running}} $SignozContainer'"
|
|
)
|
|
$runtimeOutcomeAccepted = Test-Agent99SignozTransportOutcome $runtime
|
|
$runtimeContainerId = $null
|
|
$runtimeImageId = $null
|
|
$runtimeImageRef = $null
|
|
$runtimeRunning = $false
|
|
$runtimeReadbackVerified = $false
|
|
if ($runtimeOutcomeAccepted -and [string]$runtime.stdout -match "^agent99_runtime:([0-9a-f]{64}),(sha256:[0-9a-f]{64}),([A-Za-z0-9./:_-]+),(true|false)$") {
|
|
$runtimeReadbackVerified = $true
|
|
$runtimeContainerId = [string]$Matches[1]
|
|
$runtimeImageId = [string]$Matches[2]
|
|
$runtimeImageRef = [string]$Matches[3]
|
|
$runtimeRunning = [bool]([string]$Matches[4] -eq "true")
|
|
}
|
|
$runtimeImageExact = [bool](
|
|
$runtimeRunning -and
|
|
$runtimeImageId -eq $SignozImageId -and
|
|
$runtimeImageRef -eq $SignozImage
|
|
)
|
|
|
|
# Typed restore-startup observation. It emits only fixed booleans and a
|
|
# mount count: no environment value, source mount, secret, or raw Env array.
|
|
$restoreStartupProgram = @'
|
|
set -euo pipefail
|
|
entrypoint="$(/usr/bin/docker inspect --format='{{json .Config.Entrypoint}}' signoz)"
|
|
entrypoint_exact=0
|
|
[ "$entrypoint" = '["./signoz","server"]' ] && entrypoint_exact=1
|
|
mounts="$(/usr/bin/docker inspect --format='{{range .Mounts}}{{println .Destination .Type .RW}}{{end}}' signoz)"
|
|
mount_count="$(/usr/bin/printf '%s\n' "$mounts" | /usr/bin/awk 'NF { count += 1 } END { print count + 0 }')"
|
|
mount_exact=0
|
|
[ "$mount_count" -eq 1 ] && [ "$(/usr/bin/printf '%s\n' "$mounts" | /usr/bin/grep -Fxc '/var/lib/signoz volume true')" -eq 1 ] && mount_exact=1
|
|
dsn_present=0
|
|
dsn_class_exact=0
|
|
sqlite_present=0
|
|
sqlite_exact=0
|
|
if /usr/bin/docker exec signoz /bin/sh -c 'test "${SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN+x}" = x' >/dev/null 2>&1; then dsn_present=1; fi
|
|
if /usr/bin/docker exec signoz /bin/sh -c 'case "${SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN:-}" in tcp://clickhouse:9000*) exit 0;; *) exit 1;; esac' >/dev/null 2>&1; then dsn_class_exact=1; fi
|
|
if /usr/bin/docker exec signoz /bin/sh -c 'test "${SIGNOZ_SQLSTORE_SQLITE_PATH+x}" = x' >/dev/null 2>&1; then sqlite_present=1; fi
|
|
if /usr/bin/docker exec signoz /bin/sh -c 'test "${SIGNOZ_SQLSTORE_SQLITE_PATH:-}" = /var/lib/signoz/signoz.db' >/dev/null 2>&1; then sqlite_exact=1; fi
|
|
/usr/bin/printf 'agent99_restore_startup:%s,%s,%s,%s,%s,%s,%s' "$entrypoint_exact" "$mount_count" "$mount_exact" "$dsn_present" "$dsn_class_exact" "$sqlite_present" "$sqlite_exact"
|
|
'@
|
|
$restoreStartupEncoded = [Convert]::ToBase64String(
|
|
[Text.Encoding]::UTF8.GetBytes($restoreStartupProgram)
|
|
)
|
|
$restoreStartup = Invoke-Agent99SignozFixedRemote (
|
|
"/usr/bin/printf %s $restoreStartupEncoded | /usr/bin/base64 -d | sudo -n /bin/bash"
|
|
)
|
|
$restoreStartupReadbackVerified = $false
|
|
$restoreStartupEntrypointExact = $false
|
|
$restoreStartupMountCount = $null
|
|
$restoreStartupMountExact = $false
|
|
$restoreStartupDsnPresent = $false
|
|
$restoreStartupDsnClassExact = $false
|
|
$restoreStartupSqlitePathPresent = $false
|
|
$restoreStartupSqlitePathExact = $false
|
|
if (
|
|
(Test-Agent99SignozTransportOutcome $restoreStartup) -and
|
|
[string]$restoreStartup.stdout -match "^agent99_restore_startup:(0|1),([0-9]+),(0|1),(0|1),(0|1),(0|1),(0|1)$"
|
|
) {
|
|
$restoreStartupReadbackVerified = $true
|
|
$restoreStartupEntrypointExact = [bool]([string]$Matches[1] -eq "1")
|
|
$restoreStartupMountCount = [int]$Matches[2]
|
|
$restoreStartupMountExact = [bool]([string]$Matches[3] -eq "1")
|
|
$restoreStartupDsnPresent = [bool]([string]$Matches[4] -eq "1")
|
|
$restoreStartupDsnClassExact = [bool]([string]$Matches[5] -eq "1")
|
|
$restoreStartupSqlitePathPresent = [bool]([string]$Matches[6] -eq "1")
|
|
$restoreStartupSqlitePathExact = [bool]([string]$Matches[7] -eq "1")
|
|
}
|
|
$restoreStartupContractReady = [bool](
|
|
$restoreStartupReadbackVerified -and
|
|
$restoreStartupEntrypointExact -and
|
|
$restoreStartupMountCount -eq 1 -and
|
|
$restoreStartupMountExact -and
|
|
$restoreStartupDsnPresent -and
|
|
$restoreStartupDsnClassExact -and
|
|
$restoreStartupSqlitePathPresent -and
|
|
$restoreStartupSqlitePathExact
|
|
)
|
|
$health = Invoke-Agent99SignozFixedRemote (
|
|
"sudo -n /bin/bash -c 'set -euo pipefail; " +
|
|
"/usr/bin/printf `"agent99_health:`"; " +
|
|
"/usr/bin/curl -fsS --max-time 15 $BaseUrl/api/v1/health'"
|
|
)
|
|
$healthOk = $false
|
|
$healthReadbackVerified = $false
|
|
if (
|
|
(Test-Agent99SignozTransportOutcome $health) -and
|
|
[string]$health.stdout -match "^agent99_health:(\{.+\})$"
|
|
) {
|
|
try {
|
|
$healthPayload = [string]$Matches[1] | ConvertFrom-Json
|
|
$healthReadbackVerified = [bool]($null -ne $healthPayload.PSObject.Properties["status"])
|
|
$healthOk = [bool]([string]$healthPayload.status -eq "ok")
|
|
} catch {
|
|
$healthOk = $false
|
|
}
|
|
}
|
|
|
|
$active = Invoke-Agent99SignozFixedRemote (
|
|
"sudo -n /bin/bash -c 'set -euo pipefail; set +e; " +
|
|
"active_output=`$(/usr/bin/timeout --kill-after=5s 10s /usr/bin/pgrep -af `"([s]ignoz-metadata-export.py.*--apply|[s]ignoz-metadata-isolated-restore.py.*--apply|[s]ignoz-metadata-restore-drill.py.*--apply)`" ); " +
|
|
"active_rc=`$?; set -e; " +
|
|
"case `"`$active_rc`" in " +
|
|
"0) active_count=`$(/usr/bin/printf `"%s\n`" `"`$active_output`" | /usr/bin/awk `"NF { count += 1 } END { print count + 0 }`");; " +
|
|
"1) [ -z `"`$active_output`" ]; active_count=0;; " +
|
|
"*) exit 1;; esac; " +
|
|
"/usr/bin/printf `"agent99_active:%s`" `"`$active_count`"'"
|
|
)
|
|
$activeOperationCount = $null
|
|
$activeOutcomeAccepted = Test-Agent99SignozTransportOutcome $active
|
|
if ($activeOutcomeAccepted -and [string]$active.stdout -match "^agent99_active:([0-9]+)$") {
|
|
$activeOperationCount = [int]$Matches[1]
|
|
}
|
|
$activeReadbackVerified = [bool]($null -ne $activeOperationCount)
|
|
$activeOperationsClear = [bool](
|
|
$activeReadbackVerified -and
|
|
$activeOperationCount -eq 0
|
|
)
|
|
|
|
$outputState = Invoke-Agent99SignozFixedRemote (
|
|
New-Agent99SignozPathStateCommand $OutputDir "directory"
|
|
)
|
|
$receiptState = Invoke-Agent99SignozFixedRemote (
|
|
New-Agent99SignozPathStateCommand $RemoteReceipt "file"
|
|
)
|
|
$outputStateValue = if (
|
|
(Test-Agent99SignozTransportOutcome $outputState) -and
|
|
[string]$outputState.stdout -match "^agent99_path_state:(absent|directory|other|symlink)$"
|
|
) { [string]$Matches[1] } else { "unverified" }
|
|
$receiptStateValue = if (
|
|
(Test-Agent99SignozTransportOutcome $receiptState) -and
|
|
[string]$receiptState.stdout -match "^agent99_path_state:(absent|file|other|symlink)$"
|
|
) { [string]$Matches[1] } else { "unverified" }
|
|
$outputStateReadbackVerified = [bool]($outputStateValue -ne "unverified")
|
|
$receiptStateReadbackVerified = [bool]($receiptStateValue -ne "unverified")
|
|
$outputPresent = if ($outputStateReadbackVerified) {
|
|
[bool]($outputStateValue -ne "absent")
|
|
} else { $null }
|
|
$receiptPresent = if ($receiptStateReadbackVerified) {
|
|
[bool]($receiptStateValue -ne "absent")
|
|
} else { $null }
|
|
$stateReadable = [bool](
|
|
$outputStateReadbackVerified -and
|
|
$receiptStateReadbackVerified
|
|
)
|
|
$artifactStateReady = if ($ExpectedArtifactState -eq "present") {
|
|
[bool]($outputStateValue -eq "directory" -and $receiptStateValue -eq "file")
|
|
} else {
|
|
[bool]($outputStateValue -eq "absent" -and $receiptStateValue -eq "absent")
|
|
}
|
|
$transportReadbackVerified = [bool](
|
|
$credentialReadbackVerified -and
|
|
$toolchainReadbackVerified -and
|
|
$imageReadbackVerified -and
|
|
$runtimeReadbackVerified -and
|
|
$restoreStartupReadbackVerified -and
|
|
$healthReadbackVerified -and
|
|
$activeReadbackVerified -and
|
|
$stateReadable
|
|
)
|
|
$credentialReady = [bool](-not $RequireCredential -or $credentialMetadataValid)
|
|
$ready = [bool](
|
|
$transportReadbackVerified -and
|
|
$credentialReady -and
|
|
$toolchainExact -and
|
|
$imageExact -and
|
|
$runtimeImageExact -and
|
|
$restoreStartupContractReady -and
|
|
$healthOk -and
|
|
$activeOperationsClear -and
|
|
$stateReadable -and
|
|
$artifactStateReady
|
|
)
|
|
|
|
return [pscustomobject]@{
|
|
ready = $ready
|
|
transportReadbackVerified = $transportReadbackVerified
|
|
credentialReadbackVerified = $credentialReadbackVerified
|
|
credentialState = $credentialState
|
|
credentialReferencePresent = $credentialReferencePresent
|
|
credentialMetadataValid = $credentialMetadataValid
|
|
credentialOwnerUid = $credentialOwner
|
|
credentialMode = $credentialMode
|
|
credentialSizeBytes = $credentialSize
|
|
credentialValueRead = $false
|
|
credentialValueOutput = $false
|
|
toolchainBundleId = $BundleId
|
|
toolchainFileCount = $ToolchainFiles.Count
|
|
toolchainReadbackVerified = $toolchainReadbackVerified
|
|
toolchainExact = $toolchainExact
|
|
signozImage = $SignozImage
|
|
signozImageId = if ($imageReadbackVerified) { $imageId } else { $null }
|
|
signozImageReadbackVerified = $imageReadbackVerified
|
|
signozImageExact = $imageExact
|
|
signozContainer = $SignozContainer
|
|
signozRuntimeContainerId = $runtimeContainerId
|
|
signozRuntimeImageId = $runtimeImageId
|
|
signozRuntimeImageRef = $runtimeImageRef
|
|
signozRuntimeRunning = $runtimeRunning
|
|
signozRuntimeReadbackVerified = $runtimeReadbackVerified
|
|
signozRuntimeImageExact = $runtimeImageExact
|
|
restoreStartupContractReadbackVerified = $restoreStartupReadbackVerified
|
|
restoreStartupContractReady = $restoreStartupContractReady
|
|
restoreStartupEntrypointExact = $restoreStartupEntrypointExact
|
|
restoreStartupMountDestination = if ($restoreStartupMountExact) { "/var/lib/signoz" } else { $null }
|
|
restoreStartupMountCount = $restoreStartupMountCount
|
|
restoreStartupMountExact = $restoreStartupMountExact
|
|
restoreStartupRequiredKeyPresence = [pscustomobject]@{
|
|
SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN = $restoreStartupDsnPresent
|
|
SIGNOZ_SQLSTORE_SQLITE_PATH = $restoreStartupSqlitePathPresent
|
|
}
|
|
restoreStartupDsnClass = if ($restoreStartupDsnClassExact) { "tcp://clickhouse:9000*" } else { "mismatch_or_unverified" }
|
|
restoreStartupSqlitePathClass = if ($restoreStartupSqlitePathExact) { "/var/lib/signoz/signoz.db" } else { "mismatch_or_unverified" }
|
|
restoreStartupEnvironmentValuesOutput = $false
|
|
restoreStartupFullEnvironmentOutput = $false
|
|
apiHealthReadbackVerified = $healthReadbackVerified
|
|
apiHealthOk = $healthOk
|
|
activeOperationCount = $activeOperationCount
|
|
activeOperationReadbackVerified = $activeReadbackVerified
|
|
activeOperationsClear = $activeOperationsClear
|
|
expectedArtifactState = $ExpectedArtifactState
|
|
outputState = $outputStateValue
|
|
outputStateReadbackVerified = $outputStateReadbackVerified
|
|
outputPresent = $outputPresent
|
|
receiptState = $receiptStateValue
|
|
receiptStateReadbackVerified = $receiptStateReadbackVerified
|
|
receiptPresent = $receiptPresent
|
|
artifactStateReady = $artifactStateReady
|
|
}
|
|
}
|
|
|
|
function Write-Agent99SignozEvidenceCollision {
|
|
$collision = [pscustomobject]@{
|
|
schemaVersion = "agent99_signoz_metadata_executor_dispatch_v1"
|
|
mode = $Mode
|
|
traceId = $TraceId
|
|
runId = $RunId
|
|
cleanupAttemptId = if ($Mode -eq "RestoreVerifyCleanup") { $AttemptId } else { $null }
|
|
toolchainReconcileAttemptId = if ($Mode -eq "ToolchainReconcile") { $AttemptId } else { $null }
|
|
workItemId = $WorkItemId
|
|
terminal = "blocked_evidence_identity_collision"
|
|
errorCode = "evidence_path_or_reservation_already_exists"
|
|
evidencePath = $EvidencePath
|
|
completionClaim = $false
|
|
}
|
|
[Console]::WriteLine(($collision | ConvertTo-Json -Compress))
|
|
}
|
|
|
|
function New-Agent99SignozEvidenceReservation {
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null
|
|
if (
|
|
(Test-Path -LiteralPath $EvidencePath) -or
|
|
(Test-Path -LiteralPath $EvidenceReservationPath)
|
|
) {
|
|
return $false
|
|
}
|
|
$reservationStream = $null
|
|
try {
|
|
$reservationStream = [IO.File]::Open(
|
|
$EvidenceReservationPath,
|
|
[IO.FileMode]::CreateNew,
|
|
[IO.FileAccess]::Write,
|
|
[IO.FileShare]::None
|
|
)
|
|
$reservationPayload = [Text.Encoding]::UTF8.GetBytes(
|
|
"$TraceId|$RunId|$WorkItemId|$Mode|$AttemptId" + [Environment]::NewLine
|
|
)
|
|
$reservationStream.Write($reservationPayload, 0, $reservationPayload.Length)
|
|
$reservationStream.Flush($true)
|
|
$script:EvidenceReserved = $true
|
|
return $true
|
|
} catch {
|
|
return $false
|
|
} finally {
|
|
if ($reservationStream) { $reservationStream.Dispose() }
|
|
}
|
|
}
|
|
|
|
function Write-Agent99SignozTerminal {
|
|
param(
|
|
[string]$Terminal,
|
|
[string]$ErrorCode,
|
|
[object]$Preflight,
|
|
[object]$CheckReceipt,
|
|
[object]$ApplyReceipt,
|
|
[object]$VerifierReceipt,
|
|
[object]$ArtifactWritePerformed
|
|
)
|
|
|
|
$isRestoreMode = [bool]($Mode -like "Restore*")
|
|
$isToolchainMode = [bool]($Mode -like "Toolchain*")
|
|
$restoreCompleted = [bool](
|
|
$Terminal -eq "portable_metadata_restore_independently_verified_zero_residue"
|
|
)
|
|
$zeroResidueVerified = [bool](
|
|
$Terminal -in @(
|
|
"portable_metadata_restore_verified_zero_residue_pending_independent_verifier",
|
|
"portable_metadata_restore_independently_verified_zero_residue",
|
|
"restore_zero_residue_verified_no_completion"
|
|
)
|
|
)
|
|
$controlledApplyPhases = [ordered]@{
|
|
sensorSource = if ($Preflight) { "observed" } else { "blocked" }
|
|
normalizedAsset = if ($Preflight) { $CanonicalAsset } else { "blocked" }
|
|
sourceTruthDiff = if ($isToolchainMode -and $Preflight) { "typed_target_state_readback" } elseif ($Preflight -and $Preflight.toolchainExact) { "exact" } else { "blocked_or_drift" }
|
|
aiDecision = if ($isToolchainMode) { "fixed_source_bound_inactive_toolchain_deploy" } elseif ($isRestoreMode) { "fixed_isolated_portable_metadata_restore" } else { "fixed_allowlisted_metadata_export" }
|
|
riskPolicy = if ($isToolchainMode) { "high_bounded_additive_inactive_no_active_pointer" } elseif ($isRestoreMode) { "high_bounded_ephemeral_internal_only_zero_residue" } else { "high_bounded_read_only_source_private_artifact_no_raw_sqlite" }
|
|
checkMode = if ($CheckReceipt) { [string]$CheckReceipt.terminal } elseif ($Preflight -and $Preflight.ready) { "preflight_pass" } else { "blocked" }
|
|
boundedExecution = if ($ApplyReceipt) { [string]$ApplyReceipt.terminal } elseif ($Mode -eq "Apply") { "not_verified" } else { "not_run" }
|
|
independentVerifier = if ($VerifierReceipt) { [string]$VerifierReceipt.terminal } else { "not_run" }
|
|
rollbackOrNoWrite = if ($Terminal -eq "restore_check_failed_unexpected_write_requires_reconcile") { "unexpected_receipt_write_requires_reconcile" } elseif ($Mode -in @("Check", "Verify", "RestoreCheck", "ToolchainCheck", "ToolchainVerify")) { "no_write" } elseif ($Terminal -eq "toolchain_deployed_inactive_verified") { "not_required_inactive_content_addressed_bundle" } elseif ($Terminal -eq "toolchain_reconcile_exact_deployment_verified") { "owned_residue_absent_shared_target_exact" } elseif ($Terminal -eq "toolchain_reconcile_zero_owned_residue") { "owned_live_residue_removed_new_apply_required" } elseif ($Mode -in @("ToolchainApply", "ToolchainReconcile")) { "reconcile_required_or_shared_target_preserved" } elseif ($zeroResidueVerified) { "ephemeral_resources_cleaned_zero_residue" } elseif ($Mode -eq "RestoreVerifyCleanup") { "reconciliation_failed_or_incomplete" } elseif ($Terminal -eq "export_verified_restore_pending") { "not_required_additive_immutable_artifact" } else { "artifact_preserved_no_delete" }
|
|
learningWriteback = "pending_telegram_km_rag_mcp_playbook_ack"
|
|
}
|
|
$payload = [pscustomobject]@{
|
|
schemaVersion = "agent99_signoz_metadata_executor_dispatch_v1"
|
|
canonicalAsset = $CanonicalAsset
|
|
domain = "backup_observability"
|
|
executor = "agent99_signoz_metadata_executor"
|
|
verifier = if ($isToolchainMode) { "signoz_metadata_toolchain_independent_target_readback" } elseif ($Mode -eq "RestoreVerifyCleanup") { "signoz_metadata_restore_cleanup_verifier" } elseif ($isRestoreMode) { "pending_distinct_restore_cleanup_verifier" } else { "signoz_metadata_export_independent_verifier" }
|
|
executionHub = "host:192.168.0.99"
|
|
dispatchPath = "windows99_agent99_to_host110_fixed_metadata_toolchain"
|
|
dispatchOnly = $true
|
|
mode = $Mode
|
|
traceId = $TraceId
|
|
runId = $RunId
|
|
cleanupAttemptId = if ($Mode -eq "RestoreVerifyCleanup") { $AttemptId } else { $null }
|
|
toolchainReconcileAttemptId = if ($Mode -eq "ToolchainReconcile") { $AttemptId } else { $null }
|
|
workItemId = $WorkItemId
|
|
sourceRevision = $SourceRevision
|
|
targetHost = $TargetHost
|
|
toolchainBundleId = $BundleId
|
|
credentialReference = $CredentialFile
|
|
restoreReceipt = if ($isRestoreMode) { $RestoreRemoteReceipt } else { $null }
|
|
targetOperationLock = $OperationLock
|
|
targetOperationLockRequiredExistingNonSymlink = $true
|
|
targetTimeoutKillAfterSeconds = $TargetKillAfterSeconds
|
|
targetGuardWorstCaseSeconds = $TargetGuardWorstCaseSeconds
|
|
targetProgramTimeoutSeconds = $TargetProgramTimeoutSeconds
|
|
targetWorstCaseSeconds = $TargetWorstCaseSeconds
|
|
controllerWaitSeconds = $ControllerWaitSeconds
|
|
controllerTransportReserveSeconds = $ControllerTransportReserveSeconds
|
|
targetTimeoutFinishesBeforeControllerWait = [bool](
|
|
$TargetWorstCaseSeconds + $ControllerTransportReserveSeconds -lt $ControllerWaitSeconds
|
|
)
|
|
terminal = $Terminal
|
|
errorCode = if ($ErrorCode) { $ErrorCode } else { $null }
|
|
preflight = $Preflight
|
|
checkReceipt = $CheckReceipt
|
|
applyReceipt = $ApplyReceipt
|
|
verifierReceipt = $VerifierReceipt
|
|
controlledApplyPhases = [pscustomobject]$controlledApplyPhases
|
|
artifactWritePerformed = $ArtifactWritePerformed
|
|
productionRuntimeMutationPerformed = $false
|
|
productionSourceMutationAttempted = $script:ToolchainSourceWriteAttempted
|
|
productionSourceMutationPerformed = $script:ToolchainSourceWritePerformed
|
|
productionSourceMutationState = if ($script:ToolchainSourceWritePerformed) {
|
|
"performed"
|
|
} elseif ($script:ToolchainSourceWriteAttempted) {
|
|
"attempted_unverified"
|
|
} else {
|
|
"none"
|
|
}
|
|
isolatedEphemeralRuntimeCreated = [bool]($Mode -eq "RestoreApply" -and $script:IsolatedApplyDispatchAttempted)
|
|
rawSqliteRead = $false
|
|
rawVolumeRead = $false
|
|
secretValueReadByController = $false
|
|
secretValueTransmitted = $false
|
|
secretValuePersistedInEvidence = $false
|
|
credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted
|
|
credentialReferenceConsumedOnTarget = if (-not $script:ExportCredentialDispatchAttempted) {
|
|
$false
|
|
} elseif (Test-Agent99SignozReceiptIdentity $ApplyReceipt "terminal" @("partial_degraded")) {
|
|
$true
|
|
} else {
|
|
$null
|
|
}
|
|
arbitraryCommandAllowed = $false
|
|
crossDomainFallbackAllowed = $false
|
|
restoreTerminal = if ($restoreCompleted) { "portable_metadata_restore_verified" } elseif ($isRestoreMode) { $Terminal } else { "pending_isolated_same_version_target" }
|
|
zeroResidueTerminal = if ($zeroResidueVerified) { "verified" } elseif ($isRestoreMode) { "not_verified" } else { "pending" }
|
|
completionScope = if ($restoreCompleted) { "portable_assets_only" } else { $null }
|
|
completionClaim = $restoreCompleted
|
|
fullSqliteCompletionClaim = $false
|
|
rolesPreferencesGlobalConfigRestoreClaim = $false
|
|
secretBearingAssetsRestoreClaim = $false
|
|
telegramAcknowledged = $false
|
|
kmRagMcpPlaybookAcknowledged = $false
|
|
transientTransportOutputUsed = $script:TransientTransportOutputUsed
|
|
transientTransportOutputDeletedBeforeReceipt = [bool](
|
|
$script:TransientTransportOutputUsed -and $script:TransportCleanupSucceeded
|
|
)
|
|
rawTransportOutputPersisted = [bool](
|
|
$script:TransientTransportOutputUsed -and -not $script:TransportCleanupSucceeded
|
|
)
|
|
toolchainEnvelopeTransportPayloadUsed = [bool]($Mode -eq "ToolchainApply")
|
|
toolchainEnvelopeTransportPayloadDeletedBeforeExecutor = [bool](
|
|
$Mode -eq "ToolchainApply" -and $ToolchainEnvelopeDeletedBeforeExecutor
|
|
)
|
|
toolchainEnvelopeTransportPayloadContainsSecrets = $false
|
|
evidenceReservedBeforeDispatch = $script:EvidenceReserved
|
|
evidencePath = $EvidencePath
|
|
}
|
|
$json = $payload | ConvertTo-Json -Compress -Depth 14
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null
|
|
if (
|
|
(Test-Path -LiteralPath $EvidencePath) -or
|
|
(-not $script:EvidenceReserved -and (Test-Path -LiteralPath $EvidenceReservationPath))
|
|
) {
|
|
Write-Agent99SignozEvidenceCollision
|
|
return $false
|
|
}
|
|
$tempEvidencePath = "$EvidencePath.$PID.$([Guid]::NewGuid().ToString('N')).tmp"
|
|
try {
|
|
[IO.File]::WriteAllText(
|
|
$tempEvidencePath,
|
|
$json + [Environment]::NewLine,
|
|
(New-Object Text.UTF8Encoding($false))
|
|
)
|
|
Move-Item -LiteralPath $tempEvidencePath -Destination $EvidencePath
|
|
if ($script:EvidenceReserved) {
|
|
Remove-Item -LiteralPath $EvidenceReservationPath -Force -ErrorAction SilentlyContinue
|
|
$script:EvidenceReserved = $false
|
|
}
|
|
} finally {
|
|
Remove-Item -LiteralPath $tempEvidencePath -Force -ErrorAction SilentlyContinue
|
|
}
|
|
[Console]::WriteLine($json)
|
|
return $true
|
|
}
|
|
|
|
function Stop-Agent99SignozRun {
|
|
param(
|
|
[string]$Terminal,
|
|
[string]$ErrorCode,
|
|
[object]$Preflight,
|
|
[object]$CheckReceipt,
|
|
[object]$ApplyReceipt,
|
|
[object]$VerifierReceipt,
|
|
[object]$ArtifactWritePerformed,
|
|
[int]$ExitCode
|
|
)
|
|
if (-not (Write-Agent99SignozTerminal $Terminal $ErrorCode $Preflight $CheckReceipt $ApplyReceipt $VerifierReceipt $ArtifactWritePerformed)) {
|
|
exit 74
|
|
}
|
|
exit $ExitCode
|
|
}
|
|
|
|
foreach ($identity in @($TraceId, $RunId, $WorkItemId)) {
|
|
if ($identity -notmatch $SafeIdPattern) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "identity_missing_or_invalid" $null $null $null $null $false 64
|
|
}
|
|
}
|
|
if ($Mode -in @("RestoreVerifyCleanup", "ToolchainReconcile")) {
|
|
if ($AttemptId -notmatch $SafeIdPattern) {
|
|
$terminal = if ($Mode -eq "ToolchainReconcile") { "toolchain_reconcile_blocked_with_safe_next_action" } else { "restore_cleanup_blocked_with_safe_next_action" }
|
|
$errorCode = if ($Mode -eq "ToolchainReconcile") { "reconcile_attempt_id_missing_or_invalid" } else { "cleanup_attempt_id_missing_or_invalid" }
|
|
Stop-Agent99SignozRun $terminal $errorCode $null $null $null $null $false 64
|
|
}
|
|
} elseif ($AttemptId) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "attempt_id_not_allowed_for_mode" $null $null $null $null $false 64
|
|
}
|
|
if ($Mode -eq "ToolchainApply") {
|
|
if (-not $ToolchainEnvelopeJson) {
|
|
Stop-Agent99SignozRun "toolchain_blocked_with_safe_next_action" "toolchain_envelope_required_for_apply" $null $null $null $null $false 64
|
|
}
|
|
if (-not $ToolchainEnvelopeDeletedBeforeExecutor) {
|
|
Stop-Agent99SignozRun "toolchain_blocked_with_safe_next_action" "toolchain_envelope_delete_attestation_required" $null $null $null $null $false 64
|
|
}
|
|
} elseif ($ToolchainEnvelopeJson) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "toolchain_envelope_not_allowed_for_mode" $null $null $null $null $false 64
|
|
} elseif ($ToolchainEnvelopeDeletedBeforeExecutor) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "toolchain_envelope_delete_attestation_not_allowed_for_mode" $null $null $null $null $false 64
|
|
}
|
|
if ($WorkItemId -ne "P0-OBS-002") {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "work_item_not_allowlisted" $null $null $null $null $false 64
|
|
}
|
|
$SourceRevision = $SourceRevision.ToLowerInvariant()
|
|
if ($SourceRevision -notmatch "^[0-9a-f]{40}$") {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "source_revision_missing_or_invalid" $null $null $null $null $false 64
|
|
}
|
|
if ($TargetWorstCaseSeconds + $ControllerTransportReserveSeconds -ge $ControllerWaitSeconds) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "target_timeout_not_bounded_before_controller_wait" $null $null $null $null $false 64
|
|
}
|
|
if (-not (New-Agent99SignozEvidenceReservation)) {
|
|
Write-Agent99SignozEvidenceCollision
|
|
exit 66
|
|
}
|
|
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "dedicated_identity_unavailable" $null $null $null $null $false 65
|
|
}
|
|
$runtimeSourceBinding = Get-Agent99RuntimeSourceBinding
|
|
if (-not $runtimeSourceBinding.ok) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "runtime_source_binding_failed" $runtimeSourceBinding $null $null $null $false 65
|
|
}
|
|
|
|
if ($Mode -eq "ToolchainReconcile") {
|
|
$stageDir = "/tmp/awoooi-signoz-metadata-toolchain.$RunId"
|
|
$reconcile = Invoke-Agent99SignozToolchainTarget "reconcile" $stageDir $ToolchainApplyControllerWaitSeconds
|
|
Update-Agent99SignozToolchainMutationState $reconcile.receipt
|
|
$allowedReconcileTerminals = @(
|
|
"reconcile_exact_deployment_verified",
|
|
"reconcile_zero_owned_residue",
|
|
"reconcile_exact_target_preserved_unverified",
|
|
"reconcile_shared_target_drift_preserved"
|
|
)
|
|
if (-not (Test-Agent99SignozToolchainReceipt $reconcile $allowedReconcileTerminals)) {
|
|
Stop-Agent99SignozRun "toolchain_reconcile_failed_or_unverified" "toolchain_target_reconcile_failed" $runtimeSourceBinding $null $reconcile.receipt $null $script:ToolchainSourceWritePerformed 1
|
|
}
|
|
$reconcileTerminal = [string]$reconcile.receipt.terminal
|
|
if ($reconcileTerminal -eq "reconcile_exact_deployment_verified") {
|
|
$toolchainVerify = Invoke-Agent99SignozToolchainTarget "verify" "" $ToolchainCheckControllerWaitSeconds
|
|
if (
|
|
-not (Test-Agent99SignozToolchainReceipt $toolchainVerify @("verify_pass")) -or
|
|
$toolchainVerify.receipt.durable_deploy_receipt_valid -ne $true
|
|
) {
|
|
Stop-Agent99SignozRun "toolchain_reconcile_failed_or_unverified" "toolchain_post_reconcile_verify_failed" $runtimeSourceBinding $null $reconcile.receipt $toolchainVerify.receipt $script:ToolchainSourceWritePerformed 1
|
|
}
|
|
Stop-Agent99SignozRun "toolchain_reconcile_exact_deployment_verified" "" $runtimeSourceBinding $null $reconcile.receipt $toolchainVerify.receipt $script:ToolchainSourceWritePerformed 0
|
|
}
|
|
if ($reconcileTerminal -eq "reconcile_zero_owned_residue") {
|
|
Stop-Agent99SignozRun "toolchain_reconcile_zero_owned_residue" "" $runtimeSourceBinding $null $reconcile.receipt $null $script:ToolchainSourceWritePerformed 0
|
|
}
|
|
Stop-Agent99SignozRun "toolchain_reconcile_preserved_unverified" $reconcileTerminal $runtimeSourceBinding $null $reconcile.receipt $null $script:ToolchainSourceWritePerformed 1
|
|
}
|
|
|
|
if ($Mode -in @("ToolchainCheck", "ToolchainApply", "ToolchainVerify")) {
|
|
$toolchainCheck = Invoke-Agent99SignozToolchainTarget "check" "" $ToolchainCheckControllerWaitSeconds
|
|
if (-not (Test-Agent99SignozToolchainReceipt $toolchainCheck @("check_pass_no_write"))) {
|
|
Stop-Agent99SignozRun "toolchain_blocked_with_safe_next_action" "toolchain_remote_check_failed" $toolchainCheck $null $null $null $false 1
|
|
}
|
|
$toolchainState = [string]$toolchainCheck.receipt.target_state
|
|
if ($toolchainState -notin @("absent", "exact")) {
|
|
Stop-Agent99SignozRun "toolchain_blocked_with_safe_next_action" "toolchain_remote_state_invalid" $toolchainCheck $null $null $null $false 1
|
|
}
|
|
if ($Mode -eq "ToolchainCheck") {
|
|
Stop-Agent99SignozRun "toolchain_check_ready" "" $toolchainCheck $null $null $null $false 0
|
|
}
|
|
if ($Mode -eq "ToolchainVerify") {
|
|
$toolchainVerify = Invoke-Agent99SignozToolchainTarget "verify" "" $ToolchainCheckControllerWaitSeconds
|
|
if (
|
|
-not (Test-Agent99SignozToolchainReceipt $toolchainVerify @("verify_pass")) -or
|
|
$toolchainVerify.receipt.durable_deploy_receipt_valid -ne $true
|
|
) {
|
|
Stop-Agent99SignozRun "toolchain_verification_failed" "toolchain_independent_verify_failed" $toolchainCheck $null $null $toolchainVerify $false 1
|
|
}
|
|
Stop-Agent99SignozRun "toolchain_verify_pass" "" $toolchainCheck $null $null $toolchainVerify $false 0
|
|
}
|
|
|
|
$decodedEnvelope = $null
|
|
$toolchainApplyReceipt = $null
|
|
$toolchainVerifyReceipt = $null
|
|
$toolchainError = ""
|
|
$stageDir = "/tmp/awoooi-signoz-metadata-toolchain.$RunId"
|
|
$toolchainApplyDispatched = $false
|
|
try {
|
|
$decodedEnvelope = Expand-Agent99SignozToolchainEnvelope $ToolchainEnvelopeJson
|
|
if ($toolchainState -eq "absent") {
|
|
$prepare = Invoke-Agent99SignozToolchainTarget "prepare" $stageDir $ToolchainCheckControllerWaitSeconds
|
|
$toolchainApplyReceipt = $prepare.receipt
|
|
Update-Agent99SignozToolchainMutationState $prepare.receipt
|
|
if (-not (Test-Agent99SignozToolchainReceipt $prepare @("stage_prepared", "stage_not_required_exact"))) {
|
|
throw "toolchain_stage_prepare_failed"
|
|
}
|
|
if ([string]$prepare.receipt.terminal -eq "stage_prepared") {
|
|
foreach ($row in @($decodedEnvelope.files)) {
|
|
$script:ToolchainSourceWriteAttempted = $true
|
|
if (-not (Invoke-Agent99SignozToolchainScp ([string]$row.path) "$stageDir/$([string]$row.name)")) {
|
|
throw "toolchain_transport_failed"
|
|
}
|
|
$script:ToolchainSourceWritePerformed = $true
|
|
}
|
|
}
|
|
}
|
|
$toolchainApplyDispatched = $true
|
|
$toolchainApply = Invoke-Agent99SignozToolchainTarget "apply" $stageDir $ToolchainApplyControllerWaitSeconds
|
|
$toolchainApplyReceipt = $toolchainApply.receipt
|
|
Update-Agent99SignozToolchainMutationState $toolchainApply.receipt
|
|
if (-not (Test-Agent99SignozToolchainReceipt $toolchainApply @(
|
|
"apply_pass_deployed_inactive",
|
|
"apply_pass_idempotent_exact"
|
|
))) {
|
|
throw "toolchain_target_apply_failed"
|
|
}
|
|
$toolchainVerify = Invoke-Agent99SignozToolchainTarget "verify" "" $ToolchainCheckControllerWaitSeconds
|
|
$toolchainVerifyReceipt = $toolchainVerify.receipt
|
|
if (-not (Test-Agent99SignozToolchainReceipt $toolchainVerify @("verify_pass"))) {
|
|
throw "toolchain_post_apply_verify_failed"
|
|
}
|
|
if ($toolchainVerifyReceipt.durable_deploy_receipt_valid -ne $true) {
|
|
throw "toolchain_durable_deploy_receipt_unverified"
|
|
}
|
|
} catch {
|
|
$toolchainError = [string]$_.Exception.Message
|
|
if ($toolchainApplyDispatched) {
|
|
$uncertainVerify = Invoke-Agent99SignozToolchainTarget "verify" "" $ToolchainCheckControllerWaitSeconds
|
|
if (
|
|
(Test-Agent99SignozToolchainReceipt $uncertainVerify @("verify_pass")) -and
|
|
$uncertainVerify.receipt.durable_deploy_receipt_valid -eq $true
|
|
) {
|
|
$toolchainVerifyReceipt = $uncertainVerify.receipt
|
|
$script:ToolchainSourceWritePerformed = $true
|
|
$toolchainError = ""
|
|
} else {
|
|
$toolchainVerifyReceipt = $uncertainVerify.receipt
|
|
$toolchainError = "${toolchainError}_toolchain_reconcile_required"
|
|
}
|
|
} elseif ($script:ToolchainSourceWriteAttempted) {
|
|
$toolchainError = "${toolchainError}_toolchain_reconcile_required"
|
|
}
|
|
} finally {
|
|
if ($decodedEnvelope -and $decodedEnvelope.root) {
|
|
Remove-Item -LiteralPath ([string]$decodedEnvelope.root) -Recurse -Force -ErrorAction SilentlyContinue
|
|
if (Test-Path -LiteralPath ([string]$decodedEnvelope.root)) {
|
|
$script:TransportCleanupSucceeded = $false
|
|
if (-not $toolchainError) { $toolchainError = "toolchain_local_payload_cleanup_failed" }
|
|
}
|
|
}
|
|
}
|
|
if ($toolchainError) {
|
|
Stop-Agent99SignozRun "toolchain_apply_failed_or_unverified" $toolchainError $toolchainCheck $null $toolchainApplyReceipt $toolchainVerifyReceipt $script:ToolchainSourceWritePerformed 1
|
|
}
|
|
Stop-Agent99SignozRun "toolchain_deployed_inactive_verified" "" $toolchainCheck $null $toolchainApplyReceipt $toolchainVerifyReceipt $script:ToolchainSourceWritePerformed 0
|
|
}
|
|
|
|
if ($Mode -eq "RestoreVerifyCleanup") {
|
|
$preflight = Get-Agent99SignozCleanupPreflight
|
|
if (-not $preflight.transportReadbackVerified -or -not $preflight.ready) {
|
|
Stop-Agent99SignozRun "restore_cleanup_blocked_with_safe_next_action" "restore_cleanup_toolchain_or_docker_preflight_failed" $preflight $null $null $null $false 1
|
|
}
|
|
$restoreReceiptStateBefore = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file"
|
|
if ($restoreReceiptStateBefore -eq "unverified" -or $restoreReceiptStateBefore -in @("symlink", "other", "directory")) {
|
|
Stop-Agent99SignozRun "restore_cleanup_blocked_with_safe_next_action" "restore_receipt_state_unverified_or_unsafe" $preflight $null $null $null $false 1
|
|
}
|
|
$restoreProgram = (
|
|
"/usr/bin/python3 $IsolatedRestorePath --verify-cleanup " +
|
|
"--bundle-dir $OutputDir --policy $PolicyPath " +
|
|
"--cluster-config $IsolatedClusterConfigPath " +
|
|
"--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId"
|
|
)
|
|
if ($restoreReceiptStateBefore -eq "file") {
|
|
$restoreProgram += " --apply-receipt-file $RestoreRemoteReceipt"
|
|
}
|
|
$restoreCommand = New-Agent99SignozCleanupLockedRemoteCommand $restoreProgram $RestoreCleanupTargetTimeoutSeconds
|
|
$restoreRemote = Invoke-Agent99SignozFixedRemote $restoreCommand $RestoreCleanupControllerWaitSeconds
|
|
$restoreReceipt = Convert-Agent99JsonReceipt $restoreRemote
|
|
$transportAccepted = Test-Agent99SignozTransportOutcome $restoreRemote
|
|
$failureReceiptAccepted = Test-Agent99SignozReceiptIdentity $restoreReceipt "terminal" @(
|
|
"failed_zero_residue_verified_no_completion",
|
|
"failed_cleanup_reconciliation_required"
|
|
)
|
|
if (-not $transportAccepted) {
|
|
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_controller_transport_failed_or_timed_out" $preflight $null $null $restoreReceipt $false 1
|
|
}
|
|
if ($failureReceiptAccepted) {
|
|
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_controller_failed" $preflight $null $null $restoreReceipt $false 1
|
|
}
|
|
$independentCompletion = Test-Agent99SignozReceiptIdentity $restoreReceipt "independent_restore_verifier" @(
|
|
"portable_metadata_restore_independently_verified_zero_residue"
|
|
)
|
|
$cleanupOnlyPass = Test-Agent99SignozReceiptIdentity $restoreReceipt "independent_restore_verifier" @(
|
|
"pass_zero_residue_verified"
|
|
)
|
|
if (-not $independentCompletion -and -not $cleanupOnlyPass) {
|
|
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_terminal_invalid" $preflight $null $null $restoreReceipt $false 1
|
|
}
|
|
if ($independentCompletion -and $restoreReceipt.completionClaim -ne $true) {
|
|
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_completion_claim_invalid" $preflight $null $null $restoreReceipt $false 1
|
|
}
|
|
if ($cleanupOnlyPass -and $restoreReceipt.completionClaim -ne $false) {
|
|
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_only_claim_invalid" $preflight $null $null $restoreReceipt $false 1
|
|
}
|
|
$restoreReceiptStateAfter = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file"
|
|
if ($restoreReceiptStateAfter -ne $restoreReceiptStateBefore) {
|
|
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_changed_durable_receipt_state" $preflight $null $null $restoreReceipt $false 1
|
|
}
|
|
$postflight = Get-Agent99SignozCleanupPreflight
|
|
if (-not $postflight.transportReadbackVerified -or -not $postflight.ready) {
|
|
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_postflight_failed" $postflight $null $null $restoreReceipt $false 1
|
|
}
|
|
if ($independentCompletion) {
|
|
Stop-Agent99SignozRun "portable_metadata_restore_independently_verified_zero_residue" "" $postflight $null $null $restoreReceipt $false 0
|
|
}
|
|
Stop-Agent99SignozRun "restore_zero_residue_verified_no_completion" "" $postflight $null $null $restoreReceipt $false 0
|
|
}
|
|
|
|
if ($Mode -in @("RestoreCheck", "RestoreApply")) {
|
|
$preflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
|
if (-not $preflight.transportReadbackVerified) {
|
|
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_preflight_transport_unverified" $preflight $null $null $null $false 1
|
|
}
|
|
if (-not $preflight.ready) {
|
|
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_preflight_failed" $preflight $null $null $null $false 1
|
|
}
|
|
|
|
$restoreReceiptStateBefore = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file"
|
|
if ($restoreReceiptStateBefore -eq "unverified" -or $restoreReceiptStateBefore -in @("symlink", "other", "directory")) {
|
|
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_receipt_state_unverified_or_unsafe" $preflight $null $null $null $false 1
|
|
}
|
|
if ($Mode -in @("RestoreCheck", "RestoreApply") -and $restoreReceiptStateBefore -ne "absent") {
|
|
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_receipt_identity_collision" $preflight $null $null $null $false 66
|
|
}
|
|
|
|
$restoreModeArgument = switch ($Mode) {
|
|
"RestoreCheck" { "--check" }
|
|
"RestoreApply" { "--apply" }
|
|
default { "" }
|
|
}
|
|
if (-not $restoreModeArgument) {
|
|
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_mode_not_routable" $preflight $null $null $null $false 64
|
|
}
|
|
$restoreProgram = (
|
|
"/usr/bin/python3 $IsolatedRestorePath $restoreModeArgument " +
|
|
"--bundle-dir $OutputDir --policy $PolicyPath " +
|
|
"--cluster-config $IsolatedClusterConfigPath " +
|
|
"--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId"
|
|
)
|
|
if ($Mode -eq "RestoreApply") {
|
|
$restoreProgram += " --receipt-file $RestoreRemoteReceipt"
|
|
$script:IsolatedApplyDispatchAttempted = $true
|
|
}
|
|
$restoreCommand = New-Agent99SignozLockedRemoteCommand $restoreProgram $TargetProgramTimeoutSeconds "present"
|
|
$restoreRemote = Invoke-Agent99SignozFixedRemote $restoreCommand $ControllerWaitSeconds
|
|
$restoreReceipt = Convert-Agent99JsonReceipt $restoreRemote
|
|
$transportAccepted = Test-Agent99SignozTransportOutcome $restoreRemote
|
|
$failureReceiptAccepted = Test-Agent99SignozReceiptIdentity $restoreReceipt "terminal" @(
|
|
"failed_zero_residue_verified_no_completion",
|
|
"failed_cleanup_reconciliation_required"
|
|
)
|
|
if (-not $transportAccepted) {
|
|
$transportTerminal = if ($Mode -eq "RestoreCheck") { "restore_check_failed_no_write" } else { "restore_failed_cleanup_reconciliation_required" }
|
|
Stop-Agent99SignozRun $transportTerminal "restore_controller_transport_failed_or_timed_out" $preflight $null $restoreReceipt $null $false 1
|
|
}
|
|
if ($failureReceiptAccepted) {
|
|
$failureTerminal = if ($Mode -eq "RestoreCheck") {
|
|
"restore_check_failed_no_write"
|
|
} elseif ([string]$restoreReceipt.terminal -eq "failed_zero_residue_verified_no_completion") {
|
|
"restore_failed_zero_residue_verified_no_completion"
|
|
} else {
|
|
"restore_failed_cleanup_reconciliation_required"
|
|
}
|
|
Stop-Agent99SignozRun $failureTerminal "isolated_restore_controller_failed" $preflight $null $restoreReceipt $null $false 1
|
|
}
|
|
|
|
$allowedRestoreTerminal = switch ($Mode) {
|
|
"RestoreCheck" { "check_pass_no_write" }
|
|
"RestoreApply" { "portable_metadata_restore_verified_zero_residue_pending_independent_verifier" }
|
|
}
|
|
if (-not (Test-Agent99SignozReceiptIdentity $restoreReceipt "terminal" @($allowedRestoreTerminal))) {
|
|
$invalidTerminal = if ($Mode -eq "RestoreCheck") { "restore_check_failed_no_write" } else { "restore_failed_cleanup_reconciliation_required" }
|
|
Stop-Agent99SignozRun $invalidTerminal "restore_controller_terminal_invalid" $preflight $null $restoreReceipt $null $false 1
|
|
}
|
|
|
|
$restoreReceiptStateAfter = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file"
|
|
if ($Mode -eq "RestoreApply" -and $restoreReceiptStateAfter -ne "file") {
|
|
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_durable_receipt_missing" $preflight $null $restoreReceipt $null $false 1
|
|
}
|
|
if ($Mode -eq "RestoreCheck" -and $restoreReceiptStateAfter -ne "absent") {
|
|
Stop-Agent99SignozRun "restore_check_failed_unexpected_write_requires_reconcile" "restore_check_wrote_receipt" $preflight $restoreReceipt $null $null $false 1
|
|
}
|
|
$postflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
|
if (-not $postflight.transportReadbackVerified -or -not $postflight.ready) {
|
|
$postflightTerminal = if ($Mode -eq "RestoreCheck") { "restore_check_failed_no_write" } else { "restore_failed_cleanup_reconciliation_required" }
|
|
Stop-Agent99SignozRun $postflightTerminal "restore_postflight_runtime_or_artifact_drift" $postflight $null $restoreReceipt $null $false 1
|
|
}
|
|
switch ($Mode) {
|
|
"RestoreCheck" {
|
|
Stop-Agent99SignozRun "restore_check_ready" "" $postflight $restoreReceipt $null $null $false 0
|
|
}
|
|
"RestoreApply" {
|
|
Stop-Agent99SignozRun "portable_metadata_restore_verified_zero_residue_pending_independent_verifier" "" $postflight $null $restoreReceipt $null $false 0
|
|
}
|
|
}
|
|
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_mode_fallthrough" $preflight $null $null $null $false 64
|
|
}
|
|
|
|
if ($Mode -eq "Verify") {
|
|
$preflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
|
if (-not $preflight.transportReadbackVerified) {
|
|
Stop-Agent99SignozRun "verification_failed" "preflight_transport_unverified" $preflight $null $null $null $false 1
|
|
}
|
|
if (-not $preflight.ready) {
|
|
Stop-Agent99SignozRun "verification_failed" "verified_bundle_preflight_failed" $preflight $null $null $null $false 1
|
|
}
|
|
$verifyProgram = (
|
|
"/usr/bin/python3 $VerifierPath " +
|
|
"--bundle-dir $OutputDir --policy $PolicyPath " +
|
|
"--expected-trace-id $TraceId --expected-run-id $RunId " +
|
|
"--expected-work-item-id $WorkItemId"
|
|
)
|
|
$verifyCommand = New-Agent99SignozLockedRemoteCommand $verifyProgram $CheckVerifyTargetTimeoutSeconds "present"
|
|
$verify = Invoke-Agent99SignozFixedRemote $verifyCommand $CheckVerifyControllerWaitSeconds
|
|
$verifyReceipt = Convert-Agent99JsonReceipt $verify
|
|
if (-not (Test-Agent99SignozTransportOutcome $verify) -or -not (Test-Agent99SignozReceiptIdentity $verifyReceipt "independent_export_verifier" @("pass_export_verified_restore_pending"))) {
|
|
Stop-Agent99SignozRun "verification_failed" "independent_export_verifier_failed" $preflight $null $null $verifyReceipt $false 1
|
|
}
|
|
$postflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
|
if (-not $postflight.transportReadbackVerified) {
|
|
Stop-Agent99SignozRun "verification_failed" "post_verifier_transport_unverified" $postflight $null $null $verifyReceipt $false 1
|
|
}
|
|
if (-not $postflight.ready) {
|
|
Stop-Agent99SignozRun "verification_failed" "post_verifier_runtime_or_artifact_drift" $postflight $null $null $verifyReceipt $false 1
|
|
}
|
|
Stop-Agent99SignozRun "export_verified_restore_pending" "" $postflight $null $null $verifyReceipt $false 0
|
|
}
|
|
|
|
$preflight = Get-Agent99SignozPreflight -ExpectedArtifactState "absent" -RequireCredential $true
|
|
if (-not $preflight.credentialReadbackVerified) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "credential_reference_readback_unverified" $preflight $null $null $null $false 2
|
|
}
|
|
if (-not $preflight.credentialReferencePresent) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "credential_reference_absent" $preflight $null $null $null $false 2
|
|
}
|
|
if (-not $preflight.credentialMetadataValid) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "credential_reference_metadata_invalid" $preflight $null $null $null $false 2
|
|
}
|
|
if (-not $preflight.transportReadbackVerified) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "preflight_transport_unverified" $preflight $null $null $null $false 1
|
|
}
|
|
if (-not $preflight.ready) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "metadata_export_preflight_failed" $preflight $null $null $null $false 1
|
|
}
|
|
|
|
$checkProgram = (
|
|
"/usr/bin/python3 $ExporterPath --check " +
|
|
"--base-url $BaseUrl --output-dir $OutputDir --credential-file $CredentialFile " +
|
|
"--policy $PolicyPath --trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId"
|
|
)
|
|
$checkCommand = New-Agent99SignozLockedRemoteCommand $checkProgram $CheckVerifyTargetTimeoutSeconds "absent"
|
|
$check = Invoke-Agent99SignozFixedRemote $checkCommand $CheckVerifyControllerWaitSeconds
|
|
$checkReceipt = Convert-Agent99JsonReceipt $check
|
|
if (-not (Test-Agent99SignozTransportOutcome $check) -or -not (Test-Agent99SignozReceiptIdentity $checkReceipt "terminal" @("check_pass_no_write"))) {
|
|
Stop-Agent99SignozRun "blocked_with_safe_next_action" "metadata_export_check_failed" $preflight $checkReceipt $null $null $false 1
|
|
}
|
|
if ($Mode -eq "Check") {
|
|
Stop-Agent99SignozRun "check_ready" "" $preflight $checkReceipt $null $null $false 0
|
|
}
|
|
|
|
$applyProgram = (
|
|
"/usr/bin/python3 $ExporterPath --apply " +
|
|
"--base-url $BaseUrl --output-dir $OutputDir --credential-file $CredentialFile " +
|
|
"--policy $PolicyPath --trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId " +
|
|
"--receipt-file $RemoteReceipt"
|
|
)
|
|
$applyCommand = New-Agent99SignozLockedRemoteCommand $applyProgram $ApplyTargetTimeoutSeconds "absent"
|
|
$script:ExportCredentialDispatchAttempted = $true
|
|
$apply = Invoke-Agent99SignozFixedRemote $applyCommand $ApplyControllerWaitSeconds
|
|
$applyReceipt = Convert-Agent99JsonReceipt $apply
|
|
if (-not (Test-Agent99SignozTransportOutcome $apply) -or -not (Test-Agent99SignozReceiptIdentity $applyReceipt "terminal" @("partial_degraded"))) {
|
|
$artifactReadback = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
|
if ($artifactReadback.outputPresent -eq $true -or $artifactReadback.receiptPresent -eq $true) {
|
|
Stop-Agent99SignozRun "apply_failed_artifact_preserved_for_readback" "bounded_export_failed" $artifactReadback $checkReceipt $applyReceipt $null $true 1
|
|
}
|
|
if ($artifactReadback.outputPresent -eq $false -and $artifactReadback.receiptPresent -eq $false) {
|
|
Stop-Agent99SignozRun "apply_failed_no_durable_artifact_observed" "bounded_export_failed_no_final_artifact" $artifactReadback $checkReceipt $applyReceipt $null $null 1
|
|
}
|
|
Stop-Agent99SignozRun "apply_failed_artifact_state_unverified" "bounded_export_failed_artifact_readback_unverified" $artifactReadback $checkReceipt $applyReceipt $null $null 1
|
|
}
|
|
|
|
$verifyProgram = (
|
|
"/usr/bin/python3 $VerifierPath " +
|
|
"--bundle-dir $OutputDir --policy $PolicyPath " +
|
|
"--expected-trace-id $TraceId --expected-run-id $RunId " +
|
|
"--expected-work-item-id $WorkItemId"
|
|
)
|
|
$verifyCommand = New-Agent99SignozLockedRemoteCommand $verifyProgram $CheckVerifyTargetTimeoutSeconds "present"
|
|
$verify = Invoke-Agent99SignozFixedRemote $verifyCommand $CheckVerifyControllerWaitSeconds
|
|
$verifyReceipt = Convert-Agent99JsonReceipt $verify
|
|
if (-not (Test-Agent99SignozTransportOutcome $verify) -or -not (Test-Agent99SignozReceiptIdentity $verifyReceipt "independent_export_verifier" @("pass_export_verified_restore_pending"))) {
|
|
$artifactReadback = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
|
Stop-Agent99SignozRun "verification_failed_artifact_preserved" "independent_export_verifier_failed" $artifactReadback $checkReceipt $applyReceipt $verifyReceipt $true 1
|
|
}
|
|
|
|
$postflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
|
if (-not $postflight.transportReadbackVerified) {
|
|
Stop-Agent99SignozRun "verification_failed_artifact_preserved" "post_verifier_transport_unverified" $postflight $checkReceipt $applyReceipt $verifyReceipt $true 1
|
|
}
|
|
if (-not $postflight.ready) {
|
|
Stop-Agent99SignozRun "verification_failed_artifact_preserved" "post_verifier_runtime_or_artifact_drift" $postflight $checkReceipt $applyReceipt $verifyReceipt $true 1
|
|
}
|
|
Stop-Agent99SignozRun "export_verified_restore_pending" "" $postflight $checkReceipt $applyReceipt $verifyReceipt $true 0
|