diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 58dd712d6..cd8569f59 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -226,6 +226,7 @@ $agentFiles = @( "agent99-signoz-metadata-executor.ps1", "agent99-signoz-credential-provisioner.ps1", "agent99-signoz-credential-target.py", + "agent99-signoz-toolchain-target.py", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 8bc9a8ced..1ed2dbde6 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -22,6 +22,7 @@ $requiredFiles = @( "agent99-signoz-metadata-executor.ps1", "agent99-signoz-credential-provisioner.ps1", "agent99-signoz-credential-target.py", + "agent99-signoz-toolchain-target.py", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -192,7 +193,7 @@ $signozMetadataExecutorContract = [bool]( $signozMetadataExecutor.Contains('agent99_active:') -and $signozMetadataExecutor.Contains('$RemoteResult.completed -ne $true') -and $signozMetadataExecutor.Contains('Get-Agent99RuntimeSourceBinding') -and - $signozMetadataExecutor.Contains('$result.fileCount -eq 20') -and + $signozMetadataExecutor.Contains('$result.fileCount -eq 21') -and $signozMetadataExecutor.Contains('pass_export_verified_restore_pending') -and $signozMetadataExecutor.Contains('productionRuntimeMutationPerformed = $false') -and $signozMetadataExecutor.Contains('rawSqliteRead = $false') -and diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index f33412cc1..883ed7db2 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -37,6 +37,7 @@ $runtimeFiles = @( "agent99-signoz-metadata-executor.ps1", "agent99-signoz-credential-provisioner.ps1", "agent99-signoz-credential-target.py", + "agent99-signoz-toolchain-target.py", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", diff --git a/agent99-signoz-credential-provisioner.ps1 b/agent99-signoz-credential-provisioner.ps1 index 03c1646f4..54b61a88b 100644 --- a/agent99-signoz-credential-provisioner.ps1 +++ b/agent99-signoz-credential-provisioner.ps1 @@ -18,7 +18,7 @@ $EntrypointPath = [string]$MyInvocation.MyCommand.Path $TargetProgram = Join-Path (Split-Path -Parent ([string]$MyInvocation.MyCommand.Path)) "agent99-signoz-credential-target.py" $SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" $ExpectedWorkItemId = "P0-OBS-002" -$ExpectedRuntimeFileCount = 20 +$ExpectedRuntimeFileCount = 21 $EvidenceRunId = if ($RunId -match $SafeIdPattern) { $RunId } else { "invalid-$PID" } $EvidencePath = Join-Path $EvidenceDir "agent99-signoz-credential-$($Mode.ToLowerInvariant())-$EvidenceRunId.json" diff --git a/agent99-signoz-metadata-executor.ps1 b/agent99-signoz-metadata-executor.ps1 index d2e64b43b..230f8eb12 100644 --- a/agent99-signoz-metadata-executor.ps1 +++ b/agent99-signoz-metadata-executor.ps1 @@ -1,5 +1,5 @@ param( - [ValidateSet("Check", "Apply", "Verify", "RestoreCheck", "RestoreApply", "RestoreVerifyCleanup")] + [ValidateSet("Check", "Apply", "Verify", "RestoreCheck", "RestoreApply", "RestoreVerifyCleanup", "ToolchainCheck", "ToolchainApply", "ToolchainVerify", "ToolchainReconcile")] [string]$Mode = "Check", [Parameter(Mandatory = $true)] [string]$TraceId, @@ -9,6 +9,9 @@ param( [string]$WorkItemId, [Parameter(Mandatory = $true)] [string]$SourceRevision, + [string]$AttemptId = "", + [string]$ToolchainEnvelopeJson = "", + [switch]$ToolchainEnvelopeDeletedBeforeExecutor, [string]$AgentRoot = "C:\Wooo\Agent99" ) @@ -21,8 +24,9 @@ $IdentityFile = Join-Path $AgentRoot "keys\agent99_ed25519" $EvidenceDir = Join-Path $AgentRoot "evidence" $RuntimeManifestPath = Join-Path $AgentRoot "state\runtime-manifest.json" $EntrypointPath = [string]$MyInvocation.MyCommand.Path +$ToolchainTargetPath = Join-Path (Split-Path -Parent $EntrypointPath) "agent99-signoz-toolchain-target.py" $SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$" -$BundleId = "ed4e5fbfc5f732b64d6a78f6ab26b90118e1d42a4f38b40d583876445c32e3aa" +$BundleId = "b83bfc898cd43cf8ec5688d75ec20231c2faa32de5be3fa886db33038d620dff" $ToolchainRoot = "/backup/toolchains/signoz-metadata/$BundleId" $PolicyPath = "$ToolchainRoot/metadata-export-policy.json" $ExporterPath = "$ToolchainRoot/signoz-metadata-export.py" @@ -48,17 +52,25 @@ $ApplyTargetTimeoutSeconds = 720 $RestoreCheckTargetTimeoutSeconds = 180 $RestoreApplyTargetTimeoutSeconds = 1800 $RestoreCleanupTargetTimeoutSeconds = 300 +$ToolchainCheckTargetTimeoutSeconds = 180 +$ToolchainApplyTargetTimeoutSeconds = 600 $CheckVerifyControllerWaitSeconds = 300 $ApplyControllerWaitSeconds = 900 $RestoreCheckControllerWaitSeconds = 420 $RestoreApplyControllerWaitSeconds = 2550 $RestoreCleanupControllerWaitSeconds = 1050 +$ToolchainCheckControllerWaitSeconds = 420 +$ToolchainApplyControllerWaitSeconds = 900 $ControllerTransportReserveSeconds = 30 $TargetProgramTimeoutSeconds = switch ($Mode) { "Apply" { $ApplyTargetTimeoutSeconds } "RestoreCheck" { $RestoreCheckTargetTimeoutSeconds } "RestoreApply" { $RestoreApplyTargetTimeoutSeconds } "RestoreVerifyCleanup" { $RestoreCleanupTargetTimeoutSeconds } + "ToolchainApply" { $ToolchainApplyTargetTimeoutSeconds } + "ToolchainReconcile" { $ToolchainApplyTargetTimeoutSeconds } + "ToolchainCheck" { $ToolchainCheckTargetTimeoutSeconds } + "ToolchainVerify" { $ToolchainCheckTargetTimeoutSeconds } default { $CheckVerifyTargetTimeoutSeconds } } $ControllerWaitSeconds = switch ($Mode) { @@ -66,6 +78,10 @@ $ControllerWaitSeconds = switch ($Mode) { "RestoreCheck" { $RestoreCheckControllerWaitSeconds } "RestoreApply" { $RestoreApplyControllerWaitSeconds } "RestoreVerifyCleanup" { $RestoreCleanupControllerWaitSeconds } + "ToolchainApply" { $ToolchainApplyControllerWaitSeconds } + "ToolchainReconcile" { $ToolchainApplyControllerWaitSeconds } + "ToolchainCheck" { $ToolchainCheckControllerWaitSeconds } + "ToolchainVerify" { $ToolchainCheckControllerWaitSeconds } default { $CheckVerifyControllerWaitSeconds } } $TargetKillAfterSeconds = if ($Mode -in @("RestoreApply", "RestoreVerifyCleanup")) { @@ -75,23 +91,40 @@ $TargetKillAfterSeconds = if ($Mode -in @("RestoreApply", "RestoreVerifyCleanup" } $TargetWorstCaseSeconds = $TargetGuardWorstCaseSeconds + $TargetProgramTimeoutSeconds + $TargetKillAfterSeconds $EvidenceRunId = if ($RunId -match $SafeIdPattern) { $RunId } else { "invalid-$PID" } -$EvidencePath = Join-Path $EvidenceDir "agent99-signoz-metadata-$($Mode.ToLowerInvariant())-$EvidenceRunId.json" +$EvidenceAttemptId = if ($AttemptId -match $SafeIdPattern) { $AttemptId } else { "invalid-$PID" } +$EvidenceIdentity = if ($Mode -in @("RestoreVerifyCleanup", "ToolchainReconcile")) { + "$EvidenceRunId-attempt-$EvidenceAttemptId" +} else { + $EvidenceRunId +} +$EvidencePath = Join-Path $EvidenceDir "agent99-signoz-metadata-$($Mode.ToLowerInvariant())-$EvidenceIdentity.json" $EvidenceReservationPath = "$EvidencePath.reserve" $script:TransientTransportOutputUsed = $false $script:TransportCleanupSucceeded = $true $script:EvidenceReserved = $false $script:ExportCredentialDispatchAttempted = $false $script:IsolatedApplyDispatchAttempted = $false +$script:ToolchainSourceWritePerformed = $false +$script:ToolchainSourceWriteAttempted = $false $ToolchainFiles = [ordered]@{ - $PolicyPath = "1365c6406b522f2678693cebe99914255d44b4f219a78a4625ac2c2b5eec2557" - $ExporterPath = "8cd5cabf245e3814da32e33920c431ec28ed7180edc0495a2d0b9379fb4025fb" + $PolicyPath = "a050943d50631d34d6eb2580db484197d6c7101b3b9cc2b47bb7ee3d87dd3822" $SharedContractPath = "d2b2281da9ec89afc44d5153711b2d61e972545ad107611b74a5e6dfb01defe8" + $ExporterPath = "8cd5cabf245e3814da32e33920c431ec28ed7180edc0495a2d0b9379fb4025fb" $VerifierPath = "0bac6d8ef8063f3584fc744cf205de08fc50d1085b292ffbd061f6090c344c86" $RestoreDriverPath = "eb766f1209af08c3b339756d58653165d55b3c23bc8dbd66438dfb46d8098156" - $IsolatedRestorePath = "a91fed22ae0c2682c8298eb075d6ede5c37608c83ac15f03c55e4bad52e93946" + $IsolatedRestorePath = "dc57604919e220454f25955c1c652b7bd9dea632210019b0b4de527969e03c52" $IsolatedClusterConfigPath = "e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b" } +$ToolchainPayloadSpec = @( + [pscustomobject]@{ name = "metadata-export-policy.json"; path = $PolicyPath; sha256 = $ToolchainFiles[$PolicyPath]; mode = "0644" }, + [pscustomobject]@{ name = "signoz_metadata_contract.py"; path = $SharedContractPath; sha256 = $ToolchainFiles[$SharedContractPath]; mode = "0644" }, + [pscustomobject]@{ name = "signoz-metadata-export.py"; path = $ExporterPath; sha256 = $ToolchainFiles[$ExporterPath]; mode = "0755" }, + [pscustomobject]@{ name = "verify-signoz-metadata-export.py"; path = $VerifierPath; sha256 = $ToolchainFiles[$VerifierPath]; mode = "0755" }, + [pscustomobject]@{ name = "signoz-metadata-restore-drill.py"; path = $RestoreDriverPath; sha256 = $ToolchainFiles[$RestoreDriverPath]; mode = "0755" }, + [pscustomobject]@{ name = "signoz-metadata-isolated-restore.py"; path = $IsolatedRestorePath; sha256 = $ToolchainFiles[$IsolatedRestorePath]; mode = "0755" }, + [pscustomobject]@{ name = "isolated-cluster.xml"; path = $IsolatedClusterConfigPath; sha256 = $ToolchainFiles[$IsolatedClusterConfigPath]; mode = "0644" } +) function Invoke-Agent99SignozFixedRemote { param( @@ -205,6 +238,287 @@ function Convert-Agent99JsonReceipt { } } +function Get-Agent99SignozSha256Bytes { + param([byte[]]$Bytes) + $sha = [Security.Cryptography.SHA256]::Create() + try { + return ([BitConverter]::ToString($sha.ComputeHash($Bytes))).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +function Get-Agent99SignozToolchainManifestBase64 { + $rows = @($ToolchainPayloadSpec | ForEach-Object { + [ordered]@{ + name = [string]$_.name + sha256 = [string]$_.sha256 + mode = [string]$_.mode + } + }) + $json = $rows | ConvertTo-Json -Compress + return [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($json)) +} + +function Expand-Agent99SignozToolchainEnvelope { + param([Parameter(Mandatory = $true)][string]$EnvelopeJson) + + if (-not $EnvelopeJson -or $EnvelopeJson.Length -gt 4194304) { + throw "toolchain_envelope_size_invalid" + } + try { $envelope = $EnvelopeJson | ConvertFrom-Json } catch { + throw "toolchain_envelope_json_invalid" + } + if ( + [string]$envelope.schemaVersion -ne "awoooi_signoz_metadata_toolchain_envelope_v1" -or + [string]$envelope.traceId -ne $TraceId -or + [string]$envelope.runId -ne $RunId -or + [string]$envelope.workItemId -ne $WorkItemId -or + [string]$envelope.sourceRevision -ne $SourceRevision -or + [string]$envelope.bundleId -ne $BundleId + ) { throw "toolchain_envelope_identity_invalid" } + $files = @($envelope.files) + if ($files.Count -ne $ToolchainPayloadSpec.Count) { + throw "toolchain_envelope_file_count_invalid" + } + $manifestText = "" + $decodedRows = @() + $tempRoot = Join-Path $env:TEMP "agent99-signoz-toolchain-$RunId-$PID-$([Guid]::NewGuid().ToString('N'))" + if (Test-Path -LiteralPath $tempRoot) { throw "toolchain_temp_identity_collision" } + New-Item -ItemType Directory -Path $tempRoot | Out-Null + try { + for ($index = 0; $index -lt $ToolchainPayloadSpec.Count; $index += 1) { + $expected = $ToolchainPayloadSpec[$index] + $row = $files[$index] + if ( + [string]$row.name -ne [string]$expected.name -or + [string]$row.sha256 -ne [string]$expected.sha256 -or + [string]$row.mode -ne [string]$expected.mode + ) { throw "toolchain_envelope_file_identity_invalid" } + try { $bytes = [Convert]::FromBase64String([string]$row.contentBase64) } catch { + throw "toolchain_envelope_content_base64_invalid" + } + if ($bytes.Length -le 0 -or $bytes.Length -gt 2097152) { + throw "toolchain_envelope_content_size_invalid" + } + if ((Get-Agent99SignozSha256Bytes $bytes) -ne [string]$expected.sha256) { + throw "toolchain_envelope_content_hash_mismatch" + } + $destination = Join-Path $tempRoot ([string]$expected.name) + $stream = [IO.File]::Open( + $destination, + [IO.FileMode]::CreateNew, + [IO.FileAccess]::Write, + [IO.FileShare]::None + ) + try { + $stream.Write($bytes, 0, $bytes.Length) + $stream.Flush($true) + } finally { + $stream.Dispose() + } + $decodedRows += [pscustomobject]@{ + name = [string]$expected.name + path = $destination + } + $manifestText += "$([string]$expected.name)`t$([string]$expected.sha256)`t$([string]$expected.mode)`n" + } + $bundleMaterial = (($ToolchainPayloadSpec | ForEach-Object { "$([string]$_.sha256)`n" }) -join "") + $computedBundle = Get-Agent99SignozSha256Bytes ([Text.Encoding]::ASCII.GetBytes($bundleMaterial)) + $computedManifest = Get-Agent99SignozSha256Bytes ([Text.Encoding]::UTF8.GetBytes($manifestText)) + if ($computedBundle -ne $BundleId) { throw "toolchain_envelope_bundle_digest_invalid" } + if ([string]$envelope.manifestSha256 -ne $computedManifest) { + throw "toolchain_envelope_manifest_digest_invalid" + } + return [pscustomobject]@{ + root = $tempRoot + files = $decodedRows + manifestSha256 = $computedManifest + } + } catch { + Remove-Item -LiteralPath $tempRoot -Recurse -Force -ErrorAction SilentlyContinue + throw + } +} + +function Invoke-Agent99SignozToolchainTarget { + param( + [ValidateSet("check", "prepare", "apply", "verify", "reconcile")][string]$TargetMode, + [string]$StageDir = "", + [int]$TimeoutSeconds = 420 + ) + if (-not (Test-Path -LiteralPath $ToolchainTargetPath -PathType Leaf)) { + return [pscustomobject]@{ completed = $false; timedOut = $false; exitCode = -1; receipt = $null; errorCode = "toolchain_target_missing" } + } + $manifestBase64 = Get-Agent99SignozToolchainManifestBase64 + $targetCommand = ( + "sudo -n /usr/bin/python3 - --mode $TargetMode " + + "--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId " + + "--source-revision $SourceRevision --bundle-id $BundleId " + + "--manifest-b64 $manifestBase64" + ) + if ($StageDir) { $targetCommand += " --stage-dir $StageDir" } + if ($TargetMode -eq "reconcile") { $targetCommand += " --attempt-id $AttemptId" } + $arguments = @( + "-i", $IdentityFile, + "-o", "BatchMode=yes", + "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=yes", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=8", + "-o", "ServerAliveInterval=20", + "-o", "ServerAliveCountMax=3", + "-l", $RemoteUser, + $TargetHost, + $targetCommand + ) + $token = [Guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "agent99-signoz-toolchain-target-$token.out" + $stderrPath = Join-Path $env:TEMP "agent99-signoz-toolchain-target-$token.err" + $script:TransientTransportOutputUsed = $true + try { + $process = Start-Process -FilePath "ssh.exe" -ArgumentList $arguments ` + -NoNewWindow -RedirectStandardInput $ToolchainTargetPath ` + -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {} + return [pscustomobject]@{ completed = $false; timedOut = $true; receipt = $null; errorCode = "toolchain_target_timeout" } + } + $process.WaitForExit() | Out-Null + $stdout = if (Test-Path -LiteralPath $stdoutPath) { + ([IO.File]::ReadAllText($stdoutPath, [Text.Encoding]::UTF8)).Trim() + } else { "" } + if ($stdout.Length -gt 16384) { + return [pscustomobject]@{ completed = $false; timedOut = $false; receipt = $null; errorCode = "toolchain_target_output_too_large" } + } + try { $receipt = $stdout | ConvertFrom-Json } catch { $receipt = $null } + return [pscustomobject]@{ + completed = $true + timedOut = $false + receipt = $receipt + errorCode = if ($receipt) { "" } else { "toolchain_target_receipt_invalid" } + } + } finally { + Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue + if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) { + $script:TransportCleanupSucceeded = $false + } + } +} + +function Test-Agent99SignozToolchainReceipt { + param( + [object]$Result, + [string[]]$AllowedTerminals + ) + $value = if ($Result) { $Result.receipt } else { $null } + $mutationContract = [bool]( + $value -and ( + ( + [string]$value.terminal -in @("check_pass_no_write", "verify_pass", "stage_not_required_exact") -and + $value.source_write_attempted -eq $false -and + $value.source_write_performed -eq $false + ) -or ( + [string]$value.terminal -eq "stage_prepared" -and + $value.source_write_attempted -eq $true -and + $value.source_write_performed -eq $true + ) -or ( + [string]$value.terminal -in @("apply_pass_deployed_inactive", "apply_pass_idempotent_exact") -and + $value.source_write_attempted -eq $true -and + $value.source_write_performed -eq $true -and + $value.durable_deploy_receipt_valid -eq $true -and + $value.quarantine_performed -eq $false + ) -or ( + [string]$value.terminal -eq "reconcile_exact_deployment_verified" -and + [string]$value.attempt_id -eq $AttemptId -and + $value.durable_deploy_receipt_valid -eq $true -and + $value.owned_live_residue_verified_absent -eq $true + ) -or ( + [string]$value.terminal -eq "reconcile_zero_owned_residue" -and + [string]$value.attempt_id -eq $AttemptId -and + $value.durable_deploy_receipt_valid -eq $false -and + $value.owned_live_residue_verified_absent -eq $true -and + $value.shared_target_preserved -eq $false + ) -or ( + [string]$value.terminal -in @( + "reconcile_exact_target_preserved_unverified", + "reconcile_shared_target_drift_preserved" + ) -and + [string]$value.attempt_id -eq $AttemptId -and + $value.durable_deploy_receipt_valid -eq $false -and + $value.owned_live_residue_verified_absent -eq $true -and + $value.shared_target_preserved -eq $true + ) + ) + ) + return [bool]( + $Result -and $Result.completed -eq $true -and -not $Result.timedOut -and + $value -and + [string]$value.schema -eq "awoooi_agent99_signoz_toolchain_target_v1" -and + [string]$value.trace_id -eq $TraceId -and + [string]$value.run_id -eq $RunId -and + [string]$value.work_item_id -eq $WorkItemId -and + [string]$value.source_revision -eq $SourceRevision -and + [string]$value.bundle_id -eq $BundleId -and + [string]$value.terminal -in $AllowedTerminals -and + $value.production_service_mutation_performed -eq $false -and + $value.active_pointer_created -eq $false -and + $value.credential_value_read -eq $false -and + $value.completion_claim -eq $false -and + $mutationContract + ) +} + +function Update-Agent99SignozToolchainMutationState { + param([object]$Receipt) + if (-not $Receipt) { return } + if ($Receipt.source_write_attempted -eq $true) { + $script:ToolchainSourceWriteAttempted = $true + } + if ($Receipt.source_write_performed -eq $true) { + $script:ToolchainSourceWritePerformed = $true + } +} + +function Invoke-Agent99SignozToolchainScp { + param( + [Parameter(Mandatory = $true)][string]$LocalPath, + [Parameter(Mandatory = $true)][string]$RemotePath + ) + $arguments = @( + "-q", + "-i", $IdentityFile, + "-o", "BatchMode=yes", + "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=yes", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=8", + $LocalPath, + "$RemoteUser@$TargetHost`:$RemotePath" + ) + $token = [Guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "agent99-signoz-toolchain-scp-$token.out" + $stderrPath = Join-Path $env:TEMP "agent99-signoz-toolchain-scp-$token.err" + $script:TransientTransportOutputUsed = $true + try { + $process = Start-Process -FilePath "scp.exe" -ArgumentList $arguments ` + -NoNewWindow -RedirectStandardOutput $stdoutPath ` + -RedirectStandardError $stderrPath -PassThru + if (-not $process.WaitForExit(180000)) { + try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {} + return $false + } + $process.WaitForExit() | Out-Null + $process.Refresh() + return [bool]($process.HasExited -and [int]$process.ExitCode -eq 0) + } finally { + Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue + if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) { + $script:TransportCleanupSucceeded = $false + } + } +} + function New-Agent99SignozLockedRemoteCommand { param( [Parameter(Mandatory = $true)][string]$FixedCommand, @@ -275,6 +589,8 @@ function Get-Agent99RuntimeSourceBinding { sourceRevision = "" entrypointTracked = $false entrypointHashMatched = $false + toolchainTargetTracked = $false + toolchainTargetHashMatched = $false } if (-not (Test-Path -LiteralPath $RuntimeManifestPath -PathType Leaf)) { return [pscustomobject]$result @@ -296,13 +612,25 @@ function Get-Agent99RuntimeSourceBinding { [string]$rows[0].runtimeSha256 -eq $actualHash ) } + $targetRows = @($manifest.files | Where-Object { + [string]$_.name -eq "agent99-signoz-toolchain-target.py" + }) + $result.toolchainTargetTracked = [bool]($targetRows.Count -eq 1) + if ($targetRows.Count -eq 1 -and (Test-Path -LiteralPath $ToolchainTargetPath -PathType Leaf)) { + $targetHash = (Get-FileHash -LiteralPath $ToolchainTargetPath -Algorithm SHA256).Hash.ToLowerInvariant() + $result.toolchainTargetHashMatched = [bool]( + [string]$targetRows[0].runtimeSha256 -eq $targetHash + ) + } $result.ok = [bool]( $result.runtimeMatched -and - $result.fileCount -eq 20 -and + $result.fileCount -eq 21 -and $result.mismatchCount -eq 0 -and $result.sourceRevision -eq $SourceRevision -and $result.entrypointTracked -and - $result.entrypointHashMatched + $result.entrypointHashMatched -and + $result.toolchainTargetTracked -and + $result.toolchainTargetHashMatched ) } catch { $result.ok = $false @@ -726,6 +1054,8 @@ function Write-Agent99SignozEvidenceCollision { mode = $Mode traceId = $TraceId runId = $RunId + cleanupAttemptId = if ($Mode -eq "RestoreVerifyCleanup") { $AttemptId } else { $null } + toolchainReconcileAttemptId = if ($Mode -eq "ToolchainReconcile") { $AttemptId } else { $null } workItemId = $WorkItemId terminal = "blocked_evidence_identity_collision" errorCode = "evidence_path_or_reservation_already_exists" @@ -752,7 +1082,7 @@ function New-Agent99SignozEvidenceReservation { [IO.FileShare]::None ) $reservationPayload = [Text.Encoding]::UTF8.GetBytes( - "$TraceId|$RunId|$WorkItemId|$Mode" + [Environment]::NewLine + "$TraceId|$RunId|$WorkItemId|$Mode|$AttemptId" + [Environment]::NewLine ) $reservationStream.Write($reservationPayload, 0, $reservationPayload.Length) $reservationStream.Flush($true) @@ -777,6 +1107,7 @@ function Write-Agent99SignozTerminal { ) $isRestoreMode = [bool]($Mode -like "Restore*") + $isToolchainMode = [bool]($Mode -like "Toolchain*") $restoreCompleted = [bool]( $Terminal -eq "portable_metadata_restore_independently_verified_zero_residue" ) @@ -790,13 +1121,13 @@ function Write-Agent99SignozTerminal { $controlledApplyPhases = [ordered]@{ sensorSource = if ($Preflight) { "observed" } else { "blocked" } normalizedAsset = if ($Preflight) { $CanonicalAsset } else { "blocked" } - sourceTruthDiff = if ($Preflight -and $Preflight.toolchainExact) { "exact" } else { "blocked_or_drift" } - aiDecision = if ($isRestoreMode) { "fixed_isolated_portable_metadata_restore" } else { "fixed_allowlisted_metadata_export" } - riskPolicy = if ($isRestoreMode) { "high_bounded_ephemeral_internal_only_zero_residue" } else { "high_bounded_read_only_source_private_artifact_no_raw_sqlite" } + sourceTruthDiff = if ($isToolchainMode -and $Preflight) { "typed_target_state_readback" } elseif ($Preflight -and $Preflight.toolchainExact) { "exact" } else { "blocked_or_drift" } + aiDecision = if ($isToolchainMode) { "fixed_source_bound_inactive_toolchain_deploy" } elseif ($isRestoreMode) { "fixed_isolated_portable_metadata_restore" } else { "fixed_allowlisted_metadata_export" } + riskPolicy = if ($isToolchainMode) { "high_bounded_additive_inactive_no_active_pointer" } elseif ($isRestoreMode) { "high_bounded_ephemeral_internal_only_zero_residue" } else { "high_bounded_read_only_source_private_artifact_no_raw_sqlite" } checkMode = if ($CheckReceipt) { [string]$CheckReceipt.terminal } elseif ($Preflight -and $Preflight.ready) { "preflight_pass" } else { "blocked" } boundedExecution = if ($ApplyReceipt) { [string]$ApplyReceipt.terminal } elseif ($Mode -eq "Apply") { "not_verified" } else { "not_run" } independentVerifier = if ($VerifierReceipt) { [string]$VerifierReceipt.terminal } else { "not_run" } - rollbackOrNoWrite = if ($Mode -in @("Check", "Verify", "RestoreCheck")) { "no_write" } elseif ($zeroResidueVerified) { "ephemeral_resources_cleaned_zero_residue" } elseif ($Mode -eq "RestoreVerifyCleanup") { "reconciliation_failed_or_incomplete" } elseif ($Terminal -eq "export_verified_restore_pending") { "not_required_additive_immutable_artifact" } else { "artifact_preserved_no_delete" } + rollbackOrNoWrite = if ($Terminal -eq "restore_check_failed_unexpected_write_requires_reconcile") { "unexpected_receipt_write_requires_reconcile" } elseif ($Mode -in @("Check", "Verify", "RestoreCheck", "ToolchainCheck", "ToolchainVerify")) { "no_write" } elseif ($Terminal -eq "toolchain_deployed_inactive_verified") { "not_required_inactive_content_addressed_bundle" } elseif ($Terminal -eq "toolchain_reconcile_exact_deployment_verified") { "owned_residue_absent_shared_target_exact" } elseif ($Terminal -eq "toolchain_reconcile_zero_owned_residue") { "owned_live_residue_removed_new_apply_required" } elseif ($Mode -in @("ToolchainApply", "ToolchainReconcile")) { "reconcile_required_or_shared_target_preserved" } elseif ($zeroResidueVerified) { "ephemeral_resources_cleaned_zero_residue" } elseif ($Mode -eq "RestoreVerifyCleanup") { "reconciliation_failed_or_incomplete" } elseif ($Terminal -eq "export_verified_restore_pending") { "not_required_additive_immutable_artifact" } else { "artifact_preserved_no_delete" } learningWriteback = "pending_telegram_km_rag_mcp_playbook_ack" } $payload = [pscustomobject]@{ @@ -804,13 +1135,15 @@ function Write-Agent99SignozTerminal { canonicalAsset = $CanonicalAsset domain = "backup_observability" executor = "agent99_signoz_metadata_executor" - verifier = if ($Mode -eq "RestoreVerifyCleanup") { "signoz_metadata_restore_cleanup_verifier" } elseif ($isRestoreMode) { "pending_distinct_restore_cleanup_verifier" } else { "signoz_metadata_export_independent_verifier" } + verifier = if ($isToolchainMode) { "signoz_metadata_toolchain_independent_target_readback" } elseif ($Mode -eq "RestoreVerifyCleanup") { "signoz_metadata_restore_cleanup_verifier" } elseif ($isRestoreMode) { "pending_distinct_restore_cleanup_verifier" } else { "signoz_metadata_export_independent_verifier" } executionHub = "host:192.168.0.99" dispatchPath = "windows99_agent99_to_host110_fixed_metadata_toolchain" dispatchOnly = $true mode = $Mode traceId = $TraceId runId = $RunId + cleanupAttemptId = if ($Mode -eq "RestoreVerifyCleanup") { $AttemptId } else { $null } + toolchainReconcileAttemptId = if ($Mode -eq "ToolchainReconcile") { $AttemptId } else { $null } workItemId = $WorkItemId sourceRevision = $SourceRevision targetHost = $TargetHost @@ -837,6 +1170,15 @@ function Write-Agent99SignozTerminal { controlledApplyPhases = [pscustomobject]$controlledApplyPhases artifactWritePerformed = $ArtifactWritePerformed productionRuntimeMutationPerformed = $false + productionSourceMutationAttempted = $script:ToolchainSourceWriteAttempted + productionSourceMutationPerformed = $script:ToolchainSourceWritePerformed + productionSourceMutationState = if ($script:ToolchainSourceWritePerformed) { + "performed" + } elseif ($script:ToolchainSourceWriteAttempted) { + "attempted_unverified" + } else { + "none" + } isolatedEphemeralRuntimeCreated = [bool]($Mode -eq "RestoreApply" -and $script:IsolatedApplyDispatchAttempted) rawSqliteRead = $false rawVolumeRead = $false @@ -869,6 +1211,11 @@ function Write-Agent99SignozTerminal { rawTransportOutputPersisted = [bool]( $script:TransientTransportOutputUsed -and -not $script:TransportCleanupSucceeded ) + toolchainEnvelopeTransportPayloadUsed = [bool]($Mode -eq "ToolchainApply") + toolchainEnvelopeTransportPayloadDeletedBeforeExecutor = [bool]( + $Mode -eq "ToolchainApply" -and $ToolchainEnvelopeDeletedBeforeExecutor + ) + toolchainEnvelopeTransportPayloadContainsSecrets = $false evidenceReservedBeforeDispatch = $script:EvidenceReserved evidencePath = $EvidencePath } @@ -922,6 +1269,27 @@ foreach ($identity in @($TraceId, $RunId, $WorkItemId)) { Stop-Agent99SignozRun "blocked_with_safe_next_action" "identity_missing_or_invalid" $null $null $null $null $false 64 } } +if ($Mode -in @("RestoreVerifyCleanup", "ToolchainReconcile")) { + if ($AttemptId -notmatch $SafeIdPattern) { + $terminal = if ($Mode -eq "ToolchainReconcile") { "toolchain_reconcile_blocked_with_safe_next_action" } else { "restore_cleanup_blocked_with_safe_next_action" } + $errorCode = if ($Mode -eq "ToolchainReconcile") { "reconcile_attempt_id_missing_or_invalid" } else { "cleanup_attempt_id_missing_or_invalid" } + Stop-Agent99SignozRun $terminal $errorCode $null $null $null $null $false 64 + } +} elseif ($AttemptId) { + Stop-Agent99SignozRun "blocked_with_safe_next_action" "attempt_id_not_allowed_for_mode" $null $null $null $null $false 64 +} +if ($Mode -eq "ToolchainApply") { + if (-not $ToolchainEnvelopeJson) { + Stop-Agent99SignozRun "toolchain_blocked_with_safe_next_action" "toolchain_envelope_required_for_apply" $null $null $null $null $false 64 + } + if (-not $ToolchainEnvelopeDeletedBeforeExecutor) { + Stop-Agent99SignozRun "toolchain_blocked_with_safe_next_action" "toolchain_envelope_delete_attestation_required" $null $null $null $null $false 64 + } +} elseif ($ToolchainEnvelopeJson) { + Stop-Agent99SignozRun "blocked_with_safe_next_action" "toolchain_envelope_not_allowed_for_mode" $null $null $null $null $false 64 +} elseif ($ToolchainEnvelopeDeletedBeforeExecutor) { + Stop-Agent99SignozRun "blocked_with_safe_next_action" "toolchain_envelope_delete_attestation_not_allowed_for_mode" $null $null $null $null $false 64 +} if ($WorkItemId -ne "P0-OBS-002") { Stop-Agent99SignozRun "blocked_with_safe_next_action" "work_item_not_allowlisted" $null $null $null $null $false 64 } @@ -944,6 +1312,135 @@ if (-not $runtimeSourceBinding.ok) { Stop-Agent99SignozRun "blocked_with_safe_next_action" "runtime_source_binding_failed" $runtimeSourceBinding $null $null $null $false 65 } +if ($Mode -eq "ToolchainReconcile") { + $stageDir = "/tmp/awoooi-signoz-metadata-toolchain.$RunId" + $reconcile = Invoke-Agent99SignozToolchainTarget "reconcile" $stageDir $ToolchainApplyControllerWaitSeconds + Update-Agent99SignozToolchainMutationState $reconcile.receipt + $allowedReconcileTerminals = @( + "reconcile_exact_deployment_verified", + "reconcile_zero_owned_residue", + "reconcile_exact_target_preserved_unverified", + "reconcile_shared_target_drift_preserved" + ) + if (-not (Test-Agent99SignozToolchainReceipt $reconcile $allowedReconcileTerminals)) { + Stop-Agent99SignozRun "toolchain_reconcile_failed_or_unverified" "toolchain_target_reconcile_failed" $runtimeSourceBinding $null $reconcile.receipt $null $script:ToolchainSourceWritePerformed 1 + } + $reconcileTerminal = [string]$reconcile.receipt.terminal + if ($reconcileTerminal -eq "reconcile_exact_deployment_verified") { + $toolchainVerify = Invoke-Agent99SignozToolchainTarget "verify" "" $ToolchainCheckControllerWaitSeconds + if ( + -not (Test-Agent99SignozToolchainReceipt $toolchainVerify @("verify_pass")) -or + $toolchainVerify.receipt.durable_deploy_receipt_valid -ne $true + ) { + Stop-Agent99SignozRun "toolchain_reconcile_failed_or_unverified" "toolchain_post_reconcile_verify_failed" $runtimeSourceBinding $null $reconcile.receipt $toolchainVerify.receipt $script:ToolchainSourceWritePerformed 1 + } + Stop-Agent99SignozRun "toolchain_reconcile_exact_deployment_verified" "" $runtimeSourceBinding $null $reconcile.receipt $toolchainVerify.receipt $script:ToolchainSourceWritePerformed 0 + } + if ($reconcileTerminal -eq "reconcile_zero_owned_residue") { + Stop-Agent99SignozRun "toolchain_reconcile_zero_owned_residue" "" $runtimeSourceBinding $null $reconcile.receipt $null $script:ToolchainSourceWritePerformed 0 + } + Stop-Agent99SignozRun "toolchain_reconcile_preserved_unverified" $reconcileTerminal $runtimeSourceBinding $null $reconcile.receipt $null $script:ToolchainSourceWritePerformed 1 +} + +if ($Mode -in @("ToolchainCheck", "ToolchainApply", "ToolchainVerify")) { + $toolchainCheck = Invoke-Agent99SignozToolchainTarget "check" "" $ToolchainCheckControllerWaitSeconds + if (-not (Test-Agent99SignozToolchainReceipt $toolchainCheck @("check_pass_no_write"))) { + Stop-Agent99SignozRun "toolchain_blocked_with_safe_next_action" "toolchain_remote_check_failed" $toolchainCheck $null $null $null $false 1 + } + $toolchainState = [string]$toolchainCheck.receipt.target_state + if ($toolchainState -notin @("absent", "exact")) { + Stop-Agent99SignozRun "toolchain_blocked_with_safe_next_action" "toolchain_remote_state_invalid" $toolchainCheck $null $null $null $false 1 + } + if ($Mode -eq "ToolchainCheck") { + Stop-Agent99SignozRun "toolchain_check_ready" "" $toolchainCheck $null $null $null $false 0 + } + if ($Mode -eq "ToolchainVerify") { + $toolchainVerify = Invoke-Agent99SignozToolchainTarget "verify" "" $ToolchainCheckControllerWaitSeconds + if ( + -not (Test-Agent99SignozToolchainReceipt $toolchainVerify @("verify_pass")) -or + $toolchainVerify.receipt.durable_deploy_receipt_valid -ne $true + ) { + Stop-Agent99SignozRun "toolchain_verification_failed" "toolchain_independent_verify_failed" $toolchainCheck $null $null $toolchainVerify $false 1 + } + Stop-Agent99SignozRun "toolchain_verify_pass" "" $toolchainCheck $null $null $toolchainVerify $false 0 + } + + $decodedEnvelope = $null + $toolchainApplyReceipt = $null + $toolchainVerifyReceipt = $null + $toolchainError = "" + $stageDir = "/tmp/awoooi-signoz-metadata-toolchain.$RunId" + $toolchainApplyDispatched = $false + try { + $decodedEnvelope = Expand-Agent99SignozToolchainEnvelope $ToolchainEnvelopeJson + if ($toolchainState -eq "absent") { + $prepare = Invoke-Agent99SignozToolchainTarget "prepare" $stageDir $ToolchainCheckControllerWaitSeconds + $toolchainApplyReceipt = $prepare.receipt + Update-Agent99SignozToolchainMutationState $prepare.receipt + if (-not (Test-Agent99SignozToolchainReceipt $prepare @("stage_prepared", "stage_not_required_exact"))) { + throw "toolchain_stage_prepare_failed" + } + if ([string]$prepare.receipt.terminal -eq "stage_prepared") { + foreach ($row in @($decodedEnvelope.files)) { + $script:ToolchainSourceWriteAttempted = $true + if (-not (Invoke-Agent99SignozToolchainScp ([string]$row.path) "$stageDir/$([string]$row.name)")) { + throw "toolchain_transport_failed" + } + $script:ToolchainSourceWritePerformed = $true + } + } + } + $toolchainApplyDispatched = $true + $toolchainApply = Invoke-Agent99SignozToolchainTarget "apply" $stageDir $ToolchainApplyControllerWaitSeconds + $toolchainApplyReceipt = $toolchainApply.receipt + Update-Agent99SignozToolchainMutationState $toolchainApply.receipt + if (-not (Test-Agent99SignozToolchainReceipt $toolchainApply @( + "apply_pass_deployed_inactive", + "apply_pass_idempotent_exact" + ))) { + throw "toolchain_target_apply_failed" + } + $toolchainVerify = Invoke-Agent99SignozToolchainTarget "verify" "" $ToolchainCheckControllerWaitSeconds + $toolchainVerifyReceipt = $toolchainVerify.receipt + if (-not (Test-Agent99SignozToolchainReceipt $toolchainVerify @("verify_pass"))) { + throw "toolchain_post_apply_verify_failed" + } + if ($toolchainVerifyReceipt.durable_deploy_receipt_valid -ne $true) { + throw "toolchain_durable_deploy_receipt_unverified" + } + } catch { + $toolchainError = [string]$_.Exception.Message + if ($toolchainApplyDispatched) { + $uncertainVerify = Invoke-Agent99SignozToolchainTarget "verify" "" $ToolchainCheckControllerWaitSeconds + if ( + (Test-Agent99SignozToolchainReceipt $uncertainVerify @("verify_pass")) -and + $uncertainVerify.receipt.durable_deploy_receipt_valid -eq $true + ) { + $toolchainVerifyReceipt = $uncertainVerify.receipt + $script:ToolchainSourceWritePerformed = $true + $toolchainError = "" + } else { + $toolchainVerifyReceipt = $uncertainVerify.receipt + $toolchainError = "${toolchainError}_toolchain_reconcile_required" + } + } elseif ($script:ToolchainSourceWriteAttempted) { + $toolchainError = "${toolchainError}_toolchain_reconcile_required" + } + } finally { + if ($decodedEnvelope -and $decodedEnvelope.root) { + Remove-Item -LiteralPath ([string]$decodedEnvelope.root) -Recurse -Force -ErrorAction SilentlyContinue + if (Test-Path -LiteralPath ([string]$decodedEnvelope.root)) { + $script:TransportCleanupSucceeded = $false + if (-not $toolchainError) { $toolchainError = "toolchain_local_payload_cleanup_failed" } + } + } + } + if ($toolchainError) { + Stop-Agent99SignozRun "toolchain_apply_failed_or_unverified" $toolchainError $toolchainCheck $null $toolchainApplyReceipt $toolchainVerifyReceipt $script:ToolchainSourceWritePerformed 1 + } + Stop-Agent99SignozRun "toolchain_deployed_inactive_verified" "" $toolchainCheck $null $toolchainApplyReceipt $toolchainVerifyReceipt $script:ToolchainSourceWritePerformed 0 +} + if ($Mode -eq "RestoreVerifyCleanup") { $preflight = Get-Agent99SignozCleanupPreflight if (-not $preflight.transportReadbackVerified -or -not $preflight.ready) { @@ -955,7 +1452,8 @@ if ($Mode -eq "RestoreVerifyCleanup") { } $restoreProgram = ( "/usr/bin/python3 $IsolatedRestorePath --verify-cleanup " + - "--policy $PolicyPath --cluster-config $IsolatedClusterConfigPath " + + "--bundle-dir $OutputDir --policy $PolicyPath " + + "--cluster-config $IsolatedClusterConfigPath " + "--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId" ) if ($restoreReceiptStateBefore -eq "file") { @@ -1048,10 +1546,13 @@ if ($Mode -in @("RestoreCheck", "RestoreApply")) { "failed_cleanup_reconciliation_required" ) if (-not $transportAccepted) { - Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_controller_transport_failed_or_timed_out" $preflight $null $restoreReceipt $null $false 1 + $transportTerminal = if ($Mode -eq "RestoreCheck") { "restore_check_failed_no_write" } else { "restore_failed_cleanup_reconciliation_required" } + Stop-Agent99SignozRun $transportTerminal "restore_controller_transport_failed_or_timed_out" $preflight $null $restoreReceipt $null $false 1 } if ($failureReceiptAccepted) { - $failureTerminal = if ([string]$restoreReceipt.terminal -eq "failed_zero_residue_verified_no_completion") { + $failureTerminal = if ($Mode -eq "RestoreCheck") { + "restore_check_failed_no_write" + } elseif ([string]$restoreReceipt.terminal -eq "failed_zero_residue_verified_no_completion") { "restore_failed_zero_residue_verified_no_completion" } else { "restore_failed_cleanup_reconciliation_required" @@ -1064,7 +1565,8 @@ if ($Mode -in @("RestoreCheck", "RestoreApply")) { "RestoreApply" { "portable_metadata_restore_verified_zero_residue_pending_independent_verifier" } } if (-not (Test-Agent99SignozReceiptIdentity $restoreReceipt "terminal" @($allowedRestoreTerminal))) { - Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_controller_terminal_invalid" $preflight $null $restoreReceipt $null $false 1 + $invalidTerminal = if ($Mode -eq "RestoreCheck") { "restore_check_failed_no_write" } else { "restore_failed_cleanup_reconciliation_required" } + Stop-Agent99SignozRun $invalidTerminal "restore_controller_terminal_invalid" $preflight $null $restoreReceipt $null $false 1 } $restoreReceiptStateAfter = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file" @@ -1072,11 +1574,12 @@ if ($Mode -in @("RestoreCheck", "RestoreApply")) { Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_durable_receipt_missing" $preflight $null $restoreReceipt $null $false 1 } if ($Mode -eq "RestoreCheck" -and $restoreReceiptStateAfter -ne "absent") { - Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_check_wrote_receipt" $preflight $restoreReceipt $null $null $false 1 + Stop-Agent99SignozRun "restore_check_failed_unexpected_write_requires_reconcile" "restore_check_wrote_receipt" $preflight $restoreReceipt $null $null $false 1 } $postflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false if (-not $postflight.transportReadbackVerified -or -not $postflight.ready) { - Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_postflight_runtime_or_artifact_drift" $postflight $null $restoreReceipt $null $false 1 + $postflightTerminal = if ($Mode -eq "RestoreCheck") { "restore_check_failed_no_write" } else { "restore_failed_cleanup_reconciliation_required" } + Stop-Agent99SignozRun $postflightTerminal "restore_postflight_runtime_or_artifact_drift" $postflight $null $restoreReceipt $null $false 1 } switch ($Mode) { "RestoreCheck" { diff --git a/agent99-signoz-toolchain-target.py b/agent99-signoz-toolchain-target.py new file mode 100644 index 000000000..c4cfddebc --- /dev/null +++ b/agent99-signoz-toolchain-target.py @@ -0,0 +1,916 @@ +#!/usr/bin/env python3 +"""Fixed Host110 target for Agent99 SigNoz metadata toolchain deployment.""" + +from __future__ import annotations + +import argparse +import base64 +import fcntl +import hashlib +import json +import os +import re +import shutil +import signal +import stat +import subprocess # nosec B404 - fixed executable and argv vectors only +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +LABELS = ( + "metadata-export-policy.json", + "signoz_metadata_contract.py", + "signoz-metadata-export.py", + "verify-signoz-metadata-export.py", + "signoz-metadata-restore-drill.py", + "signoz-metadata-isolated-restore.py", + "isolated-cluster.xml", +) +MODES = (0o644, 0o644, 0o755, 0o755, 0o755, 0o755, 0o644) +ROOT_UID = 0 +REMOTE_ROOT = Path("/backup/toolchains/signoz-metadata") +RECEIPT_ROOT = Path("/backup/deploy-receipts/signoz-metadata-toolchain") +OPERATION_LOCK = Path("/tmp/awoooi-signoz-backup-operation.lock") +SIGNOZ_IMAGE_REF = "signoz/signoz:v0.113.0" +SIGNOZ_IMAGE_ID = ( + "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b" +) +ACTIVE_PATTERN = ( + r"([s]ignoz-metadata-export[.]py|[s]ignoz-metadata-restore-drill[.]py|" + r"[s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])" + r"([b]ackup-signoz|[c]lickhouse-native-backup|" + r"[c]lickhouse-native-restore-drill|[r]un-signoz-backup-canary)" + r"[.]sh([[:space:]]|$))" +) +MUTATION_STATE = { + "source_write_attempted": False, + "source_write_performed": False, + "quarantine_performed": False, + "stage_cleanup_performed": False, +} + + +class TargetError(RuntimeError): + pass + + +def canonical_bytes(value: Any) -> bytes: + return ( + json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + + "\n" + ).encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--mode", + choices=("check", "prepare", "apply", "verify", "reconcile"), + required=True, + ) + parser.add_argument("--trace-id", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--work-item-id", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--bundle-id", required=True) + parser.add_argument("--manifest-b64", required=True) + parser.add_argument("--stage-dir") + parser.add_argument("--attempt-id", default="") + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> list[dict[str, Any]]: + for value in (args.trace_id, args.run_id, args.work_item_id): + if not SAFE_ID.fullmatch(value): + raise TargetError("identity_invalid") + if args.mode == "reconcile": + if not SAFE_ID.fullmatch(args.attempt_id): + raise TargetError("reconcile_attempt_id_invalid") + elif args.attempt_id: + raise TargetError("attempt_id_not_allowed") + if args.work_item_id != "P0-OBS-002": + raise TargetError("work_item_not_allowlisted") + if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision): + raise TargetError("source_revision_invalid") + if not SHA256.fullmatch(args.bundle_id): + raise TargetError("bundle_id_invalid") + try: + raw = base64.b64decode(args.manifest_b64, validate=True) + manifest = json.loads(raw) + except (ValueError, json.JSONDecodeError) as exc: + raise TargetError("manifest_invalid") from exc + if not isinstance(manifest, list) or len(manifest) != len(LABELS): + raise TargetError("manifest_shape_invalid") + normalized: list[dict[str, Any]] = [] + for index, row in enumerate(manifest): + if not isinstance(row, dict): + raise TargetError("manifest_row_invalid") + expected = { + "name": LABELS[index], + "sha256": row.get("sha256"), + "mode": f"{MODES[index]:04o}", + } + if ( + row.get("name") != expected["name"] + or not isinstance(expected["sha256"], str) + or not SHA256.fullmatch(expected["sha256"]) + or row.get("mode") != expected["mode"] + or set(row) != {"name", "sha256", "mode"} + ): + raise TargetError("manifest_row_contract_invalid") + normalized.append(expected) + bundle_id = sha256_bytes( + "".join(f"{row['sha256']}\n" for row in normalized).encode("ascii") + ) + if bundle_id != args.bundle_id: + raise TargetError("bundle_id_manifest_mismatch") + expected_stage = Path(f"/tmp/awoooi-signoz-metadata-toolchain.{args.run_id}") + if args.mode in {"prepare", "apply", "reconcile"}: + if args.stage_dir != str(expected_stage): + raise TargetError("stage_path_not_exact") + elif args.stage_dir: + raise TargetError("stage_path_not_allowed") + return normalized + + +def run(argv: list[str], timeout: int = 20) -> str: + try: + result = subprocess.run( # nosec B603 + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=timeout, + check=False, + env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LANG": "C.UTF-8"}, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise TargetError("bounded_command_failed") from exc + if result.returncode != 0: + raise TargetError("bounded_command_failed") + return result.stdout.strip() + + +def production_snapshot() -> dict[str, Any]: + output = run(["/usr/bin/docker", "inspect", "signoz"]) + try: + rows = json.loads(output) + except json.JSONDecodeError as exc: + raise TargetError("signoz_runtime_readback_invalid") from exc + if not isinstance(rows, list) or len(rows) != 1 or not isinstance(rows[0], dict): + raise TargetError("signoz_runtime_readback_invalid") + row = rows[0] + config = row.get("Config") + state = row.get("State") + value = { + "id": row.get("Id"), + "image_id": row.get("Image"), + "image_ref": config.get("Image") if isinstance(config, dict) else None, + "running": state.get("Running") if isinstance(state, dict) else None, + "started_at": state.get("StartedAt") if isinstance(state, dict) else None, + "restart_count": row.get("RestartCount"), + } + if ( + not isinstance(value["id"], str) + or len(value["id"]) != 64 + or value["image_id"] != SIGNOZ_IMAGE_ID + or value["image_ref"] != SIGNOZ_IMAGE_REF + or value["running"] is not True + or not isinstance(value["started_at"], str) + or not isinstance(value["restart_count"], int) + ): + raise TargetError("signoz_runtime_identity_drift") + return value + + +def require_health() -> None: + request = urllib.request.Request("http://127.0.0.1:8080/api/v1/health") + try: + with urllib.request.urlopen(request, timeout=10) as response: # nosec B310 + if response.status != 200: + raise TargetError("signoz_health_not_200") + response.read(4097) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise TargetError("signoz_health_unavailable") from exc + + +def require_no_active_operations() -> None: + try: + result = subprocess.run( # nosec B603 + ["/usr/bin/pgrep", "-af", ACTIVE_PATTERN], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=10, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise TargetError("active_operation_readback_failed") from exc + if result.returncode != 1 or result.stdout.strip(): + raise TargetError("active_operation_present_or_unverified") + + +def require_directory( + path: Path, + *, + owner: int | None = ROOT_UID, + mode: int | tuple[int, ...] | None = None, +) -> None: + try: + metadata = path.lstat() + except OSError as exc: + raise TargetError("directory_unavailable") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise TargetError("directory_not_safe") + if owner is not None and metadata.st_uid != owner: + raise TargetError("directory_owner_invalid") + if mode is not None: + allowed_modes = (mode,) if isinstance(mode, int) else mode + if stat.S_IMODE(metadata.st_mode) not in allowed_modes: + raise TargetError("directory_mode_invalid") + + +def ensure_private_root(path: Path) -> None: + if os.path.lexists(path): + require_directory(path, owner=ROOT_UID, mode=0o700) + return + parent = path.parent + require_directory(parent, owner=ROOT_UID) + path.mkdir(mode=0o700) + + +def target_state(bundle_id: str, manifest: list[dict[str, Any]]) -> str: + if os.path.lexists(REMOTE_ROOT / "current"): + raise TargetError("active_pointer_must_remain_absent") + target = REMOTE_ROOT / bundle_id + if not os.path.lexists(target): + return "absent" + require_directory(target, owner=ROOT_UID, mode=0o750) + if {item.name for item in target.iterdir()} != set(LABELS): + raise TargetError("target_tree_drift") + for row in manifest: + path = target / row["name"] + metadata = path.lstat() + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != ROOT_UID + or stat.S_IMODE(metadata.st_mode) != int(row["mode"], 8) + or sha256_file(path) != row["sha256"] + ): + raise TargetError("target_file_drift") + return "exact" + + +def base_receipt( + args: argparse.Namespace, terminal: str, **extra: Any +) -> dict[str, Any]: + value = { + "schema": "awoooi_agent99_signoz_toolchain_target_v1", + "trace_id": args.trace_id, + "run_id": args.run_id, + "work_item_id": args.work_item_id, + "source_revision": args.source_revision, + "mode": args.mode, + "bundle_id": args.bundle_id, + "terminal": terminal, + "production_service_mutation_performed": False, + "active_pointer_created": False, + "credential_value_read": False, + "completion_claim": False, + **MUTATION_STATE, + } + if args.attempt_id: + value["attempt_id"] = args.attempt_id + value.update(extra) + return value + + +def reset_mutation_state() -> None: + for key in MUTATION_STATE: + MUTATION_STATE[key] = False + + +def write_private_new(path: Path, value: bytes, mode: int) -> None: + descriptor = os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), + mode, + ) + try: + with os.fdopen(descriptor, "wb", closefd=False) as stream: + stream.write(value) + stream.flush() + os.fsync(stream.fileno()) + finally: + os.close(descriptor) + os.chmod(path, mode) + + +def validate_stage(stage: Path, manifest: list[dict[str, Any]]) -> None: + sudo_uid = os.environ.get("SUDO_UID", "") + if not sudo_uid.isdigit() or int(sudo_uid) <= 0: + raise TargetError("sudo_transport_identity_unavailable") + require_directory(stage, owner=int(sudo_uid), mode=0o700) + if {item.name for item in stage.iterdir()} != set(LABELS): + raise TargetError("stage_tree_invalid") + for row in manifest: + path = stage / row["name"] + metadata = path.lstat() + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_size <= 0 + or metadata.st_size > 2 * 1024 * 1024 + or sha256_file(path) != row["sha256"] + ): + raise TargetError("stage_file_invalid") + + +def validate_candidate(candidate: Path) -> None: + for name in LABELS: + if name.endswith(".py"): + source = (candidate / name).read_text(encoding="utf-8") + compile(source, str(candidate / name), "exec") + policy = json.loads((candidate / "metadata-export-policy.json").read_text("utf-8")) + if ( + policy.get("work_item_id") != "P0-OBS-002" + or policy.get("source_runtime", {}).get("raw_database_access_allowed") + is not False + or policy.get("source_runtime", {}).get("raw_volume_access_allowed") + is not False + or policy.get("completion_contract", {}).get( + "full_sqlite_completion_claim_allowed" + ) + is not False + ): + raise TargetError("candidate_policy_contract_invalid") + + +def acquire_operation_lock() -> Any: + metadata = OPERATION_LOCK.lstat() + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISREG(metadata.st_mode): + raise TargetError("operation_lock_unsafe") + stream = OPERATION_LOCK.open("rb") + try: + fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + stream.close() + raise TargetError("operation_lock_busy") from exc + return stream + + +def prepare_stage( + args: argparse.Namespace, manifest: list[dict[str, Any]] +) -> dict[str, Any]: + lock = acquire_operation_lock() + try: + before = production_snapshot() + require_health() + require_no_active_operations() + state = target_state(args.bundle_id, manifest) + if state == "exact": + return base_receipt(args, "stage_not_required_exact", target_state=state) + stage = Path(args.stage_dir) + if os.path.lexists(stage): + raise TargetError("stage_identity_collision") + sudo_uid = os.environ.get("SUDO_UID", "") + sudo_gid = os.environ.get("SUDO_GID", "") + if ( + not sudo_uid.isdigit() + or not sudo_gid.isdigit() + or int(sudo_uid) <= 0 + or int(sudo_gid) <= 0 + ): + raise TargetError("sudo_transport_identity_unavailable") + MUTATION_STATE["source_write_attempted"] = True + stage.mkdir(mode=0o700) + os.chown(stage, int(sudo_uid), int(sudo_gid)) + MUTATION_STATE["source_write_performed"] = True + try: + if production_snapshot() != before: + raise TargetError("production_identity_changed_during_prepare") + require_health() + except Exception: + shutil.rmtree(stage) + MUTATION_STATE["stage_cleanup_performed"] = True + raise + return base_receipt(args, "stage_prepared", target_state=state) + finally: + lock.close() + + +def read_private_json(path: Path, error_prefix: str) -> dict[str, Any]: + metadata = path.lstat() + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != ROOT_UID + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_size <= 0 + or metadata.st_size > 65536 + ): + raise TargetError(f"{error_prefix}_metadata_invalid") + try: + raw = path.read_bytes() + value = json.loads(raw) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise TargetError(f"{error_prefix}_invalid") from exc + if not isinstance(value, dict) or raw != canonical_bytes(value): + raise TargetError(f"{error_prefix}_not_canonical") + return value + + +def apply_intent( + args: argparse.Namespace, state: str, continuity: str +) -> dict[str, Any]: + return { + "schema": "awoooi_agent99_signoz_toolchain_apply_intent_v1", + "trace_id": args.trace_id, + "run_id": args.run_id, + "work_item_id": args.work_item_id, + "source_revision": args.source_revision, + "bundle_id": args.bundle_id, + "target_state_before": state, + "production_continuity_sha256": continuity, + } + + +def load_apply_intent(args: argparse.Namespace) -> dict[str, Any] | None: + path = RECEIPT_ROOT / args.run_id / "apply-intent.json" + if not os.path.lexists(path): + return None + value = read_private_json(path, "apply_intent") + expected = { + "schema": "awoooi_agent99_signoz_toolchain_apply_intent_v1", + "trace_id": args.trace_id, + "run_id": args.run_id, + "work_item_id": args.work_item_id, + "source_revision": args.source_revision, + "bundle_id": args.bundle_id, + } + if ( + set(value) + != set(expected) | {"target_state_before", "production_continuity_sha256"} + or any( + value.get(key) != expected_value for key, expected_value in expected.items() + ) + or value.get("target_state_before") not in {"absent", "exact"} + or not isinstance(value.get("production_continuity_sha256"), str) + or not SHA256.fullmatch(value["production_continuity_sha256"]) + ): + raise TargetError("apply_intent_identity_invalid") + return value + + +def deployment_receipt( + args: argparse.Namespace, + *, + continuity: str, + method: str, + target_mutated: bool, +) -> dict[str, Any]: + return { + "schema": "awoooi_agent99_signoz_toolchain_deployment_v1", + "trace_id": args.trace_id, + "run_id": args.run_id, + "work_item_id": args.work_item_id, + "source_revision": args.source_revision, + "bundle_id": args.bundle_id, + "target_state": "exact", + "deployment_method": method, + "finalized_by_mode": args.mode, + "reconcile_attempt_id": args.attempt_id if args.mode == "reconcile" else "", + "production_continuity_sha256": continuity, + "production_service_mutation_performed": False, + "content_addressed_target_mutation_performed": target_mutated, + "active_pointer_created": False, + "credential_value_read": False, + "stage_cleanup_verified": True, + "candidate_cleanup_verified": True, + "completion_claim": False, + } + + +def write_deployment_receipt( + args: argparse.Namespace, + *, + continuity: str, + method: str, + target_mutated: bool, +) -> dict[str, Any]: + value = deployment_receipt( + args, + continuity=continuity, + method=method, + target_mutated=target_mutated, + ) + path = RECEIPT_ROOT / args.run_id / "deploy-receipt.json" + write_private_new(path, canonical_bytes(value), 0o600) + return value + + +def load_deploy_receipt(args: argparse.Namespace) -> dict[str, Any] | None: + path = RECEIPT_ROOT / args.run_id / "deploy-receipt.json" + if not os.path.lexists(path): + return None + value = read_private_json(path, "deploy_receipt") + expected = { + "schema": "awoooi_agent99_signoz_toolchain_deployment_v1", + "trace_id": args.trace_id, + "run_id": args.run_id, + "work_item_id": args.work_item_id, + "source_revision": args.source_revision, + "bundle_id": args.bundle_id, + "target_state": "exact", + "production_service_mutation_performed": False, + "active_pointer_created": False, + "credential_value_read": False, + "stage_cleanup_verified": True, + "candidate_cleanup_verified": True, + "completion_claim": False, + } + additional = { + "deployment_method", + "finalized_by_mode", + "reconcile_attempt_id", + "production_continuity_sha256", + "content_addressed_target_mutation_performed", + } + if ( + set(value) != set(expected) | additional + or any( + value.get(key) != expected_value for key, expected_value in expected.items() + ) + or value.get("deployment_method") + not in {"created", "adopted_existing_exact", "reconciled_exact"} + or value.get("finalized_by_mode") not in {"apply", "reconcile"} + or not isinstance( + value.get("content_addressed_target_mutation_performed"), bool + ) + or not isinstance(value.get("production_continuity_sha256"), str) + or not SHA256.fullmatch(value["production_continuity_sha256"]) + ): + raise TargetError("deploy_receipt_identity_invalid") + attempt_id = value.get("reconcile_attempt_id") + if value["finalized_by_mode"] == "apply": + if attempt_id != "": + raise TargetError("deploy_receipt_attempt_identity_invalid") + elif not isinstance(attempt_id, str) or not SAFE_ID.fullmatch(attempt_id): + raise TargetError("deploy_receipt_attempt_identity_invalid") + return value + + +def quarantine_owned_directory( + source: Path, + destination: Path, + *, + owner: int, + modes: tuple[int, ...] = (0o700,), +) -> None: + if not os.path.lexists(source): + if os.path.lexists(destination): + require_directory(destination, owner=owner, mode=modes) + return + require_directory(source, owner=owner, mode=modes) + if os.path.lexists(destination): + raise TargetError("quarantine_identity_collision") + MUTATION_STATE["source_write_attempted"] = True + os.rename(source, destination) + MUTATION_STATE["source_write_performed"] = True + MUTATION_STATE["quarantine_performed"] = True + + +def apply_bundle( + args: argparse.Namespace, manifest: list[dict[str, Any]] +) -> dict[str, Any]: + stage = Path(args.stage_dir) + target = REMOTE_ROOT / args.bundle_id + candidate = REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}" + receipt_dir = RECEIPT_ROOT / args.run_id + lock = acquire_operation_lock() + try: + before = production_snapshot() + continuity = sha256_bytes(canonical_bytes(before)) + require_health() + require_no_active_operations() + state = target_state(args.bundle_id, manifest) + ensure_private_root(REMOTE_ROOT) + ensure_private_root(RECEIPT_ROOT) + if os.path.lexists(receipt_dir) or os.path.lexists(candidate): + raise TargetError("deployment_identity_collision") + MUTATION_STATE["source_write_attempted"] = True + receipt_dir.mkdir(mode=0o700) + MUTATION_STATE["source_write_performed"] = True + write_private_new( + receipt_dir / "apply-intent.json", + canonical_bytes(apply_intent(args, state, continuity)), + 0o600, + ) + try: + if state == "exact": + if os.path.lexists(stage): + raise TargetError("idempotent_apply_stage_must_be_absent") + after = production_snapshot() + require_health() + if after != before: + raise TargetError("production_identity_changed_during_apply") + durable = write_deployment_receipt( + args, + continuity=continuity, + method="adopted_existing_exact", + target_mutated=False, + ) + return base_receipt( + args, + "apply_pass_idempotent_exact", + target_state="exact", + durable_deploy_receipt_valid=True, + durable_deploy_receipt_sha256=sha256_bytes( + canonical_bytes(durable) + ), + production_continuity_sha256=continuity, + ) + + candidate.mkdir(mode=0o700) + validate_stage(stage, manifest) + for row in manifest: + source = stage / row["name"] + destination = candidate / row["name"] + write_private_new(destination, source.read_bytes(), int(row["mode"], 8)) + validate_candidate(candidate) + # Publish the final directory mode atomically with the rename so a + # signal cannot expose an otherwise exact shared target as drift. + os.chmod(candidate, 0o750) + os.rename(candidate, target) + if target_state(args.bundle_id, manifest) != "exact": + raise TargetError("post_apply_target_not_exact") + after = production_snapshot() + require_health() + if after != before: + raise TargetError("production_identity_changed_during_apply") + manifest_rows = "".join( + f"{row['name']}\t{row['sha256']}\t{row['mode']}\n" for row in manifest + ).encode("ascii") + write_private_new( + receipt_dir / "deployed-manifest.tsv", manifest_rows, 0o600 + ) + shutil.rmtree(stage) + if os.path.lexists(stage) or os.path.lexists(candidate): + raise TargetError("owned_live_residue_after_apply") + MUTATION_STATE["stage_cleanup_performed"] = True + durable = write_deployment_receipt( + args, + continuity=continuity, + method="created", + target_mutated=True, + ) + return base_receipt( + args, + "apply_pass_deployed_inactive", + target_state="exact", + durable_deploy_receipt_valid=True, + durable_deploy_receipt_sha256=sha256_bytes(canonical_bytes(durable)), + production_continuity_sha256=continuity, + ) + except Exception: + sudo_uid = os.environ.get("SUDO_UID", "") + if receipt_dir.is_dir() and sudo_uid.isdigit() and int(sudo_uid) > 0: + quarantine_owned_directory( + stage, + receipt_dir / "transport-stage", + owner=int(sudo_uid), + ) + quarantine_owned_directory( + candidate, + receipt_dir / "quarantine-candidate", + owner=ROOT_UID, + modes=(0o700, 0o750), + ) + # A content-addressed target is shared once visible. Never move or delete it. + raise + finally: + lock.close() + + +def reconcile_apply( + args: argparse.Namespace, manifest: list[dict[str, Any]] +) -> dict[str, Any]: + stage = Path(args.stage_dir) + target = REMOTE_ROOT / args.bundle_id + candidate = REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}" + receipt_dir = RECEIPT_ROOT / args.run_id + lock = acquire_operation_lock() + try: + before = production_snapshot() + continuity = sha256_bytes(canonical_bytes(before)) + require_health() + require_no_active_operations() + ensure_private_root(RECEIPT_ROOT) + if not os.path.lexists(receipt_dir): + MUTATION_STATE["source_write_attempted"] = True + receipt_dir.mkdir(mode=0o700) + MUTATION_STATE["source_write_performed"] = True + else: + require_directory(receipt_dir, owner=ROOT_UID, mode=0o700) + + state_error = "" + try: + state = target_state(args.bundle_id, manifest) + except TargetError as exc: + state = "drift_or_unverified" + state_error = str(exc) + + intent_error = "" + try: + intent = load_apply_intent(args) + except TargetError as exc: + intent = None + intent_error = str(exc) + + deploy_error = "" + try: + durable = load_deploy_receipt(args) + except TargetError as exc: + durable = None + deploy_error = str(exc) + + sudo_uid = os.environ.get("SUDO_UID", "") + if not sudo_uid.isdigit() or int(sudo_uid) <= 0: + raise TargetError("sudo_transport_identity_unavailable") + quarantine_owned_directory( + stage, + receipt_dir / "transport-stage", + owner=int(sudo_uid), + ) + quarantine_owned_directory( + candidate, + receipt_dir / "quarantine-candidate", + owner=ROOT_UID, + modes=(0o700, 0o750), + ) + if os.path.lexists(stage) or os.path.lexists(candidate): + raise TargetError("reconcile_owned_live_residue_remaining") + + if deploy_error: + invalid = receipt_dir / "invalid-deploy-receipt.json" + current = receipt_dir / "deploy-receipt.json" + if os.path.lexists(current): + if os.path.lexists(invalid): + raise TargetError("invalid_deploy_receipt_quarantine_collision") + os.rename(current, invalid) + MUTATION_STATE["source_write_attempted"] = True + MUTATION_STATE["source_write_performed"] = True + MUTATION_STATE["quarantine_performed"] = True + + if production_snapshot() != before: + raise TargetError("production_identity_changed_during_reconcile") + require_health() + if intent is not None and intent["production_continuity_sha256"] != continuity: + raise TargetError("apply_intent_production_continuity_mismatch") + + durable_sha = "" + if state == "exact" and durable is not None: + if durable["production_continuity_sha256"] != continuity: + raise TargetError("deploy_receipt_production_continuity_mismatch") + terminal = "reconcile_exact_deployment_verified" + durable_sha = sha256_bytes(canonical_bytes(durable)) + durable_valid = True + elif state == "exact" and intent is not None and not intent_error: + durable = write_deployment_receipt( + args, + continuity=continuity, + method="reconciled_exact", + target_mutated=intent["target_state_before"] == "absent", + ) + terminal = "reconcile_exact_deployment_verified" + durable_sha = sha256_bytes(canonical_bytes(durable)) + durable_valid = True + elif state == "exact": + terminal = "reconcile_exact_target_preserved_unverified" + durable_valid = False + elif state == "absent": + terminal = "reconcile_zero_owned_residue" + durable_valid = False + else: + terminal = "reconcile_shared_target_drift_preserved" + durable_valid = False + + result = base_receipt( + args, + terminal, + target_state=state, + target_state_error=state_error, + apply_intent_error=intent_error, + deploy_receipt_error=deploy_error, + durable_deploy_receipt_valid=durable_valid, + durable_deploy_receipt_sha256=durable_sha, + production_continuity_sha256=continuity, + shared_target_preserved=os.path.lexists(target), + owned_live_residue_verified_absent=True, + ) + reconcile_path = receipt_dir / f"reconcile-{args.attempt_id}.json" + write_private_new(reconcile_path, canonical_bytes(result), 0o600) + return result + finally: + lock.close() + + +def main() -> int: + args = parse_args() + reset_mutation_state() + old_handlers: dict[int, Any] = {} + + def abort_on_signal(signum: int, _frame: Any) -> None: + for handled in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): + signal.signal(handled, signal.SIG_IGN) + raise TargetError(f"signal_{signum}_reconcile_required") + + try: + manifest = validate_args(args) + if args.mode in {"prepare", "apply", "reconcile"}: + for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): + old_handlers[signum] = signal.getsignal(signum) + signal.signal(signum, abort_on_signal) + if args.mode == "prepare": + result = prepare_stage(args, manifest) + elif args.mode == "apply": + result = apply_bundle(args, manifest) + elif args.mode == "reconcile": + result = reconcile_apply(args, manifest) + else: + before = production_snapshot() + require_health() + require_no_active_operations() + state = target_state(args.bundle_id, manifest) + deploy_receipt = None + if args.mode == "verify": + if state != "exact": + raise TargetError("verify_target_not_exact") + expected_stage = Path( + f"/tmp/awoooi-signoz-metadata-toolchain.{args.run_id}" + ) + expected_candidate = REMOTE_ROOT / ( + f".{args.bundle_id}.candidate.{args.run_id}" + ) + if os.path.lexists(expected_stage) or os.path.lexists( + expected_candidate + ): + raise TargetError("verify_owned_residue_present") + deploy_receipt = load_deploy_receipt(args) + if deploy_receipt is not None and deploy_receipt[ + "production_continuity_sha256" + ] != sha256_bytes(canonical_bytes(before)): + raise TargetError("deploy_receipt_production_continuity_mismatch") + if production_snapshot() != before: + raise TargetError("production_identity_changed_during_readback") + require_health() + result = base_receipt( + args, + "verify_pass" if args.mode == "verify" else "check_pass_no_write", + target_state=state, + durable_deploy_receipt_valid=(deploy_receipt is not None), + production_continuity_sha256=sha256_bytes(canonical_bytes(before)), + ) + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 + except (TargetError, OSError, UnicodeError, json.JSONDecodeError) as exc: + failure = base_receipt( + args, + "failed", + error_code=str(exc), + ) + if args.mode == "apply": + failure_path = RECEIPT_ROOT / args.run_id / "failure-receipt.json" + try: + if failure_path.parent.is_dir() and not os.path.lexists(failure_path): + write_private_new(failure_path, canonical_bytes(failure), 0o600) + except OSError: + pass + print(json.dumps(failure, sort_keys=True, separators=(",", ":"))) + return 1 + finally: + for signum, handler in old_handlers.items(): + signal.signal(signum, handler) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/config/signoz/metadata-export-policy.json b/config/signoz/metadata-export-policy.json index 944bcf16c..cd245bfa8 100644 --- a/config/signoz/metadata-export-policy.json +++ b/config/signoz/metadata-export-policy.json @@ -315,6 +315,40 @@ "private_workspace_parent": "/tmp", "server_container_port": 8080, "loopback_publish_host": "127.0.0.1", + "resource_limits": { + "zookeeper": { + "cpus": "0.50", + "nano_cpus": 500000000, + "memory": "512m", + "memory_bytes": 536870912, + "memory_swap": "512m", + "memory_swap_bytes": 536870912, + "pids_limit": 128 + }, + "clickhouse": { + "cpus": "2.00", + "nano_cpus": 2000000000, + "memory": "2g", + "memory_bytes": 2147483648, + "memory_swap": "2g", + "memory_swap_bytes": 2147483648, + "pids_limit": 512 + }, + "signoz": { + "cpus": "1.00", + "nano_cpus": 1000000000, + "memory": "1g", + "memory_bytes": 1073741824, + "memory_swap": "1g", + "memory_swap_bytes": 1073741824, + "pids_limit": 256 + } + }, + "resource_log_config": { + "type": "local", + "max_size": "10m", + "max_file": "2" + }, "isolated_cluster_config": "isolated-cluster.xml", "production_continuity": { "signoz": { diff --git a/scripts/backup/signoz-metadata-isolated-restore.py b/scripts/backup/signoz-metadata-isolated-restore.py index f9ad99a55..c122e6ccf 100755 --- a/scripts/backup/signoz-metadata-isolated-restore.py +++ b/scripts/backup/signoz-metadata-isolated-restore.py @@ -43,7 +43,6 @@ from signoz_metadata_contract import ( write_new_private_atomic, ) - DOCKER = "/usr/bin/docker" PYTHON = "/usr/bin/python3" RESTORE_DRIVER = Path(__file__).with_name("signoz-metadata-restore-drill.py") @@ -80,6 +79,48 @@ EXPECTED_SERVER_ENV = { "SIGNOZ_SQLSTORE_SQLITE_PATH": "/var/lib/signoz/signoz.db", } EXPECTED_SERVER_MOUNT = "/var/lib/signoz" +RESOURCE_LIMITS = { + "zookeeper": { + "cpus": "0.50", + "nano_cpus": 500_000_000, + "memory": "512m", + "memory_bytes": 512 * 1024 * 1024, + "memory_swap": "512m", + "memory_swap_bytes": 512 * 1024 * 1024, + "pids_limit": 128, + }, + "clickhouse": { + "cpus": "2.00", + "nano_cpus": 2_000_000_000, + "memory": "2g", + "memory_bytes": 2 * 1024 * 1024 * 1024, + "memory_swap": "2g", + "memory_swap_bytes": 2 * 1024 * 1024 * 1024, + "pids_limit": 512, + }, + "signoz": { + "cpus": "1.00", + "nano_cpus": 1_000_000_000, + "memory": "1g", + "memory_bytes": 1024 * 1024 * 1024, + "memory_swap": "1g", + "memory_swap_bytes": 1024 * 1024 * 1024, + "pids_limit": 256, + }, +} +RESOURCE_LOG_CONFIG = { + "type": "local", + "max_size": "10m", + "max_file": "2", +} +PORTABLE_ASSET_IDS = ( + "dashboards", + "alert_rules", + "explorer_views_logs", + "explorer_views_meter", + "explorer_views_metrics", + "explorer_views_traces", +) PORT_RETRIES = 120 PORT_RETRY_INTERVAL_SECONDS = 1.0 CLICKHOUSE_RETRIES = 90 @@ -254,9 +295,7 @@ def validate_restore_policy(policy: dict[str, Any]) -> dict[str, Any]: raise ContractError("restore_signoz_image_id_not_exact") if isolation.get("resource_prefix") != "awoooi-signoz-md-restore-": raise ContractError("restore_resource_prefix_not_exact") - if isolation.get("canonical_id_prefix") != ( - "ephemeral-signoz-metadata-restore-" - ): + if isolation.get("canonical_id_prefix") != ("ephemeral-signoz-metadata-restore-"): raise ContractError("restore_canonical_prefix_not_exact") if isolation.get("resource_label_key") != PRIMARY_LABEL: raise ContractError("restore_primary_label_not_exact") @@ -268,6 +307,17 @@ def validate_restore_policy(policy: dict[str, Any]) -> dict[str, Any]: raise ContractError("restore_server_port_not_exact") if isolation.get("loopback_publish_host") != "127.0.0.1": raise ContractError("restore_publish_host_not_exact") + if isolation.get("resource_limits") != RESOURCE_LIMITS: + raise ContractError("restore_resource_limits_not_exact") + if isolation.get("resource_log_config") != RESOURCE_LOG_CONFIG: + raise ContractError("restore_resource_log_config_not_exact") + portable_ids = tuple( + item.get("id") + for item in policy.get("assets", []) + if isinstance(item, dict) and item.get("classification") == "portable" + ) + if portable_ids != PORTABLE_ASSET_IDS: + raise ContractError("restore_portable_asset_ids_not_exact") if any( isolation.get(key) is not expected for key, expected in ( @@ -303,9 +353,10 @@ def validate_restore_policy(policy: dict[str, Any]) -> dict[str, Any]: raise ContractError("restore_production_key_presence_not_exact") if startup.get("production_dsn_class") != "tcp://clickhouse:9000*": raise ContractError("restore_production_dsn_class_not_exact") - if startup.get("production_sqlite_path") != EXPECTED_SERVER_ENV[ - "SIGNOZ_SQLSTORE_SQLITE_PATH" - ]: + if ( + startup.get("production_sqlite_path") + != EXPECTED_SERVER_ENV["SIGNOZ_SQLSTORE_SQLITE_PATH"] + ): raise ContractError("restore_production_sqlite_path_not_exact") if startup.get("secret_value_output_allowed") is not False: raise ContractError("restore_secret_output_must_be_forbidden") @@ -360,6 +411,26 @@ def label_args(plan: ResourcePlan, run_id: str) -> list[str]: ] +def resource_limit_args(role: str) -> list[str]: + limits = RESOURCE_LIMITS[role] + return [ + "--cpus", + str(limits["cpus"]), + "--memory", + str(limits["memory"]), + "--memory-swap", + str(limits["memory_swap"]), + "--pids-limit", + str(limits["pids_limit"]), + "--log-driver", + str(RESOURCE_LOG_CONFIG["type"]), + "--log-opt", + f"max-size={RESOURCE_LOG_CONFIG['max_size']}", + "--log-opt", + f"max-file={RESOURCE_LOG_CONFIG['max_file']}", + ] + + def inspect_image_id(runner: Runner, ref: str) -> str: return run_required( runner, @@ -488,8 +559,7 @@ def inspect_resource_labels( if labels is None: return {} if not isinstance(labels, dict) or not all( - isinstance(key, str) and isinstance(value, str) - for key, value in labels.items() + isinstance(key, str) and isinstance(value, str) for key, value in labels.items() ): raise ContractError(f"docker_{kind}_labels_invalid") return labels @@ -510,7 +580,10 @@ def assert_absent_or_owned( ) if labels is None: return False - if labels.get(PRIMARY_LABEL) != run_id or labels.get(IDENTITY_LABEL) != plan.identity: + if ( + labels.get(PRIMARY_LABEL) != run_id + or labels.get(IDENTITY_LABEL) != plan.identity + ): raise ContractError(f"restore_{kind}_name_collision_not_owned") return True @@ -532,7 +605,7 @@ def require_no_preexisting_resources( raise ContractError("restore_private_workspace_residue_requires_reconcile") -def verify_bundle(args: argparse.Namespace, runner: Runner) -> None: +def verify_bundle(args: argparse.Namespace, runner: Runner) -> str: if not args.bundle_dir: raise ContractError("bundle_dir_required") bundle = Path(args.bundle_dir) @@ -560,6 +633,23 @@ def verify_bundle(args: argparse.Namespace, runner: Runner) -> None: raise ContractError("independent_bundle_verifier_receipt_invalid") from exc if value.get("terminal") != "pass_export_verified_restore_pending": raise ContractError("independent_bundle_verifier_terminal_invalid") + manifest_path = bundle / "manifest.json" + require_no_symlink_components(manifest_path, label="export_manifest") + try: + metadata = manifest_path.lstat() + raw_manifest = manifest_path.read_bytes() + except OSError as exc: + raise ContractError("export_manifest_unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.geteuid() + or stat.S_IMODE(metadata.st_mode) != 0o600 + or not raw_manifest + or len(raw_manifest) > 1024 * 1024 + ): + raise ContractError("export_manifest_owner_mode_or_size_invalid") + return sha256_bytes(raw_manifest) def create_workspace(plan: ResourcePlan) -> None: @@ -625,6 +715,7 @@ def create_resources( "--detach", "--pull=never", "--restart=no", + *resource_limit_args("zookeeper"), "--name", plan.zookeeper_container, "--network", @@ -654,6 +745,7 @@ def create_resources( "--detach", "--pull=never", "--restart=no", + *resource_limit_args("clickhouse"), "--name", plan.clickhouse_container, "--network", @@ -686,6 +778,7 @@ def create_resources( "--detach", "--pull=never", "--restart=no", + *resource_limit_args("signoz"), "--name", plan.server_container, "--network", @@ -728,22 +821,37 @@ def wait_for_clickhouse(runner: Runner, plan: ResourcePlan) -> None: def verify_runtime_container_images(runner: Runner, plan: ResourcePlan) -> None: expected = { - plan.server_container: EXPECTED_IMAGES["signoz"], - plan.clickhouse_container: EXPECTED_IMAGES["clickhouse"], - plan.zookeeper_container: EXPECTED_IMAGES["zookeeper"], + plan.server_container: ("signoz", EXPECTED_IMAGES["signoz"]), + plan.clickhouse_container: ("clickhouse", EXPECTED_IMAGES["clickhouse"]), + plan.zookeeper_container: ("zookeeper", EXPECTED_IMAGES["zookeeper"]), } - for name, (expected_ref, expected_id) in expected.items(): + for name, (role, (expected_ref, expected_id)) in expected.items(): data = inspect_json(runner, "container", name) if data is None: raise ContractError("restore_runtime_container_missing") config = data.get("Config", {}) host_config = data.get("HostConfig", {}) + limits = RESOURCE_LIMITS[role] if ( data.get("Image") != expected_id or config.get("Image") != expected_ref or host_config.get("RestartPolicy", {}).get("Name") not in {"", "no"} + or host_config.get("NanoCpus") != limits["nano_cpus"] + or host_config.get("Memory") != limits["memory_bytes"] + or host_config.get("MemorySwap") != limits["memory_swap_bytes"] + or host_config.get("PidsLimit") != limits["pids_limit"] + or host_config.get("LogConfig") + != { + "Type": RESOURCE_LOG_CONFIG["type"], + "Config": { + "max-file": RESOURCE_LOG_CONFIG["max_file"], + "max-size": RESOURCE_LOG_CONFIG["max_size"], + }, + } ): - raise ContractError("restore_runtime_container_image_or_restart_drift") + raise ContractError( + "restore_runtime_container_image_restart_or_resource_limit_drift" + ) def discover_loopback_port(runner: Runner, plan: ResourcePlan) -> int: @@ -919,6 +1027,7 @@ def invoke_restore_driver( token_file: Path, mode: str, ) -> dict[str, Any]: + child_receipt: Path | None = None argv = [ PYTHON, str(RESTORE_DRIVER), @@ -942,6 +1051,13 @@ def invoke_restore_driver( "--work-item-id", args.work_item_id, ] + if mode == "--apply": + child_receipt = isolation_receipt.parent / "restore-driver-apply-receipt.json" + validate_new_private_destination( + child_receipt, + label="restore_driver_apply_receipt", + ) + argv.extend(["--receipt-file", str(child_receipt)]) output = run_required( runner, argv, @@ -959,6 +1075,59 @@ def invoke_restore_driver( ) if value.get("terminal") != expected: raise ContractError("isolated_restore_driver_terminal_invalid") + for key, expected_value in ( + ("schema", "awoooi_signoz_metadata_receipt_v1"), + ("trace_id", args.trace_id), + ("run_id", args.run_id), + ("work_item_id", args.work_item_id), + ("phase", "terminal"), + ): + if value.get(key) != expected_value: + raise ContractError("isolated_restore_driver_identity_invalid") + if mode == "--apply": + if child_receipt is None: + raise ContractError("restore_driver_apply_receipt_path_missing") + try: + metadata = child_receipt.lstat() + raw_receipt = child_receipt.read_bytes() + except OSError as exc: + raise ContractError("restore_driver_apply_receipt_unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or metadata.st_uid != os.geteuid() + or stat.S_IMODE(metadata.st_mode) != 0o600 + or raw_receipt != canonical_json_bytes(value) + ): + raise ContractError("restore_driver_apply_receipt_invalid") + counts = value.get("restored_asset_counts") + asset_digests = value.get("restored_asset_semantic_sha256") + semantic_digest = value.get("portable_semantic_sha256") + if ( + not isinstance(counts, dict) + or set(counts) != set(PORTABLE_ASSET_IDS) + or not all( + isinstance(count, int) and not isinstance(count, bool) and count >= 0 + for count in counts.values() + ) + or not isinstance(asset_digests, dict) + or set(asset_digests) != set(PORTABLE_ASSET_IDS) + or not all( + isinstance(digest, str) + and len(digest) == 64 + and all(character in "0123456789abcdef" for character in digest) + for digest in asset_digests.values() + ) + or not isinstance(semantic_digest, str) + or len(semantic_digest) != 64 + or any(character not in "0123456789abcdef" for character in semantic_digest) + or value.get("completion_scope") != "portable_assets_only" + or value.get("completion_claim") is not False + or value.get("full_sqlite_completion_claim") is not False + or value.get("roles_preferences_global_config_restore_claim") is not False + or value.get("secret_bearing_assets_restore_claim") is not False + ): + raise ContractError("restore_driver_apply_semantic_receipt_invalid") return value @@ -980,7 +1149,10 @@ def remove_owned_resource( labels = inspect_resource_labels(runner, kind, name) if labels is None: return False - if labels.get(PRIMARY_LABEL) != run_id or labels.get(IDENTITY_LABEL) != plan.identity: + if ( + labels.get(PRIMARY_LABEL) != run_id + or labels.get(IDENTITY_LABEL) != plan.identity + ): raise ContractError(f"cleanup_{kind}_ownership_mismatch") if kind == "container": argv = [DOCKER, "rm", "--force", name] @@ -1071,11 +1243,14 @@ def cleanup_and_verify( plan: ResourcePlan, run_id: str, state: dict[str, Any] | None, + continuity_evidence: dict[str, Any] | None = None, ) -> list[str]: baseline_errors: list[str] = [] listener: dict[str, Any] | None = None production_before: dict[str, Any] | None = None image_inventory_before: list[str] | None = None + production_digest: str | None = None + image_inventory_digest: str | None = None if state is not None: state_listener = state.get("listener") if isinstance(state_listener, dict): @@ -1088,19 +1263,55 @@ def cleanup_and_verify( isinstance(item, str) for item in state_images ): image_inventory_before = state_images + if continuity_evidence is not None: + evidence_listener = continuity_evidence.get("listenerBinding") + if listener is None and isinstance(evidence_listener, dict): + listener = evidence_listener + evidence_production_digest = continuity_evidence.get( + "productionContinuitySha256" + ) + if isinstance(evidence_production_digest, str): + production_digest = evidence_production_digest + evidence_inventory_digest = continuity_evidence.get("imageInventorySha256") + if isinstance(evidence_inventory_digest, str): + image_inventory_digest = evidence_inventory_digest + if production_before is not None: + state_production_digest = sha256_bytes(canonical_json_bytes(production_before)) + if ( + production_digest is not None + and production_digest != state_production_digest + ): + baseline_errors.append("cleanup_production_evidence_digest_mismatch") + production_digest = state_production_digest + if image_inventory_before is not None: + state_inventory_digest = sha256_bytes( + canonical_json_bytes(image_inventory_before) + ) + if ( + image_inventory_digest is not None + and image_inventory_digest != state_inventory_digest + ): + baseline_errors.append("cleanup_image_inventory_evidence_digest_mismatch") + image_inventory_digest = state_inventory_digest try: if listener is None: - listener = capture_owned_listener_before_cleanup( - runner, plan, run_id - ) + listener = capture_owned_listener_before_cleanup(runner, plan, run_id) if production_before is None: production_before = production_snapshot( runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS ) + if production_digest is None: + production_digest = sha256_bytes( + canonical_json_bytes(production_before) + ) if image_inventory_before is None: image_inventory_before = image_inventory( runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS ) + if image_inventory_digest is None: + image_inventory_digest = sha256_bytes( + canonical_json_bytes(image_inventory_before) + ) except ContractError as exc: baseline_errors.append(str(exc)) @@ -1138,31 +1349,38 @@ def cleanup_and_verify( attempt_errors.append(str(exc)) try: - require_daemon_readable( - runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS - ) + require_daemon_readable(runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS) for name in plan.containers: - if inspect_json( - runner, - "container", - name, - timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, - ) is not None: + if ( + inspect_json( + runner, + "container", + name, + timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, + ) + is not None + ): attempt_errors.append("cleanup_exact_container_residue") for name in plan.volumes: - if inspect_json( - runner, - "volume", - name, - timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, - ) is not None: + if ( + inspect_json( + runner, + "volume", + name, + timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, + ) + is not None + ): attempt_errors.append("cleanup_exact_volume_residue") - if inspect_json( - runner, - "network", - plan.network, - timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, - ) is not None: + if ( + inspect_json( + runner, + "network", + plan.network, + timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS, + ) + is not None + ): attempt_errors.append("cleanup_exact_network_residue") for label in ( f"{PRIMARY_LABEL}={run_id}", @@ -1182,19 +1400,27 @@ def cleanup_and_verify( attempt_errors.append("cleanup_tcp_listener_still_present") else: attempt_errors.append("cleanup_listener_state_invalid") - if production_before is None or image_inventory_before is None: + if production_digest is None or image_inventory_digest is None: attempt_errors.append("cleanup_continuity_baseline_unavailable") else: try: - if production_snapshot( - runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS - ) != production_before: + current_production_digest = sha256_bytes( + canonical_json_bytes( + production_snapshot( + runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS + ) + ) + ) + if current_production_digest != production_digest: attempt_errors.append( "cleanup_production_identity_continuity_failed" ) - if image_inventory( - runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS - ) != image_inventory_before: + current_inventory_digest = sha256_bytes( + canonical_json_bytes( + image_inventory(runner, timeout=CLEANUP_COMMAND_TIMEOUT_SECONDS) + ) + ) + if current_inventory_digest != image_inventory_digest: attempt_errors.append("cleanup_image_inventory_drift") except ContractError as exc: attempt_errors.append(str(exc)) @@ -1234,7 +1460,9 @@ def load_state_if_safe(plan: ResourcePlan) -> dict[str, Any] | None: if not isinstance(value.get("production_before"), dict): raise ContractError("restore_state_production_baseline_invalid") images = value.get("image_inventory_before") - if not isinstance(images, list) or not all(isinstance(item, str) for item in images): + if not isinstance(images, list) or not all( + isinstance(item, str) for item in images + ): raise ContractError("restore_state_image_baseline_invalid") listener = value.get("listener") if listener is not None and ( @@ -1257,7 +1485,7 @@ def run_check( ) -> dict[str, Any]: validate_restore_policy(policy) require_regular_cluster_config(cluster_config) - verify_bundle(args, runner) + export_manifest_sha256 = verify_bundle(args, runner) require_daemon_readable(runner) verify_exact_images(runner) production_snapshot(runner) @@ -1278,6 +1506,7 @@ def run_check( "resourceWritePerformed": False, "secretValueRead": False, "completionClaim": False, + "exportBundleManifestSha256": export_manifest_sha256, } ) return value @@ -1290,7 +1519,7 @@ def run_apply( policy: dict[str, Any], cluster_config: Path, ) -> dict[str, Any]: - run_check(args, runner, plan, policy, cluster_config) + check_result = run_check(args, runner, plan, policy, cluster_config) create_workspace(plan) state_path = plan.workspace / "state.json" images = verify_exact_images(runner) @@ -1355,6 +1584,10 @@ def run_apply( ) value["restoreSemanticDigest"] = restore.get("portable_semantic_sha256") value["restoredAssetCounts"] = restore.get("restored_asset_counts") + value["listenerBinding"] = dict(state["listener"]) + value["productionContinuitySha256"] = sha256_bytes(canonical_json_bytes(before)) + value["imageInventorySha256"] = sha256_bytes(canonical_json_bytes(inventory_before)) + value["exportBundleManifestSha256"] = check_result["exportBundleManifestSha256"] value["completionClaim"] = False return value @@ -1383,11 +1616,7 @@ def executor_success( "identity": plan.identity, "resourcePrefix": plan.prefix, "completionScope": "portable_assets_only", - "portableAssetIds": [ - "dashboards", - "alert_rules", - "explorer_views", - ], + "portableAssetIds": list(PORTABLE_ASSET_IDS), "completionClaim": False, "executorZeroResidueVerified": True, "independentZeroResidueVerified": False, @@ -1401,6 +1630,17 @@ def executor_success( "secretValueInReceipt": False, "restoreSemanticDigest": restore_result.get("restoreSemanticDigest"), "restoredAssetCounts": restore_result.get("restoredAssetCounts"), + "listenerBinding": restore_result.get("listenerBinding"), + "productionContinuitySha256": restore_result.get( + "productionContinuitySha256" + ), + "imageInventorySha256": restore_result.get("imageInventorySha256"), + "exportBundleManifestSha256": restore_result.get( + "exportBundleManifestSha256" + ), + "productionContinuityVerified": True, + "imageInventoryContinuityVerified": True, + "listenerBindingUnreachableVerified": True, "zeroResidue": { "exactContainers": 0, "exactVolumes": 0, @@ -1459,19 +1699,47 @@ def validate_apply_receipt( "imageBuildPerformed": False, "secretValueInArguments": False, "secretValueInReceipt": False, + "productionContinuityVerified": True, + "imageInventoryContinuityVerified": True, + "listenerBindingUnreachableVerified": True, } - if any(value.get(key) != expected_value for key, expected_value in expected.items()): + if any( + value.get(key) != expected_value for key, expected_value in expected.items() + ): raise ContractError("apply_receipt_identity_or_terminal_invalid") digest = value.get("restoreSemanticDigest") counts = value.get("restoredAssetCounts") zero_residue = value.get("zeroResidue") + listener = value.get("listenerBinding") + continuity_digests = ( + value.get("productionContinuitySha256"), + value.get("imageInventorySha256"), + value.get("exportBundleManifestSha256"), + ) if ( not isinstance(digest, str) or len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest) or not isinstance(counts, dict) - or sorted(counts) != ["alert_rules", "dashboards", "explorer_views"] - or not all(isinstance(count, int) and count >= 0 for count in counts.values()) + or set(counts) != set(PORTABLE_ASSET_IDS) + or not all( + isinstance(count, int) and not isinstance(count, bool) and count >= 0 + for count in counts.values() + ) + or value.get("portableAssetIds") != list(PORTABLE_ASSET_IDS) + or not isinstance(listener, dict) + or listener.get("host") != "127.0.0.1" + or not isinstance(listener.get("port"), int) + or isinstance(listener.get("port"), bool) + or listener["port"] <= 0 + or listener["port"] > 65535 + or listener["port"] == 8080 + or not all( + isinstance(item, str) + and len(item) == 64 + and all(character in "0123456789abcdef" for character in item) + for item in continuity_digests + ) or zero_residue != { "exactContainers": 0, @@ -1503,9 +1771,7 @@ def independent_success( { "verifierIdentity": "signoz_metadata_restore_cleanup_verifier", "applyReceiptValidated": True, - "applyReceiptSha256": sha256_bytes( - canonical_json_bytes(apply_receipt) - ), + "applyReceiptSha256": sha256_bytes(canonical_json_bytes(apply_receipt)), "identity": plan.identity, "resourcePrefix": plan.prefix, "completionScope": "portable_assets_only", @@ -1517,6 +1783,13 @@ def independent_success( "secretBearingAssetsRestoreClaim": False, "restoreSemanticDigest": apply_receipt["restoreSemanticDigest"], "restoredAssetCounts": apply_receipt["restoredAssetCounts"], + "listenerBinding": apply_receipt["listenerBinding"], + "productionContinuitySha256": apply_receipt["productionContinuitySha256"], + "imageInventorySha256": apply_receipt["imageInventorySha256"], + "exportBundleManifestSha256": apply_receipt["exportBundleManifestSha256"], + "productionContinuityVerified": True, + "imageInventoryContinuityVerified": True, + "listenerBindingUnreachableVerified": True, "zeroResidue": { "stablePasses": CLEANUP_STABLE_PASSES, "exactContainers": 0, @@ -1544,7 +1817,9 @@ def main() -> int: runner = default_runner plan: ResourcePlan | None = None cleanup_errors: list[str] = [] + verifier_errors: list[str] = [] restore_result: dict[str, Any] | None = None + result: dict[str, Any] | None = None caught_error: str | None = None old_handlers: dict[int, Any] = {} cleanup_reconciliation_completed = False @@ -1568,70 +1843,90 @@ def main() -> int: signals_held_until_terminal = True try: - for value, label in ( - (args.trace_id, "trace_id"), - (args.run_id, "run_id"), - (args.work_item_id, "work_item_id"), - ): - validate_identity(value, label=label) - if args.work_item_id != "P0-OBS-002": - raise ContractError("work_item_not_allowlisted") - if args.receipt_file: - validate_new_private_destination( - Path(args.receipt_file), label="restore_receipt" - ) - plan = build_resource_plan(args.trace_id, args.run_id, args.work_item_id) - policy = load_policy(Path(args.policy)) - cluster_config = Path(args.cluster_config) - for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): - old_handlers[signum] = signal.getsignal(signum) - signal.signal(signum, handle_signal) - - if args.check: - result = run_check(args, runner, plan, policy, cluster_config) + try: + for value, label in ( + (args.trace_id, "trace_id"), + (args.run_id, "run_id"), + (args.work_item_id, "work_item_id"), + ): + validate_identity(value, label=label) + if args.work_item_id != "P0-OBS-002": + raise ContractError("work_item_not_allowlisted") if args.receipt_file: - write_terminal(Path(args.receipt_file), result) - emit(result) - return 0 - - if args.verify_cleanup: - try: - state = load_state_if_safe(plan) - except (ContractError, OSError, UnicodeError) as exc: - state = None - cleanup_errors.append(str(exc)) - reconciliation_errors = cleanup_and_verify( - runner, plan, args.run_id, state - ) - cleanup_errors.extend(reconciliation_errors) - cleanup_reconciliation_completed = True - if cleanup_errors: - raise ContractError("cleanup_reconciliation_incomplete") - if args.apply_receipt_file: - apply_receipt = validate_apply_receipt( - Path(args.apply_receipt_file), args, plan + validate_new_private_destination( + Path(args.receipt_file), label="restore_receipt" ) - result = independent_success(args, plan, apply_receipt) + plan = build_resource_plan(args.trace_id, args.run_id, args.work_item_id) + policy = load_policy(Path(args.policy)) + cluster_config = Path(args.cluster_config) + for signum in (signal.SIGHUP, signal.SIGINT, signal.SIGTERM): + old_handlers[signum] = signal.getsignal(signum) + signal.signal(signum, handle_signal) + + if args.check: + result = run_check(args, runner, plan, policy, cluster_config) + elif args.verify_cleanup: + apply_receipt: dict[str, Any] | None = None + export_manifest_sha256: str | None = None + if args.apply_receipt_file: + try: + apply_receipt = validate_apply_receipt( + Path(args.apply_receipt_file), args, plan + ) + except (ContractError, OSError, UnicodeError) as exc: + verifier_errors.append(str(exc)) + try: + export_manifest_sha256 = verify_bundle(args, runner) + except (ContractError, OSError, UnicodeError) as exc: + verifier_errors.append(str(exc)) + if ( + apply_receipt is not None + and export_manifest_sha256 is not None + and export_manifest_sha256 + != apply_receipt["exportBundleManifestSha256"] + ): + verifier_errors.append("export_bundle_manifest_digest_drift") + try: + state = load_state_if_safe(plan) + except (ContractError, OSError, UnicodeError) as exc: + state = None + cleanup_errors.append(str(exc)) + reconciliation_errors = cleanup_and_verify( + runner, + plan, + args.run_id, + state, + apply_receipt, + ) + cleanup_errors.extend(reconciliation_errors) + cleanup_reconciliation_completed = True + if cleanup_errors: + raise ContractError("cleanup_reconciliation_incomplete") + if verifier_errors: + raise ContractError(verifier_errors[0]) + if apply_receipt is not None: + result = independent_success(args, plan, apply_receipt) + else: + result = receipt( + trace_id=args.trace_id, + run_id=args.run_id, + work_item_id=args.work_item_id, + phase="independent_restore_verifier", + terminal="pass_zero_residue_verified", + detail="deterministic_reconciliation_zero_residue_pass", + ) + result["completionClaim"] = False + result["productionContinuityScope"] = "reconciliation_window_only" + result["restoreCompletionClaim"] = False else: - result = receipt( - trace_id=args.trace_id, - run_id=args.run_id, - work_item_id=args.work_item_id, - phase="independent_restore_verifier", - terminal="pass_zero_residue_verified", - detail="deterministic_reconciliation_zero_residue_pass", - ) - result["completionClaim"] = False - result["productionContinuityScope"] = ( - "reconciliation_window_only" - ) - result["restoreCompletionClaim"] = False - if args.receipt_file: - write_terminal(Path(args.receipt_file), result) - emit(result) - return 0 - - restore_result = run_apply(args, runner, plan, policy, cluster_config) + restore_result = run_apply(args, runner, plan, policy, cluster_config) + finally: + # This inner barrier closes the transition into the outer cleanup + # finalizer. If the first signal lands before this call, the signal + # handler has already ignored later signals and the outer finally + # still reconciles. Once this call runs, cleanup cannot be raised out. + if old_handlers and (args.apply or args.verify_cleanup): + hold_cleanup_signals() except (ContractError, OSError, UnicodeError) as exc: caught_error = str(exc) finally: @@ -1651,9 +1946,7 @@ def main() -> int: try: state = load_state_if_safe(plan) if state is None and args.apply: - cleanup_errors.append( - "restore_state_missing_for_apply_continuity" - ) + cleanup_errors.append("restore_state_missing_for_apply_continuity") except (ContractError, OSError, UnicodeError) as exc: state = None cleanup_errors.append(str(exc)) @@ -1665,17 +1958,14 @@ def main() -> int: cleanup_reconciliation_completed = True except (ContractError, OSError, UnicodeError) as exc: cleanup_errors.append(str(exc)) - if plan is not None and args.verify_cleanup and caught_error is not None: - # Cleanup may have completed just before a signal or verifier - # receipt validation failure. Preserve the failure receipt window - # even when no second reconciliation pass is required. - hold_cleanup_signals() if not args.apply and not signals_held_until_terminal: restore_signal_handlers() if plan is None: return 1 - if caught_error is None and not cleanup_errors and restore_result is not None: + if caught_error is None and result is not None: + exit_code = 0 + elif caught_error is None and not cleanup_errors and restore_result is not None: result = executor_success(args, plan, restore_result) exit_code = 0 else: @@ -1686,9 +1976,13 @@ def main() -> int: work_item_id=args.work_item_id, phase="terminal", terminal=( - "failed_zero_residue_verified_no_completion" - if not cleanup_errors - else "failed_cleanup_reconciliation_required" + "check_failed_no_write" + if args.check + else ( + "failed_zero_residue_verified_no_completion" + if not cleanup_errors + else "failed_cleanup_reconciliation_required" + ) ), detail=f"isolated_restore_failed_{error_code}", ) @@ -1700,6 +1994,7 @@ def main() -> int: "secretValueInReceipt": False, "zeroResidueVerified": not cleanup_errors, "cleanupErrorCodes": cleanup_errors, + "verifierErrorCodes": verifier_errors, } ) exit_code = 1 diff --git a/scripts/backup/tests/test_signoz_metadata_export_contract.py b/scripts/backup/tests/test_signoz_metadata_export_contract.py index ae9f5f974..90c0f44b9 100644 --- a/scripts/backup/tests/test_signoz_metadata_export_contract.py +++ b/scripts/backup/tests/test_signoz_metadata_export_contract.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import importlib.util import json import os import socket @@ -11,13 +12,16 @@ import urllib.parse from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from types import SimpleNamespace from typing import Any, Iterator - ROOT = Path(__file__).resolve().parents[3] EXPORTER = ROOT / "scripts" / "backup" / "signoz-metadata-export.py" VERIFIER = ROOT / "scripts" / "backup" / "verify-signoz-metadata-export.py" RESTORE = ROOT / "scripts" / "backup" / "signoz-metadata-restore-drill.py" +ISOLATED_CONTROLLER = ( + ROOT / "scripts" / "backup" / "signoz-metadata-isolated-restore.py" +) POLICY = ROOT / "config" / "signoz" / "metadata-export-policy.json" TRACE_ID = "trace-P0-OBS-002-metadata" RUN_ID = "run-P0-OBS-002-metadata" @@ -106,9 +110,7 @@ class MetadataHandler(BaseHTTPRequestHandler): if source_page not in {"logs", "meter", "metrics", "traces"}: self._send_json(400, {"error": "sourcePage required"}) return - asset_key = ( - f"{path}?{urllib.parse.urlencode({'sourcePage': source_page})}" - ) + asset_key = f"{path}?{urllib.parse.urlencode({'sourcePage': source_page})}" if asset_key not in self.assets: self._send_json(404, {"error": "not found"}) return @@ -220,6 +222,21 @@ def create_bundle(tmp_path: Path) -> tuple[Path, Path]: return bundle, credential +def load_isolated_controller() -> Any: + backup_dir = str(ROOT / "scripts" / "backup") + if backup_dir not in sys.path: + sys.path.insert(0, backup_dir) + spec = importlib.util.spec_from_file_location( + "signoz_isolated_restore_contract_integration", + ISOLATED_CONTROLLER, + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + def isolation_receipt(tmp_path: Path, target_base_url: str) -> Path: policy = json.loads(POLICY.read_text(encoding="utf-8")) identity = hashlib.sha256( @@ -373,9 +390,7 @@ def test_policy_rejects_unhashable_empty_collection_sentinel_as_contract_error( ) -> None: policy_value = json.loads(POLICY.read_text(encoding="utf-8")) explorer = next( - item - for item in policy_value["assets"] - if item["id"] == "explorer_views_logs" + item for item in policy_value["assets"] if item["id"] == "explorer_views_logs" ) explorer["empty_collection_sentinels"] = [{}] malformed_policy = tmp_path / "malformed-policy.json" @@ -549,9 +564,7 @@ def test_explorer_views_nonempty_scalar_fails_closed(tmp_path: Path) -> None: def test_explorer_view_item_must_match_scoped_source_page(tmp_path: Path) -> None: assets = json.loads(json.dumps(SOURCE_ASSETS)) - assets["/api/v1/explorer/views?sourcePage=logs"]["data"][0][ - "sourcePage" - ] = "traces" + assets["/api/v1/explorer/views?sourcePage=logs"]["data"][0]["sourcePage"] = "traces" output = tmp_path / "bundle" credential = credential_file(tmp_path) with metadata_server(assets) as (base_url, _): @@ -752,6 +765,68 @@ def test_restore_check_and_semantic_apply_are_bounded(tmp_path: Path) -> None: assert handler.post_count == 3 +def test_isolated_controller_invokes_real_restore_driver_contract( + tmp_path: Path, +) -> None: + controller = load_isolated_controller() + bundle, credential = create_bundle(tmp_path) + target_assets = { + "/api/v1/dashboards": {"data": []}, + "/api/v1/rules": {"status": "success", "data": {"rules": []}}, + "/api/v1/explorer/views?sourcePage=logs": {"data": []}, + "/api/v1/explorer/views?sourcePage=meter": {"data": None}, + "/api/v1/explorer/views?sourcePage=metrics": {"data": {}}, + "/api/v1/explorer/views?sourcePage=traces": {"data": []}, + } + args = SimpleNamespace( + bundle_dir=bundle, + policy=POLICY, + trace_id=TRACE_ID, + run_id=RUN_ID, + work_item_id=WORK_ITEM_ID, + ) + + def real_child_runner(argv: list[str], timeout: int) -> Any: + assert argv[0] == controller.PYTHON + completed = subprocess.run( + [sys.executable, *argv[1:]], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=timeout, + check=False, + ) + return controller.CommandResult(completed.returncode, completed.stdout) + + with metadata_server(target_assets) as (target_base_url, handler): + isolation = isolation_receipt(tmp_path, target_base_url) + check = controller.invoke_restore_driver( + args, + real_child_runner, + target_base_url, + isolation, + credential, + "--check", + ) + apply = controller.invoke_restore_driver( + args, + real_child_runner, + target_base_url, + isolation, + credential, + "--apply", + ) + + assert check["terminal"] == "check_pass_no_write" + assert apply["terminal"] == "partial_degraded_cleanup_pending" + assert set(apply["restored_asset_counts"]) == set(controller.PORTABLE_ASSET_IDS) + assert handler.post_count == 3 + child_receipt = tmp_path / "restore-driver-apply-receipt.json" + assert child_receipt.is_file() + assert child_receipt.read_bytes() == controller.canonical_json_bytes(apply) + + def test_restore_rejects_non_loopback_target_before_writes(tmp_path: Path) -> None: bundle, credential = create_bundle(tmp_path) target = "https://signoz.example.invalid" diff --git a/scripts/backup/tests/test_signoz_metadata_isolated_restore.py b/scripts/backup/tests/test_signoz_metadata_isolated_restore.py index f5664f198..8ffd29faa 100644 --- a/scripts/backup/tests/test_signoz_metadata_isolated_restore.py +++ b/scripts/backup/tests/test_signoz_metadata_isolated_restore.py @@ -11,7 +11,6 @@ from typing import Any import pytest - ROOT = Path(__file__).resolve().parents[3] CONTROLLER = ROOT / "scripts" / "backup" / "signoz-metadata-isolated-restore.py" POLICY = ROOT / "config" / "signoz" / "metadata-export-policy.json" @@ -91,6 +90,12 @@ class RecordingRunner: if argv[1:2] == ["run"]: name = argv[argv.index("--name") + 1] ref = argv[-1] + role = next( + candidate + for candidate, (expected_ref, _image_id) in c.EXPECTED_IMAGES.items() + if expected_ref == ref + ) + limits = c.RESOURCE_LIMITS[role] expected_id = next( image_id for expected_ref, image_id in c.EXPECTED_IMAGES.values() @@ -100,16 +105,27 @@ class RecordingRunner: "Id": (name.encode().hex() + "0" * 64)[:64], "Image": expected_id, "Config": {"Image": ref, "Labels": labels_from_command(argv)}, - "HostConfig": {"RestartPolicy": {"Name": "no"}}, + "HostConfig": { + "RestartPolicy": {"Name": "no"}, + "NanoCpus": limits["nano_cpus"], + "Memory": limits["memory_bytes"], + "MemorySwap": limits["memory_swap_bytes"], + "PidsLimit": limits["pids_limit"], + "LogConfig": { + "Type": c.RESOURCE_LOG_CONFIG["type"], + "Config": { + "max-file": c.RESOURCE_LOG_CONFIG["max_file"], + "max-size": c.RESOURCE_LOG_CONFIG["max_size"], + }, + }, + }, "State": {"Running": True}, "NetworkSettings": { - "Ports": { - "8080/tcp": [ - {"HostIp": "127.0.0.1", "HostPort": "49173"} - ] - } - if name.endswith("-server") - else {} + "Ports": ( + {"8080/tcp": [{"HostIp": "127.0.0.1", "HostPort": "49173"}]} + if name.endswith("-server") + else {} + ) }, "RestartCount": 0, } @@ -161,9 +177,7 @@ def labels_from_command(argv: list[str]) -> dict[str, str]: def test_identity_and_exact_resource_names(controller: Any) -> None: plan = controller.build_resource_plan(TRACE_ID, RUN_ID, WORK_ITEM_ID) - assert plan.identity == controller.derive_identity( - TRACE_ID, RUN_ID, WORK_ITEM_ID - ) + assert plan.identity == controller.derive_identity(TRACE_ID, RUN_ID, WORK_ITEM_ID) assert len(plan.identity) == 64 assert plan.prefix == f"awoooi-signoz-md-restore-{plan.identity[:16]}" assert plan.canonical_id == ( @@ -180,9 +194,10 @@ def test_identity_and_exact_resource_names(controller: Any) -> None: def test_policy_requires_reviewed_exact_startup_and_images(controller: Any) -> None: policy = json.loads(POLICY.read_text(encoding="utf-8")) - assert controller.validate_restore_policy(policy)["startup_contract"][ - "reviewed" - ] is True + isolation = controller.validate_restore_policy(policy) + assert isolation["startup_contract"]["reviewed"] is True + assert isolation["resource_limits"] == controller.RESOURCE_LIMITS + assert isolation["resource_log_config"] == controller.RESOURCE_LOG_CONFIG policy["restore_isolation"]["startup_contract"]["reviewed"] = False with pytest.raises( @@ -191,14 +206,19 @@ def test_policy_requires_reviewed_exact_startup_and_images(controller: Any) -> N controller.validate_restore_policy(policy) policy = json.loads(POLICY.read_text(encoding="utf-8")) - policy["restore_isolation"]["images"]["clickhouse"]["id"] = ( - f"sha256:{'a' * 64}" - ) + policy["restore_isolation"]["images"]["clickhouse"]["id"] = f"sha256:{'a' * 64}" with pytest.raises( controller.ContractError, match="restore_image_policy_not_exact" ): controller.validate_restore_policy(policy) + policy = json.loads(POLICY.read_text(encoding="utf-8")) + policy["assets"][0]["classification"] = "export_only" + with pytest.raises( + controller.ContractError, match="restore_portable_asset_ids_not_exact" + ): + controller.validate_restore_policy(policy) + def test_resource_creation_is_internal_no_pull_no_restart_and_loopback( controller: Any, @@ -210,7 +230,9 @@ def test_resource_creation_is_internal_no_pull_no_restart_and_loopback( controller.create_resources(runner, plan, RUN_ID, cluster) commands = runner.commands - network = next(command for command in commands if command[1:3] == ["network", "create"]) + network = next( + command for command in commands if command[1:3] == ["network", "create"] + ) assert "--internal" in network run_commands = [command for command in commands if command[1:2] == ["run"]] assert len(run_commands) == 3 @@ -220,21 +242,50 @@ def test_resource_creation_is_internal_no_pull_no_restart_and_loopback( assert f"{controller.PRIMARY_LABEL}={RUN_ID}" in command assert f"{controller.IDENTITY_LABEL}={plan.identity}" in command assert not any("signoz-clickhouse:" in value for value in command) - server = next(command for command in run_commands if plan.server_container in command) + ref = command[-1] + role = next( + candidate + for candidate, ( + expected_ref, + _image_id, + ) in controller.EXPECTED_IMAGES.items() + if expected_ref == ref + ) + limits = controller.RESOURCE_LIMITS[role] + assert command[command.index("--cpus") + 1] == limits["cpus"] + assert command[command.index("--memory") + 1] == limits["memory"] + assert command[command.index("--memory-swap") + 1] == limits["memory_swap"] + assert command[command.index("--pids-limit") + 1] == str(limits["pids_limit"]) + assert command[command.index("--log-driver") + 1] == "local" + assert "max-size=10m" in command + assert "max-file=2" in command + server = next( + command for command in run_commands if plan.server_container in command + ) assert "127.0.0.1::8080" in server assert f"{plan.server_data_volume}:/var/lib/signoz" in server assert "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://clickhouse:9000" in server clickhouse_run_index = commands.index( - next(command for command in run_commands if plan.clickhouse_container in command) + next( + command for command in run_commands if plan.clickhouse_container in command + ) ) readiness_index = next( index for index, command in enumerate(commands) - if command[1:2] == ["exec"] - and command[-2:] == ["--query", "SELECT 1"] + if command[1:2] == ["exec"] and command[-2:] == ["--query", "SELECT 1"] ) server_run_index = commands.index(server) assert clickhouse_run_index < readiness_index < server_run_index + controller.verify_runtime_container_images(runner, plan) + + server_runtime = runner.resources[("container", plan.server_container)] + server_runtime["HostConfig"]["Memory"] += 1 + with pytest.raises( + controller.ContractError, + match="restore_runtime_container_image_restart_or_resource_limit_drift", + ): + controller.verify_runtime_container_images(runner, plan) def test_exact_image_drift_fails_before_resource_creation(controller: Any) -> None: @@ -320,8 +371,7 @@ def test_cleanup_reconciles_without_listener_state_after_interruption( controller.create_workspace(plan) if server_created: cluster = ( - ROOT - / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" ) controller.create_resources(runner, plan, RUN_ID, cluster) state = { @@ -390,8 +440,7 @@ def test_cleanup_reconciles_resource_materialized_after_first_inventory( removals = [ command for command in runner.commands - if command[1:3] == ["volume", "rm"] - and command[-1] == plan.server_data_volume + if command[1:3] == ["volume", "rm"] and command[-1] == plan.server_data_volume ] assert len(removals) == 1 @@ -441,8 +490,15 @@ def test_distinct_verifier_requires_immutable_apply_receipt( "restoredAssetCounts": { "dashboards": 1, "alert_rules": 2, - "explorer_views": 3, + "explorer_views_logs": 3, + "explorer_views_meter": 0, + "explorer_views_metrics": 0, + "explorer_views_traces": 0, }, + "listenerBinding": {"host": "127.0.0.1", "port": 49173}, + "productionContinuitySha256": "b" * 64, + "imageInventorySha256": "c" * 64, + "exportBundleManifestSha256": "d" * 64, }, ) path = tmp_path / "apply.json" @@ -453,12 +509,152 @@ def test_distinct_verifier_requires_immutable_apply_receipt( assert executor_receipt["completionClaim"] is False assert executor_receipt["executorZeroResidueVerified"] is True + assert executor_receipt["portableAssetIds"] == [ + "dashboards", + "alert_rules", + "explorer_views_logs", + "explorer_views_meter", + "explorer_views_metrics", + "explorer_views_traces", + ] assert verified["phase"] == "independent_restore_verifier" assert verified["completionClaim"] is True assert verified["independentZeroResidueVerified"] is True assert verified["restoreSemanticDigest"] == "a" * 64 assert verified["applyReceiptValidated"] is True assert len(verified["applyReceiptSha256"]) == 64 + assert verified["listenerBinding"] == { + "host": "127.0.0.1", + "port": 49173, + } + assert verified["productionContinuityVerified"] is True + assert verified["imageInventoryContinuityVerified"] is True + assert verified["listenerBindingUnreachableVerified"] is True + + stale_receipt = json.loads(json.dumps(executor_receipt)) + del stale_receipt["restoredAssetCounts"]["explorer_views_meter"] + stale_path = tmp_path / "stale-apply.json" + controller.write_terminal(stale_path, stale_receipt) + with pytest.raises( + controller.ContractError, + match="apply_receipt_semantic_evidence_invalid", + ): + controller.validate_apply_receipt(stale_path, args, plan) + + +def test_independent_cleanup_uses_apply_listener_and_continuity_after_workspace_gone( + controller: Any, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runner = RecordingRunner(controller) + plan = controller.build_resource_plan( + TRACE_ID, RUN_ID, WORK_ITEM_ID, workspace_parent=tmp_path + ) + production_digest = controller.sha256_bytes( + controller.canonical_json_bytes(controller.production_snapshot(runner)) + ) + inventory_digest = controller.sha256_bytes( + controller.canonical_json_bytes(controller.image_inventory(runner)) + ) + continuity = { + "listenerBinding": {"host": "127.0.0.1", "port": 49173}, + "productionContinuitySha256": production_digest, + "imageInventorySha256": inventory_digest, + } + probed: list[tuple[str, int]] = [] + + def probe(host: str, port: int) -> bool: + probed.append((host, port)) + return False + + monkeypatch.setattr(controller, "tcp_connectable", probe) + monkeypatch.setattr(controller.time, "sleep", lambda _seconds: None) + + errors = controller.cleanup_and_verify( + runner, + plan, + RUN_ID, + None, + continuity, + ) + + assert errors == [] + assert probed == [("127.0.0.1", 49173)] * controller.CLEANUP_STABLE_PASSES + assert not plan.workspace.exists() + + +def test_restore_driver_apply_uses_and_validates_private_child_receipt( + controller: Any, + tmp_path: Path, +) -> None: + args = SimpleNamespace( + bundle_dir=tmp_path / "bundle", + policy=POLICY, + trace_id=TRACE_ID, + run_id=RUN_ID, + work_item_id=WORK_ITEM_ID, + ) + isolation_path = tmp_path / "isolation-receipt.json" + isolation_path.write_text("{}", encoding="utf-8") + token_path = tmp_path / "session-access-token" + token_path.write_text("test-only-token\n", encoding="ascii") + commands: list[list[str]] = [] + + def child_runner(argv: list[str], _timeout: int) -> Any: + commands.append(argv[:]) + receipt_path = Path(argv[argv.index("--receipt-file") + 1]) + value = controller.receipt( + trace_id=TRACE_ID, + run_id=RUN_ID, + work_item_id=WORK_ITEM_ID, + phase="terminal", + terminal="partial_degraded_cleanup_pending", + detail="isolated_restore_actions_3_semantic_readback_pass_cleanup_pending", + ) + value.update( + { + "restored_asset_counts": { + "dashboards": 1, + "alert_rules": 1, + "explorer_views_logs": 1, + "explorer_views_meter": 0, + "explorer_views_metrics": 0, + "explorer_views_traces": 0, + }, + "restored_asset_semantic_sha256": { + asset_id: "a" * 64 for asset_id in controller.PORTABLE_ASSET_IDS + }, + "portable_semantic_sha256": "b" * 64, + "completion_scope": "portable_assets_only", + "completion_claim": False, + "full_sqlite_completion_claim": False, + "roles_preferences_global_config_restore_claim": False, + "secret_bearing_assets_restore_claim": False, + } + ) + receipt_path.write_bytes(controller.canonical_json_bytes(value)) + receipt_path.chmod(0o600) + return controller.CommandResult( + 0, + json.dumps(value, sort_keys=True, separators=(",", ":")), + ) + + result = controller.invoke_restore_driver( + args, + child_runner, + "http://127.0.0.1:49173", + isolation_path, + token_path, + "--apply", + ) + + assert result["portable_semantic_sha256"] == "b" * 64 + assert len(commands) == 1 + assert "--receipt-file" in commands[0] + child_path = Path(commands[0][commands[0].index("--receipt-file") + 1]) + assert child_path == tmp_path / "restore-driver-apply-receipt.json" + assert stat.S_IMODE(child_path.stat().st_mode) == 0o600 def test_interrupted_verify_cleanup_reconciles_again_before_zero_residue_receipt( @@ -477,6 +673,7 @@ def test_interrupted_verify_cleanup_reconciles_again_before_zero_residue_receipt _plan: Any, _run_id: str, _state: dict[str, Any] | None, + _continuity: dict[str, Any] | None = None, ) -> list[str]: nonlocal calls calls += 1 @@ -514,6 +711,94 @@ def test_interrupted_verify_cleanup_reconciles_again_before_zero_residue_receipt assert "signal_15_cleanup_required" in emitted[-1]["detail"] +def test_signal_at_cleanup_transition_still_reconciles_and_writes_terminal( + controller: Any, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + plan = controller.build_resource_plan( + TRACE_ID, RUN_ID, WORK_ITEM_ID, workspace_parent=tmp_path + ) + receipt_path = tmp_path / "transition-terminal.json" + args = SimpleNamespace( + check=False, + apply=True, + verify_cleanup=False, + bundle_dir=str(tmp_path / "bundle"), + policy=str(POLICY), + cluster_config=str( + ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + ), + trace_id=TRACE_ID, + run_id=RUN_ID, + work_item_id=WORK_ITEM_ID, + receipt_file=str(receipt_path), + apply_receipt_file=None, + ) + state = { + "production_before": {}, + "image_inventory_before": ["sha256:test|test:fixture"], + "listener": {"host": "127.0.0.1", "port": 49173}, + } + cleanup_calls = 0 + installed: dict[int, Any] = {} + transition_signal_sent = False + + def fake_getsignal(signum: int) -> Any: + return installed.get(signum, "original-handler") + + def fake_signal(signum: int, handler: Any) -> Any: + nonlocal transition_signal_sent + previous = installed.get(signum, "original-handler") + installed[signum] = handler + if ( + handler is controller.signal.SIG_IGN + and not transition_signal_sent + and callable(previous) + ): + transition_signal_sent = True + previous(controller.signal.SIGTERM, None) + return previous + + def cleanup_once( + _runner: Any, + _plan: Any, + _run_id: str, + _state: dict[str, Any] | None, + _continuity: dict[str, Any] | None = None, + ) -> list[str]: + nonlocal cleanup_calls + cleanup_calls += 1 + return [] + + monkeypatch.setattr(controller, "parse_args", lambda: args) + monkeypatch.setattr(controller, "build_resource_plan", lambda *_args: plan) + monkeypatch.setattr(controller, "load_policy", lambda _path: {}) + monkeypatch.setattr( + controller, + "run_apply", + lambda *_args: { + "restoreSemanticDigest": "a" * 64, + "restoredAssetCounts": {}, + }, + ) + monkeypatch.setattr(controller, "load_state_if_safe", lambda _plan: state) + monkeypatch.setattr(controller, "cleanup_and_verify", cleanup_once) + monkeypatch.setattr(controller.signal, "getsignal", fake_getsignal) + monkeypatch.setattr(controller.signal, "signal", fake_signal) + + exit_code = controller.main() + + terminal = json.loads(receipt_path.read_text(encoding="utf-8")) + assert exit_code == 1 + assert transition_signal_sent is True + assert cleanup_calls == 1 + assert terminal["terminal"] == "failed_zero_residue_verified_no_completion" + assert terminal["zeroResidueVerified"] is True + assert terminal["cleanupErrorCodes"] == [] + assert "signal_15_cleanup_required" in terminal["detail"] + + def test_bootstrap_secrets_stay_in_mode_0400_files( controller: Any, tmp_path: Path, @@ -548,9 +833,7 @@ def test_bootstrap_secrets_stay_in_mode_0400_files( assert token == tmp_path / "session-access-token" assert stat.S_IMODE(token.stat().st_mode) == 0o400 - assert sorted(path.name for path in tmp_path.iterdir()) == [ - "session-access-token" - ] + assert sorted(path.name for path in tmp_path.iterdir()) == ["session-access-token"] assert calls[0][0].endswith("/api/v1/register") assert calls[1][0].endswith("/api/v2/sessions/email_password") assert capsys.readouterr().out == "" @@ -571,9 +854,7 @@ def test_bootstrap_accepts_live_root_response_envelope( monkeypatch.setattr(controller, "request_json", fake_request) - token = controller.bootstrap_isolated_session( - "http://127.0.0.1:49173", tmp_path - ) + token = controller.bootstrap_isolated_session("http://127.0.0.1:49173", tmp_path) assert token.read_text(encoding="ascii").strip() == "a" * 64 assert stat.S_IMODE(token.stat().st_mode) == 0o400 @@ -589,8 +870,22 @@ def test_controller_source_has_signal_cleanup_and_no_secret_argv() -> None: assert '"--pull=never"' in source assert '"--restart=no"' in source assert '"--auth-mode"' in source - assert "access_token" not in source[source.index("argv = [") : source.index("output = run_required", source.index("argv = ["))] - assert "password" not in source[source.index("argv = [") : source.index("output = run_required", source.index("argv = ["))] + assert ( + "access_token" + not in source[ + source.index("argv = [") : source.index( + "output = run_required", source.index("argv = [") + ) + ] + ) + assert ( + "password" + not in source[ + source.index("argv = [") : source.index( + "output = run_required", source.index("argv = [") + ) + ] + ) durable_write = source.rindex("write_terminal(Path(args.receipt_file), result)") final_handler_restore = source.rindex("restore_signal_handlers()") assert durable_write < final_handler_restore diff --git a/scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh b/scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh new file mode 100755 index 000000000..138440ade --- /dev/null +++ b/scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh @@ -0,0 +1,485 @@ +#!/usr/bin/env bash +# Source-bound controller -> Windows99/Agent99 -> Host110 SigNoz toolchain route. + +set -euo pipefail + +readonly SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly DEFAULT_SOURCE_ROOT="$(CDPATH= cd -- "${SCRIPT_DIR}/../.." && pwd)" +readonly TARGET="Administrator@192.168.0.99" +readonly GITEA_REMOTE="gitea" +readonly GITEA_URL_HTTPS="https://gitea.wooo.work/wooo/awoooi.git" +readonly GITEA_URL_HTTP_INTERNAL="http://192.168.0.110:3000/wooo/awoooi.git" +readonly GITEA_URL_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git" +readonly EXECUTOR_RELATIVE="agent99-signoz-metadata-executor.ps1" +readonly TARGET_RELATIVE="agent99-signoz-toolchain-target.py" +readonly WRAPPER_RELATIVE="scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh" + +LABELS=( + "metadata-export-policy.json" + "signoz_metadata_contract.py" + "signoz-metadata-export.py" + "verify-signoz-metadata-export.py" + "signoz-metadata-restore-drill.py" + "signoz-metadata-isolated-restore.py" + "isolated-cluster.xml" +) +TOOLCHAIN_SOURCES=( + "config/signoz/metadata-export-policy.json" + "scripts/backup/signoz_metadata_contract.py" + "scripts/backup/signoz-metadata-export.py" + "scripts/backup/verify-signoz-metadata-export.py" + "scripts/backup/signoz-metadata-restore-drill.py" + "scripts/backup/signoz-metadata-isolated-restore.py" + "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" +) +MODES=("0644" "0644" "0755" "0755" "0755" "0755" "0644") + +MODE="check" +MODE_SELECTED="" +TRACE_ID="" +RUN_ID="" +WORK_ITEM_ID="" +ATTEMPT_ID="" +SOURCE_ROOT="${DEFAULT_SOURCE_ROOT}" +SOURCE_REVISION="" +IDENTITY_FILE="" +KNOWN_HOSTS_FILE="${HOME:-}/.ssh/known_hosts" +CONNECT_TIMEOUT_SECONDS="8" +REMOTE_EXECUTION_TIMEOUT_SECONDS="1200" +REMOTE_PAYLOAD_PREPARED="false" +REMOTE_PAYLOAD_PATH_WINDOWS="" + +usage() { + printf '%s\n' \ + "Usage: $0 [--check|--apply|--verify|--reconcile] [options]" \ + "" \ + "Fixed route: local fresh Gitea main -> Windows99 Agent99 -> Host110." \ + "No local connection to Host110 and no active toolchain pointer are allowed." \ + "" \ + " --trace-id ID Required controlled-apply trace identity." \ + " --run-id ID Required original deployment run identity." \ + " --work-item-id P0-OBS-002 Required fixed work item." \ + " --attempt-id ID Required for reconcile; optional apply recovery ID." \ + " --source-root PATH Checkout root (default: repository root)." \ + " --source-revision SHA Must equal freshly fetched gitea/main." \ + " --identity-file PATH Optional Windows99 SSH identity path." \ + " --known-hosts-file PATH Pinned known_hosts file." \ + " --connect-timeout SECONDS 1 through 30 (default: 8)." +} + +fail() { + printf 'signoz_toolchain_agent99_error=%s\n' "$1" >&2 + exit 64 +} + +require_value() { + [[ $# -ge 2 && -n "$2" ]] || fail "missing_option_value" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --check|--apply|--verify|--reconcile) + [[ -z "${MODE_SELECTED}" ]] || fail "multiple_mode_flags" + MODE="${1#--}" + MODE_SELECTED="true" + shift + ;; + --trace-id) + require_value "$@"; TRACE_ID="$2"; shift 2 ;; + --run-id) + require_value "$@"; RUN_ID="$2"; shift 2 ;; + --work-item-id) + require_value "$@"; WORK_ITEM_ID="$2"; shift 2 ;; + --attempt-id) + require_value "$@"; ATTEMPT_ID="$2"; shift 2 ;; + --source-root) + require_value "$@"; SOURCE_ROOT="$2"; shift 2 ;; + --source-revision) + require_value "$@"; SOURCE_REVISION="$2"; shift 2 ;; + --identity-file) + require_value "$@"; IDENTITY_FILE="$2"; shift 2 ;; + --known-hosts-file) + require_value "$@"; KNOWN_HOSTS_FILE="$2"; shift 2 ;; + --connect-timeout) + require_value "$@"; CONNECT_TIMEOUT_SECONDS="$2"; shift 2 ;; + -h|--help) + usage; exit 0 ;; + *) fail "unsupported_option" ;; + esac +done + +readonly SAFE_ID_PATTERN='^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$' +[[ "${TRACE_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "trace_id_missing_or_invalid" +[[ "${RUN_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "run_id_missing_or_invalid" +[[ "${WORK_ITEM_ID}" == "P0-OBS-002" ]] || fail "work_item_id_not_allowlisted" +if [[ "${MODE}" == "reconcile" ]]; then + [[ "${ATTEMPT_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "reconcile_attempt_id_missing_or_invalid" +elif [[ -n "${ATTEMPT_ID}" && "${MODE}" != "apply" ]]; then + fail "attempt_id_not_allowed_for_mode" +fi +[[ "${CONNECT_TIMEOUT_SECONDS}" =~ ^[0-9]+$ ]] || fail "invalid_connect_timeout" +(( CONNECT_TIMEOUT_SECONDS >= 1 && CONNECT_TIMEOUT_SECONDS <= 30 )) || fail "invalid_connect_timeout" + +for dependency in git python3 ssh scp mktemp timeout; do + command -v "${dependency}" >/dev/null 2>&1 || fail "required_command_missing_${dependency}" +done +[[ -f "${KNOWN_HOSTS_FILE}" && ! -L "${KNOWN_HOSTS_FILE}" ]] || fail "known_hosts_file_missing_or_unsafe" +if [[ -n "${IDENTITY_FILE}" ]]; then + [[ -f "${IDENTITY_FILE}" && ! -L "${IDENTITY_FILE}" ]] || fail "identity_file_missing_or_unsafe" +fi + +[[ -d "${SOURCE_ROOT}" ]] || fail "source_root_missing" +SOURCE_ROOT="$(CDPATH= cd -- "${SOURCE_ROOT}" && pwd)" +GITEA_URL="$(git -C "${SOURCE_ROOT}" remote get-url "${GITEA_REMOTE}" 2>/dev/null)" \ + || fail "gitea_remote_missing" +case "${GITEA_URL}" in + "${GITEA_URL_HTTPS}"|"${GITEA_URL_HTTP_INTERNAL}"|"${GITEA_URL_SSH_INTERNAL}") ;; + *) fail "gitea_remote_not_fixed_internal" ;; +esac +git -C "${SOURCE_ROOT}" fetch --quiet --no-tags "${GITEA_REMOTE}" \ + '+refs/heads/main:refs/remotes/gitea/main' || fail "fresh_gitea_main_fetch_failed" +GITEA_MAIN_REVISION="$(git -C "${SOURCE_ROOT}" rev-parse --verify 'refs/remotes/gitea/main^{commit}')" \ + || fail "fresh_gitea_main_revision_unavailable" +GITEA_MAIN_REVISION="$(printf '%s' "${GITEA_MAIN_REVISION}" | tr '[:upper:]' '[:lower:]')" +if [[ -z "${SOURCE_REVISION}" ]]; then + SOURCE_REVISION="${GITEA_MAIN_REVISION}" +fi +SOURCE_REVISION="$(printf '%s' "${SOURCE_REVISION}" | tr '[:upper:]' '[:lower:]')" +[[ "${SOURCE_REVISION}" =~ ^[0-9a-f]{40}$ ]] || fail "source_revision_must_be_full_commit_sha" +[[ "${SOURCE_REVISION}" == "${GITEA_MAIN_REVISION}" ]] || fail "source_revision_is_not_fresh_gitea_main" + +SOURCE_BOUND_FILES=( + "${TOOLCHAIN_SOURCES[@]}" + "${EXECUTOR_RELATIVE}" + "${TARGET_RELATIVE}" + "${WRAPPER_RELATIVE}" +) +for relative_path in "${SOURCE_BOUND_FILES[@]}"; do + [[ -f "${SOURCE_ROOT}/${relative_path}" && ! -L "${SOURCE_ROOT}/${relative_path}" ]] \ + || fail "source_bound_file_missing_or_unsafe" + git -C "${SOURCE_ROOT}" ls-files --error-unmatch -- "${relative_path}" >/dev/null 2>&1 \ + || fail "source_bound_file_untracked" +done +git -C "${SOURCE_ROOT}" diff --quiet "${SOURCE_REVISION}" -- "${SOURCE_BOUND_FILES[@]}" \ + || fail "source_bundle_differs_from_revision" + +umask 077 +WORK_DIR="$(mktemp -d /tmp/awoooi-signoz-toolchain-agent99.XXXXXXXX)" +ENVELOPE_PATH="${WORK_DIR}/toolchain-envelope.json" +REMOTE_STDOUT_PATH="${WORK_DIR}/remote.stdout" +REMOTE_STDERR_PATH="${WORK_DIR}/remote.stderr" + +SSH_OPTIONS=( + -T + -o BatchMode=yes + -o PreferredAuthentications=publickey + -o PubkeyAuthentication=yes + -o PasswordAuthentication=no + -o KbdInteractiveAuthentication=no + -o NumberOfPasswordPrompts=0 + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=${KNOWN_HOSTS_FILE}" + -o "ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}" + -o ConnectionAttempts=1 + -o ServerAliveInterval=10 + -o ServerAliveCountMax=3 + -o LogLevel=ERROR + -o IdentitiesOnly=yes +) +SCP_OPTIONS=("${SSH_OPTIONS[@]:1}") +if [[ -n "${IDENTITY_FILE}" ]]; then + SSH_OPTIONS+=( -i "${IDENTITY_FILE}" ) + SCP_OPTIONS+=( -i "${IDENTITY_FILE}" ) +fi + +make_encoded_command() { + local executor_mode="$1" payload_path="$2" attempt_id="$3" + python3 - "${executor_mode}" "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ + "${SOURCE_REVISION}" "${payload_path}" "${attempt_id}" <<'PY' +from __future__ import annotations + +import base64 +import sys + +mode, trace_id, run_id, work_item_id, revision, payload_path, attempt_id = sys.argv[1:] +values = [mode, trace_id, run_id, work_item_id, revision, payload_path, attempt_id] +if any("'" in value or "\r" in value or "\n" in value for value in values): + raise SystemExit("unsafe_remote_command_value") +script = rf'''$ErrorActionPreference = 'Stop' +$executor = 'C:\Wooo\Agent99\bin\agent99-signoz-metadata-executor.ps1' +if (-not (Test-Path -LiteralPath $executor -PathType Leaf)) {{ throw 'executor_missing' }} +$parameters = @{{ + Mode = '{mode}' + TraceId = '{trace_id}' + RunId = '{run_id}' + WorkItemId = '{work_item_id}' + SourceRevision = '{revision}' +}} +''' +if mode == "ToolchainApply": + script += rf'''$payloadPath = '{payload_path}' +$envelope = '' +try {{ + $item = Get-Item -LiteralPath $payloadPath -ErrorAction Stop + if (($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0 -or $item.Length -le 0 -or $item.Length -gt 4194304) {{ + throw 'toolchain_envelope_transport_file_invalid' + }} + $envelope = [IO.File]::ReadAllText($payloadPath, [Text.Encoding]::UTF8) +}} finally {{ + Remove-Item -LiteralPath $payloadPath -Force -ErrorAction SilentlyContinue +}} +if (Test-Path -LiteralPath $payloadPath) {{ throw 'toolchain_envelope_transport_delete_failed' }} +$parameters.ToolchainEnvelopeJson = $envelope +$parameters.ToolchainEnvelopeDeletedBeforeExecutor = $true +''' +elif mode == "ToolchainReconcile": + script += f"$parameters.AttemptId = '{attempt_id}'\r\n" +script += "& $executor @parameters\r\nexit [int]$LASTEXITCODE\r\n" +encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii") +print( + "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass " + f"-EncodedCommand {encoded}" +) +PY +} + +make_file_command() { + local action="$1" path="$2" + python3 - "${action}" "${path}" <<'PY' +from __future__ import annotations + +import base64 +import sys + +action, path = sys.argv[1:] +if "'" in path or "\r" in path or "\n" in path: + raise SystemExit("unsafe_transport_path") +if action == "prepare": + script = rf'''$ErrorActionPreference='Stop' +$directory='C:\Wooo\Agent99\deploy\transport-inbox' +$path='{path}' +New-Item -ItemType Directory -Force -Path $directory | Out-Null +Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue +if (Test-Path -LiteralPath $path) {{ throw 'stale_transport_payload_cleanup_failed' }} +''' +else: + script = rf'''$ErrorActionPreference='Stop' +$path='{path}' +Remove-Item -LiteralPath $path -Force -ErrorAction SilentlyContinue +if (Test-Path -LiteralPath $path) {{ throw 'transport_payload_cleanup_failed' }} +''' +encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii") +print( + "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass " + f"-EncodedCommand {encoded}" +) +PY +} + +cleanup() { + if [[ "${REMOTE_PAYLOAD_PREPARED}" == "true" && -n "${REMOTE_PAYLOAD_PATH_WINDOWS}" ]]; then + cleanup_command="$(make_file_command cleanup "${REMOTE_PAYLOAD_PATH_WINDOWS}" 2>/dev/null || true)" + if [[ -n "${cleanup_command}" ]]; then + timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${cleanup_command}" \ + >/dev/null 2>&1 || true + fi + fi + rm -rf -- "${WORK_DIR}" +} +trap cleanup EXIT HUP INT TERM + +build_envelope() { + python3 - "${SOURCE_ROOT}" "${SOURCE_REVISION}" "${TRACE_ID}" "${RUN_ID}" \ + "${WORK_ITEM_ID}" "${ENVELOPE_PATH}" \ + "${#TOOLCHAIN_SOURCES[@]}" \ + "${TOOLCHAIN_SOURCES[@]}" \ + "${LABELS[@]}" \ + "${MODES[@]}" <<'PY' +from __future__ import annotations + +import base64 +import hashlib +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +revision, trace_id, run_id, work_item_id = sys.argv[2:6] +output = Path(sys.argv[6]) +count = int(sys.argv[7]) +sources = sys.argv[8 : 8 + count] +labels = sys.argv[8 + count : 8 + 2 * count] +modes = sys.argv[8 + 2 * count :] +if count != 7 or len(set(labels)) != 7 or len(modes) != 7: + raise SystemExit("fixed_toolchain_file_contract_failed") +files = [] +manifest_rows = [] +hashes = [] +for source, label, mode in zip(sources, labels, modes, strict=True): + content = (root / source).read_bytes() + digest = hashlib.sha256(content).hexdigest() + hashes.append(digest) + manifest_rows.append(f"{label}\t{digest}\t{mode}\n") + files.append( + { + "name": label, + "sha256": digest, + "mode": mode, + "contentBase64": base64.b64encode(content).decode("ascii"), + } + ) +bundle_id = hashlib.sha256( + "".join(f"{digest}\n" for digest in hashes).encode("ascii") +).hexdigest() +manifest_sha256 = hashlib.sha256("".join(manifest_rows).encode()).hexdigest() +value = { + "schemaVersion": "awoooi_signoz_metadata_toolchain_envelope_v1", + "transport": "fresh_gitea_main_to_windows99_agent99_to_fixed_host110", + "traceId": trace_id, + "runId": run_id, + "workItemId": work_item_id, + "sourceRevision": revision, + "bundleId": bundle_id, + "manifestSha256": manifest_sha256, + "fileCount": 7, + "transportPayloadContainsSecrets": False, + "files": files, +} +output.write_text(json.dumps(value, separators=(",", ":")), encoding="utf-8") +PY +} + +invoke_remote() { + local executor_mode="$1" payload_path="$2" attempt_id="$3" + local command + command="$(make_encoded_command "${executor_mode}" "${payload_path}" "${attempt_id}")" + : >"${REMOTE_STDOUT_PATH}" + : >"${REMOTE_STDERR_PATH}" + set +e + timeout --kill-after=20 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}" \ + ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${command}" \ + >"${REMOTE_STDOUT_PATH}" 2>"${REMOTE_STDERR_PATH}" + LAST_REMOTE_RC=$? + set -e + if [[ ! -s "${REMOTE_STDOUT_PATH}" || $(wc -c <"${REMOTE_STDOUT_PATH}") -gt 65536 ]]; then + LAST_REMOTE_OUTPUT="" + return + fi + LAST_REMOTE_OUTPUT="$(tr -d '\r' <"${REMOTE_STDOUT_PATH}")" +} + +validate_receipt() { + local value="$1" executor_mode="$2" allowed_csv="$3" + python3 - "${executor_mode}" "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ + "${SOURCE_REVISION}" "${allowed_csv}" "${value}" <<'PY' +from __future__ import annotations + +import json +import sys + +mode, trace_id, run_id, work_item_id, revision, allowed_csv, raw = sys.argv[1:] +try: + value = json.loads(raw) +except json.JSONDecodeError as exc: + raise SystemExit("agent99_receipt_json_invalid") from exc +allowed = set(allowed_csv.split(",")) +required = { + "schemaVersion": "agent99_signoz_metadata_executor_dispatch_v1", + "mode": mode, + "traceId": trace_id, + "runId": run_id, + "workItemId": work_item_id, + "sourceRevision": revision, + "targetHost": "192.168.0.110", + "dispatchPath": "windows99_agent99_to_host110_fixed_metadata_toolchain", + "productionRuntimeMutationPerformed": False, + "completionClaim": False, +} +if any(value.get(key) != expected for key, expected in required.items()): + raise SystemExit("agent99_receipt_identity_or_boundary_invalid") +if value.get("terminal") not in allowed: + raise SystemExit("agent99_receipt_terminal_invalid") +if mode == "ToolchainApply" and ( + value.get("toolchainEnvelopeTransportPayloadUsed") is not True + or value.get("toolchainEnvelopeTransportPayloadDeletedBeforeExecutor") is not True + or value.get("toolchainEnvelopeTransportPayloadContainsSecrets") is not False +): + raise SystemExit("agent99_envelope_transport_receipt_invalid") +PY +} + +run_checked_mode() { + local executor_mode="$1" payload_path="$2" attempt_id="$3" allowed="$4" + invoke_remote "${executor_mode}" "${payload_path}" "${attempt_id}" + [[ -n "${LAST_REMOTE_OUTPUT}" ]] || fail "${executor_mode}_receipt_missing" + validate_receipt "${LAST_REMOTE_OUTPUT}" "${executor_mode}" "${allowed}" \ + || fail "${executor_mode}_receipt_invalid" + printf '%s\n' "${LAST_REMOTE_OUTPUT}" +} + +case "${MODE}" in + check) + run_checked_mode "ToolchainCheck" "" "" "toolchain_check_ready" + [[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_check_failed" + ;; + verify) + run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass" + [[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed" + ;; + reconcile) + run_checked_mode "ToolchainReconcile" "" "${ATTEMPT_ID}" \ + "toolchain_reconcile_exact_deployment_verified,toolchain_reconcile_zero_owned_residue,toolchain_reconcile_preserved_unverified,toolchain_reconcile_failed_or_unverified" + [[ ${LAST_REMOTE_RC} -eq 0 ]] || exit 75 + ;; + apply) + run_checked_mode "ToolchainCheck" "" "" "toolchain_check_ready" + [[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_check_failed_before_apply" + build_envelope + PAYLOAD_TOKEN="$(python3 - "${ENVELOPE_PATH}" <<'PY' +import hashlib +import sys +from pathlib import Path +print(hashlib.sha256(Path(sys.argv[1]).read_bytes()).hexdigest()[:24]) +PY +)" + REMOTE_PAYLOAD_PATH_WINDOWS="C:\\Wooo\\Agent99\\deploy\\transport-inbox\\signoz-toolchain-${PAYLOAD_TOKEN}.json" + REMOTE_PAYLOAD_PATH_SCP="C:/Wooo/Agent99/deploy/transport-inbox/signoz-toolchain-${PAYLOAD_TOKEN}.json" + prepare_command="$(make_file_command prepare "${REMOTE_PAYLOAD_PATH_WINDOWS}")" + timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${prepare_command}" \ + >/dev/null 2>"${REMOTE_STDERR_PATH}" || fail "remote_payload_prepare_failed" + REMOTE_PAYLOAD_PREPARED="true" + timeout --kill-after=10 120 scp -q "${SCP_OPTIONS[@]}" "${ENVELOPE_PATH}" \ + "${TARGET}:${REMOTE_PAYLOAD_PATH_SCP}" >/dev/null 2>"${REMOTE_STDERR_PATH}" \ + || fail "remote_payload_transport_failed" + invoke_remote "ToolchainApply" "${REMOTE_PAYLOAD_PATH_WINDOWS}" "" + apply_output="${LAST_REMOTE_OUTPUT}" + apply_rc="${LAST_REMOTE_RC}" + cleanup_command="$(make_file_command cleanup "${REMOTE_PAYLOAD_PATH_WINDOWS}")" + timeout --kill-after=10 60 ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${cleanup_command}" \ + >/dev/null 2>&1 || true + REMOTE_PAYLOAD_PREPARED="false" + if [[ -n "${apply_output}" ]] && validate_receipt "${apply_output}" "ToolchainApply" \ + "toolchain_deployed_inactive_verified,toolchain_apply_failed_or_unverified"; then + printf '%s\n' "${apply_output}" + fi + if [[ ${apply_rc} -eq 0 ]]; then + validate_receipt "${apply_output}" "ToolchainApply" "toolchain_deployed_inactive_verified" \ + || fail "toolchain_apply_success_receipt_invalid" + run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass" + [[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed_after_apply" + exit 0 + fi + if [[ -z "${ATTEMPT_ID}" ]]; then + ATTEMPT_ID="reconcile-$(date -u +%Y%m%dT%H%M%SZ)-$$" + fi + [[ "${ATTEMPT_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "generated_reconcile_attempt_id_invalid" + run_checked_mode "ToolchainReconcile" "" "${ATTEMPT_ID}" \ + "toolchain_reconcile_exact_deployment_verified,toolchain_reconcile_zero_owned_residue,toolchain_reconcile_preserved_unverified,toolchain_reconcile_failed_or_unverified" + if [[ ${LAST_REMOTE_RC} -eq 0 ]] && [[ "${LAST_REMOTE_OUTPUT}" == *'"terminal":"toolchain_reconcile_exact_deployment_verified"'* ]]; then + run_checked_mode "ToolchainVerify" "" "" "toolchain_verify_pass" + [[ ${LAST_REMOTE_RC} -eq 0 ]] || fail "toolchain_verify_failed_after_reconcile" + exit 0 + fi + exit 75 + ;; +esac diff --git a/scripts/ops/deploy-signoz-metadata-toolchain.sh b/scripts/ops/deploy-signoz-metadata-toolchain.sh index c26ae2b7f..81927b56d 100755 --- a/scripts/ops/deploy-signoz-metadata-toolchain.sh +++ b/scripts/ops/deploy-signoz-metadata-toolchain.sh @@ -1,548 +1,8 @@ #!/usr/bin/env bash -# P0-OBS-002 - deploy an immutable, inactive SigNoz metadata toolchain to host110. -# -# The deployer never reads credentials, SQLite, or Docker volumes. It does not -# change an active pointer or invoke an authenticated metadata export. Failed -# candidates are moved into the run receipt directory rather than deleted. +# Historical compatibility entrypoint. Direct controller -> Host110 mutation is retired. set -euo pipefail -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -TARGET_HOST="${SIGNOZ_METADATA_TARGET_HOST:-wooo@192.168.0.110}" -REMOTE_ROOT="${SIGNOZ_METADATA_REMOTE_ROOT:-/backup/toolchains/signoz-metadata}" -RECEIPT_ROOT="${SIGNOZ_METADATA_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-metadata-toolchain}" -STAGE_ROOT="${SIGNOZ_METADATA_STAGE_ROOT:-/tmp}" -SSH_TIMEOUT_SECONDS="${SIGNOZ_METADATA_SSH_TIMEOUT_SECONDS:-120}" -SCP_TIMEOUT_SECONDS="${SIGNOZ_METADATA_SCP_TIMEOUT_SECONDS:-120}" -REMOTE_TIMEOUT_SECONDS="${SIGNOZ_METADATA_REMOTE_TIMEOUT_SECONDS:-60}" -KILL_AFTER_SECONDS="${SIGNOZ_METADATA_KILL_AFTER_SECONDS:-10}" -LOCAL_PYTHON_BIN="${SIGNOZ_METADATA_LOCAL_PYTHON_BIN:-python3.11}" -SSH_IDENTITY_FILE="${SIGNOZ_METADATA_SSH_IDENTITY_FILE:-}" -SSH_KNOWN_HOSTS_FILE="${SIGNOZ_METADATA_KNOWN_HOSTS_FILE:-}" - -LABELS=( - metadata-export-policy.json - signoz_metadata_contract.py - signoz-metadata-export.py - verify-signoz-metadata-export.py - signoz-metadata-restore-drill.py - signoz-metadata-isolated-restore.py - isolated-cluster.xml -) -LOCAL_SOURCES=( - "${ROOT_DIR}/config/signoz/metadata-export-policy.json" - "${ROOT_DIR}/scripts/backup/signoz_metadata_contract.py" - "${ROOT_DIR}/scripts/backup/signoz-metadata-export.py" - "${ROOT_DIR}/scripts/backup/verify-signoz-metadata-export.py" - "${ROOT_DIR}/scripts/backup/signoz-metadata-restore-drill.py" - "${ROOT_DIR}/scripts/backup/signoz-metadata-isolated-restore.py" - "${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" -) -MODES=(0644 0644 0755 0755 0755 0755 0644) -SSH_OPTIONS=( - -o BatchMode=yes - -o IdentitiesOnly=yes - -o StrictHostKeyChecking=yes - -o ConnectTimeout=8 - -o ConnectionAttempts=1 - -o ServerAliveInterval=5 - -o ServerAliveCountMax=2 -) -if [ -n "${SSH_IDENTITY_FILE}" ]; then - [ -f "${SSH_IDENTITY_FILE}" ] && [ ! -L "${SSH_IDENTITY_FILE}" ] || { - echo "invalid SSH identity file" >&2 - exit 64 - } - SSH_OPTIONS+=( -i "${SSH_IDENTITY_FILE}" ) -fi -if [ -n "${SSH_KNOWN_HOSTS_FILE}" ]; then - [ -f "${SSH_KNOWN_HOSTS_FILE}" ] && [ ! -L "${SSH_KNOWN_HOSTS_FILE}" ] || { - echo "invalid SSH known_hosts file" >&2 - exit 64 - } - SSH_OPTIONS+=( -o "UserKnownHostsFile=${SSH_KNOWN_HOSTS_FILE}" ) -fi - -usage() { - cat <<'USAGE' -Usage: deploy-signoz-metadata-toolchain.sh [--check|--apply] - -Required environment: - TRACE_ID Path-safe controlled-apply trace identifier - RUN_ID Unique path-safe execution identifier - WORK_ITEM_ID Must be P0-OBS-002 - -Optional fixed transport paths: - SIGNOZ_METADATA_SSH_IDENTITY_FILE - SIGNOZ_METADATA_KNOWN_HOSTS_FILE - ---check validates the seven local sources and performs a no-write host110 diff. ---apply creates one immutable exact-hash directory. It does not create or -change an active pointer and does not run an authenticated metadata export. -USAGE -} - -MODE="${1:---check}" -case "${MODE}" in - --check|--apply) ;; - -h|--help) usage; exit 0 ;; - *) usage >&2; exit 64 ;; -esac -[ "$#" -le 1 ] || { usage >&2; exit 64; } - -TRACE_ID="${TRACE_ID:-}" -RUN_ID="${RUN_ID:-}" -WORK_ITEM_ID="${WORK_ITEM_ID:-}" - -valid_identity() { - [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] -} - -valid_positive_integer() { - [[ "$1" =~ ^[1-9][0-9]*$ ]] -} - -safe_absolute_path() { - local path="$1" - [[ "${path}" == /* ]] || return 1 - [[ "${path}" != / && "${path}" != */ ]] || return 1 - [[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1 - case "/${path#/}/" in - *'//'*|*'/../'*|*'/./'*) return 1 ;; - esac -} - -valid_identity "${TRACE_ID}" || { echo "invalid TRACE_ID" >&2; exit 64; } -valid_identity "${RUN_ID}" || { echo "invalid RUN_ID" >&2; exit 64; } -[ "${WORK_ITEM_ID}" = "P0-OBS-002" ] || { - echo "WORK_ITEM_ID must be P0-OBS-002" >&2 - exit 64 -} -for value in "${SSH_TIMEOUT_SECONDS}" "${SCP_TIMEOUT_SECONDS}" \ - "${REMOTE_TIMEOUT_SECONDS}" "${KILL_AFTER_SECONDS}"; do - valid_positive_integer "${value}" || { echo "invalid timeout" >&2; exit 64; } -done -for path in "${REMOTE_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do - safe_absolute_path "${path}" || { echo "unsafe remote path" >&2; exit 64; } -done - -bounded_ssh() { - timeout --kill-after="${KILL_AFTER_SECONDS}" "${SSH_TIMEOUT_SECONDS}" ssh "$@" -} - -bounded_scp() { - timeout --kill-after="${KILL_AFTER_SECONDS}" "${SCP_TIMEOUT_SECONDS}" scp "$@" -} - -hash_file() { - sha256sum "$1" | awk '{print $1}' -} - -emit() { - local phase="$1" terminal="$2" detail="$3" - printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-metadata-toolchain","phase":"%s","terminal":"%s","detail":"%s"}\n' \ - "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${terminal}" "${detail}" -} - -validate_local_sources() { - local source - command -v "${LOCAL_PYTHON_BIN}" >/dev/null 2>&1 - "${LOCAL_PYTHON_BIN}" -c 'import sys; assert sys.version_info >= (3, 10)' - for source in "${LOCAL_SOURCES[@]}"; do - [ -f "${source}" ] && [ ! -L "${source}" ] && [ -s "${source}" ] - done - "${LOCAL_PYTHON_BIN}" - "${LOCAL_SOURCES[@]:1}" <<'PY' -from pathlib import Path -import sys - -for value in sys.argv[1:]: - path = Path(value) - if path.suffix == ".py": - compile(path.read_text(encoding="utf-8"), str(path), "exec") -PY - PYTHONPATH="${ROOT_DIR}/scripts/backup" "${LOCAL_PYTHON_BIN}" - \ - "${LOCAL_SOURCES[0]}" <<'PY' -from pathlib import Path -import sys -from signoz_metadata_contract import load_policy - -policy = load_policy(Path(sys.argv[1])) -assert policy["work_item_id"] == "P0-OBS-002" -assert policy["source_runtime"]["raw_database_access_allowed"] is False -assert policy["source_runtime"]["raw_volume_access_allowed"] is False -assert policy["completion_contract"]["full_sqlite_completion_claim_allowed"] is False -PY -} - -validate_local_sources -SOURCE_HASHES=() -SOURCE_DETAIL="" -for index in "${!LOCAL_SOURCES[@]}"; do - source_hash="$(hash_file "${LOCAL_SOURCES[index]}")" - SOURCE_HASHES+=("${source_hash}") - SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_" -done -SOURCE_DETAIL="${SOURCE_DETAIL%_}" -BUNDLE_ID="$(printf '%s\n' "${SOURCE_HASHES[@]}" | sha256sum | awk '{print $1}')" -TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}" -STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-metadata-toolchain.${RUN_ID}" - -emit sensor_source pass "local_exact_sources_7" -emit normalized_asset_identity pass "bundle_${BUNDLE_ID}" -emit ai_decision pass "candidate_additive_immutable_inactive_source_deploy" -emit risk_policy_decision pass "medium_no_active_pointer_no_secret_no_raw_sqlite" - -remote_check() { - # The immutable root is deliberately 0700/root-owned. Run this bounded - # verifier as root so it can distinguish exact from absent without relaxing - # directory permissions. REMOTE_CHECK contains only stat/hash/process/API - # reads and performs no writes. - bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" sudo -n bash -s -- \ - "${REMOTE_ROOT}" "${BUNDLE_ID}" "${REMOTE_TIMEOUT_SECONDS}" \ - "${KILL_AFTER_SECONDS}" "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK' -set -euo pipefail - -REMOTE_ROOT="$1" -BUNDLE_ID="$2" -REMOTE_TIMEOUT_SECONDS="$3" -KILL_AFTER_SECONDS="$4" -shift 4 -EXPECTED_HASHES=("$@") -LABELS=( - metadata-export-policy.json - signoz_metadata_contract.py - signoz-metadata-export.py - verify-signoz-metadata-export.py - signoz-metadata-restore-drill.py - signoz-metadata-isolated-restore.py - isolated-cluster.xml -) -MODES=(0644 0644 0755 0755 0755 0755 0644) -TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}" - -[ "${#EXPECTED_HASHES[@]}" -eq 7 ] -[ -d /backup ] && [ ! -L /backup ] -[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ] -for command_name in curl docker pgrep python3 sha256sum stat timeout; do - command -v "${command_name}" >/dev/null 2>&1 -done -python3 -c 'import sys; assert sys.version_info >= (3, 10)' - -container_state="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \ - "${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \ - '{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)" -api_status="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \ - "${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \ - http://127.0.0.1:8080/api/v1/health)" -[ "${api_status}" = "200" ] - -active_operations="$( - (pgrep -af '([s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$))' || true) \ - | wc -l | tr -d ' ' -)" -[ "${active_operations}" = "0" ] - -state=absent -drift=0 -if [ -e "${TARGET_DIR}" ] || [ -L "${TARGET_DIR}" ]; then - [ -d "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ] || drift=$((drift + 1)) - for index in "${!LABELS[@]}"; do - target="${TARGET_DIR}/${LABELS[index]}" - if [ ! -f "${target}" ] || [ -L "${target}" ]; then - drift=$((drift + 1)) - continue - fi - [ "$(sha256sum "${target}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ] || drift=$((drift + 1)) - [ "0$(stat -c '%a' "${target}")" = "${MODES[index]}" ] || drift=$((drift + 1)) - done - state=exact - [ "${drift}" -eq 0 ] || state=drift -fi - -candidate_count=0 -for candidate in "${REMOTE_ROOT}"/."${BUNDLE_ID}".candidate.*; do - [ -e "${candidate}" ] || [ -L "${candidate}" ] || continue - candidate_count=$((candidate_count + 1)) -done -[ "${candidate_count}" -eq 0 ] -[ "${drift}" -eq 0 ] -printf 'REMOTE_CHECK terminal=check_pass_no_write state=%s drift=%s candidate_count=%s api=%s active_operations=%s container_state_sha256=%s\n' \ - "${state}" "${drift}" "${candidate_count}" "${api_status}" "${active_operations}" \ - "$(printf '%s' "${container_state}" | sha256sum | awk '{print $1}')" -REMOTE_CHECK -} - -quarantine_transport_stage() { - bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" sudo -n bash -s -- \ - "${STAGE_DIR}" "${RECEIPT_ROOT}" "${RUN_ID}" <<'REMOTE_QUARANTINE' -set -euo pipefail -stage_dir="$1" -receipt_root="$2" -run_id="$3" -receipt_dir="${receipt_root}/${run_id}" -if [ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then - exit 0 -fi -[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ] -mkdir -p "${receipt_root}" -if [ ! -e "${receipt_dir}" ] && [ ! -L "${receipt_dir}" ]; then - mkdir -m 0700 "${receipt_dir}" -fi -[ -d "${receipt_dir}" ] && [ ! -L "${receipt_dir}" ] -[ ! -e "${receipt_dir}/transport-stage" ] && [ ! -L "${receipt_dir}/transport-stage" ] -mv "${stage_dir}" "${receipt_dir}/transport-stage" -REMOTE_QUARANTINE -} - -if ! REMOTE_CHECK_OUTPUT="$(remote_check)"; then - emit source_of_truth_diff failed "remote_read_only_check_failed" - emit terminal failed_no_write "remote_check_failed" - exit 1 -fi -case "${REMOTE_CHECK_OUTPUT}" in - *"terminal=check_pass_no_write state=absent "*) REMOTE_STATE=absent ;; - *"terminal=check_pass_no_write state=exact "*) REMOTE_STATE=exact ;; - *) - emit source_of_truth_diff failed "remote_check_receipt_invalid" - emit terminal failed_no_write "remote_check_receipt_invalid" - exit 1 - ;; -esac -emit source_of_truth_diff pass "remote_state_${REMOTE_STATE}_drift_0" - -if [ "${MODE}" = "--check" ]; then - emit check_mode check_pass_no_write "bundle_${BUNDLE_ID}_remote_state_${REMOTE_STATE}" - emit rollback no_write "active_pointer_unchanged" - emit terminal check_pass_no_write "source_ready_remote_state_${REMOTE_STATE}" - exit 0 -fi - -if [ "${REMOTE_STATE}" = "exact" ]; then - emit check_mode pass "immutable_bundle_already_exact" - emit bounded_execution idempotent_no_write "bundle_${BUNDLE_ID}" - emit independent_post_verifier pass "remote_exact_hash_mode_api_identity" - emit rollback no_write "active_pointer_absent" - emit learning_writeback partial_degraded "runtime_export_not_executed" - emit terminal pass_source_deployed_inactive "idempotent_runtime_export_pending" - exit 0 -fi - -if ! bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" bash -s -- \ - "${STAGE_DIR}" <<'REMOTE_STAGE'; then -set -euo pipefail -stage_dir="$1" -[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ] -stage_root="${stage_dir%/*}" -[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ] -umask 077 -mkdir -m 0700 "${stage_dir}" -REMOTE_STAGE - emit bounded_execution failed "transport_stage_create_failed" - emit terminal failed_no_active_pointer_write "stage_create_failed" - exit 1 -fi - -for index in "${!LOCAL_SOURCES[@]}"; do - if ! bounded_scp -q "${SSH_OPTIONS[@]}" "${LOCAL_SOURCES[index]}" \ - "${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"; then - quarantine_transport_stage >/dev/null 2>&1 || true - emit bounded_execution failed "transport_failed_index_${index}" - emit terminal failed_no_active_pointer_write "transport_stage_quarantine_attempted" - exit 1 - fi -done - -if ! REMOTE_APPLY_OUTPUT="$(bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" \ - sudo -n bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ - "${REMOTE_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" "${BUNDLE_ID}" \ - "${REMOTE_TIMEOUT_SECONDS}" "${KILL_AFTER_SECONDS}" \ - "${SOURCE_HASHES[@]}" <<'REMOTE_APPLY' -set -euo pipefail - -TRACE_ID="$1" -RUN_ID="$2" -WORK_ITEM_ID="$3" -REMOTE_ROOT="$4" -RECEIPT_ROOT="$5" -STAGE_DIR="$6" -BUNDLE_ID="$7" -REMOTE_TIMEOUT_SECONDS="$8" -KILL_AFTER_SECONDS="$9" -shift 9 -EXPECTED_HASHES=("$@") -LABELS=( - metadata-export-policy.json - signoz_metadata_contract.py - signoz-metadata-export.py - verify-signoz-metadata-export.py - signoz-metadata-restore-drill.py - signoz-metadata-isolated-restore.py - isolated-cluster.xml -) -MODES=(0644 0644 0755 0755 0755 0755 0644) -TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}" -CANDIDATE_DIR="${REMOTE_ROOT}/.${BUNDLE_ID}.candidate.${RUN_ID}" -RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" -RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl" -DEPLOYED=0 -TERMINAL=failed - -emit_remote() { - local phase="$1" terminal="$2" detail="$3" - printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-metadata-toolchain","phase":"%s","terminal":"%s","detail":"%s"}\n' \ - "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${terminal}" "${detail}" | tee -a "${RECEIPT_LOG}" -} - -finalize() { - local rc=$? quarantine_terminal=not_required - trap - EXIT HUP INT TERM - set +e - if [ -d "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]; then - if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then - mv "${STAGE_DIR}" "${RECEIPT_DIR}/transport-stage" - fi - if [ "${rc}" -ne 0 ]; then - quarantine_terminal=inactive_candidates_preserved - if [ -d "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]; then - mv "${CANDIDATE_DIR}" "${RECEIPT_DIR}/quarantine-candidate" - fi - if [ "${DEPLOYED}" -eq 1 ] && [ -d "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]; then - mv "${TARGET_DIR}" "${RECEIPT_DIR}/quarantine-target" - fi - fi - emit_remote rollback "${quarantine_terminal}" "active_pointer_never_created" - emit_remote terminal "${TERMINAL}" "exit_${rc}_runtime_export_not_executed" - fi - exit "${rc}" -} -trap finalize EXIT HUP INT TERM - -[ "${#EXPECTED_HASHES[@]}" -eq 7 ] -[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] -[ ! -e "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ] -[ ! -e "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ] -[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ] -[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ] -python3 -c 'import sys; assert sys.version_info >= (3, 10)' -umask 077 -mkdir -p "${REMOTE_ROOT}" "${RECEIPT_ROOT}" -mkdir -m 0700 "${RECEIPT_DIR}" "${CANDIDATE_DIR}" -: > "${RECEIPT_LOG}" -chmod 0600 "${RECEIPT_LOG}" - -OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock -[ -f "${OPERATION_LOCK}" ] && [ ! -L "${OPERATION_LOCK}" ] -# The shared lock is intentionally owned by the unprivileged backup account. -# Open the existing inode read-only so Linux fs.protected_regular=2 cannot -# reject a root O_CREAT/O_APPEND open in sticky /tmp. flock(2) still provides -# the same exclusive inter-process lock without changing file contents, -# ownership, or mode. Missing/unsafe lock state fails closed. -exec 8<"${OPERATION_LOCK}" -flock -n 8 -active_operations="$( - (pgrep -af '([s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$))' || true) \ - | wc -l | tr -d ' ' -)" -[ "${active_operations}" = "0" ] - -runtime_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \ - "${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \ - '{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)" -api_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \ - "${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \ - http://127.0.0.1:8080/api/v1/health)" -[ "${api_before}" = "200" ] -emit_remote sensor_source pass "staged_sources_7_runtime_api_200" - -for index in "${!LABELS[@]}"; do - staged="${STAGE_DIR}/${LABELS[index]}" - [ -f "${staged}" ] && [ ! -L "${staged}" ] - [ "$(sha256sum "${staged}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ] - install -o root -g root -m "${MODES[index]}" "${staged}" \ - "${CANDIDATE_DIR}/${LABELS[index]}" -done - -python3 - "${CANDIDATE_DIR}" <<'PY' -from pathlib import Path -import sys - -root = Path(sys.argv[1]) -for name in ( - "signoz_metadata_contract.py", - "signoz-metadata-export.py", - "verify-signoz-metadata-export.py", - "signoz-metadata-restore-drill.py", - "signoz-metadata-isolated-restore.py", -): - path = root / name - compile(path.read_text(encoding="utf-8"), str(path), "exec") -PY -PYTHONPATH="${CANDIDATE_DIR}" python3 "${CANDIDATE_DIR}/signoz-metadata-export.py" \ - --check --base-url https://signoz.invalid \ - --output-dir "${RECEIPT_DIR}/offline-output-must-not-exist" \ - --policy "${CANDIDATE_DIR}/metadata-export-policy.json" \ - --trace-id "${TRACE_ID}" --run-id "${RUN_ID}" \ - --work-item-id "${WORK_ITEM_ID}" >/dev/null -[ ! -e "${RECEIPT_DIR}/offline-output-must-not-exist" ] -emit_remote check_mode pass "candidate_hash_mode_compile_policy_check" - -mv "${CANDIDATE_DIR}" "${TARGET_DIR}" -chmod 0750 "${TARGET_DIR}" -DEPLOYED=1 - -for index in "${!LABELS[@]}"; do - target="${TARGET_DIR}/${LABELS[index]}" - [ -f "${target}" ] && [ ! -L "${target}" ] - [ "$(sha256sum "${target}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ] - [ "0$(stat -c '%a' "${target}")" = "${MODES[index]}" ] - printf '%s\t%s\t%s\n' "${LABELS[index]}" "${EXPECTED_HASHES[index]}" "${MODES[index]}" \ - >> "${RECEIPT_DIR}/deployed-manifest.tsv" -done -chmod 0600 "${RECEIPT_DIR}/deployed-manifest.tsv" -runtime_after="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \ - "${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \ - '{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)" -api_after="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \ - "${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \ - http://127.0.0.1:8080/api/v1/health)" -[ "${runtime_after}" = "${runtime_before}" ] -[ "${api_after}" = "200" ] -emit_remote bounded_execution pass "immutable_bundle_deployed_active_pointer_0" -emit_remote independent_post_verifier pass "hash_mode_compile_api_identity_unchanged" -emit_remote learning_writeback partial_degraded "authenticated_export_restore_pending" -TERMINAL=pass_source_deployed_inactive -REMOTE_APPLY -)"; then - quarantine_transport_stage >/dev/null 2>&1 || true - emit bounded_execution failed "remote_apply_failed_inactive_candidate_quarantined" - emit terminal failed_no_active_pointer_write "runtime_export_not_executed" - exit 1 -fi -case "${REMOTE_APPLY_OUTPUT}" in - *'"phase":"terminal","terminal":"pass_source_deployed_inactive"'*) ;; - *) - emit independent_post_verifier failed "remote_apply_terminal_invalid" - emit terminal failed_no_active_pointer_write "runtime_export_not_executed" - exit 1 - ;; -esac - -if ! POSTCHECK_OUTPUT="$(remote_check)"; then - emit independent_post_verifier failed "remote_postcheck_failed" - emit terminal failed_source_deploy_unverified "active_pointer_0" - exit 1 -fi -case "${POSTCHECK_OUTPUT}" in - *"terminal=check_pass_no_write state=exact "*) ;; - *) - emit independent_post_verifier failed "remote_postcheck_receipt_invalid" - emit terminal failed_source_deploy_unverified "active_pointer_0" - exit 1 - ;; -esac -emit check_mode pass "precheck_absent" -emit bounded_execution pass "immutable_bundle_${BUNDLE_ID}_active_pointer_0" -emit independent_post_verifier pass "postcheck_exact_hash_mode_api_identity" -emit rollback not_required "inactive_additive_bundle" -emit learning_writeback partial_degraded "runtime_export_restore_zero_residue_pending" -emit terminal pass_source_deployed_inactive "runtime_export_not_executed" +printf '%s\n' \ + '{"schema":"awoooi_signoz_metadata_toolchain_route_retirement_v1","terminal":"blocked_direct_host110_route_retired","replacement":"scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh","production_mutation_performed":false,"completion_claim":false}' +exit 78 diff --git a/scripts/ops/tests/test_agent99_signoz_credential_provisioner.py b/scripts/ops/tests/test_agent99_signoz_credential_provisioner.py index 5612b7425..a1f382ca0 100644 --- a/scripts/ops/tests/test_agent99_signoz_credential_provisioner.py +++ b/scripts/ops/tests/test_agent99_signoz_credential_provisioner.py @@ -149,7 +149,7 @@ def test_agent99_entrypoint_is_fixed_and_never_transports_secret() -> None: assert '[ValidateSet("Check", "Apply", "Verify")]' in source assert '$TargetHost = "192.168.0.110"' in source assert '$ExpectedWorkItemId = "P0-OBS-002"' in source - assert '$ExpectedRuntimeFileCount = 20' in source + assert '$ExpectedRuntimeFileCount = 21' in source assert '"sudo -n /usr/bin/python3 - --mode $($Mode.ToLowerInvariant())"' in source assert "$process.Refresh()" in source assert 'if (-not $process.HasExited) { throw "target_exit_state_unavailable" }' in source diff --git a/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py b/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py index 1169397bc..79f7dd1d3 100644 --- a/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py +++ b/scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py @@ -4,7 +4,6 @@ import hashlib import re from pathlib import Path - ROOT = Path(__file__).resolve().parents[3] ENTRYPOINT = ROOT / "agent99-signoz-metadata-executor.ps1" @@ -20,7 +19,8 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None: assert ( '[ValidateSet("Check", "Apply", "Verify", "RestoreCheck", ' - '"RestoreApply", "RestoreVerifyCleanup")]' in source + '"RestoreApply", "RestoreVerifyCleanup", "ToolchainCheck", ' + '"ToolchainApply", "ToolchainVerify", "ToolchainReconcile")]' in source ) assert '$CanonicalAsset = "signoz-110-control-plane-metadata"' in source assert '$TargetHost = "192.168.0.110"' in source @@ -28,24 +28,26 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None: assert '$WorkItemId -ne "P0-OBS-002"' in source assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source - assert '$EvidenceRunId = if ($RunId -match $SafeIdPattern)' in source + assert "$EvidenceRunId = if ($RunId -match $SafeIdPattern)" in source assert "Get-Agent99RuntimeSourceBinding" in source - assert "$result.fileCount -eq 20" in source + assert "$result.fileCount -eq 21" in source assert "$result.sourceRevision -eq $SourceRevision" in source assert "$result.entrypointHashMatched" in source assert '$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"' in source assert ( - '$BundleId = "ed4e5fbfc5f732b64d6a78f6ab26b90118e1d42a4f38b40d583876445c32e3aa"' + '$BundleId = "b83bfc898cd43cf8ec5688d75ec20231c2faa32de5be3fa886db33038d620dff"' in source ) assert re.search(r'\$BundleId = "[0-9a-f]{64}"', source) - assert '$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"' in source - assert '$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"' in source - assert '$BaseUrl = "http://127.0.0.1:8080"' in source assert ( - '$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"' + '$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"' in source ) + assert ( + '$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"' in source + ) + assert '$BaseUrl = "http://127.0.0.1:8080"' in source + assert '$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"' in source assert '$OutputDir = "/backup/signoz-metadata-export-$RunId"' not in source assert '$SignozImage = "signoz/signoz:v0.113.0"' in source assert '$SignozContainer = "signoz"' in source @@ -63,11 +65,13 @@ def test_entrypoint_pins_current_seven_file_toolchain_bundle() -> None: ("ExporterPath", ROOT / "scripts/backup/signoz-metadata-export.py"), ("VerifierPath", ROOT / "scripts/backup/verify-signoz-metadata-export.py"), ("RestoreDriverPath", ROOT / "scripts/backup/signoz-metadata-restore-drill.py"), - ("IsolatedRestorePath", ROOT / "scripts/backup/signoz-metadata-isolated-restore.py"), + ( + "IsolatedRestorePath", + ROOT / "scripts/backup/signoz-metadata-isolated-restore.py", + ), ( "IsolatedClusterConfigPath", - ROOT - / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml", + ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml", ), ) hashes = [hashlib.sha256(path.read_bytes()).hexdigest() for _, path in rows] @@ -86,8 +90,9 @@ def test_entrypoint_pins_current_seven_file_toolchain_bundle() -> None: def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> None: source = ENTRYPOINT.read_text(encoding="utf-8") lock_helper = source[ - source.index("function New-Agent99SignozLockedRemoteCommand") : - source.index("function Get-Agent99RuntimeSourceBinding") + source.index("function New-Agent99SignozLockedRemoteCommand") : source.index( + "function Get-Agent99RuntimeSourceBinding" + ) ] assert 'Start-Process -FilePath "ssh.exe"' in source @@ -95,9 +100,9 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> assert '"-o", "IdentitiesOnly=yes"' in source assert '"-o", "StrictHostKeyChecking=yes"' in source assert '"-o", "NumberOfPasswordPrompts=0"' in source - assert 'windows99_agent99_to_host110_fixed_metadata_toolchain' in source - assert 'arbitraryCommandAllowed = $false' in source - assert 'crossDomainFallbackAllowed = $false' in source + assert "windows99_agent99_to_host110_fixed_metadata_toolchain" in source + assert "arbitraryCommandAllowed = $false" in source + assert "crossDomainFallbackAllowed = $false" in source assert "Invoke-Expression" not in source assert "-EncodedCommand" not in source assert "cmd.exe" not in source @@ -128,9 +133,9 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> ) assert "targetTimeoutFinishesBeforeControllerWait" in source assert "target_timeout_not_bounded_before_controller_wait" in source - assert '--kill-after=5s 15s /usr/bin/docker inspect' in source - assert '--kill-after=5s 15s /usr/bin/curl' in source - assert '--kill-after=5s 10s /usr/bin/pgrep' in lock_helper + assert "--kill-after=5s 15s /usr/bin/docker inspect" in source + assert "--kill-after=5s 15s /usr/bin/curl" in source + assert "--kill-after=5s 10s /usr/bin/pgrep" in lock_helper assert "active_rc=`$?; set -e" in lock_helper assert '[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]' in lock_helper assert "$bashProgram = (" in lock_helper @@ -142,49 +147,52 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> assert "sudo -n /bin/bash -c '" not in lock_helper assert "pgrep -af" in lock_helper assert "|| true" not in lock_helper - assert '--signal=TERM --kill-after=$($TargetKillAfterSeconds)s' in source + assert "--signal=TERM --kill-after=$($TargetKillAfterSeconds)s" in source def test_entrypoint_uses_typed_stdout_without_windows_process_exit_code() -> None: source = ENTRYPOINT.read_text(encoding="utf-8") transport = source[ - source.index("function Invoke-Agent99SignozFixedRemote") : - source.index("function New-Agent99SignozLockedRemoteCommand") + source.index("function Invoke-Agent99SignozFixedRemote") : source.index( + "function Test-Agent99SignozTransportOutcome" + ) ] assert "$process.ExitCode" not in transport assert "exitCodeObserved" not in transport assert "AllowedExitCodes" not in transport - assert "function Test-Agent99SignozTransportOutcome" in transport - assert "$Result.completed -eq $true" in transport - assert "-not $Result.timedOut" in transport - assert '$process.WaitForExit($TimeoutSeconds * 1000)' in transport + assert "function Test-Agent99SignozTransportOutcome" in source + assert "$Result.completed -eq $true" in source + assert "-not $Result.timedOut" in source + assert "$process.WaitForExit($TimeoutSeconds * 1000)" in transport assert "taskkill.exe /PID $process.Id /T /F" in transport -def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contracts() -> None: +def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contracts() -> ( + None +): source = ENTRYPOINT.read_text(encoding="utf-8") - assert 'function New-Agent99SignozPathStateCommand' in source - assert 'agent99_credential:symlink' in source - assert 'agent99_credential:absent' in source - assert 'agent99_credential:present:%u:%a:%s' in source - assert 'agent99_path_state:$ExpectedKind' in source - assert 'agent99_path_state:absent' in source - assert 'agent99_hashes_begin' in source - assert 'agent99_hashes_end' in source - assert '/usr/bin/echo agent99_hashes_begin' in source - assert '/usr/bin/echo agent99_hashes_end' in source - assert 'agent99_hashes_begin\\n' not in source - assert 'agent99_image:' in source - assert 'agent99_runtime:' in source - assert 'agent99_health:' in source - assert 'agent99_active:' in source + assert "function New-Agent99SignozPathStateCommand" in source + assert "agent99_credential:symlink" in source + assert "agent99_credential:absent" in source + assert "agent99_credential:present:%u:%a:%s" in source + assert "agent99_path_state:$ExpectedKind" in source + assert "agent99_path_state:absent" in source + assert "agent99_hashes_begin" in source + assert "agent99_hashes_end" in source + assert "/usr/bin/echo agent99_hashes_begin" in source + assert "/usr/bin/echo agent99_hashes_end" in source + assert "agent99_hashes_begin\\n" not in source + assert "agent99_image:" in source + assert "agent99_runtime:" in source + assert "agent99_health:" in source + assert "agent99_active:" in source assert '$credentialState -in @("absent", "symlink", "other")' in source - assert 'credentialReadbackVerified = $credentialReadbackVerified' in source - assert 'transportReadbackVerified = $transportReadbackVerified' in source - assert 'outputStateReadbackVerified = $outputStateReadbackVerified' in source - assert 'receiptStateReadbackVerified = $receiptStateReadbackVerified' in source + assert "credentialReadbackVerified = $credentialReadbackVerified" in source + assert "transportReadbackVerified = $transportReadbackVerified" in source + assert "outputStateReadbackVerified = $outputStateReadbackVerified" in source + assert "receiptStateReadbackVerified = $receiptStateReadbackVerified" in source assert '"credential_reference_readback_unverified"' in source assert '"preflight_transport_unverified"' in source assert source.count("Test-Agent99SignozTransportOutcome $verify") == 2 @@ -195,24 +203,29 @@ def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contrac def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> None: source = ENTRYPOINT.read_text(encoding="utf-8") preflight = source[ - source.index("function Get-Agent99SignozPreflight") : - source.index("function Write-Agent99SignozEvidenceCollision") + source.index("function Get-Agent99SignozPreflight") : source.index( + "function Write-Agent99SignozEvidenceCollision" + ) ] main = source[ source.index( - '$preflight = Get-Agent99SignozPreflight ' - '-ExpectedArtifactState "absent"' + "$preflight = Get-Agent99SignozPreflight " '-ExpectedArtifactState "absent"' ) : ] assert "$hashLines.Count -eq $ToolchainFiles.Count + 2" in preflight - assert "$hashLines[0] -eq \"agent99_hashes_begin\"" in preflight + assert '$hashLines[0] -eq "agent99_hashes_begin"' in preflight assert '$hashLines[$hashLines.Count - 1] -eq "agent99_hashes_end"' in preflight assert "$actualHashes.ContainsKey($path)" in preflight assert "-not $ToolchainFiles.Contains($path)" in preflight assert '"^agent99_image:(sha256:[0-9a-f]{64})$"' in preflight - assert '--format={{.Id}},{{.Image}},{{.Config.Image}},{{.State.Running}}' in preflight - assert '--format=`"{{.Id}}|{{.Image}}|{{.Config.Image}}|{{.State.Running}}`"' not in preflight + assert ( + "--format={{.Id}},{{.Image}},{{.Config.Image}},{{.State.Running}}" in preflight + ) + assert ( + '--format=`"{{.Id}}|{{.Image}}|{{.Config.Image}}|{{.State.Running}}`"' + not in preflight + ) assert '"^agent99_runtime:([0-9a-f]{64}),(sha256:' in preflight assert '"^agent99_active:([0-9]+)$"' in preflight assert '[ `"`$active_rc`" -eq 0 ] || [ `"`$active_rc`" -eq 1 ]' in preflight @@ -258,69 +271,72 @@ def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> No def test_entrypoint_never_reads_or_persists_credential_value_in_controller() -> None: source = ENTRYPOINT.read_text(encoding="utf-8") - assert '/usr/bin/stat -c `"agent99_credential:present:%u:%a:%s`" `"$CredentialFile`"' in source - assert 'credentialValueRead = $false' in source - assert 'credentialValueOutput = $false' in source - assert 'secretValueReadByController = $false' in source - assert 'secretValueTransmitted = $false' in source - assert 'secretValuePersistedInEvidence = $false' in source + assert ( + '/usr/bin/stat -c `"agent99_credential:present:%u:%a:%s`" `"$CredentialFile`"' + in source + ) + assert "credentialValueRead = $false" in source + assert "credentialValueOutput = $false" in source + assert "secretValueReadByController = $false" in source + assert "secretValueTransmitted = $false" in source + assert "secretValuePersistedInEvidence = $false" in source assert "credentialReferenceDispatchAttempted" in source assert "$script:ExportCredentialDispatchAttempted = $true" in source assert "$script:IsolatedApplyDispatchAttempted = $true" in source - assert "credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted" in source + assert ( + "credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted" + in source + ) assert "credentialReferenceConsumedOnTarget = if" in source assert "elseif (Test-Agent99SignozReceiptIdentity $ApplyReceipt" in source - assert 'rawSqliteRead = $false' in source - assert 'rawVolumeRead = $false' in source + assert "rawSqliteRead = $false" in source + assert "rawVolumeRead = $false" in source assert "Get-Content -LiteralPath $CredentialFile" not in source assert "ReadAllText($CredentialFile" not in source assert "ReadAllBytes($CredentialFile" not in source -def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal() -> None: +def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal() -> ( + None +): source = ENTRYPOINT.read_text(encoding="utf-8") assert 'Receipt.schema -eq "awoooi_signoz_metadata_receipt_v1"' in source assert "awoooi_controlled_apply_receipt_v1" not in source - assert '[string]$Receipt.phase -eq $ExpectedPhase' in source + assert "[string]$Receipt.phase -eq $ExpectedPhase" in source assert ( - 'Test-Agent99SignozReceiptIdentity $verifyReceipt ' + "Test-Agent99SignozReceiptIdentity $verifyReceipt " '"independent_export_verifier"' in source ) - assert ( - 'Test-Agent99SignozReceiptIdentity $checkReceipt "terminal"' in source - ) - assert ( - 'Test-Agent99SignozReceiptIdentity $applyReceipt "terminal"' in source - ) + assert 'Test-Agent99SignozReceiptIdentity $checkReceipt "terminal"' in source + assert 'Test-Agent99SignozReceiptIdentity $applyReceipt "terminal"' in source assert "$ExporterPath --check" in source assert "$ExporterPath --apply" in source - assert '--receipt-file $RemoteReceipt' in source - assert 'pass_export_verified_restore_pending' in source - assert 'export_verified_restore_pending' in source + assert "--receipt-file $RemoteReceipt" in source + assert "pass_export_verified_restore_pending" in source + assert "export_verified_restore_pending" in source assert 'else { "pending_isolated_same_version_target" }' in source assert 'else { "pending" }' in source - assert 'completionClaim = $false' in source + assert "completionClaim = $false" in source assert "controlledApplyPhases = [pscustomobject]$controlledApplyPhases" in source - assert 'credential_reference_absent' in source - assert 'blocked_with_safe_next_action' in source - assert 'apply_failed_artifact_preserved_for_readback' in source - assert 'post_verifier_runtime_or_artifact_drift' in source + assert "credential_reference_absent" in source + assert "blocked_with_safe_next_action" in source + assert "apply_failed_artifact_preserved_for_readback" in source + assert "post_verifier_runtime_or_artifact_drift" in source def test_restore_modes_are_bounded_distinct_and_cleanup_remains_available() -> None: source = ENTRYPOINT.read_text(encoding="utf-8") main_start = source.index("foreach ($identity") - cleanup_start = source.index( - 'if ($Mode -eq "RestoreVerifyCleanup")', main_start - ) + cleanup_start = source.index('if ($Mode -eq "RestoreVerifyCleanup")', main_start) restore_apply_start = source.index( 'if ($Mode -in @("RestoreCheck", "RestoreApply"))', cleanup_start ) export_verify_start = source.index('if ($Mode -eq "Verify")', restore_apply_start) cleanup_helper = source[ - source.index("function New-Agent99SignozCleanupLockedRemoteCommand") : - source.index("function Get-Agent99RuntimeSourceBinding") + source.index( + "function New-Agent99SignozCleanupLockedRemoteCommand" + ) : source.index("function Get-Agent99RuntimeSourceBinding") ] cleanup_branch = source[cleanup_start:restore_apply_start] restore_apply_branch = source[restore_apply_start:export_verify_start] @@ -328,18 +344,82 @@ def test_restore_modes_are_bounded_distinct_and_cleanup_remains_available() -> N assert "Get-Agent99SignozCleanupPreflight" in cleanup_branch assert "New-Agent99SignozCleanupLockedRemoteCommand" in cleanup_branch assert "Get-Agent99SignozPreflight" not in cleanup_branch - assert "$OutputDir" not in cleanup_branch + assert "--bundle-dir $OutputDir" in cleanup_branch assert "$BaseUrl/api/v1/health" not in cleanup_helper assert "ExpectedArtifactState" not in cleanup_helper assert "/usr/bin/docker info" in cleanup_helper assert "--apply-receipt-file $RestoreRemoteReceipt" in cleanup_branch assert '"independent_restore_verifier"' in cleanup_branch - assert '"portable_metadata_restore_independently_verified_zero_residue"' in cleanup_branch - assert '"portable_metadata_restore_verified_zero_residue_pending_independent_verifier"' in restore_apply_branch + assert ( + '"portable_metadata_restore_independently_verified_zero_residue"' + in cleanup_branch + ) + assert ( + '"portable_metadata_restore_verified_zero_residue_pending_independent_verifier"' + in restore_apply_branch + ) assert cleanup_start < restore_apply_start < export_verify_start assert "$RestoreCleanupKillAfterSeconds = 600" in source assert "$RestoreApplyControllerWaitSeconds = 2550" in source assert "$RestoreCleanupControllerWaitSeconds = 1050" in source + assert '[string]$AttemptId = ""' in source + assert '"$EvidenceRunId-attempt-$EvidenceAttemptId"' in source + assert 'cleanupAttemptId = if ($Mode -eq "RestoreVerifyCleanup")' in source + assert "cleanup_attempt_id_missing_or_invalid" in source + assert "attempt_id_not_allowed_for_mode" in source + + +def test_toolchain_modes_use_separate_reconcile_attempt_and_never_fall_through() -> ( + None +): + source = ENTRYPOINT.read_text(encoding="utf-8") + main_start = source.index("foreach ($identity") + reconcile_start = source.index('if ($Mode -eq "ToolchainReconcile")', main_start) + toolchain_start = source.index( + 'if ($Mode -in @("ToolchainCheck", "ToolchainApply", "ToolchainVerify"))', + reconcile_start, + ) + restore_start = source.index( + 'if ($Mode -eq "RestoreVerifyCleanup")', toolchain_start + ) + reconcile_branch = source[reconcile_start:toolchain_start] + toolchain_branch = source[toolchain_start:restore_start] + + assert '"$EvidenceRunId-attempt-$EvidenceAttemptId"' in source + assert 'toolchainReconcileAttemptId = if ($Mode -eq "ToolchainReconcile")' in source + assert 'Invoke-Agent99SignozToolchainTarget "reconcile"' in reconcile_branch + assert 'Invoke-Agent99SignozToolchainTarget "reconcile"' not in toolchain_branch + assert '"toolchain_reconcile_exact_deployment_verified"' in reconcile_branch + assert '"toolchain_reconcile_zero_owned_residue"' in reconcile_branch + assert "_toolchain_reconcile_required" in toolchain_branch + assert "Get-Agent99SignozPreflight" not in reconcile_branch + assert "New-Agent99SignozLockedRemoteCommand" not in reconcile_branch + assert "Get-Agent99SignozPreflight" not in toolchain_branch + assert "$ToolchainEnvelopeDeletedBeforeExecutor" in source + assert "toolchainEnvelopeTransportPayloadDeletedBeforeExecutor" in source + assert "productionSourceMutationState" in source + + +def test_restore_check_failure_terminals_preserve_no_write_semantics() -> None: + source = ENTRYPOINT.read_text(encoding="utf-8") + cleanup_start = source.index( + 'if ($Mode -eq "RestoreVerifyCleanup")', source.index("foreach ($identity") + ) + restore_start = source.index( + 'if ($Mode -in @("RestoreCheck", "RestoreApply"))', cleanup_start + ) + restore_end = source.index('if ($Mode -eq "Verify")', restore_start) + restore_branch = source[restore_start:restore_end] + + assert restore_branch.count('"restore_check_failed_no_write"') >= 4 + assert ( + '"restore_check_failed_unexpected_write_requires_reconcile"' in restore_branch + ) + assert '"restore_check_wrote_receipt"' in restore_branch + assert ( + 'if ($Terminal -eq "restore_check_failed_unexpected_write_requires_reconcile") ' + '{ "unexpected_receipt_write_requires_reconcile" }' in source + ) def test_entrypoint_reserves_durable_evidence_before_any_remote_dispatch() -> None: @@ -349,7 +429,7 @@ def test_entrypoint_reserves_durable_evidence_before_any_remote_dispatch() -> No reservation = main.index("New-Agent99SignozEvidenceReservation") first_remote = main.index("Get-Agent99RuntimeSourceBinding") assert reservation < first_remote - assert '[IO.FileMode]::CreateNew' in source + assert "[IO.FileMode]::CreateNew" in source assert '$EvidenceReservationPath = "$EvidencePath.reserve"' in source assert "blocked_evidence_identity_collision" in source assert "evidence_path_or_reservation_already_exists" in source @@ -379,8 +459,7 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None: ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh" ).read_text(encoding="utf-8") receiver = ( - ROOT - / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" + ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" ).read_text(encoding="utf-8") assert expected in _quoted_array(bootstrap, "$agentFiles = @(", ")") @@ -388,8 +467,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None: assert expected in _quoted_array(contract, "$requiredFiles = @(", ")") assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")") assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")") - assert 'expectedRuntimeFileCount": 20' in sender - assert '$runtimeManifest.fileCount -eq 20' in receiver + assert 'expectedRuntimeFileCount": 21' in sender + assert "$runtimeManifest.fileCount -eq 21" in receiver def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None: diff --git a/scripts/ops/tests/test_agent99_signoz_toolchain_target.py b/scripts/ops/tests/test_agent99_signoz_toolchain_target.py new file mode 100644 index 000000000..6667c8a73 --- /dev/null +++ b/scripts/ops/tests/test_agent99_signoz_toolchain_target.py @@ -0,0 +1,357 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import os +import stat +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +ROOT = Path(__file__).resolve().parents[3] +TARGET = ROOT / "agent99-signoz-toolchain-target.py" + + +def _load_target() -> ModuleType: + spec = importlib.util.spec_from_file_location( + "agent99_signoz_toolchain_target", TARGET + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _payloads() -> dict[str, bytes]: + policy = { + "work_item_id": "P0-OBS-002", + "source_runtime": { + "raw_database_access_allowed": False, + "raw_volume_access_allowed": False, + }, + "completion_contract": {"full_sqlite_completion_claim_allowed": False}, + } + values: dict[str, bytes] = { + "metadata-export-policy.json": ( + json.dumps(policy, sort_keys=True, separators=(",", ":")) + "\n" + ).encode(), + "isolated-cluster.xml": b"\n", + } + for name in ( + "signoz_metadata_contract.py", + "signoz-metadata-export.py", + "verify-signoz-metadata-export.py", + "signoz-metadata-restore-drill.py", + "signoz-metadata-isolated-restore.py", + ): + values[name] = b"VALUE = 'bounded-toolchain-test'\n" + return values + + +def _manifest(module: ModuleType, values: dict[str, bytes]) -> list[dict[str, str]]: + return [ + { + "name": name, + "sha256": hashlib.sha256(values[name]).hexdigest(), + "mode": f"{mode:04o}", + } + for name, mode in zip(module.LABELS, module.MODES, strict=True) + ] + + +def _args( + module: ModuleType, + manifest: list[dict[str, str]], + *, + mode: str, + run_id: str = "run-001", + attempt_id: str = "", + stage: Path | None = None, +) -> SimpleNamespace: + bundle_id = module.sha256_bytes( + "".join(f"{row['sha256']}\n" for row in manifest).encode("ascii") + ) + return SimpleNamespace( + mode=mode, + trace_id="trace-001", + run_id=run_id, + work_item_id="P0-OBS-002", + source_revision="1" * 40, + bundle_id=bundle_id, + stage_dir=str(stage) if stage else None, + attempt_id=attempt_id, + ) + + +@pytest.fixture +def target_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + module = _load_target() + uid = os.getuid() + gid = os.getgid() + remote_parent = tmp_path / "backup" / "toolchains" + receipt_parent = tmp_path / "backup" / "deploy-receipts" + remote_parent.mkdir(parents=True) + receipt_parent.mkdir(parents=True) + lock = tmp_path / "operation.lock" + lock.write_bytes(b"") + os.chmod(lock, 0o600) + snapshot = { + "id": "a" * 64, + "image_id": module.SIGNOZ_IMAGE_ID, + "image_ref": module.SIGNOZ_IMAGE_REF, + "running": True, + "started_at": "2026-07-22T00:00:00Z", + "restart_count": 0, + } + monkeypatch.setattr(module, "ROOT_UID", uid) + monkeypatch.setattr(module, "REMOTE_ROOT", remote_parent / "signoz-metadata") + monkeypatch.setattr( + module, + "RECEIPT_ROOT", + receipt_parent / "signoz-metadata-toolchain", + ) + module.REMOTE_ROOT.mkdir(mode=0o700) + module.RECEIPT_ROOT.mkdir(mode=0o700) + monkeypatch.setattr(module, "OPERATION_LOCK", lock) + monkeypatch.setattr(module, "production_snapshot", lambda: dict(snapshot)) + monkeypatch.setattr(module, "require_health", lambda: None) + monkeypatch.setattr(module, "require_no_active_operations", lambda: None) + monkeypatch.setenv("SUDO_UID", str(uid)) + monkeypatch.setenv("SUDO_GID", str(gid)) + values = _payloads() + manifest = _manifest(module, values) + return module, values, manifest, snapshot + + +def _write_tree( + root: Path, + manifest: list[dict[str, str]], + values: dict[str, bytes], + *, + directory_mode: int, +) -> None: + root.mkdir(mode=directory_mode, parents=True) + for row in manifest: + path = root / row["name"] + path.write_bytes(values[row["name"]]) + os.chmod(path, int(row["mode"], 8)) + + +def test_target_state_distinguishes_absent_exact_drift_and_active_pointer( + target_env, +) -> None: + module, values, manifest, _snapshot = target_env + args = _args(module, manifest, mode="check") + assert module.target_state(args.bundle_id, manifest) == "absent" + + target = module.REMOTE_ROOT / args.bundle_id + _write_tree(target, manifest, values, directory_mode=0o750) + assert module.target_state(args.bundle_id, manifest) == "exact" + + (target / manifest[0]["name"]).write_bytes(b"drift") + with pytest.raises(module.TargetError, match="target_file_drift"): + module.target_state(args.bundle_id, manifest) + + (target / manifest[0]["name"]).write_bytes(values[manifest[0]["name"]]) + os.chmod(target / manifest[0]["name"], int(manifest[0]["mode"], 8)) + (module.REMOTE_ROOT / "current").symlink_to(target) + with pytest.raises(module.TargetError, match="active_pointer_must_remain_absent"): + module.target_state(args.bundle_id, manifest) + + +def test_prepare_creates_private_transport_owned_stage( + target_env, tmp_path: Path +) -> None: + module, _values, manifest, _snapshot = target_env + stage = tmp_path / "transport-stage" + args = _args(module, manifest, mode="prepare", stage=stage) + + module.reset_mutation_state() + receipt = module.prepare_stage(args, manifest) + + metadata = stage.lstat() + assert receipt["terminal"] == "stage_prepared" + assert metadata.st_uid == os.getuid() + assert stat.S_IMODE(metadata.st_mode) == 0o700 + assert receipt["source_write_attempted"] is True + assert receipt["source_write_performed"] is True + + +def test_apply_writes_receipt_only_after_exact_target_and_live_stage_cleanup( + target_env, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module, values, manifest, _snapshot = target_env + stage = tmp_path / "transport-stage" + _write_tree(stage, manifest, values, directory_mode=0o700) + args = _args(module, manifest, mode="apply", stage=stage) + original_write = module.write_deployment_receipt + observed: list[tuple[bool, str]] = [] + + def guarded_write(*call_args, **call_kwargs): + observed.append( + ( + os.path.lexists(stage), + module.target_state(args.bundle_id, manifest), + ) + ) + return original_write(*call_args, **call_kwargs) + + monkeypatch.setattr(module, "write_deployment_receipt", guarded_write) + module.reset_mutation_state() + receipt = module.apply_bundle(args, manifest) + + assert receipt["terminal"] == "apply_pass_deployed_inactive" + assert observed == [(False, "exact")] + assert not os.path.lexists(stage) + assert not os.path.lexists( + module.REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}" + ) + durable = module.load_deploy_receipt(args) + assert durable is not None + assert durable["deployment_method"] == "created" + assert ( + durable["production_continuity_sha256"] + == receipt["production_continuity_sha256"] + ) + assert not os.path.lexists(module.REMOTE_ROOT / "current") + + +def test_reconcile_retries_half_quarantine_and_preserves_shared_target( + target_env, tmp_path: Path +) -> None: + module, values, manifest, _snapshot = target_env + stage = tmp_path / "transport-stage" + args = _args( + module, + manifest, + mode="reconcile", + run_id="run-half", + attempt_id="attempt-002", + stage=stage, + ) + receipt_dir = module.RECEIPT_ROOT / args.run_id + receipt_dir.mkdir(mode=0o700, parents=True) + # Model a signal after the stage was moved but before the candidate move. + (receipt_dir / "transport-stage").mkdir(mode=0o700) + candidate = module.REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}" + candidate.mkdir(mode=0o700, parents=True) + target = module.REMOTE_ROOT / args.bundle_id + _write_tree(target, manifest, values, directory_mode=0o750) + + module.reset_mutation_state() + receipt = module.reconcile_apply(args, manifest) + + assert receipt["terminal"] == "reconcile_exact_target_preserved_unverified" + assert receipt["owned_live_residue_verified_absent"] is True + assert target.is_dir() + assert (receipt_dir / "transport-stage").is_dir() + assert (receipt_dir / "quarantine-candidate").is_dir() + assert not candidate.exists() + + +def test_reconcile_finalizes_interrupted_exact_apply_from_durable_intent( + target_env, tmp_path: Path +) -> None: + module, values, manifest, snapshot = target_env + stage = tmp_path / "transport-stage" + args = _args( + module, + manifest, + mode="reconcile", + run_id="run-interrupted", + attempt_id="attempt-003", + stage=stage, + ) + receipt_dir = module.RECEIPT_ROOT / args.run_id + receipt_dir.mkdir(mode=0o700, parents=True) + continuity = module.sha256_bytes(module.canonical_bytes(snapshot)) + intent = module.apply_intent(args, "absent", continuity) + module.write_private_new( + receipt_dir / "apply-intent.json", module.canonical_bytes(intent), 0o600 + ) + target = module.REMOTE_ROOT / args.bundle_id + _write_tree(target, manifest, values, directory_mode=0o750) + + module.reset_mutation_state() + receipt = module.reconcile_apply(args, manifest) + + assert receipt["terminal"] == "reconcile_exact_deployment_verified" + assert receipt["durable_deploy_receipt_valid"] is True + durable = module.load_deploy_receipt(args) + assert durable is not None + assert durable["deployment_method"] == "reconciled_exact" + assert durable["reconcile_attempt_id"] == "attempt-003" + assert target.is_dir() + + +def test_reconcile_zero_live_residue_is_repeatable_with_new_attempt( + target_env, tmp_path: Path +) -> None: + module, _values, manifest, _snapshot = target_env + stage = tmp_path / "transport-stage" + first = _args( + module, + manifest, + mode="reconcile", + run_id="run-cleanup", + attempt_id="attempt-004", + stage=stage, + ) + stage.mkdir(mode=0o700) + + module.reset_mutation_state() + first_receipt = module.reconcile_apply(first, manifest) + second = _args( + module, + manifest, + mode="reconcile", + run_id=first.run_id, + attempt_id="attempt-005", + stage=stage, + ) + module.reset_mutation_state() + second_receipt = module.reconcile_apply(second, manifest) + + assert first_receipt["terminal"] == "reconcile_zero_owned_residue" + assert second_receipt["terminal"] == "reconcile_zero_owned_residue" + assert not stage.exists() + assert (module.RECEIPT_ROOT / first.run_id / "transport-stage").is_dir() + + +def test_reconcile_fails_closed_on_production_continuity_change( + target_env, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module, values, manifest, snapshot = target_env + stage = tmp_path / "transport-stage" + args = _args( + module, + manifest, + mode="reconcile", + run_id="run-continuity", + attempt_id="attempt-006", + stage=stage, + ) + receipt_dir = module.RECEIPT_ROOT / args.run_id + receipt_dir.mkdir(mode=0o700, parents=True) + intent = module.apply_intent( + args, + "absent", + module.sha256_bytes(module.canonical_bytes(snapshot)), + ) + module.write_private_new( + receipt_dir / "apply-intent.json", module.canonical_bytes(intent), 0o600 + ) + _write_tree( + module.REMOTE_ROOT / args.bundle_id, manifest, values, directory_mode=0o750 + ) + changed = dict(snapshot) + changed["restart_count"] = 1 + monkeypatch.setattr(module, "production_snapshot", lambda: dict(changed)) + + module.reset_mutation_state() + with pytest.raises( + module.TargetError, match="apply_intent_production_continuity_mismatch" + ): + module.reconcile_apply(args, manifest) diff --git a/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py b/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py index 54587138c..399c52821 100644 --- a/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py +++ b/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py @@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None: assert "$dbExecutorRecoveryContract" in contract assert "awoooi_db_bounded_executor_receipt_v2" in contract assert "Receipt.database_sqlstate" in contract - assert 'expectedRuntimeFileCount": 20' in transport - assert "expectedRuntimeFileCount=20" in transport - assert "expectedRuntimeFileCount -ne 20" in receiver - assert "fileCount = 20" in receiver + assert 'expectedRuntimeFileCount": 21' in transport + assert "expectedRuntimeFileCount=21" in transport + assert "expectedRuntimeFileCount -ne 21" in receiver + assert "fileCount = 21" in receiver diff --git a/scripts/ops/tests/test_signoz_metadata_toolchain_agent99_transport.py b/scripts/ops/tests/test_signoz_metadata_toolchain_agent99_transport.py new file mode 100644 index 000000000..034ad8769 --- /dev/null +++ b/scripts/ops/tests/test_signoz_metadata_toolchain_agent99_transport.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +SENDER = ROOT / "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh" + + +def _array(source: str, name: str) -> tuple[str, ...]: + match = re.search(rf"{re.escape(name)}=\(\n(?P.*?)\n\)", source, re.DOTALL) + assert match is not None + return tuple(re.findall(r'"([^"\n]+)"', match.group("body"))) + + +def test_sender_is_syntax_valid_and_routes_only_through_windows99_agent99() -> None: + source = SENDER.read_text(encoding="utf-8") + result = subprocess.run( + ["bash", "-n", str(SENDER)], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert 'readonly TARGET="Administrator@192.168.0.99"' in source + assert "wooo@192.168.0.110" not in source + assert ( + 'dispatchPath": "windows99_agent99_to_host110_fixed_metadata_toolchain"' + in source + ) + assert "C:\\Wooo\\Agent99\\bin\\agent99-signoz-metadata-executor.ps1" in source + assert "Invoke-Expression" not in source + assert "github.com" not in source + assert "ghcr.io" not in source + + +def test_sender_binds_exact_fresh_gitea_main_and_all_ten_transport_sources() -> None: + source = SENDER.read_text(encoding="utf-8") + toolchain_sources = _array(source, "TOOLCHAIN_SOURCES") + + assert len(toolchain_sources) == 7 + assert len(set(toolchain_sources)) == 7 + assert 'readonly GITEA_REMOTE="gitea"' in source + assert "+refs/heads/main:refs/remotes/gitea/main" in source + assert 'SOURCE_REVISION}" == "${GITEA_MAIN_REVISION}' in source + assert 'git -C "${SOURCE_ROOT}" diff --quiet "${SOURCE_REVISION}"' in source + assert '"${TOOLCHAIN_SOURCES[@]}"' in source + assert '"${EXECUTOR_RELATIVE}"' in source + assert '"${TARGET_RELATIVE}"' in source + assert '"${WRAPPER_RELATIVE}"' in source + assert "source_bound_file_untracked" in source + assert "source_bundle_differs_from_revision" in source + + +def test_sender_envelope_is_exact_seven_file_secret_free_and_deleted_before_executor() -> ( + None +): + source = SENDER.read_text(encoding="utf-8") + + assert '"schemaVersion": "awoooi_signoz_metadata_toolchain_envelope_v1"' in source + assert '"fileCount": 7' in source + assert '"transportPayloadContainsSecrets": False' in source + assert '"contentBase64": base64.b64encode(content).decode("ascii")' in source + assert "fixed_toolchain_file_contract_failed" in source + assert "toolchain_envelope_transport_file_invalid" in source + assert "toolchain_envelope_transport_delete_failed" in source + delete = source.index("Remove-Item -LiteralPath $payloadPath") + invoke = source.index("& $executor @parameters") + assert delete < invoke + assert "$parameters.ToolchainEnvelopeDeletedBeforeExecutor = $true" in source + assert "REMOTE_PAYLOAD_PREPARED" in source + assert "transport_payload_cleanup_failed" in source + + +def test_sender_has_bounded_check_apply_verify_and_attempt_scoped_reconcile() -> None: + source = SENDER.read_text(encoding="utf-8") + apply_branch = source[source.index(" apply)") :] + + for mode in ( + "ToolchainCheck", + "ToolchainApply", + "ToolchainVerify", + "ToolchainReconcile", + ): + assert mode in source + assert "reconcile_attempt_id_missing_or_invalid" in source + assert "reconcile-$(date -u +%Y%m%dT%H%M%SZ)-$$" in source + assert apply_branch.index('run_checked_mode "ToolchainCheck"') < apply_branch.index( + 'invoke_remote "ToolchainApply"' + ) + assert apply_branch.index('invoke_remote "ToolchainApply"') < apply_branch.index( + 'run_checked_mode "ToolchainVerify"' + ) + assert 'run_checked_mode "ToolchainReconcile"' in apply_branch + assert "toolchain_reconcile_zero_owned_residue" in source + assert "exit 75" in source + assert 'timeout --kill-after=20 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}"' in source diff --git a/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py b/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py index 3c8782c41..5e4cf2dd6 100644 --- a/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py +++ b/scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py @@ -1,267 +1,62 @@ from __future__ import annotations import json -import os import subprocess from pathlib import Path - ROOT = Path(__file__).resolve().parents[3] -DEPLOYER = ROOT / "scripts" / "ops" / "deploy-signoz-metadata-toolchain.sh" +DEPLOYER = ROOT / "scripts/ops/deploy-signoz-metadata-toolchain.sh" -def _write_executable(path: Path, text: str) -> None: - path.write_text(text, encoding="utf-8") - path.chmod(0o755) - - -def fake_environment(tmp_path: Path, *, mode: str) -> dict[str, str]: - fake_bin = tmp_path / "bin" - fake_bin.mkdir() - event_log = tmp_path / "events.log" - ssh_count = tmp_path / "ssh.count" - event_log.touch() - ssh_count.write_text("0\n", encoding="utf-8") - - _write_executable( - fake_bin / "timeout", - """#!/bin/bash -set -eu -while [[ "${1:-}" == --kill-after=* ]]; do shift; done -shift -exec "$@" -""", +def test_historical_direct_host110_deployer_is_syntax_valid_and_fail_closed() -> None: + syntax = subprocess.run( + ["bash", "-n", str(DEPLOYER)], + check=False, + capture_output=True, + text=True, ) - _write_executable( - fake_bin / "ssh", - """#!/bin/bash -set -eu -count="$(cat "${TEST_SSH_COUNT:?}")" -count=$((count + 1)) -printf '%s\n' "$count" > "${TEST_SSH_COUNT:?}" -printf 'ssh %s\n' "$*" >> "${TEST_EVENT_LOG:?}" -cat >/dev/null || true -case "${FAKE_SSH_MODE:?}" in - check_absent) - printf 'REMOTE_CHECK terminal=check_pass_no_write state=absent drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0 - ;; - check_drift) - exit 3 - ;; - apply) - case "$count" in - 1) - printf 'REMOTE_CHECK terminal=check_pass_no_write state=absent drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0 - ;; - 2) ;; - 3) - printf '{"phase":"terminal","terminal":"pass_source_deployed_inactive"}\n' - ;; - 4) - printf 'REMOTE_CHECK terminal=check_pass_no_write state=exact drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0 - ;; - *) exit 90 ;; - esac - ;; -esac -""", + result = subprocess.run( + ["bash", str(DEPLOYER), "--apply"], + check=False, + capture_output=True, + text=True, ) - _write_executable( - fake_bin / "scp", - """#!/bin/bash -set -eu -printf 'scp %s\n' "$*" >> "${TEST_EVENT_LOG:?}" -""", - ) - return { - **os.environ, - "PATH": f"{fake_bin}:{os.environ['PATH']}", - "TEST_EVENT_LOG": str(event_log), - "TEST_SSH_COUNT": str(ssh_count), - "FAKE_SSH_MODE": mode, - "TRACE_ID": "trace-P0-OBS-002-metadata-deploy", - "RUN_ID": f"run-{mode}", - "WORK_ITEM_ID": "P0-OBS-002", + + assert syntax.returncode == 0, syntax.stderr + assert result.returncode == 78 + receipt = json.loads(result.stdout) + assert receipt == { + "schema": "awoooi_signoz_metadata_toolchain_route_retirement_v1", + "terminal": "blocked_direct_host110_route_retired", + "replacement": "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh", + "production_mutation_performed": False, + "completion_claim": False, } -def run_deployer( - tmp_path: Path, - *, - mode: str, - apply: bool, - env_overrides: dict[str, str] | None = None, -) -> subprocess.CompletedProcess[str]: - env = fake_environment(tmp_path, mode=mode) - if env_overrides: - env.update(env_overrides) - return subprocess.run( - ["bash", str(DEPLOYER), "--apply" if apply else "--check"], - env=env, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=20, - check=False, - ) - - -def receipt_rows(stdout: str) -> list[dict[str, object]]: - return [json.loads(line) for line in stdout.splitlines() if line.startswith("{")] - - -def test_help_documents_inactive_immutable_contract() -> None: - result = subprocess.run( - ["bash", str(DEPLOYER), "--help"], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=10, - check=False, - ) - assert result.returncode == 0 - assert "immutable exact-hash directory" in result.stdout - assert "does not create or" in result.stdout - assert "active pointer" in result.stdout - - -def test_check_mode_is_no_write_and_ready_when_bundle_absent(tmp_path: Path) -> None: - result = run_deployer(tmp_path, mode="check_absent", apply=False) - assert result.returncode == 0, result.stderr - rows = receipt_rows(result.stdout) - assert rows[-1]["terminal"] == "check_pass_no_write" - assert rows[-1]["detail"].endswith("remote_state_absent") - events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines() - assert len([line for line in events if line.startswith("ssh ")]) == 1 - assert not [line for line in events if line.startswith("scp ")] - - -def test_check_mode_fails_closed_on_remote_drift(tmp_path: Path) -> None: - result = run_deployer(tmp_path, mode="check_drift", apply=False) - assert result.returncode == 1 - rows = receipt_rows(result.stdout) - assert rows[-1]["terminal"] == "failed_no_write" - - -def test_apply_uses_seven_transfers_and_exact_postcheck(tmp_path: Path) -> None: - result = run_deployer(tmp_path, mode="apply", apply=True) - assert result.returncode == 0, result.stderr - rows = receipt_rows(result.stdout) - assert rows[-1]["terminal"] == "pass_source_deployed_inactive" - assert rows[-1]["detail"] == "runtime_export_not_executed" - events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines() - assert len([line for line in events if line.startswith("scp ")]) == 7 - assert len([line for line in events if line.startswith("ssh ")]) == 4 - - -def test_deployer_has_no_active_pointer_or_destructive_fallback() -> None: +def test_historical_entrypoint_contains_no_transport_or_mutation_primitive() -> None: source = DEPLOYER.read_text(encoding="utf-8") - assert '"${REMOTE_ROOT}/current"' in source - assert "active_pointer_0" in source - assert "pass_source_deployed_inactive" in source - assert "quarantine-candidate" in source - assert "quarantine-target" in source - assert "docker stop" not in source - assert "docker start" not in source - assert "docker restart" not in source - assert "sqlite3" not in source - assert "github.com" not in source.casefold() - assert "curl |" not in source - assert "curl -fsSL" not in source - assert "\nrm " not in source - assert "ln -s" not in source - assert "signoz-metadata-isolated-restore.py" in source - assert "isolated-cluster.xml" in source - assert "local_exact_sources_7" in source - assert "[ \"${#EXPECTED_HASHES[@]}\" -eq 7 ]" in source - -def test_transport_propagates_strict_fixed_paths_with_spaces(tmp_path: Path) -> None: - transport_dir = tmp_path / "transport paths" - transport_dir.mkdir() - identity = transport_dir / "agent99 identity" - known_hosts = transport_dir / "known hosts" - identity.write_text("test-only-identity\n", encoding="utf-8") - known_hosts.write_text("test-only-known-host\n", encoding="utf-8") - - result = run_deployer( - tmp_path, - mode="check_absent", - apply=False, - env_overrides={ - "SIGNOZ_METADATA_SSH_IDENTITY_FILE": str(identity), - "SIGNOZ_METADATA_KNOWN_HOSTS_FILE": str(known_hosts), - }, - ) - - assert result.returncode == 0, result.stderr - event = (tmp_path / "events.log").read_text(encoding="utf-8") - assert "IdentitiesOnly=yes" in event - assert "StrictHostKeyChecking=yes" in event - assert str(identity) in event - assert f"UserKnownHostsFile={known_hosts}" in event - - -def test_transport_rejects_missing_fixed_path_before_remote_use(tmp_path: Path) -> None: - result = run_deployer( - tmp_path, - mode="check_absent", - apply=False, - env_overrides={ - "SIGNOZ_METADATA_SSH_IDENTITY_FILE": str(tmp_path / "missing identity") - }, - ) - - assert result.returncode == 64 - assert "invalid SSH identity file" in result.stderr - assert not (tmp_path / "events.log").read_text(encoding="utf-8") - - -def test_transport_rejects_symlink_fixed_path_before_remote_use(tmp_path: Path) -> None: - target = tmp_path / "known-hosts-target" - target.write_text("test-only-known-host\n", encoding="utf-8") - symlink = tmp_path / "known hosts link" - symlink.symlink_to(target) - - result = run_deployer( - tmp_path, - mode="check_absent", - apply=False, - env_overrides={"SIGNOZ_METADATA_KNOWN_HOSTS_FILE": str(symlink)}, - ) - - assert result.returncode == 64 - assert "invalid SSH known_hosts file" in result.stderr - assert not (tmp_path / "events.log").read_text(encoding="utf-8") - - -def test_shared_operation_lock_is_existing_read_only_and_fail_closed() -> None: - source = DEPLOYER.read_text(encoding="utf-8") - assert "OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock" in source - assert '[ -f "${OPERATION_LOCK}" ] && [ ! -L "${OPERATION_LOCK}" ]' in source - assert 'exec 8<"${OPERATION_LOCK}"' in source - assert "flock -n 8" in source - assert "exec 8>>/tmp/awoooi-signoz-backup-operation.lock" not in source - - -def test_remote_check_uses_privileged_read_only_visibility() -> None: - source = DEPLOYER.read_text(encoding="utf-8") - remote_check = source.split("remote_check() {", 1)[1].split("\n}", 1)[0] - assert '"${TARGET_HOST}" sudo -n bash -s --' in remote_check - assert "REMOTE_CHECK contains only stat/hash/process/API" in remote_check - - -def test_exact_seven_file_supply_chain_and_modes_are_fixed() -> None: - source = DEPLOYER.read_text(encoding="utf-8") - for path in ( - "config/signoz/metadata-export-policy.json", - "scripts/backup/signoz_metadata_contract.py", - "scripts/backup/signoz-metadata-export.py", - "scripts/backup/verify-signoz-metadata-export.py", - "scripts/backup/signoz-metadata-restore-drill.py", - "scripts/backup/signoz-metadata-isolated-restore.py", - "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml", + for forbidden in ( + "192.168.0.110", + "ssh ", + "scp ", + "docker ", + "sudo ", + "curl ", + "rm ", + "mv ", + "ln ", ): - assert path in source - assert "MODES=(0644 0644 0755 0755 0755 0755 0644)" in source - assert '[ "${#EXPECTED_HASHES[@]}" -eq 7 ]' in source + assert forbidden not in source + assert "blocked_direct_host110_route_retired" in source + assert "production_mutation_performed" in source + assert "completion_claim" in source + + +def test_historical_entrypoint_points_only_to_agent99_replacement() -> None: + source = DEPLOYER.read_text(encoding="utf-8") + + assert "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh" in source + assert "github.com" not in source.casefold() + assert "ghcr.io" not in source.casefold() diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index a913ab191..950837166 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -1221,7 +1221,7 @@ $decision = Get-AgentLivePreflightDecision ` -AgentRootPresent ([bool](Test-Path $AgentRoot)) ` -Manifest $manifest ` -ManifestCheckRequired ([bool]($ReadinessScope -ne "PromotionReserve")) ` - -ExpectedRuntimeFileCount 20 ` + -ExpectedRuntimeFileCount 21 ` -TaskFailureCount $taskFailures.Count ` -RelayListenerCount $relayListeners.Count ` -RequiredEvidenceStaleCount $requiredEvidenceStale.Count ` diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index ca42fbc5e..be8aee12b 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -38,6 +38,7 @@ $FixedRuntimeFiles = @( "agent99-signoz-metadata-executor.ps1", "agent99-signoz-credential-provisioner.ps1", "agent99-signoz-credential-target.py", + "agent99-signoz-toolchain-target.py", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -1080,7 +1081,7 @@ try { } $sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant() if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" } - if ([int]$envelope.expectedRuntimeFileCount -ne 20) { throw "invalid_expected_runtime_file_count" } + if ([int]$envelope.expectedRuntimeFileCount -ne 21) { throw "invalid_expected_runtime_file_count" } $traceId = [string]$envelope.traceId $runId = [string]$envelope.runId @@ -1115,7 +1116,7 @@ try { $sourceManifest = [pscustomobject]@{ schemaVersion = "agent99_remote_source_manifest_v1" sourceRevision = $sourceRevision - fileCount = 20 + fileCount = 21 manifestSha256 = $manifestDigest files = @($FixedRuntimeFiles | ForEach-Object { [pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() } @@ -1152,7 +1153,7 @@ try { status = "check_ready" mode = "check" sourceRevision = $sourceRevision - expectedRuntimeFileCount = 20 + expectedRuntimeFileCount = 21 manifestSha256 = $manifestDigest plannedStagingPath = $stagePath runtimeManifest = $currentManifest @@ -1239,7 +1240,7 @@ try { if ( [string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or [string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or - [int]$existingSourceManifest.fileCount -ne 20 -or + [int]$existingSourceManifest.fileCount -ne 21 -or [string]$existingSourceManifest.manifestSha256 -ne $manifestDigest ) { throw "existing_staging_manifest_mismatch" } } catch { @@ -1266,7 +1267,7 @@ try { [string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and $live.runtimeMatched -and $live.sourceRevision -eq $sourceRevision -and - $live.fileCount -eq 20 -and + $live.fileCount -eq 21 -and $live.mismatchCount -eq 0 -and $liveLauncherContract.ok -and [string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256 @@ -1594,7 +1595,7 @@ try { $runtimeManifest.exists -and $runtimeManifest.runtimeMatched -and $runtimeManifest.sourceRevision -eq $sourceRevision -and - $runtimeManifest.fileCount -eq 20 -and + $runtimeManifest.fileCount -eq 21 -and $runtimeManifest.mismatchCount -eq 0 -and $launcherContract.ok -and [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index 5fcce3a39..a161f5847 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -20,6 +20,7 @@ RUNTIME_FILES=( "agent99-signoz-metadata-executor.ps1" "agent99-signoz-credential-provisioner.ps1" "agent99-signoz-credential-target.py" + "agent99-signoz-toolchain-target.py" "agent99-deploy.ps1" "agent99-register-tasks.ps1" "agent99-alertmanager-alertchain-poll.ps1" @@ -229,7 +230,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7] output_path = Path(sys.argv[7]) runtime_names = sys.argv[8:] -if len(runtime_names) != 20 or len(set(runtime_names)) != 20: +if len(runtime_names) != 21 or len(set(runtime_names)) != 21: raise SystemExit("fixed_runtime_file_contract_failed") files: list[dict[str, str]] = [] @@ -269,7 +270,7 @@ envelope = { "runId": run_id, "workItemId": work_item_id, "sourceRevision": source_revision, - "expectedRuntimeFileCount": 20, + "expectedRuntimeFileCount": 21, "manifestSha256": manifest_sha256, "files": files, "livePreflight": { @@ -421,14 +422,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20] script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue' $a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json' $r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''} -if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 20-and$m.mismatchCount-eq 0-and$rows.Count-eq 20-and$seen.Count-eq 20-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}} +if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 21-and$m.mismatchCount-eq 0-and$rows.Count-eq 21-and$seen.Count-eq 21-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}} $c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''} try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name} $z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false} $ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0) $status=if($ready){'check_ready'}else{'check_failed_no_apply'} $next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'} -[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=20;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 +[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=21;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 if(-not$ready){exit 65}''' script = ( script.replace("__SOURCE_REVISION__", source_revision) diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index 05b9a0b81..42b80487a 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -105,7 +105,7 @@ def _base_case() -> dict[str, Any]: "exists": True, "sourceRevision": "a" * 40, "runtimeMatched": True, - "fileCount": 20, + "fileCount": 21, "mismatchCount": 0, "parseError": "", }, @@ -154,7 +154,7 @@ $parameters = @{{ AgentRootPresent = [bool]$case.agentRootPresent Manifest = $case.manifest ManifestCheckRequired = [bool]$case.manifestCheckRequired - ExpectedRuntimeFileCount = 20 + ExpectedRuntimeFileCount = 21 TaskFailureCount = [int]$case.taskFailureCount RelayListenerCount = [int]$case.relayListenerCount RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount @@ -208,7 +208,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No ) assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision assert "preflightGreen = [bool]$decision.deploymentEligible" in source - assert "-ExpectedRuntimeFileCount 20" in source + assert "-ExpectedRuntimeFileCount 21" in source assert "warningReasons = $warningReasons" in source assert "selfHealthPerformanceIssues = @(" in source assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 357bc871d..40401228d 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -28,6 +28,7 @@ EXPECTED_RUNTIME_FILES = ( "agent99-signoz-metadata-executor.ps1", "agent99-signoz-credential-provisioner.ps1", "agent99-signoz-credential-target.py", + "agent99-signoz-toolchain-target.py", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -90,7 +91,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None ) assert preflight_count is not None assert int(preflight_count.group(1)) == len(EXPECTED_RUNTIME_FILES) - assert "expectedRuntimeFileCount\": 20" in sender + assert "expectedRuntimeFileCount\": 21" in sender assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver assert "required_runtime_file_missing" in receiver assert "duplicate_runtime_file" in receiver @@ -265,7 +266,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() ) assert "$livePreflight.sourceRevision -eq $sourceRevision" in source assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source - assert '$runtimeManifest.fileCount -eq 20' in source + assert '$runtimeManifest.fileCount -eq 21' in source assert "function Invoke-AgentWindowsControlBaseline" in source assert "function Get-AgentRunEvidenceToken" in source assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source