feat(signoz): add isolated metadata restore controller
This commit is contained in:
Your Name
@@ -1,5 +1,5 @@
|
||||
param(
|
||||
[ValidateSet("Check", "Apply", "Verify")]
|
||||
[ValidateSet("Check", "Apply", "Verify", "RestoreCheck", "RestoreApply", "RestoreVerifyCleanup")]
|
||||
[string]$Mode = "Check",
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$TraceId,
|
||||
@@ -22,30 +22,57 @@ $EvidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$RuntimeManifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
||||
$EntrypointPath = [string]$MyInvocation.MyCommand.Path
|
||||
$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"
|
||||
$BundleId = "d1318c412aa303c5d8a5795f22ee62dce16ca79354c5a15e95bfad41a5e35533"
|
||||
$BundleId = "ed4e5fbfc5f732b64d6a78f6ab26b90118e1d42a4f38b40d583876445c32e3aa"
|
||||
$ToolchainRoot = "/backup/toolchains/signoz-metadata/$BundleId"
|
||||
$PolicyPath = "$ToolchainRoot/metadata-export-policy.json"
|
||||
$ExporterPath = "$ToolchainRoot/signoz-metadata-export.py"
|
||||
$VerifierPath = "$ToolchainRoot/verify-signoz-metadata-export.py"
|
||||
$RestoreDriverPath = "$ToolchainRoot/signoz-metadata-restore-drill.py"
|
||||
$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"
|
||||
$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"
|
||||
$SharedContractPath = "$ToolchainRoot/signoz_metadata_contract.py"
|
||||
$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"
|
||||
$BaseUrl = "http://127.0.0.1:8080"
|
||||
$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"
|
||||
$RemoteReceipt = "/backup/deploy-receipts/signoz-metadata-toolchain/export-$RunId.json"
|
||||
$RestoreRemoteReceipt = "/backup/deploy-receipts/signoz-metadata-toolchain/restore-$RunId.json"
|
||||
$SignozImage = "signoz/signoz:v0.113.0"
|
||||
$SignozImageId = "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b"
|
||||
$SignozContainer = "signoz"
|
||||
$OperationLock = "/tmp/awoooi-signoz-backup-operation.lock"
|
||||
$TargetKillAfterSeconds = 15
|
||||
$DefaultTargetKillAfterSeconds = 15
|
||||
$RestoreCleanupKillAfterSeconds = 600
|
||||
$TargetGuardWorstCaseSeconds = 95
|
||||
$CheckVerifyTargetTimeoutSeconds = 90
|
||||
$ApplyTargetTimeoutSeconds = 720
|
||||
$RestoreCheckTargetTimeoutSeconds = 180
|
||||
$RestoreApplyTargetTimeoutSeconds = 1800
|
||||
$RestoreCleanupTargetTimeoutSeconds = 300
|
||||
$CheckVerifyControllerWaitSeconds = 300
|
||||
$ApplyControllerWaitSeconds = 900
|
||||
$RestoreCheckControllerWaitSeconds = 420
|
||||
$RestoreApplyControllerWaitSeconds = 2550
|
||||
$RestoreCleanupControllerWaitSeconds = 1050
|
||||
$ControllerTransportReserveSeconds = 30
|
||||
$TargetProgramTimeoutSeconds = if ($Mode -eq "Apply") { $ApplyTargetTimeoutSeconds } else { $CheckVerifyTargetTimeoutSeconds }
|
||||
$ControllerWaitSeconds = if ($Mode -eq "Apply") { $ApplyControllerWaitSeconds } else { $CheckVerifyControllerWaitSeconds }
|
||||
$TargetProgramTimeoutSeconds = switch ($Mode) {
|
||||
"Apply" { $ApplyTargetTimeoutSeconds }
|
||||
"RestoreCheck" { $RestoreCheckTargetTimeoutSeconds }
|
||||
"RestoreApply" { $RestoreApplyTargetTimeoutSeconds }
|
||||
"RestoreVerifyCleanup" { $RestoreCleanupTargetTimeoutSeconds }
|
||||
default { $CheckVerifyTargetTimeoutSeconds }
|
||||
}
|
||||
$ControllerWaitSeconds = switch ($Mode) {
|
||||
"Apply" { $ApplyControllerWaitSeconds }
|
||||
"RestoreCheck" { $RestoreCheckControllerWaitSeconds }
|
||||
"RestoreApply" { $RestoreApplyControllerWaitSeconds }
|
||||
"RestoreVerifyCleanup" { $RestoreCleanupControllerWaitSeconds }
|
||||
default { $CheckVerifyControllerWaitSeconds }
|
||||
}
|
||||
$TargetKillAfterSeconds = if ($Mode -in @("RestoreApply", "RestoreVerifyCleanup")) {
|
||||
$RestoreCleanupKillAfterSeconds
|
||||
} else {
|
||||
$DefaultTargetKillAfterSeconds
|
||||
}
|
||||
$TargetWorstCaseSeconds = $TargetGuardWorstCaseSeconds + $TargetProgramTimeoutSeconds + $TargetKillAfterSeconds
|
||||
$EvidenceRunId = if ($RunId -match $SafeIdPattern) { $RunId } else { "invalid-$PID" }
|
||||
$EvidencePath = Join-Path $EvidenceDir "agent99-signoz-metadata-$($Mode.ToLowerInvariant())-$EvidenceRunId.json"
|
||||
@@ -53,14 +80,17 @@ $EvidenceReservationPath = "$EvidencePath.reserve"
|
||||
$script:TransientTransportOutputUsed = $false
|
||||
$script:TransportCleanupSucceeded = $true
|
||||
$script:EvidenceReserved = $false
|
||||
$script:ApplyDispatchAttempted = $false
|
||||
$script:ExportCredentialDispatchAttempted = $false
|
||||
$script:IsolatedApplyDispatchAttempted = $false
|
||||
|
||||
$ToolchainFiles = [ordered]@{
|
||||
$PolicyPath = "9eac4e39515ef007763faacdb4c386e030aeb142872cec7a7a8d2a842befc2f0"
|
||||
$PolicyPath = "1365c6406b522f2678693cebe99914255d44b4f219a78a4625ac2c2b5eec2557"
|
||||
$ExporterPath = "8cd5cabf245e3814da32e33920c431ec28ed7180edc0495a2d0b9379fb4025fb"
|
||||
$SharedContractPath = "d2b2281da9ec89afc44d5153711b2d61e972545ad107611b74a5e6dfb01defe8"
|
||||
$VerifierPath = "0bac6d8ef8063f3584fc744cf205de08fc50d1085b292ffbd061f6090c344c86"
|
||||
$RestoreDriverPath = "5688f3212f970b3a0d0f93dfdb1c59c3147729bb5da09610572cd12da5dabd36"
|
||||
$RestoreDriverPath = "eb766f1209af08c3b339756d58653165d55b3c23bc8dbd66438dfb46d8098156"
|
||||
$IsolatedRestorePath = "a91fed22ae0c2682c8298eb075d6ede5c37608c83ac15f03c55e4bad52e93946"
|
||||
$IsolatedClusterConfigPath = "e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b"
|
||||
}
|
||||
|
||||
function Invoke-Agent99SignozFixedRemote {
|
||||
@@ -143,6 +173,23 @@ function New-Agent99SignozPathStateCommand {
|
||||
)
|
||||
}
|
||||
|
||||
function Get-Agent99SignozRemotePathState {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$Path,
|
||||
[ValidateSet("directory", "file")][string]$ExpectedKind
|
||||
)
|
||||
$remote = Invoke-Agent99SignozFixedRemote (
|
||||
New-Agent99SignozPathStateCommand $Path $ExpectedKind
|
||||
)
|
||||
if (
|
||||
(Test-Agent99SignozTransportOutcome $remote) -and
|
||||
[string]$remote.stdout -match "^agent99_path_state:(absent|directory|file|other|symlink)$"
|
||||
) {
|
||||
return [string]$Matches[1]
|
||||
}
|
||||
return "unverified"
|
||||
}
|
||||
|
||||
function Convert-Agent99JsonReceipt {
|
||||
param([object]$RemoteResult)
|
||||
if (
|
||||
@@ -180,7 +227,31 @@ function New-Agent99SignozLockedRemoteCommand {
|
||||
"[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker inspect --format={{.Config.Image}} $SignozContainer)`" = `"$SignozImage`" ]; " +
|
||||
"[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker inspect --format={{.State.Running}} $SignozContainer)`" = `"true`" ]; " +
|
||||
"[ `"`$(/usr/bin/timeout --kill-after=5s 15s /usr/bin/curl -sS -o /dev/null -w `"%{http_code}`" $BaseUrl/api/v1/health)`" = `"200`" ]; " +
|
||||
"set +e; active_output=`$(/usr/bin/timeout --kill-after=5s 10s /usr/bin/pgrep -af `"([s]ignoz-metadata-export[.]py|[s]ignoz-metadata-restore-drill[.]py|(^|/|[[:space:]])([b]ackup-signoz|[c]lickhouse-native-backup|[c]lickhouse-native-restore-drill|[r]un-signoz-backup-canary)[.]sh([[:space:]]|$))`"); active_rc=`$?; set -e; " +
|
||||
"set +e; active_output=`$(/usr/bin/timeout --kill-after=5s 10s /usr/bin/pgrep -af `"([s]ignoz-metadata-export[.]py|[s]ignoz-metadata-restore-drill[.]py|[s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])([b]ackup-signoz|[c]lickhouse-native-backup|[c]lickhouse-native-restore-drill|[r]un-signoz-backup-canary)[.]sh([[:space:]]|$))`"); active_rc=`$?; set -e; " +
|
||||
"[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]; " +
|
||||
"exec /usr/bin/timeout --signal=TERM --kill-after=$($TargetKillAfterSeconds)s " +
|
||||
"$($TargetTimeoutSeconds)s $FixedCommand"
|
||||
)
|
||||
$encodedProgram = [Convert]::ToBase64String(
|
||||
[Text.Encoding]::UTF8.GetBytes($bashProgram)
|
||||
)
|
||||
return (
|
||||
"/usr/bin/printf %s $encodedProgram | /usr/bin/base64 -d | " +
|
||||
"sudo -n /bin/bash"
|
||||
)
|
||||
}
|
||||
|
||||
function New-Agent99SignozCleanupLockedRemoteCommand {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$FixedCommand,
|
||||
[Parameter(Mandatory = $true)][int]$TargetTimeoutSeconds
|
||||
)
|
||||
$bashProgram = (
|
||||
"set -euo pipefail; " +
|
||||
"[ -f `"$OperationLock`" ] && [ ! -L `"$OperationLock`" ]; " +
|
||||
"exec 8<`"$OperationLock`"; /usr/bin/flock -n 8; " +
|
||||
"/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker info >/dev/null; " +
|
||||
"set +e; active_output=`$(/usr/bin/timeout --kill-after=5s 10s /usr/bin/pgrep -af `"([s]ignoz-metadata-export[.]py|[s]ignoz-metadata-restore-drill[.]py|[s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])([b]ackup-signoz|[c]lickhouse-native-backup|[c]lickhouse-native-restore-drill|[r]un-signoz-backup-canary)[.]sh([[:space:]]|$))`"); active_rc=`$?; set -e; " +
|
||||
"[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]; " +
|
||||
"exec /usr/bin/timeout --signal=TERM --kill-after=$($TargetKillAfterSeconds)s " +
|
||||
"$($TargetTimeoutSeconds)s $FixedCommand"
|
||||
@@ -256,6 +327,66 @@ function Test-Agent99SignozReceiptIdentity {
|
||||
)
|
||||
}
|
||||
|
||||
function Get-Agent99SignozCleanupPreflight {
|
||||
$program = (
|
||||
"set -euo pipefail; " +
|
||||
"/usr/bin/echo agent99_hashes_begin; " +
|
||||
"/usr/bin/sha256sum " + (($ToolchainFiles.Keys) -join " ") + "; " +
|
||||
"/usr/bin/echo agent99_hashes_end; " +
|
||||
"/usr/bin/printf agent99_docker:; " +
|
||||
"/usr/bin/timeout --kill-after=5s 15s /usr/bin/docker info --format={{.ServerVersion}}"
|
||||
)
|
||||
$encoded = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($program))
|
||||
$remote = Invoke-Agent99SignozFixedRemote (
|
||||
"/usr/bin/printf %s $encoded | /usr/bin/base64 -d | sudo -n /bin/bash"
|
||||
)
|
||||
$lines = if (Test-Agent99SignozTransportOutcome $remote) {
|
||||
@(([string]$remote.stdout -split "`r?`n") | Where-Object { [string]$_ -ne "" })
|
||||
} else { @() }
|
||||
$shapeValid = [bool](
|
||||
$lines.Count -eq $ToolchainFiles.Count + 3 -and
|
||||
[string]$lines[0] -eq "agent99_hashes_begin" -and
|
||||
[string]$lines[$lines.Count - 2] -eq "agent99_hashes_end" -and
|
||||
[string]$lines[$lines.Count - 1] -match "^agent99_docker:[A-Za-z0-9._+-]+$"
|
||||
)
|
||||
$actualHashes = @{}
|
||||
if ($shapeValid) {
|
||||
foreach ($line in @($lines[1..($lines.Count - 3)])) {
|
||||
if ($line -match "^([0-9a-f]{64})\s+(.+)$") {
|
||||
$path = [string]$Matches[2]
|
||||
if ($actualHashes.ContainsKey($path) -or -not $ToolchainFiles.Contains($path)) {
|
||||
$shapeValid = $false
|
||||
break
|
||||
}
|
||||
$actualHashes[$path] = [string]$Matches[1]
|
||||
} else {
|
||||
$shapeValid = $false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
$toolchainExact = [bool]($shapeValid -and $actualHashes.Count -eq $ToolchainFiles.Count)
|
||||
foreach ($entry in $ToolchainFiles.GetEnumerator()) {
|
||||
if (-not $actualHashes.ContainsKey([string]$entry.Key) -or $actualHashes[[string]$entry.Key] -ne [string]$entry.Value) {
|
||||
$toolchainExact = $false
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
ready = [bool]($shapeValid -and $toolchainExact)
|
||||
transportReadbackVerified = $shapeValid
|
||||
toolchainBundleId = $BundleId
|
||||
toolchainFileCount = $ToolchainFiles.Count
|
||||
toolchainReadbackVerified = $shapeValid
|
||||
toolchainExact = $toolchainExact
|
||||
dockerDaemonReadbackVerified = $shapeValid
|
||||
credentialValueRead = $false
|
||||
credentialValueOutput = $false
|
||||
productionHealthRequired = $false
|
||||
exportArtifactRequired = $false
|
||||
deterministicOwnershipEnforcedByController = $true
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Agent99SignozPreflight {
|
||||
param(
|
||||
[ValidateSet("absent", "present")][string]$ExpectedArtifactState,
|
||||
@@ -379,6 +510,65 @@ function Get-Agent99SignozPreflight {
|
||||
$runtimeImageId -eq $SignozImageId -and
|
||||
$runtimeImageRef -eq $SignozImage
|
||||
)
|
||||
|
||||
# Typed restore-startup observation. It emits only fixed booleans and a
|
||||
# mount count: no environment value, source mount, secret, or raw Env array.
|
||||
$restoreStartupProgram = @'
|
||||
set -euo pipefail
|
||||
entrypoint="$(/usr/bin/docker inspect --format='{{json .Config.Entrypoint}}' signoz)"
|
||||
entrypoint_exact=0
|
||||
[ "$entrypoint" = '["./signoz","server"]' ] && entrypoint_exact=1
|
||||
mounts="$(/usr/bin/docker inspect --format='{{range .Mounts}}{{println .Destination .Type .RW}}{{end}}' signoz)"
|
||||
mount_count="$(/usr/bin/printf '%s\n' "$mounts" | /usr/bin/awk 'NF { count += 1 } END { print count + 0 }')"
|
||||
mount_exact=0
|
||||
[ "$mount_count" -eq 1 ] && [ "$(/usr/bin/printf '%s\n' "$mounts" | /usr/bin/grep -Fxc '/var/lib/signoz volume true')" -eq 1 ] && mount_exact=1
|
||||
dsn_present=0
|
||||
dsn_class_exact=0
|
||||
sqlite_present=0
|
||||
sqlite_exact=0
|
||||
if /usr/bin/docker exec signoz /bin/sh -c 'test "${SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN+x}" = x' >/dev/null 2>&1; then dsn_present=1; fi
|
||||
if /usr/bin/docker exec signoz /bin/sh -c 'case "${SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN:-}" in tcp://clickhouse:9000*) exit 0;; *) exit 1;; esac' >/dev/null 2>&1; then dsn_class_exact=1; fi
|
||||
if /usr/bin/docker exec signoz /bin/sh -c 'test "${SIGNOZ_SQLSTORE_SQLITE_PATH+x}" = x' >/dev/null 2>&1; then sqlite_present=1; fi
|
||||
if /usr/bin/docker exec signoz /bin/sh -c 'test "${SIGNOZ_SQLSTORE_SQLITE_PATH:-}" = /var/lib/signoz/signoz.db' >/dev/null 2>&1; then sqlite_exact=1; fi
|
||||
/usr/bin/printf 'agent99_restore_startup:%s,%s,%s,%s,%s,%s,%s' "$entrypoint_exact" "$mount_count" "$mount_exact" "$dsn_present" "$dsn_class_exact" "$sqlite_present" "$sqlite_exact"
|
||||
'@
|
||||
$restoreStartupEncoded = [Convert]::ToBase64String(
|
||||
[Text.Encoding]::UTF8.GetBytes($restoreStartupProgram)
|
||||
)
|
||||
$restoreStartup = Invoke-Agent99SignozFixedRemote (
|
||||
"/usr/bin/printf %s $restoreStartupEncoded | /usr/bin/base64 -d | sudo -n /bin/bash"
|
||||
)
|
||||
$restoreStartupReadbackVerified = $false
|
||||
$restoreStartupEntrypointExact = $false
|
||||
$restoreStartupMountCount = $null
|
||||
$restoreStartupMountExact = $false
|
||||
$restoreStartupDsnPresent = $false
|
||||
$restoreStartupDsnClassExact = $false
|
||||
$restoreStartupSqlitePathPresent = $false
|
||||
$restoreStartupSqlitePathExact = $false
|
||||
if (
|
||||
(Test-Agent99SignozTransportOutcome $restoreStartup) -and
|
||||
[string]$restoreStartup.stdout -match "^agent99_restore_startup:(0|1),([0-9]+),(0|1),(0|1),(0|1),(0|1),(0|1)$"
|
||||
) {
|
||||
$restoreStartupReadbackVerified = $true
|
||||
$restoreStartupEntrypointExact = [bool]([string]$Matches[1] -eq "1")
|
||||
$restoreStartupMountCount = [int]$Matches[2]
|
||||
$restoreStartupMountExact = [bool]([string]$Matches[3] -eq "1")
|
||||
$restoreStartupDsnPresent = [bool]([string]$Matches[4] -eq "1")
|
||||
$restoreStartupDsnClassExact = [bool]([string]$Matches[5] -eq "1")
|
||||
$restoreStartupSqlitePathPresent = [bool]([string]$Matches[6] -eq "1")
|
||||
$restoreStartupSqlitePathExact = [bool]([string]$Matches[7] -eq "1")
|
||||
}
|
||||
$restoreStartupContractReady = [bool](
|
||||
$restoreStartupReadbackVerified -and
|
||||
$restoreStartupEntrypointExact -and
|
||||
$restoreStartupMountCount -eq 1 -and
|
||||
$restoreStartupMountExact -and
|
||||
$restoreStartupDsnPresent -and
|
||||
$restoreStartupDsnClassExact -and
|
||||
$restoreStartupSqlitePathPresent -and
|
||||
$restoreStartupSqlitePathExact
|
||||
)
|
||||
$health = Invoke-Agent99SignozFixedRemote (
|
||||
"sudo -n /bin/bash -c 'set -euo pipefail; " +
|
||||
"/usr/bin/printf `"agent99_health:`"; " +
|
||||
@@ -401,7 +591,7 @@ function Get-Agent99SignozPreflight {
|
||||
|
||||
$active = Invoke-Agent99SignozFixedRemote (
|
||||
"sudo -n /bin/bash -c 'set -euo pipefail; set +e; " +
|
||||
"active_output=`$(/usr/bin/pgrep -fc `"[s]ignoz-metadata-export.py.*--apply`" ); " +
|
||||
"active_output=`$(/usr/bin/pgrep -fc `"([s]ignoz-metadata-export.py.*--apply|[s]ignoz-metadata-isolated-restore.py.*--apply|[s]ignoz-metadata-restore-drill.py.*--apply)`" ); " +
|
||||
"active_rc=`$?; set -e; " +
|
||||
"[ `"`$active_rc`" -eq 0 ] || [ `"`$active_rc`" -eq 1 ]; " +
|
||||
"/usr/bin/printf `"%s`" `"`$active_output`" | /usr/bin/grep -Eq `"^[0-9]+$`"; " +
|
||||
@@ -454,6 +644,7 @@ function Get-Agent99SignozPreflight {
|
||||
$toolchainReadbackVerified -and
|
||||
$imageReadbackVerified -and
|
||||
$runtimeReadbackVerified -and
|
||||
$restoreStartupReadbackVerified -and
|
||||
$healthReadbackVerified -and
|
||||
$activeReadbackVerified -and
|
||||
$stateReadable
|
||||
@@ -465,6 +656,7 @@ function Get-Agent99SignozPreflight {
|
||||
$toolchainExact -and
|
||||
$imageExact -and
|
||||
$runtimeImageExact -and
|
||||
$restoreStartupContractReady -and
|
||||
$healthOk -and
|
||||
$activeOperationsClear -and
|
||||
$stateReadable -and
|
||||
@@ -498,6 +690,20 @@ function Get-Agent99SignozPreflight {
|
||||
signozRuntimeRunning = $runtimeRunning
|
||||
signozRuntimeReadbackVerified = $runtimeReadbackVerified
|
||||
signozRuntimeImageExact = $runtimeImageExact
|
||||
restoreStartupContractReadbackVerified = $restoreStartupReadbackVerified
|
||||
restoreStartupContractReady = $restoreStartupContractReady
|
||||
restoreStartupEntrypointExact = $restoreStartupEntrypointExact
|
||||
restoreStartupMountDestination = if ($restoreStartupMountExact) { "/var/lib/signoz" } else { $null }
|
||||
restoreStartupMountCount = $restoreStartupMountCount
|
||||
restoreStartupMountExact = $restoreStartupMountExact
|
||||
restoreStartupRequiredKeyPresence = [pscustomobject]@{
|
||||
SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN = $restoreStartupDsnPresent
|
||||
SIGNOZ_SQLSTORE_SQLITE_PATH = $restoreStartupSqlitePathPresent
|
||||
}
|
||||
restoreStartupDsnClass = if ($restoreStartupDsnClassExact) { "tcp://clickhouse:9000*" } else { "mismatch_or_unverified" }
|
||||
restoreStartupSqlitePathClass = if ($restoreStartupSqlitePathExact) { "/var/lib/signoz/signoz.db" } else { "mismatch_or_unverified" }
|
||||
restoreStartupEnvironmentValuesOutput = $false
|
||||
restoreStartupFullEnvironmentOutput = $false
|
||||
apiHealthReadbackVerified = $healthReadbackVerified
|
||||
apiHealthOk = $healthOk
|
||||
activeOperationCount = $activeOperationCount
|
||||
@@ -570,16 +776,27 @@ function Write-Agent99SignozTerminal {
|
||||
[object]$ArtifactWritePerformed
|
||||
)
|
||||
|
||||
$isRestoreMode = [bool]($Mode -like "Restore*")
|
||||
$restoreCompleted = [bool](
|
||||
$Terminal -eq "portable_metadata_restore_independently_verified_zero_residue"
|
||||
)
|
||||
$zeroResidueVerified = [bool](
|
||||
$Terminal -in @(
|
||||
"portable_metadata_restore_verified_zero_residue_pending_independent_verifier",
|
||||
"portable_metadata_restore_independently_verified_zero_residue",
|
||||
"restore_zero_residue_verified_no_completion"
|
||||
)
|
||||
)
|
||||
$controlledApplyPhases = [ordered]@{
|
||||
sensorSource = if ($Preflight) { "observed" } else { "blocked" }
|
||||
normalizedAsset = if ($Preflight) { $CanonicalAsset } else { "blocked" }
|
||||
sourceTruthDiff = if ($Preflight -and $Preflight.toolchainExact) { "exact" } else { "blocked_or_drift" }
|
||||
aiDecision = "fixed_allowlisted_metadata_export"
|
||||
riskPolicy = "high_bounded_read_only_source_private_artifact_no_raw_sqlite"
|
||||
aiDecision = if ($isRestoreMode) { "fixed_isolated_portable_metadata_restore" } else { "fixed_allowlisted_metadata_export" }
|
||||
riskPolicy = if ($isRestoreMode) { "high_bounded_ephemeral_internal_only_zero_residue" } else { "high_bounded_read_only_source_private_artifact_no_raw_sqlite" }
|
||||
checkMode = if ($CheckReceipt) { [string]$CheckReceipt.terminal } elseif ($Preflight -and $Preflight.ready) { "preflight_pass" } else { "blocked" }
|
||||
boundedExecution = if ($ApplyReceipt) { [string]$ApplyReceipt.terminal } elseif ($Mode -eq "Apply") { "not_verified" } else { "not_run" }
|
||||
independentVerifier = if ($VerifierReceipt) { [string]$VerifierReceipt.terminal } else { "not_run" }
|
||||
rollbackOrNoWrite = if ($Mode -in @("Check", "Verify")) { "no_write" } elseif ($Terminal -eq "export_verified_restore_pending") { "not_required_additive_immutable_artifact" } else { "artifact_preserved_no_delete" }
|
||||
rollbackOrNoWrite = if ($Mode -in @("Check", "Verify", "RestoreCheck")) { "no_write" } elseif ($zeroResidueVerified) { "ephemeral_resources_cleaned_zero_residue" } elseif ($Mode -eq "RestoreVerifyCleanup") { "reconciliation_failed_or_incomplete" } elseif ($Terminal -eq "export_verified_restore_pending") { "not_required_additive_immutable_artifact" } else { "artifact_preserved_no_delete" }
|
||||
learningWriteback = "pending_telegram_km_rag_mcp_playbook_ack"
|
||||
}
|
||||
$payload = [pscustomobject]@{
|
||||
@@ -587,7 +804,7 @@ function Write-Agent99SignozTerminal {
|
||||
canonicalAsset = $CanonicalAsset
|
||||
domain = "backup_observability"
|
||||
executor = "agent99_signoz_metadata_executor"
|
||||
verifier = "signoz_metadata_export_independent_verifier"
|
||||
verifier = if ($Mode -eq "RestoreVerifyCleanup") { "signoz_metadata_restore_cleanup_verifier" } elseif ($isRestoreMode) { "pending_distinct_restore_cleanup_verifier" } else { "signoz_metadata_export_independent_verifier" }
|
||||
executionHub = "host:192.168.0.99"
|
||||
dispatchPath = "windows99_agent99_to_host110_fixed_metadata_toolchain"
|
||||
dispatchOnly = $true
|
||||
@@ -599,6 +816,7 @@ function Write-Agent99SignozTerminal {
|
||||
targetHost = $TargetHost
|
||||
toolchainBundleId = $BundleId
|
||||
credentialReference = $CredentialFile
|
||||
restoreReceipt = if ($isRestoreMode) { $RestoreRemoteReceipt } else { $null }
|
||||
targetOperationLock = $OperationLock
|
||||
targetOperationLockRequiredExistingNonSymlink = $true
|
||||
targetTimeoutKillAfterSeconds = $TargetKillAfterSeconds
|
||||
@@ -619,13 +837,14 @@ function Write-Agent99SignozTerminal {
|
||||
controlledApplyPhases = [pscustomobject]$controlledApplyPhases
|
||||
artifactWritePerformed = $ArtifactWritePerformed
|
||||
productionRuntimeMutationPerformed = $false
|
||||
isolatedEphemeralRuntimeCreated = [bool]($Mode -eq "RestoreApply" -and $script:IsolatedApplyDispatchAttempted)
|
||||
rawSqliteRead = $false
|
||||
rawVolumeRead = $false
|
||||
secretValueReadByController = $false
|
||||
secretValueTransmitted = $false
|
||||
secretValuePersistedInEvidence = $false
|
||||
credentialReferenceDispatchAttempted = $script:ApplyDispatchAttempted
|
||||
credentialReferenceConsumedOnTarget = if (-not $script:ApplyDispatchAttempted) {
|
||||
credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted
|
||||
credentialReferenceConsumedOnTarget = if (-not $script:ExportCredentialDispatchAttempted) {
|
||||
$false
|
||||
} elseif (Test-Agent99SignozReceiptIdentity $ApplyReceipt "terminal" @("partial_degraded")) {
|
||||
$true
|
||||
@@ -634,9 +853,13 @@ function Write-Agent99SignozTerminal {
|
||||
}
|
||||
arbitraryCommandAllowed = $false
|
||||
crossDomainFallbackAllowed = $false
|
||||
restoreTerminal = "pending_isolated_same_version_target"
|
||||
zeroResidueTerminal = "pending"
|
||||
completionClaim = $false
|
||||
restoreTerminal = if ($restoreCompleted) { "portable_metadata_restore_verified" } elseif ($isRestoreMode) { $Terminal } else { "pending_isolated_same_version_target" }
|
||||
zeroResidueTerminal = if ($zeroResidueVerified) { "verified" } elseif ($isRestoreMode) { "not_verified" } else { "pending" }
|
||||
completionScope = if ($restoreCompleted) { "portable_assets_only" } else { $null }
|
||||
completionClaim = $restoreCompleted
|
||||
fullSqliteCompletionClaim = $false
|
||||
rolesPreferencesGlobalConfigRestoreClaim = $false
|
||||
secretBearingAssetsRestoreClaim = $false
|
||||
telegramAcknowledged = $false
|
||||
kmRagMcpPlaybookAcknowledged = $false
|
||||
transientTransportOutputUsed = $script:TransientTransportOutputUsed
|
||||
@@ -721,6 +944,151 @@ if (-not $runtimeSourceBinding.ok) {
|
||||
Stop-Agent99SignozRun "blocked_with_safe_next_action" "runtime_source_binding_failed" $runtimeSourceBinding $null $null $null $false 65
|
||||
}
|
||||
|
||||
if ($Mode -eq "RestoreVerifyCleanup") {
|
||||
$preflight = Get-Agent99SignozCleanupPreflight
|
||||
if (-not $preflight.transportReadbackVerified -or -not $preflight.ready) {
|
||||
Stop-Agent99SignozRun "restore_cleanup_blocked_with_safe_next_action" "restore_cleanup_toolchain_or_docker_preflight_failed" $preflight $null $null $null $false 1
|
||||
}
|
||||
$restoreReceiptStateBefore = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file"
|
||||
if ($restoreReceiptStateBefore -eq "unverified" -or $restoreReceiptStateBefore -in @("symlink", "other", "directory")) {
|
||||
Stop-Agent99SignozRun "restore_cleanup_blocked_with_safe_next_action" "restore_receipt_state_unverified_or_unsafe" $preflight $null $null $null $false 1
|
||||
}
|
||||
$restoreProgram = (
|
||||
"/usr/bin/python3 $IsolatedRestorePath --verify-cleanup " +
|
||||
"--policy $PolicyPath --cluster-config $IsolatedClusterConfigPath " +
|
||||
"--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId"
|
||||
)
|
||||
if ($restoreReceiptStateBefore -eq "file") {
|
||||
$restoreProgram += " --apply-receipt-file $RestoreRemoteReceipt"
|
||||
}
|
||||
$restoreCommand = New-Agent99SignozCleanupLockedRemoteCommand $restoreProgram $RestoreCleanupTargetTimeoutSeconds
|
||||
$restoreRemote = Invoke-Agent99SignozFixedRemote $restoreCommand $RestoreCleanupControllerWaitSeconds
|
||||
$restoreReceipt = Convert-Agent99JsonReceipt $restoreRemote
|
||||
$transportAccepted = Test-Agent99SignozTransportOutcome $restoreRemote
|
||||
$failureReceiptAccepted = Test-Agent99SignozReceiptIdentity $restoreReceipt "terminal" @(
|
||||
"failed_zero_residue_verified_no_completion",
|
||||
"failed_cleanup_reconciliation_required"
|
||||
)
|
||||
if (-not $transportAccepted) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_controller_transport_failed_or_timed_out" $preflight $null $null $restoreReceipt $false 1
|
||||
}
|
||||
if ($failureReceiptAccepted) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_controller_failed" $preflight $null $null $restoreReceipt $false 1
|
||||
}
|
||||
$independentCompletion = Test-Agent99SignozReceiptIdentity $restoreReceipt "independent_restore_verifier" @(
|
||||
"portable_metadata_restore_independently_verified_zero_residue"
|
||||
)
|
||||
$cleanupOnlyPass = Test-Agent99SignozReceiptIdentity $restoreReceipt "independent_restore_verifier" @(
|
||||
"pass_zero_residue_verified"
|
||||
)
|
||||
if (-not $independentCompletion -and -not $cleanupOnlyPass) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_terminal_invalid" $preflight $null $null $restoreReceipt $false 1
|
||||
}
|
||||
if ($independentCompletion -and $restoreReceipt.completionClaim -ne $true) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_completion_claim_invalid" $preflight $null $null $restoreReceipt $false 1
|
||||
}
|
||||
if ($cleanupOnlyPass -and $restoreReceipt.completionClaim -ne $false) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_only_claim_invalid" $preflight $null $null $restoreReceipt $false 1
|
||||
}
|
||||
$restoreReceiptStateAfter = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file"
|
||||
if ($restoreReceiptStateAfter -ne $restoreReceiptStateBefore) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_changed_durable_receipt_state" $preflight $null $null $restoreReceipt $false 1
|
||||
}
|
||||
$postflight = Get-Agent99SignozCleanupPreflight
|
||||
if (-not $postflight.transportReadbackVerified -or -not $postflight.ready) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_cleanup_postflight_failed" $postflight $null $null $restoreReceipt $false 1
|
||||
}
|
||||
if ($independentCompletion) {
|
||||
Stop-Agent99SignozRun "portable_metadata_restore_independently_verified_zero_residue" "" $postflight $null $null $restoreReceipt $false 0
|
||||
}
|
||||
Stop-Agent99SignozRun "restore_zero_residue_verified_no_completion" "" $postflight $null $null $restoreReceipt $false 0
|
||||
}
|
||||
|
||||
if ($Mode -in @("RestoreCheck", "RestoreApply")) {
|
||||
$preflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
||||
if (-not $preflight.transportReadbackVerified) {
|
||||
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_preflight_transport_unverified" $preflight $null $null $null $false 1
|
||||
}
|
||||
if (-not $preflight.ready) {
|
||||
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_preflight_failed" $preflight $null $null $null $false 1
|
||||
}
|
||||
|
||||
$restoreReceiptStateBefore = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file"
|
||||
if ($restoreReceiptStateBefore -eq "unverified" -or $restoreReceiptStateBefore -in @("symlink", "other", "directory")) {
|
||||
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_receipt_state_unverified_or_unsafe" $preflight $null $null $null $false 1
|
||||
}
|
||||
if ($Mode -in @("RestoreCheck", "RestoreApply") -and $restoreReceiptStateBefore -ne "absent") {
|
||||
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_receipt_identity_collision" $preflight $null $null $null $false 66
|
||||
}
|
||||
|
||||
$restoreModeArgument = switch ($Mode) {
|
||||
"RestoreCheck" { "--check" }
|
||||
"RestoreApply" { "--apply" }
|
||||
default { "" }
|
||||
}
|
||||
if (-not $restoreModeArgument) {
|
||||
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_mode_not_routable" $preflight $null $null $null $false 64
|
||||
}
|
||||
$restoreProgram = (
|
||||
"/usr/bin/python3 $IsolatedRestorePath $restoreModeArgument " +
|
||||
"--bundle-dir $OutputDir --policy $PolicyPath " +
|
||||
"--cluster-config $IsolatedClusterConfigPath " +
|
||||
"--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId"
|
||||
)
|
||||
if ($Mode -eq "RestoreApply") {
|
||||
$restoreProgram += " --receipt-file $RestoreRemoteReceipt"
|
||||
$script:IsolatedApplyDispatchAttempted = $true
|
||||
}
|
||||
$restoreCommand = New-Agent99SignozLockedRemoteCommand $restoreProgram $TargetProgramTimeoutSeconds "present"
|
||||
$restoreRemote = Invoke-Agent99SignozFixedRemote $restoreCommand $ControllerWaitSeconds
|
||||
$restoreReceipt = Convert-Agent99JsonReceipt $restoreRemote
|
||||
$transportAccepted = Test-Agent99SignozTransportOutcome $restoreRemote
|
||||
$failureReceiptAccepted = Test-Agent99SignozReceiptIdentity $restoreReceipt "terminal" @(
|
||||
"failed_zero_residue_verified_no_completion",
|
||||
"failed_cleanup_reconciliation_required"
|
||||
)
|
||||
if (-not $transportAccepted) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_controller_transport_failed_or_timed_out" $preflight $null $restoreReceipt $null $false 1
|
||||
}
|
||||
if ($failureReceiptAccepted) {
|
||||
$failureTerminal = if ([string]$restoreReceipt.terminal -eq "failed_zero_residue_verified_no_completion") {
|
||||
"restore_failed_zero_residue_verified_no_completion"
|
||||
} else {
|
||||
"restore_failed_cleanup_reconciliation_required"
|
||||
}
|
||||
Stop-Agent99SignozRun $failureTerminal "isolated_restore_controller_failed" $preflight $null $restoreReceipt $null $false 1
|
||||
}
|
||||
|
||||
$allowedRestoreTerminal = switch ($Mode) {
|
||||
"RestoreCheck" { "check_pass_no_write" }
|
||||
"RestoreApply" { "portable_metadata_restore_verified_zero_residue_pending_independent_verifier" }
|
||||
}
|
||||
if (-not (Test-Agent99SignozReceiptIdentity $restoreReceipt "terminal" @($allowedRestoreTerminal))) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_controller_terminal_invalid" $preflight $null $restoreReceipt $null $false 1
|
||||
}
|
||||
|
||||
$restoreReceiptStateAfter = Get-Agent99SignozRemotePathState $RestoreRemoteReceipt "file"
|
||||
if ($Mode -eq "RestoreApply" -and $restoreReceiptStateAfter -ne "file") {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_durable_receipt_missing" $preflight $null $restoreReceipt $null $false 1
|
||||
}
|
||||
if ($Mode -eq "RestoreCheck" -and $restoreReceiptStateAfter -ne "absent") {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_check_wrote_receipt" $preflight $restoreReceipt $null $null $false 1
|
||||
}
|
||||
$postflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
||||
if (-not $postflight.transportReadbackVerified -or -not $postflight.ready) {
|
||||
Stop-Agent99SignozRun "restore_failed_cleanup_reconciliation_required" "restore_postflight_runtime_or_artifact_drift" $postflight $null $restoreReceipt $null $false 1
|
||||
}
|
||||
switch ($Mode) {
|
||||
"RestoreCheck" {
|
||||
Stop-Agent99SignozRun "restore_check_ready" "" $postflight $restoreReceipt $null $null $false 0
|
||||
}
|
||||
"RestoreApply" {
|
||||
Stop-Agent99SignozRun "portable_metadata_restore_verified_zero_residue_pending_independent_verifier" "" $postflight $null $restoreReceipt $null $false 0
|
||||
}
|
||||
}
|
||||
Stop-Agent99SignozRun "restore_blocked_with_safe_next_action" "restore_mode_fallthrough" $preflight $null $null $null $false 64
|
||||
}
|
||||
|
||||
if ($Mode -eq "Verify") {
|
||||
$preflight = Get-Agent99SignozPreflight -ExpectedArtifactState "present" -RequireCredential $false
|
||||
if (-not $preflight.transportReadbackVerified) {
|
||||
@@ -790,7 +1158,7 @@ $applyProgram = (
|
||||
"--receipt-file $RemoteReceipt"
|
||||
)
|
||||
$applyCommand = New-Agent99SignozLockedRemoteCommand $applyProgram $ApplyTargetTimeoutSeconds "absent"
|
||||
$script:ApplyDispatchAttempted = $true
|
||||
$script:ExportCredentialDispatchAttempted = $true
|
||||
$apply = Invoke-Agent99SignozFixedRemote $applyCommand $ApplyControllerWaitSeconds
|
||||
$applyReceipt = Convert-Agent99JsonReceipt $apply
|
||||
if (-not (Test-Agent99SignozTransportOutcome $apply) -or -not (Test-Agent99SignozReceiptIdentity $applyReceipt "terminal" @("partial_degraded"))) {
|
||||
|
||||
@@ -270,7 +270,74 @@
|
||||
"restore_isolation": {
|
||||
"target_url_scope": "loopback_only",
|
||||
"required_image_ref": "signoz/signoz:v0.113.0",
|
||||
"required_image_id": "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b",
|
||||
"image_id_sha256_required": true,
|
||||
"images": {
|
||||
"signoz": {
|
||||
"ref": "signoz/signoz:v0.113.0",
|
||||
"id": "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b"
|
||||
},
|
||||
"clickhouse": {
|
||||
"ref": "clickhouse/clickhouse-server:25.5.6",
|
||||
"id": "sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6"
|
||||
},
|
||||
"zookeeper": {
|
||||
"ref": "signoz/zookeeper:3.7.1",
|
||||
"id": "sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0"
|
||||
}
|
||||
},
|
||||
"startup_contract": {
|
||||
"reviewed": true,
|
||||
"review_source": "agent99_typed_live_nonsecret_preflight_2026-07-22",
|
||||
"server_entrypoint": [
|
||||
"./signoz",
|
||||
"server"
|
||||
],
|
||||
"server_environment": {
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN": "tcp://clickhouse:9000",
|
||||
"SIGNOZ_SQLSTORE_SQLITE_PATH": "/var/lib/signoz/signoz.db"
|
||||
},
|
||||
"server_data_mount_destination": "/var/lib/signoz",
|
||||
"clickhouse_network_alias": "clickhouse",
|
||||
"zookeeper_network_alias": "restore-zookeeper",
|
||||
"required_production_key_presence": [
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN",
|
||||
"SIGNOZ_SQLSTORE_SQLITE_PATH"
|
||||
],
|
||||
"production_dsn_class": "tcp://clickhouse:9000*",
|
||||
"production_sqlite_path": "/var/lib/signoz/signoz.db",
|
||||
"secret_value_output_allowed": false,
|
||||
"full_environment_output_allowed": false
|
||||
},
|
||||
"resource_prefix": "awoooi-signoz-md-restore-",
|
||||
"canonical_id_prefix": "ephemeral-signoz-metadata-restore-",
|
||||
"secondary_resource_label_key": "com.awoooi.restore.identity",
|
||||
"private_workspace_parent": "/tmp",
|
||||
"server_container_port": 8080,
|
||||
"loopback_publish_host": "127.0.0.1",
|
||||
"isolated_cluster_config": "isolated-cluster.xml",
|
||||
"production_continuity": {
|
||||
"signoz": {
|
||||
"container": "signoz",
|
||||
"image_ref": "signoz/signoz:v0.113.0",
|
||||
"image_id": "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b"
|
||||
},
|
||||
"clickhouse": {
|
||||
"container": "signoz-clickhouse",
|
||||
"image_ref": "clickhouse/clickhouse-server:25.5.6",
|
||||
"image_id": "sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6"
|
||||
},
|
||||
"zookeeper": {
|
||||
"container": "signoz-zookeeper-1",
|
||||
"image_ref": "signoz/zookeeper:3.7.1",
|
||||
"image_id": "sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0"
|
||||
},
|
||||
"otel_collector": {
|
||||
"container": "signoz-otel-collector",
|
||||
"image_ref": "signoz/signoz-otel-collector:v0.144.1",
|
||||
"image_id": "sha256:47cbed488288df7bac6aab3665275bec6d6e24fe5c545c66914827a891e538d8"
|
||||
}
|
||||
},
|
||||
"image_pull_allowed": false,
|
||||
"production_network_attachment_allowed": false,
|
||||
"production_volume_attachment_allowed": false,
|
||||
|
||||
1721
scripts/backup/signoz-metadata-isolated-restore.py
Executable file
1721
scripts/backup/signoz-metadata-isolated-restore.py
Executable file
File diff suppressed because it is too large
Load Diff
@@ -4,21 +4,23 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import stat
|
||||
|
||||
# Child commands use fixed argv without a shell.
|
||||
import subprocess # nosec B404
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from signoz_metadata_contract import (
|
||||
DEFAULT_POLICY,
|
||||
SHA256,
|
||||
ApiClient,
|
||||
ContractError,
|
||||
canonical_json_bytes,
|
||||
@@ -49,6 +51,11 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--policy", default=str(DEFAULT_POLICY))
|
||||
parser.add_argument("--target-base-url", required=True)
|
||||
parser.add_argument("--credential-file")
|
||||
parser.add_argument(
|
||||
"--auth-mode",
|
||||
choices=("signoz_api_key", "isolated_session_bearer"),
|
||||
default="signoz_api_key",
|
||||
)
|
||||
parser.add_argument("--isolation-receipt", required=True)
|
||||
parser.add_argument("--trace-id", required=True)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
@@ -57,6 +64,57 @@ def parse_args() -> argparse.Namespace:
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
EXPECTED_IMAGES = {
|
||||
"signoz": {
|
||||
"ref": "signoz/signoz:v0.113.0",
|
||||
"id": "sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b",
|
||||
},
|
||||
"clickhouse": {
|
||||
"ref": "clickhouse/clickhouse-server:25.5.6",
|
||||
"id": "sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6",
|
||||
},
|
||||
"zookeeper": {
|
||||
"ref": "signoz/zookeeper:3.7.1",
|
||||
"id": "sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0",
|
||||
},
|
||||
}
|
||||
PRIMARY_LABEL = "awoooi.signoz.metadata.restore.run_id"
|
||||
IDENTITY_LABEL = "com.awoooi.restore.identity"
|
||||
|
||||
|
||||
def restore_identity(trace_id: str, run_id: str, work_item_id: str) -> str:
|
||||
return hashlib.sha256(
|
||||
trace_id.encode("utf-8")
|
||||
+ b"\0"
|
||||
+ run_id.encode("utf-8")
|
||||
+ b"\0"
|
||||
+ work_item_id.encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def expected_resources(identity: str) -> dict[str, Any]:
|
||||
short_id = identity[:16]
|
||||
prefix = f"awoooi-signoz-md-restore-{short_id}"
|
||||
return {
|
||||
"identity": identity,
|
||||
"prefix": prefix,
|
||||
"canonical_id": f"ephemeral-signoz-metadata-restore-{short_id}",
|
||||
"network": f"{prefix}-net",
|
||||
"containers": [
|
||||
f"{prefix}-server",
|
||||
f"{prefix}-clickhouse",
|
||||
f"{prefix}-zookeeper",
|
||||
],
|
||||
"volumes": [
|
||||
f"{prefix}-zk-data",
|
||||
f"{prefix}-zk-log",
|
||||
f"{prefix}-ch-data",
|
||||
f"{prefix}-server-data",
|
||||
],
|
||||
"workspace": f"/tmp/{prefix}-private",
|
||||
}
|
||||
|
||||
|
||||
def run_independent_bundle_verifier(args: argparse.Namespace) -> None:
|
||||
command = [
|
||||
sys.executable,
|
||||
@@ -122,34 +180,60 @@ def validate_isolation_receipt(
|
||||
if not isinstance(target, dict):
|
||||
raise ContractError("isolation_target_missing")
|
||||
isolation_policy = policy["restore_isolation"]
|
||||
identity = restore_identity(args.trace_id, args.run_id, args.work_item_id)
|
||||
resources = expected_resources(identity)
|
||||
target_url = validate_base_url(args.target_base_url, loopback_only=True)
|
||||
if target.get("base_url_sha256") != sha256_bytes(target_url.encode("utf-8")):
|
||||
raise ContractError("isolation_target_url_hash_mismatch")
|
||||
canonical_id = target.get("canonical_id")
|
||||
if not isinstance(canonical_id, str) or not canonical_id.startswith(
|
||||
"ephemeral-signoz-metadata-restore-"
|
||||
):
|
||||
if target.get("identity") != identity:
|
||||
raise ContractError("isolation_target_identity_mismatch")
|
||||
if target.get("prefix") != resources["prefix"]:
|
||||
raise ContractError("isolation_target_prefix_mismatch")
|
||||
if target.get("canonical_id") != resources["canonical_id"]:
|
||||
raise ContractError("isolation_target_canonical_id_invalid")
|
||||
if target.get("image_ref") != isolation_policy["required_image_ref"]:
|
||||
raise ContractError("isolation_target_image_ref_mismatch")
|
||||
image_id = target.get("image_id")
|
||||
if target.get("image_id") != isolation_policy["required_image_id"]:
|
||||
raise ContractError("isolation_target_image_id_mismatch")
|
||||
if target.get("images") != EXPECTED_IMAGES:
|
||||
raise ContractError("isolation_target_image_set_mismatch")
|
||||
if target.get("resources") != {
|
||||
"network": resources["network"],
|
||||
"containers": resources["containers"],
|
||||
"volumes": resources["volumes"],
|
||||
"workspace": resources["workspace"],
|
||||
}:
|
||||
raise ContractError("isolation_target_resource_names_mismatch")
|
||||
listener = target.get("listener")
|
||||
if (
|
||||
not isinstance(image_id, str)
|
||||
or not image_id.startswith("sha256:")
|
||||
or not SHA256.fullmatch(image_id.removeprefix("sha256:"))
|
||||
not isinstance(listener, dict)
|
||||
or listener.get("host") != "127.0.0.1"
|
||||
or not isinstance(listener.get("port"), int)
|
||||
or listener["port"] <= 0
|
||||
or listener["port"] > 65535
|
||||
):
|
||||
raise ContractError("isolation_target_image_id_invalid")
|
||||
raise ContractError("isolation_target_listener_invalid")
|
||||
expected_url = f"http://127.0.0.1:{listener['port']}"
|
||||
if target_url != expected_url:
|
||||
raise ContractError("isolation_target_listener_url_mismatch")
|
||||
if args.auth_mode != "isolated_session_bearer":
|
||||
raise ContractError("restore_auth_mode_must_be_isolated_session_bearer")
|
||||
if target.get("authentication_mode") != args.auth_mode:
|
||||
raise ContractError("isolation_target_auth_mode_mismatch")
|
||||
required_values = {
|
||||
"image_present_locally": True,
|
||||
"image_pull_performed": False,
|
||||
"image_build_performed": False,
|
||||
"production_network_attached": False,
|
||||
"production_volume_attached": False,
|
||||
"ephemeral_volumes_only": True,
|
||||
"cleanup_controller_armed": True,
|
||||
"zero_residue_verifier_armed": True,
|
||||
"network_scope": "loopback_and_internal_only",
|
||||
"resource_label_key": isolation_policy["resource_label_key"],
|
||||
"resource_label_key": PRIMARY_LABEL,
|
||||
"resource_label_value": args.run_id,
|
||||
"identity_label_key": IDENTITY_LABEL,
|
||||
"identity_label_value": identity,
|
||||
}
|
||||
for key, expected in required_values.items():
|
||||
if target.get(key) != expected:
|
||||
@@ -201,9 +285,14 @@ def make_client(
|
||||
return None
|
||||
if token is None:
|
||||
raise ContractError("credential_value_unavailable")
|
||||
if args.auth_mode == "isolated_session_bearer":
|
||||
header_name = "Authorization"
|
||||
token = f"Bearer {token}"
|
||||
else:
|
||||
header_name = policy["authentication"]["header_name"]
|
||||
return ApiClient(
|
||||
base_url=validate_base_url(args.target_base_url, loopback_only=True),
|
||||
header_name=policy["authentication"]["header_name"],
|
||||
header_name=header_name,
|
||||
token=token,
|
||||
timeout_seconds=policy["limits"]["request_timeout_seconds"],
|
||||
max_response_bytes=policy["limits"]["max_response_bytes"],
|
||||
@@ -278,6 +367,30 @@ def run_apply(
|
||||
expected = {asset["id"]: items for asset, items in assets}
|
||||
if canonical_json_bytes(after) != canonical_json_bytes(expected):
|
||||
raise ContractError("restore_semantic_readback_mismatch")
|
||||
restored_asset_counts: dict[str, int] = {}
|
||||
semantic_sha256: dict[str, str] = {}
|
||||
for asset, items in assets:
|
||||
asset_id = asset["id"]
|
||||
restored_items = after[asset_id]
|
||||
expected_digest = sha256_bytes(canonical_json_bytes(items))
|
||||
restored_digest = sha256_bytes(canonical_json_bytes(restored_items))
|
||||
if len(restored_items) != len(items):
|
||||
raise ContractError(f"restore_semantic_count_mismatch_{asset_id}")
|
||||
if restored_digest != expected_digest:
|
||||
raise ContractError(f"restore_semantic_digest_mismatch_{asset_id}")
|
||||
restored_asset_counts[asset_id] = len(restored_items)
|
||||
semantic_sha256[asset_id] = restored_digest
|
||||
portable_semantic_sha256 = sha256_bytes(
|
||||
canonical_json_bytes(
|
||||
{
|
||||
asset_id: {
|
||||
"count": restored_asset_counts[asset_id],
|
||||
"sha256": semantic_sha256[asset_id],
|
||||
}
|
||||
for asset_id in sorted(restored_asset_counts)
|
||||
}
|
||||
)
|
||||
)
|
||||
terminal = receipt(
|
||||
trace_id=args.trace_id,
|
||||
run_id=args.run_id,
|
||||
@@ -289,6 +402,14 @@ def run_apply(
|
||||
"cleanup_and_zero_residue_verifier_pending"
|
||||
),
|
||||
)
|
||||
terminal["restored_asset_counts"] = restored_asset_counts
|
||||
terminal["restored_asset_semantic_sha256"] = semantic_sha256
|
||||
terminal["portable_semantic_sha256"] = portable_semantic_sha256
|
||||
terminal["completion_scope"] = "portable_assets_only"
|
||||
terminal["completion_claim"] = False
|
||||
terminal["full_sqlite_completion_claim"] = False
|
||||
terminal["roles_preferences_global_config_restore_claim"] = False
|
||||
terminal["secret_bearing_assets_restore_claim"] = False
|
||||
terminal["phase_receipts"] = [
|
||||
receipt(
|
||||
trace_id=args.trace_id,
|
||||
@@ -397,41 +518,153 @@ def docker_label_count(resource: str, label: str) -> int:
|
||||
return len([line for line in result.stdout.splitlines() if line.strip()])
|
||||
|
||||
|
||||
def require_target_unreachable(base_url: str) -> None:
|
||||
base_url = validate_base_url(base_url, loopback_only=True)
|
||||
request = urllib.request.Request(base_url, method="GET")
|
||||
def docker_inspect(resource: str, name: str) -> dict[str, Any] | None:
|
||||
command = ["docker", resource, "inspect", name]
|
||||
try:
|
||||
# base_url is validated as loopback HTTP(S) immediately above.
|
||||
with urllib.request.urlopen( # nosec B310
|
||||
request, timeout=2
|
||||
):
|
||||
pass
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise ContractError("cleanup_target_listener_still_present") from exc
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
result = subprocess.run( # nosec B603
|
||||
command,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise ContractError(f"cleanup_{resource}_inventory_failed") from exc
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
try:
|
||||
value = json.loads(result.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ContractError(f"cleanup_{resource}_inventory_invalid") from exc
|
||||
if not isinstance(value, list) or len(value) != 1 or not isinstance(value[0], dict):
|
||||
raise ContractError(f"cleanup_{resource}_inventory_invalid")
|
||||
return value[0]
|
||||
|
||||
|
||||
def require_target_tcp_unreachable(base_url: str) -> None:
|
||||
base_url = validate_base_url(base_url, loopback_only=True)
|
||||
parsed = urllib.parse.urlsplit(base_url)
|
||||
if parsed.hostname != "127.0.0.1" or parsed.port is None:
|
||||
raise ContractError("cleanup_target_listener_invalid")
|
||||
try:
|
||||
with socket.create_connection((parsed.hostname, parsed.port), timeout=2):
|
||||
raise ContractError("cleanup_target_tcp_listener_still_present")
|
||||
except ContractError:
|
||||
raise
|
||||
except OSError:
|
||||
return
|
||||
raise ContractError("cleanup_target_listener_still_present")
|
||||
|
||||
|
||||
def cleanup_production_snapshot() -> dict[str, dict[str, Any]]:
|
||||
expected = {
|
||||
"signoz": EXPECTED_IMAGES["signoz"],
|
||||
"signoz-clickhouse": EXPECTED_IMAGES["clickhouse"],
|
||||
"signoz-zookeeper-1": EXPECTED_IMAGES["zookeeper"],
|
||||
"signoz-otel-collector": {
|
||||
"ref": "signoz/signoz-otel-collector:v0.144.1",
|
||||
"id": "sha256:47cbed488288df7bac6aab3665275bec6d6e24fe5c545c66914827a891e538d8",
|
||||
},
|
||||
}
|
||||
snapshot: dict[str, dict[str, Any]] = {}
|
||||
for name, image in expected.items():
|
||||
data = docker_inspect("container", name)
|
||||
if data is None:
|
||||
raise ContractError("cleanup_production_container_missing")
|
||||
config = data.get("Config")
|
||||
state = data.get("State")
|
||||
if not isinstance(config, dict) or not isinstance(state, dict):
|
||||
raise ContractError("cleanup_production_container_shape_invalid")
|
||||
row = {
|
||||
"id": data.get("Id"),
|
||||
"image_id": data.get("Image"),
|
||||
"image_ref": config.get("Image"),
|
||||
"running": state.get("Running"),
|
||||
"started_at": state.get("StartedAt"),
|
||||
"restart_count": data.get("RestartCount"),
|
||||
}
|
||||
if row["image_id"] != image["id"] or row["image_ref"] != image["ref"]:
|
||||
raise ContractError("cleanup_production_image_identity_invalid")
|
||||
snapshot[name] = row
|
||||
return snapshot
|
||||
|
||||
|
||||
def cleanup_image_inventory() -> list[str]:
|
||||
try:
|
||||
result = subprocess.run( # nosec B603
|
||||
[
|
||||
"docker",
|
||||
"image",
|
||||
"ls",
|
||||
"--no-trunc",
|
||||
"--format={{.ID}}|{{.Repository}}:{{.Tag}}",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||
raise ContractError("cleanup_image_inventory_failed") from exc
|
||||
if result.returncode != 0:
|
||||
raise ContractError("cleanup_image_inventory_failed")
|
||||
rows = sorted(line for line in result.stdout.splitlines() if line.strip())
|
||||
if not rows:
|
||||
raise ContractError("cleanup_image_inventory_empty")
|
||||
return rows
|
||||
|
||||
|
||||
def run_cleanup_verifier(
|
||||
args: argparse.Namespace,
|
||||
policy: dict[str, Any],
|
||||
isolation: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
label = f"{policy['restore_isolation']['resource_label_key']}={args.run_id}"
|
||||
counts = {
|
||||
resource: docker_label_count(resource, label)
|
||||
for resource in ("container", "volume", "network")
|
||||
}
|
||||
if any(counts.values()):
|
||||
raise ContractError("cleanup_resource_residue_present")
|
||||
require_target_unreachable(args.target_base_url)
|
||||
identity = restore_identity(args.trace_id, args.run_id, args.work_item_id)
|
||||
resources = expected_resources(identity)
|
||||
for name in resources["containers"]:
|
||||
if docker_inspect("container", name) is not None:
|
||||
raise ContractError("cleanup_exact_container_residue_present")
|
||||
for name in resources["volumes"]:
|
||||
if docker_inspect("volume", name) is not None:
|
||||
raise ContractError("cleanup_exact_volume_residue_present")
|
||||
if docker_inspect("network", resources["network"]) is not None:
|
||||
raise ContractError("cleanup_exact_network_residue_present")
|
||||
labels = (
|
||||
f"{PRIMARY_LABEL}={args.run_id}",
|
||||
f"{IDENTITY_LABEL}={identity}",
|
||||
)
|
||||
for label in labels:
|
||||
counts = {
|
||||
resource: docker_label_count(resource, label)
|
||||
for resource in ("container", "volume", "network")
|
||||
}
|
||||
if any(counts.values()):
|
||||
raise ContractError("cleanup_resource_label_residue_present")
|
||||
require_target_tcp_unreachable(args.target_base_url)
|
||||
workspace = Path(resources["workspace"])
|
||||
if workspace.exists() or workspace.is_symlink():
|
||||
raise ContractError("cleanup_private_workspace_residue_present")
|
||||
production_before = isolation.get("production_before")
|
||||
if not isinstance(production_before, dict):
|
||||
raise ContractError("cleanup_production_before_missing")
|
||||
if cleanup_production_snapshot() != production_before:
|
||||
raise ContractError("cleanup_production_identity_continuity_failed")
|
||||
image_inventory_before = isolation.get("image_inventory_before")
|
||||
if not isinstance(image_inventory_before, list):
|
||||
raise ContractError("cleanup_image_inventory_before_missing")
|
||||
if cleanup_image_inventory() != image_inventory_before:
|
||||
raise ContractError("cleanup_image_inventory_drift")
|
||||
return receipt(
|
||||
trace_id=args.trace_id,
|
||||
run_id=args.run_id,
|
||||
work_item_id=args.work_item_id,
|
||||
phase="terminal",
|
||||
terminal="pass_zero_residue_verified",
|
||||
detail="container_0_volume_0_network_0_listener_0",
|
||||
detail=(
|
||||
"exact_container_0_volume_0_network_0_label_inventory_0_"
|
||||
"tcp_listener_0_workspace_0_production_identity_unchanged"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -469,14 +702,14 @@ def main() -> int:
|
||||
)
|
||||
policy = load_policy(Path(args.policy))
|
||||
run_independent_bundle_verifier(args)
|
||||
validate_isolation_receipt(args, policy)
|
||||
isolation = validate_isolation_receipt(args, policy)
|
||||
assets = load_portable_assets(Path(args.bundle_dir), policy)
|
||||
if args.check:
|
||||
result = run_check(args, policy, assets)
|
||||
elif args.apply:
|
||||
result = run_apply(args, policy, assets)
|
||||
else:
|
||||
result = run_cleanup_verifier(args, policy)
|
||||
result = run_cleanup_verifier(args, policy, isolation)
|
||||
if args.receipt_file:
|
||||
write_receipt(Path(args.receipt_file), result)
|
||||
emit_result(result)
|
||||
|
||||
@@ -60,7 +60,10 @@ class MetadataHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
|
||||
def _authorized(self) -> bool:
|
||||
return self.headers.get("SIGNOZ-API-KEY") == TOKEN
|
||||
return (
|
||||
self.headers.get("SIGNOZ-API-KEY") == TOKEN
|
||||
or self.headers.get("Authorization") == f"Bearer {TOKEN}"
|
||||
)
|
||||
|
||||
def _send_json(self, status: int, value: Any) -> None:
|
||||
payload = json.dumps(value, separators=(",", ":")).encode("utf-8")
|
||||
@@ -219,20 +222,46 @@ def create_bundle(tmp_path: Path) -> tuple[Path, Path]:
|
||||
|
||||
def isolation_receipt(tmp_path: Path, target_base_url: str) -> Path:
|
||||
policy = json.loads(POLICY.read_text(encoding="utf-8"))
|
||||
identity = hashlib.sha256(
|
||||
TRACE_ID.encode("utf-8")
|
||||
+ b"\0"
|
||||
+ RUN_ID.encode("utf-8")
|
||||
+ b"\0"
|
||||
+ WORK_ITEM_ID.encode("utf-8")
|
||||
).hexdigest()
|
||||
short_id = identity[:16]
|
||||
prefix = f"awoooi-signoz-md-restore-{short_id}"
|
||||
port_text = target_base_url.rsplit(":", 1)[-1]
|
||||
port = int(port_text) if port_text.isdigit() else 443
|
||||
production_before = {
|
||||
entry["container"]: {
|
||||
"id": hashlib.sha256(entry["container"].encode()).hexdigest(),
|
||||
"image_id": entry["image_id"],
|
||||
"image_ref": entry["image_ref"],
|
||||
"running": True,
|
||||
"started_at": "2026-07-22T00:00:00Z",
|
||||
"restart_count": 0,
|
||||
}
|
||||
for entry in policy["restore_isolation"]["production_continuity"].values()
|
||||
}
|
||||
value = {
|
||||
"schema": "awoooi_signoz_metadata_restore_isolation_v1",
|
||||
"trace_id": TRACE_ID,
|
||||
"run_id": RUN_ID,
|
||||
"work_item_id": WORK_ITEM_ID,
|
||||
"target": {
|
||||
"canonical_id": f"ephemeral-signoz-metadata-restore-{RUN_ID}",
|
||||
"identity": identity,
|
||||
"prefix": prefix,
|
||||
"canonical_id": f"ephemeral-signoz-metadata-restore-{short_id}",
|
||||
"base_url_sha256": hashlib.sha256(
|
||||
target_base_url.encode("utf-8")
|
||||
).hexdigest(),
|
||||
"image_ref": "signoz/signoz:v0.113.0",
|
||||
"image_id": f"sha256:{'a' * 64}",
|
||||
"image_id": policy["restore_isolation"]["required_image_id"],
|
||||
"images": policy["restore_isolation"]["images"],
|
||||
"image_present_locally": True,
|
||||
"image_pull_performed": False,
|
||||
"image_build_performed": False,
|
||||
"production_network_attached": False,
|
||||
"production_volume_attached": False,
|
||||
"ephemeral_volumes_only": True,
|
||||
@@ -241,7 +270,28 @@ def isolation_receipt(tmp_path: Path, target_base_url: str) -> Path:
|
||||
"network_scope": "loopback_and_internal_only",
|
||||
"resource_label_key": policy["restore_isolation"]["resource_label_key"],
|
||||
"resource_label_value": RUN_ID,
|
||||
"identity_label_key": "com.awoooi.restore.identity",
|
||||
"identity_label_value": identity,
|
||||
"resources": {
|
||||
"network": f"{prefix}-net",
|
||||
"containers": [
|
||||
f"{prefix}-server",
|
||||
f"{prefix}-clickhouse",
|
||||
f"{prefix}-zookeeper",
|
||||
],
|
||||
"volumes": [
|
||||
f"{prefix}-zk-data",
|
||||
f"{prefix}-zk-log",
|
||||
f"{prefix}-ch-data",
|
||||
f"{prefix}-server-data",
|
||||
],
|
||||
"workspace": f"/tmp/{prefix}-private",
|
||||
},
|
||||
"listener": {"host": "127.0.0.1", "port": port},
|
||||
"authentication_mode": "isolated_session_bearer",
|
||||
},
|
||||
"production_before": production_before,
|
||||
"image_inventory_before": ["sha256:test|test:fixture"],
|
||||
}
|
||||
path = tmp_path / "isolation.json"
|
||||
path.write_text(json.dumps(value), encoding="utf-8")
|
||||
@@ -266,6 +316,8 @@ def restore_command(
|
||||
str(bundle),
|
||||
"--target-base-url",
|
||||
target_base_url,
|
||||
"--auth-mode",
|
||||
"isolated_session_bearer",
|
||||
"--isolation-receipt",
|
||||
str(isolation),
|
||||
"--trace-id",
|
||||
@@ -349,6 +401,23 @@ def test_policy_rejects_unhashable_empty_collection_sentinel_as_contract_error(
|
||||
assert "Traceback" not in result.stderr
|
||||
|
||||
|
||||
def test_policy_pins_reviewed_isolated_restore_runtime() -> None:
|
||||
policy = json.loads(POLICY.read_text(encoding="utf-8"))
|
||||
isolation = policy["restore_isolation"]
|
||||
assert isolation["required_image_id"] == (
|
||||
"sha256:c17991d10966aedcb0d4d83805a0baf5310f1553d90611629036fcfaee8d969b"
|
||||
)
|
||||
assert isolation["startup_contract"]["reviewed"] is True
|
||||
assert isolation["startup_contract"]["server_entrypoint"] == [
|
||||
"./signoz",
|
||||
"server",
|
||||
]
|
||||
assert isolation["startup_contract"]["server_environment"] == {
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN": "tcp://clickhouse:9000",
|
||||
"SIGNOZ_SQLSTORE_SQLITE_PATH": "/var/lib/signoz/signoz.db",
|
||||
}
|
||||
|
||||
|
||||
def test_check_mode_writes_nothing(tmp_path: Path) -> None:
|
||||
output = tmp_path / "must-not-exist"
|
||||
result = subprocess.run(
|
||||
@@ -654,7 +723,19 @@ def test_restore_check_and_semantic_apply_are_bounded(tmp_path: Path) -> None:
|
||||
assert json.loads(apply_result.stdout)["terminal"] == (
|
||||
"partial_degraded_cleanup_pending"
|
||||
)
|
||||
phase_receipts = json.loads(apply_result.stdout)["phase_receipts"]
|
||||
apply_receipt = json.loads(apply_result.stdout)
|
||||
assert apply_receipt["restored_asset_counts"] == {
|
||||
"dashboards": 1,
|
||||
"alert_rules": 1,
|
||||
"explorer_views_logs": 1,
|
||||
"explorer_views_meter": 0,
|
||||
"explorer_views_metrics": 0,
|
||||
"explorer_views_traces": 0,
|
||||
}
|
||||
assert len(apply_receipt["portable_semantic_sha256"]) == 64
|
||||
assert apply_receipt["completion_scope"] == "portable_assets_only"
|
||||
assert apply_receipt["full_sqlite_completion_claim"] is False
|
||||
phase_receipts = apply_receipt["phase_receipts"]
|
||||
assert [item["phase"] for item in phase_receipts] == [
|
||||
"sensor_source",
|
||||
"normalized_asset_identity",
|
||||
@@ -693,12 +774,74 @@ def test_restore_rejects_non_loopback_target_before_writes(tmp_path: Path) -> No
|
||||
assert "restore_target_must_be_loopback" in result.stderr
|
||||
|
||||
|
||||
def test_restore_rejects_non_exact_isolation_image_before_writes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
bundle, credential = create_bundle(tmp_path)
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
port = sock.getsockname()[1]
|
||||
target = f"http://127.0.0.1:{port}"
|
||||
isolation = isolation_receipt(tmp_path, target)
|
||||
value = json.loads(isolation.read_text(encoding="utf-8"))
|
||||
value["target"]["image_id"] = f"sha256:{'a' * 64}"
|
||||
isolation.write_text(json.dumps(value), encoding="utf-8")
|
||||
isolation.chmod(0o600)
|
||||
|
||||
result = subprocess.run(
|
||||
restore_command(
|
||||
mode="--apply",
|
||||
bundle=bundle,
|
||||
target_base_url=target,
|
||||
credential=credential,
|
||||
isolation=isolation,
|
||||
),
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "isolation_target_image_id_mismatch" in result.stderr
|
||||
|
||||
|
||||
def test_cleanup_verifier_requires_zero_labeled_residue(tmp_path: Path) -> None:
|
||||
bundle, _ = create_bundle(tmp_path)
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
docker = fake_bin / "docker"
|
||||
docker.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8")
|
||||
policy = json.loads(POLICY.read_text(encoding="utf-8"))
|
||||
production = {
|
||||
entry["container"]: {
|
||||
"Id": hashlib.sha256(entry["container"].encode()).hexdigest(),
|
||||
"Image": entry["image_id"],
|
||||
"Config": {"Image": entry["image_ref"]},
|
||||
"State": {
|
||||
"Running": True,
|
||||
"StartedAt": "2026-07-22T00:00:00Z",
|
||||
},
|
||||
"RestartCount": 0,
|
||||
}
|
||||
for entry in policy["restore_isolation"]["production_continuity"].values()
|
||||
}
|
||||
docker.write_text(
|
||||
"#!/usr/bin/env python3\n"
|
||||
"import json, sys\n"
|
||||
f"production = {production!r}\n"
|
||||
"args = sys.argv[1:]\n"
|
||||
"if args[:2] == ['container', 'inspect'] and args[2] in production:\n"
|
||||
" print(json.dumps([production[args[2]]]))\n"
|
||||
" raise SystemExit(0)\n"
|
||||
"if args[:3] == ['image', 'ls', '--no-trunc']:\n"
|
||||
" print('sha256:test|test:fixture')\n"
|
||||
" raise SystemExit(0)\n"
|
||||
"if len(args) >= 2 and args[1] == 'inspect':\n"
|
||||
" raise SystemExit(1)\n"
|
||||
"raise SystemExit(0)\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
docker.chmod(0o755)
|
||||
|
||||
with socket.socket() as sock:
|
||||
|
||||
597
scripts/backup/tests/test_signoz_metadata_isolated_restore.py
Normal file
597
scripts/backup/tests/test_signoz_metadata_isolated_restore.py
Normal file
@@ -0,0 +1,597 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
CONTROLLER = ROOT / "scripts" / "backup" / "signoz-metadata-isolated-restore.py"
|
||||
POLICY = ROOT / "config" / "signoz" / "metadata-export-policy.json"
|
||||
TRACE_ID = "trace-P0-OBS-002-isolated"
|
||||
RUN_ID = "run-P0-OBS-002-isolated"
|
||||
WORK_ITEM_ID = "P0-OBS-002"
|
||||
|
||||
|
||||
def load_controller() -> Any:
|
||||
backup_dir = str(ROOT / "scripts" / "backup")
|
||||
if backup_dir not in sys.path:
|
||||
sys.path.insert(0, backup_dir)
|
||||
spec = importlib.util.spec_from_file_location("signoz_isolated_restore", CONTROLLER)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def controller() -> Any:
|
||||
return load_controller()
|
||||
|
||||
|
||||
class RecordingRunner:
|
||||
def __init__(self, controller: Any) -> None:
|
||||
self.controller = controller
|
||||
self.commands: list[list[str]] = []
|
||||
self.resources: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
self.production = {
|
||||
name: {
|
||||
"Id": (name.encode().hex() + "0" * 64)[:64],
|
||||
"Image": image_id,
|
||||
"Config": {"Image": ref},
|
||||
"State": {
|
||||
"Running": True,
|
||||
"StartedAt": "2026-07-22T00:00:00Z",
|
||||
},
|
||||
"RestartCount": 0,
|
||||
}
|
||||
for name, (ref, image_id) in controller.EXPECTED_PRODUCTION.items()
|
||||
}
|
||||
self.image_rows = ["sha256:test|test:fixture"]
|
||||
|
||||
def __call__(self, argv: list[str], _timeout: int) -> Any:
|
||||
self.commands.append(argv[:])
|
||||
c = self.controller
|
||||
if argv[1:3] == ["image", "inspect"]:
|
||||
ref = argv[-1]
|
||||
for expected_ref, image_id in c.EXPECTED_IMAGES.values():
|
||||
if ref == expected_ref:
|
||||
return c.CommandResult(0, f"{image_id}\n")
|
||||
return c.CommandResult(1, "")
|
||||
if argv[1:3] == ["image", "ls"]:
|
||||
return c.CommandResult(0, "\n".join(self.image_rows) + "\n")
|
||||
if argv[1:2] == ["info"]:
|
||||
return c.CommandResult(0, "28.0.0\n")
|
||||
if len(argv) >= 4 and argv[2] == "inspect":
|
||||
kind, name = argv[1], argv[3]
|
||||
if kind == "container" and name in self.production:
|
||||
return c.CommandResult(0, json.dumps([self.production[name]]))
|
||||
value = self.resources.get((kind, name))
|
||||
if value is None:
|
||||
return c.CommandResult(1, "")
|
||||
return c.CommandResult(0, json.dumps([value]))
|
||||
if argv[1:3] == ["network", "create"]:
|
||||
name = argv[-1]
|
||||
labels = labels_from_command(argv)
|
||||
self.resources[("network", name)] = {"Name": name, "Labels": labels}
|
||||
return c.CommandResult(0, f"{name}\n")
|
||||
if argv[1:3] == ["volume", "create"]:
|
||||
name = argv[-1]
|
||||
labels = labels_from_command(argv)
|
||||
self.resources[("volume", name)] = {"Name": name, "Labels": labels}
|
||||
return c.CommandResult(0, f"{name}\n")
|
||||
if argv[1:2] == ["run"]:
|
||||
name = argv[argv.index("--name") + 1]
|
||||
ref = argv[-1]
|
||||
expected_id = next(
|
||||
image_id
|
||||
for expected_ref, image_id in c.EXPECTED_IMAGES.values()
|
||||
if expected_ref == ref
|
||||
)
|
||||
self.resources[("container", name)] = {
|
||||
"Id": (name.encode().hex() + "0" * 64)[:64],
|
||||
"Image": expected_id,
|
||||
"Config": {"Image": ref, "Labels": labels_from_command(argv)},
|
||||
"HostConfig": {"RestartPolicy": {"Name": "no"}},
|
||||
"State": {"Running": True},
|
||||
"NetworkSettings": {
|
||||
"Ports": {
|
||||
"8080/tcp": [
|
||||
{"HostIp": "127.0.0.1", "HostPort": "49173"}
|
||||
]
|
||||
}
|
||||
if name.endswith("-server")
|
||||
else {}
|
||||
},
|
||||
"RestartCount": 0,
|
||||
}
|
||||
return c.CommandResult(0, f"{name}\n")
|
||||
if argv[1:2] == ["port"]:
|
||||
return c.CommandResult(0, "127.0.0.1:49173\n")
|
||||
if argv[1:2] == ["exec"] and argv[-2:] == ["--query", "SELECT 1"]:
|
||||
return c.CommandResult(0, "1\n")
|
||||
if argv[1:3] == ["rm", "--force"]:
|
||||
self.resources.pop(("container", argv[-1]), None)
|
||||
return c.CommandResult(0, "")
|
||||
if argv[1:3] == ["volume", "rm"]:
|
||||
self.resources.pop(("volume", argv[-1]), None)
|
||||
return c.CommandResult(0, "")
|
||||
if argv[1:3] == ["network", "rm"]:
|
||||
self.resources.pop(("network", argv[-1]), None)
|
||||
return c.CommandResult(0, "")
|
||||
if argv[1:2] == ["ps"] or (
|
||||
len(argv) >= 3 and argv[1] in {"volume", "network"} and argv[2] == "ls"
|
||||
):
|
||||
label = argv[-1].removeprefix("label=")
|
||||
key, expected = label.split("=", 1)
|
||||
matches = []
|
||||
for (kind, name), value in self.resources.items():
|
||||
if argv[1] == "ps" and kind != "container":
|
||||
continue
|
||||
if argv[1] in {"volume", "network"} and kind != argv[1]:
|
||||
continue
|
||||
labels = (
|
||||
value.get("Config", {}).get("Labels", {})
|
||||
if kind == "container"
|
||||
else value.get("Labels", {})
|
||||
)
|
||||
if labels.get(key) == expected:
|
||||
matches.append(name)
|
||||
return c.CommandResult(0, "\n".join(matches))
|
||||
return c.CommandResult(1, "")
|
||||
|
||||
|
||||
def labels_from_command(argv: list[str]) -> dict[str, str]:
|
||||
labels: dict[str, str] = {}
|
||||
for index, value in enumerate(argv):
|
||||
if value == "--label":
|
||||
key, label_value = argv[index + 1].split("=", 1)
|
||||
labels[key] = label_value
|
||||
return labels
|
||||
|
||||
|
||||
def test_identity_and_exact_resource_names(controller: Any) -> None:
|
||||
plan = controller.build_resource_plan(TRACE_ID, RUN_ID, WORK_ITEM_ID)
|
||||
|
||||
assert plan.identity == controller.derive_identity(
|
||||
TRACE_ID, RUN_ID, WORK_ITEM_ID
|
||||
)
|
||||
assert len(plan.identity) == 64
|
||||
assert plan.prefix == f"awoooi-signoz-md-restore-{plan.identity[:16]}"
|
||||
assert plan.canonical_id == (
|
||||
f"ephemeral-signoz-metadata-restore-{plan.identity[:16]}"
|
||||
)
|
||||
assert plan.containers == (
|
||||
f"{plan.prefix}-server",
|
||||
f"{plan.prefix}-clickhouse",
|
||||
f"{plan.prefix}-zookeeper",
|
||||
)
|
||||
assert len(plan.volumes) == 4
|
||||
assert plan.workspace == Path(f"/tmp/{plan.prefix}-private")
|
||||
|
||||
|
||||
def test_policy_requires_reviewed_exact_startup_and_images(controller: Any) -> None:
|
||||
policy = json.loads(POLICY.read_text(encoding="utf-8"))
|
||||
assert controller.validate_restore_policy(policy)["startup_contract"][
|
||||
"reviewed"
|
||||
] is True
|
||||
|
||||
policy["restore_isolation"]["startup_contract"]["reviewed"] = False
|
||||
with pytest.raises(
|
||||
controller.ContractError, match="restore_startup_contract_not_reviewed"
|
||||
):
|
||||
controller.validate_restore_policy(policy)
|
||||
|
||||
policy = json.loads(POLICY.read_text(encoding="utf-8"))
|
||||
policy["restore_isolation"]["images"]["clickhouse"]["id"] = (
|
||||
f"sha256:{'a' * 64}"
|
||||
)
|
||||
with pytest.raises(
|
||||
controller.ContractError, match="restore_image_policy_not_exact"
|
||||
):
|
||||
controller.validate_restore_policy(policy)
|
||||
|
||||
|
||||
def test_resource_creation_is_internal_no_pull_no_restart_and_loopback(
|
||||
controller: Any,
|
||||
) -> None:
|
||||
runner = RecordingRunner(controller)
|
||||
plan = controller.build_resource_plan(TRACE_ID, RUN_ID, WORK_ITEM_ID)
|
||||
cluster = ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
|
||||
controller.create_resources(runner, plan, RUN_ID, cluster)
|
||||
|
||||
commands = runner.commands
|
||||
network = next(command for command in commands if command[1:3] == ["network", "create"])
|
||||
assert "--internal" in network
|
||||
run_commands = [command for command in commands if command[1:2] == ["run"]]
|
||||
assert len(run_commands) == 3
|
||||
for command in run_commands:
|
||||
assert "--pull=never" in command
|
||||
assert "--restart=no" in command
|
||||
assert f"{controller.PRIMARY_LABEL}={RUN_ID}" in command
|
||||
assert f"{controller.IDENTITY_LABEL}={plan.identity}" in command
|
||||
assert not any("signoz-clickhouse:" in value for value in command)
|
||||
server = next(command for command in run_commands if plan.server_container in command)
|
||||
assert "127.0.0.1::8080" in server
|
||||
assert f"{plan.server_data_volume}:/var/lib/signoz" in server
|
||||
assert "SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://clickhouse:9000" in server
|
||||
clickhouse_run_index = commands.index(
|
||||
next(command for command in run_commands if plan.clickhouse_container in command)
|
||||
)
|
||||
readiness_index = next(
|
||||
index
|
||||
for index, command in enumerate(commands)
|
||||
if command[1:2] == ["exec"]
|
||||
and command[-2:] == ["--query", "SELECT 1"]
|
||||
)
|
||||
server_run_index = commands.index(server)
|
||||
assert clickhouse_run_index < readiness_index < server_run_index
|
||||
|
||||
|
||||
def test_exact_image_drift_fails_before_resource_creation(controller: Any) -> None:
|
||||
runner = RecordingRunner(controller)
|
||||
|
||||
def drift(argv: list[str], timeout: int) -> Any:
|
||||
result = runner(argv, timeout)
|
||||
if argv[-1] == controller.EXPECTED_IMAGES["clickhouse"][0]:
|
||||
return controller.CommandResult(0, f"sha256:{'b' * 64}\n")
|
||||
return result
|
||||
|
||||
with pytest.raises(
|
||||
controller.ContractError, match="restore_image_id_mismatch_clickhouse"
|
||||
):
|
||||
controller.verify_exact_images(drift)
|
||||
assert not runner.resources
|
||||
|
||||
|
||||
def test_cleanup_refuses_same_name_with_wrong_ownership(controller: Any) -> None:
|
||||
runner = RecordingRunner(controller)
|
||||
plan = controller.build_resource_plan(TRACE_ID, RUN_ID, WORK_ITEM_ID)
|
||||
runner.resources[("container", plan.server_container)] = {
|
||||
"Config": {
|
||||
"Labels": {
|
||||
controller.PRIMARY_LABEL: "some-other-run",
|
||||
controller.IDENTITY_LABEL: plan.identity,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
with pytest.raises(
|
||||
controller.ContractError, match="cleanup_container_ownership_mismatch"
|
||||
):
|
||||
controller.remove_owned_resource(
|
||||
runner,
|
||||
plan,
|
||||
RUN_ID,
|
||||
"container",
|
||||
plan.server_container,
|
||||
)
|
||||
assert ("container", plan.server_container) in runner.resources
|
||||
assert not any(command[1:3] == ["rm", "--force"] for command in runner.commands)
|
||||
|
||||
|
||||
def test_cleanup_removes_exact_resources_and_proves_continuity(
|
||||
controller: Any,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner = RecordingRunner(controller)
|
||||
plan = controller.build_resource_plan(
|
||||
TRACE_ID, RUN_ID, WORK_ITEM_ID, workspace_parent=tmp_path
|
||||
)
|
||||
cluster = ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
controller.create_workspace(plan)
|
||||
controller.create_resources(runner, plan, RUN_ID, cluster)
|
||||
state = {
|
||||
"production_before": controller.production_snapshot(runner),
|
||||
"image_inventory_before": controller.image_inventory(runner),
|
||||
"listener": {"host": "127.0.0.1", "port": 49173},
|
||||
}
|
||||
monkeypatch.setattr(controller, "tcp_connectable", lambda _host, _port: False)
|
||||
monkeypatch.setattr(controller.time, "sleep", lambda _seconds: None)
|
||||
|
||||
errors = controller.cleanup_and_verify(runner, plan, RUN_ID, state)
|
||||
|
||||
assert errors == []
|
||||
assert not runner.resources
|
||||
assert not plan.workspace.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("server_created", [False, True])
|
||||
def test_cleanup_reconciles_without_listener_state_after_interruption(
|
||||
controller: Any,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
server_created: bool,
|
||||
) -> None:
|
||||
runner = RecordingRunner(controller)
|
||||
plan = controller.build_resource_plan(
|
||||
TRACE_ID, RUN_ID, WORK_ITEM_ID, workspace_parent=tmp_path
|
||||
)
|
||||
controller.create_workspace(plan)
|
||||
if server_created:
|
||||
cluster = (
|
||||
ROOT
|
||||
/ "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
)
|
||||
controller.create_resources(runner, plan, RUN_ID, cluster)
|
||||
state = {
|
||||
"production_before": controller.production_snapshot(runner),
|
||||
"image_inventory_before": controller.image_inventory(runner),
|
||||
"listener": None,
|
||||
}
|
||||
monkeypatch.setattr(controller, "tcp_connectable", lambda _host, _port: False)
|
||||
monkeypatch.setattr(controller.time, "sleep", lambda _seconds: None)
|
||||
|
||||
errors = controller.cleanup_and_verify(runner, plan, RUN_ID, state)
|
||||
|
||||
assert errors == []
|
||||
assert not runner.resources
|
||||
assert not plan.workspace.exists()
|
||||
|
||||
|
||||
def test_cleanup_already_clean_without_state_uses_fresh_continuity(
|
||||
controller: Any,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner = RecordingRunner(controller)
|
||||
plan = controller.build_resource_plan(
|
||||
TRACE_ID, RUN_ID, WORK_ITEM_ID, workspace_parent=tmp_path
|
||||
)
|
||||
monkeypatch.setattr(controller, "tcp_connectable", lambda _host, _port: False)
|
||||
monkeypatch.setattr(controller.time, "sleep", lambda _seconds: None)
|
||||
|
||||
errors = controller.cleanup_and_verify(runner, plan, RUN_ID, None)
|
||||
|
||||
assert errors == []
|
||||
assert not plan.workspace.exists()
|
||||
|
||||
|
||||
def test_cleanup_reconciles_resource_materialized_after_first_inventory(
|
||||
controller: Any,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
runner = RecordingRunner(controller)
|
||||
plan = controller.build_resource_plan(
|
||||
TRACE_ID, RUN_ID, WORK_ITEM_ID, workspace_parent=tmp_path
|
||||
)
|
||||
sleeps = 0
|
||||
|
||||
def materialize_after_first_pass(_seconds: float) -> None:
|
||||
nonlocal sleeps
|
||||
sleeps += 1
|
||||
if sleeps == 1:
|
||||
runner.resources[("volume", plan.server_data_volume)] = {
|
||||
"Name": plan.server_data_volume,
|
||||
"Labels": {
|
||||
controller.PRIMARY_LABEL: RUN_ID,
|
||||
controller.IDENTITY_LABEL: plan.identity,
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(controller, "tcp_connectable", lambda _host, _port: False)
|
||||
monkeypatch.setattr(controller.time, "sleep", materialize_after_first_pass)
|
||||
|
||||
errors = controller.cleanup_and_verify(runner, plan, RUN_ID, None)
|
||||
|
||||
assert errors == []
|
||||
assert not runner.resources
|
||||
removals = [
|
||||
command
|
||||
for command in runner.commands
|
||||
if command[1:3] == ["volume", "rm"]
|
||||
and command[-1] == plan.server_data_volume
|
||||
]
|
||||
assert len(removals) == 1
|
||||
|
||||
|
||||
def test_apply_cli_requires_durable_receipt_before_runtime_dispatch() -> None:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(CONTROLLER),
|
||||
"--apply",
|
||||
"--trace-id",
|
||||
TRACE_ID,
|
||||
"--run-id",
|
||||
RUN_ID,
|
||||
"--work-item-id",
|
||||
WORK_ITEM_ID,
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 2
|
||||
assert "receipt_file_required_for_restore_apply" in result.stderr
|
||||
assert result.stdout == ""
|
||||
|
||||
|
||||
def test_distinct_verifier_requires_immutable_apply_receipt(
|
||||
controller: Any,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
args = SimpleNamespace(
|
||||
trace_id=TRACE_ID,
|
||||
run_id=RUN_ID,
|
||||
work_item_id=WORK_ITEM_ID,
|
||||
)
|
||||
plan = controller.build_resource_plan(
|
||||
TRACE_ID, RUN_ID, WORK_ITEM_ID, workspace_parent=tmp_path
|
||||
)
|
||||
executor_receipt = controller.executor_success(
|
||||
args,
|
||||
plan,
|
||||
{
|
||||
"restoreSemanticDigest": "a" * 64,
|
||||
"restoredAssetCounts": {
|
||||
"dashboards": 1,
|
||||
"alert_rules": 2,
|
||||
"explorer_views": 3,
|
||||
},
|
||||
},
|
||||
)
|
||||
path = tmp_path / "apply.json"
|
||||
controller.write_terminal(path, executor_receipt)
|
||||
|
||||
validated = controller.validate_apply_receipt(path, args, plan)
|
||||
verified = controller.independent_success(args, plan, validated)
|
||||
|
||||
assert executor_receipt["completionClaim"] is False
|
||||
assert executor_receipt["executorZeroResidueVerified"] is True
|
||||
assert verified["phase"] == "independent_restore_verifier"
|
||||
assert verified["completionClaim"] is True
|
||||
assert verified["independentZeroResidueVerified"] is True
|
||||
assert verified["restoreSemanticDigest"] == "a" * 64
|
||||
assert verified["applyReceiptValidated"] is True
|
||||
assert len(verified["applyReceiptSha256"]) == 64
|
||||
|
||||
|
||||
def test_interrupted_verify_cleanup_reconciles_again_before_zero_residue_receipt(
|
||||
controller: Any,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
plan = controller.build_resource_plan(
|
||||
TRACE_ID, RUN_ID, WORK_ITEM_ID, workspace_parent=tmp_path
|
||||
)
|
||||
calls = 0
|
||||
emitted: list[dict[str, Any]] = []
|
||||
|
||||
def interrupt_once(
|
||||
_runner: Any,
|
||||
_plan: Any,
|
||||
_run_id: str,
|
||||
_state: dict[str, Any] | None,
|
||||
) -> list[str]:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
raise controller.SignalAbort("signal_15_cleanup_required")
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(
|
||||
sys,
|
||||
"argv",
|
||||
[
|
||||
str(CONTROLLER),
|
||||
"--verify-cleanup",
|
||||
"--trace-id",
|
||||
TRACE_ID,
|
||||
"--run-id",
|
||||
RUN_ID,
|
||||
"--work-item-id",
|
||||
WORK_ITEM_ID,
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(controller, "build_resource_plan", lambda *_args: plan)
|
||||
monkeypatch.setattr(controller, "load_policy", lambda _path: {})
|
||||
monkeypatch.setattr(controller, "load_state_if_safe", lambda _plan: None)
|
||||
monkeypatch.setattr(controller, "cleanup_and_verify", interrupt_once)
|
||||
monkeypatch.setattr(controller, "emit", emitted.append)
|
||||
|
||||
exit_code = controller.main()
|
||||
|
||||
assert exit_code == 1
|
||||
assert calls == 2
|
||||
assert emitted[-1]["terminal"] == "failed_zero_residue_verified_no_completion"
|
||||
assert emitted[-1]["zeroResidueVerified"] is True
|
||||
assert emitted[-1]["cleanupErrorCodes"] == []
|
||||
assert "signal_15_cleanup_required" in emitted[-1]["detail"]
|
||||
|
||||
|
||||
def test_bootstrap_secrets_stay_in_mode_0400_files(
|
||||
controller: Any,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
calls: list[tuple[str, dict[str, Any] | None]] = []
|
||||
|
||||
def fake_request(
|
||||
url: str,
|
||||
*,
|
||||
method: str = "GET",
|
||||
payload: dict[str, Any] | None = None,
|
||||
authorization: str | None = None,
|
||||
timeout: int = 10,
|
||||
) -> dict[str, Any]:
|
||||
assert authorization is None
|
||||
calls.append((url, payload))
|
||||
if url.endswith("/api/v1/register"):
|
||||
return {"data": {"orgId": "isolated-org"}, "status": "success"}
|
||||
return {
|
||||
"data": {
|
||||
"accessToken": "a" * 64,
|
||||
"refreshToken": "r" * 64,
|
||||
},
|
||||
"status": "success",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(controller, "request_json", fake_request)
|
||||
|
||||
token = controller.bootstrap_isolated_session("http://127.0.0.1:49173", tmp_path)
|
||||
|
||||
assert token == tmp_path / "session-access-token"
|
||||
assert stat.S_IMODE(token.stat().st_mode) == 0o400
|
||||
assert sorted(path.name for path in tmp_path.iterdir()) == [
|
||||
"session-access-token"
|
||||
]
|
||||
assert calls[0][0].endswith("/api/v1/register")
|
||||
assert calls[1][0].endswith("/api/v2/sessions/email_password")
|
||||
assert capsys.readouterr().out == ""
|
||||
|
||||
|
||||
def test_bootstrap_accepts_live_root_response_envelope(
|
||||
controller: Any,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
def fake_request(
|
||||
url: str,
|
||||
**_kwargs: Any,
|
||||
) -> dict[str, Any]:
|
||||
if url.endswith("/api/v1/register"):
|
||||
return {"orgId": "isolated-org"}
|
||||
return {"accessToken": "a" * 64, "refreshToken": "r" * 64}
|
||||
|
||||
monkeypatch.setattr(controller, "request_json", fake_request)
|
||||
|
||||
token = controller.bootstrap_isolated_session(
|
||||
"http://127.0.0.1:49173", tmp_path
|
||||
)
|
||||
|
||||
assert token.read_text(encoding="ascii").strip() == "a" * 64
|
||||
assert stat.S_IMODE(token.stat().st_mode) == 0o400
|
||||
|
||||
|
||||
def test_controller_source_has_signal_cleanup_and_no_secret_argv() -> None:
|
||||
source = CONTROLLER.read_text(encoding="utf-8")
|
||||
assert "signal.SIGHUP" in source
|
||||
assert "signal.SIGINT" in source
|
||||
assert "signal.SIGTERM" in source
|
||||
assert "finally:" in source
|
||||
assert "cleanup_and_verify(" in source
|
||||
assert '"--pull=never"' in source
|
||||
assert '"--restart=no"' in source
|
||||
assert '"--auth-mode"' in source
|
||||
assert "access_token" not in source[source.index("argv = [") : source.index("output = run_required", source.index("argv = ["))]
|
||||
assert "password" not in source[source.index("argv = [") : source.index("output = run_required", source.index("argv = ["))]
|
||||
durable_write = source.rindex("write_terminal(Path(args.receipt_file), result)")
|
||||
final_handler_restore = source.rindex("restore_signal_handlers()")
|
||||
assert durable_write < final_handler_restore
|
||||
assert "if not args.apply and not signals_held_until_terminal:" in source
|
||||
@@ -26,6 +26,8 @@ LABELS=(
|
||||
signoz-metadata-export.py
|
||||
verify-signoz-metadata-export.py
|
||||
signoz-metadata-restore-drill.py
|
||||
signoz-metadata-isolated-restore.py
|
||||
isolated-cluster.xml
|
||||
)
|
||||
LOCAL_SOURCES=(
|
||||
"${ROOT_DIR}/config/signoz/metadata-export-policy.json"
|
||||
@@ -33,8 +35,10 @@ LOCAL_SOURCES=(
|
||||
"${ROOT_DIR}/scripts/backup/signoz-metadata-export.py"
|
||||
"${ROOT_DIR}/scripts/backup/verify-signoz-metadata-export.py"
|
||||
"${ROOT_DIR}/scripts/backup/signoz-metadata-restore-drill.py"
|
||||
"${ROOT_DIR}/scripts/backup/signoz-metadata-isolated-restore.py"
|
||||
"${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
)
|
||||
MODES=(0644 0644 0755 0755 0755)
|
||||
MODES=(0644 0644 0755 0755 0755 0755 0644)
|
||||
SSH_OPTIONS=(
|
||||
-o BatchMode=yes
|
||||
-o IdentitiesOnly=yes
|
||||
@@ -72,7 +76,7 @@ Optional fixed transport paths:
|
||||
SIGNOZ_METADATA_SSH_IDENTITY_FILE
|
||||
SIGNOZ_METADATA_KNOWN_HOSTS_FILE
|
||||
|
||||
--check validates the five local sources and performs a no-write host110 diff.
|
||||
--check validates the seven local sources and performs a no-write host110 diff.
|
||||
--apply creates one immutable exact-hash directory. It does not create or
|
||||
change an active pointer and does not run an authenticated metadata export.
|
||||
USAGE
|
||||
@@ -153,7 +157,8 @@ import sys
|
||||
|
||||
for value in sys.argv[1:]:
|
||||
path = Path(value)
|
||||
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
||||
if path.suffix == ".py":
|
||||
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
||||
PY
|
||||
PYTHONPATH="${ROOT_DIR}/scripts/backup" "${LOCAL_PYTHON_BIN}" - \
|
||||
"${LOCAL_SOURCES[0]}" <<'PY'
|
||||
@@ -182,7 +187,7 @@ BUNDLE_ID="$(printf '%s\n' "${SOURCE_HASHES[@]}" | sha256sum | awk '{print $1}')
|
||||
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
||||
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-metadata-toolchain.${RUN_ID}"
|
||||
|
||||
emit sensor_source pass "local_exact_sources_5"
|
||||
emit sensor_source pass "local_exact_sources_7"
|
||||
emit normalized_asset_identity pass "bundle_${BUNDLE_ID}"
|
||||
emit ai_decision pass "candidate_additive_immutable_inactive_source_deploy"
|
||||
emit risk_policy_decision pass "medium_no_active_pointer_no_secret_no_raw_sqlite"
|
||||
@@ -209,11 +214,13 @@ LABELS=(
|
||||
signoz-metadata-export.py
|
||||
verify-signoz-metadata-export.py
|
||||
signoz-metadata-restore-drill.py
|
||||
signoz-metadata-isolated-restore.py
|
||||
isolated-cluster.xml
|
||||
)
|
||||
MODES=(0644 0644 0755 0755 0755)
|
||||
MODES=(0644 0644 0755 0755 0755 0755 0644)
|
||||
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
||||
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 5 ]
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
|
||||
[ -d /backup ] && [ ! -L /backup ]
|
||||
[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ]
|
||||
for command_name in curl docker pgrep python3 sha256sum stat timeout; do
|
||||
@@ -230,7 +237,7 @@ api_status="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
[ "${api_status}" = "200" ]
|
||||
|
||||
active_operations="$(
|
||||
(pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \
|
||||
(pgrep -af '([s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$))' || true) \
|
||||
| wc -l | tr -d ' '
|
||||
)"
|
||||
[ "${active_operations}" = "0" ]
|
||||
@@ -369,8 +376,10 @@ LABELS=(
|
||||
signoz-metadata-export.py
|
||||
verify-signoz-metadata-export.py
|
||||
signoz-metadata-restore-drill.py
|
||||
signoz-metadata-isolated-restore.py
|
||||
isolated-cluster.xml
|
||||
)
|
||||
MODES=(0644 0644 0755 0755 0755)
|
||||
MODES=(0644 0644 0755 0755 0755 0755 0644)
|
||||
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
||||
CANDIDATE_DIR="${REMOTE_ROOT}/.${BUNDLE_ID}.candidate.${RUN_ID}"
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
@@ -408,7 +417,7 @@ finalize() {
|
||||
}
|
||||
trap finalize EXIT HUP INT TERM
|
||||
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 5 ]
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
|
||||
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
||||
[ ! -e "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]
|
||||
[ ! -e "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]
|
||||
@@ -431,7 +440,7 @@ OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock
|
||||
exec 8<"${OPERATION_LOCK}"
|
||||
flock -n 8
|
||||
active_operations="$(
|
||||
(pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \
|
||||
(pgrep -af '([s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$))' || true) \
|
||||
| wc -l | tr -d ' '
|
||||
)"
|
||||
[ "${active_operations}" = "0" ]
|
||||
@@ -443,7 +452,7 @@ api_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
|
||||
http://127.0.0.1:8080/api/v1/health)"
|
||||
[ "${api_before}" = "200" ]
|
||||
emit_remote sensor_source pass "staged_sources_5_runtime_api_200"
|
||||
emit_remote sensor_source pass "staged_sources_7_runtime_api_200"
|
||||
|
||||
for index in "${!LABELS[@]}"; do
|
||||
staged="${STAGE_DIR}/${LABELS[index]}"
|
||||
@@ -463,6 +472,7 @@ for name in (
|
||||
"signoz-metadata-export.py",
|
||||
"verify-signoz-metadata-export.py",
|
||||
"signoz-metadata-restore-drill.py",
|
||||
"signoz-metadata-isolated-restore.py",
|
||||
):
|
||||
path = root / name
|
||||
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
@@ -17,13 +18,16 @@ def _quoted_array(source: str, start: str, end: str) -> tuple[str, ...]:
|
||||
def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
|
||||
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
|
||||
assert (
|
||||
'[ValidateSet("Check", "Apply", "Verify", "RestoreCheck", '
|
||||
'"RestoreApply", "RestoreVerifyCleanup")]' in source
|
||||
)
|
||||
assert '$CanonicalAsset = "signoz-110-control-plane-metadata"' in source
|
||||
assert '$TargetHost = "192.168.0.110"' in source
|
||||
assert '$RemoteUser = "wooo"' in source
|
||||
assert '$WorkItemId -ne "P0-OBS-002"' in source
|
||||
assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source
|
||||
assert "._+" not in source
|
||||
assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source
|
||||
assert '$EvidenceRunId = if ($RunId -match $SafeIdPattern)' in source
|
||||
assert "Get-Agent99RuntimeSourceBinding" in source
|
||||
assert "$result.fileCount -eq 20" in source
|
||||
@@ -31,9 +35,12 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
|
||||
assert "$result.entrypointHashMatched" in source
|
||||
assert '$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"' in source
|
||||
assert (
|
||||
'$BundleId = "d1318c412aa303c5d8a5795f22ee62dce16ca79354c5a15e95bfad41a5e35533"'
|
||||
'$BundleId = "ed4e5fbfc5f732b64d6a78f6ab26b90118e1d42a4f38b40d583876445c32e3aa"'
|
||||
in source
|
||||
)
|
||||
assert re.search(r'\$BundleId = "[0-9a-f]{64}"', source)
|
||||
assert '$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"' in source
|
||||
assert '$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"' in source
|
||||
assert '$BaseUrl = "http://127.0.0.1:8080"' in source
|
||||
assert (
|
||||
'$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"'
|
||||
@@ -48,6 +55,34 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_entrypoint_pins_current_seven_file_toolchain_bundle() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
rows = (
|
||||
("PolicyPath", ROOT / "config/signoz/metadata-export-policy.json"),
|
||||
("SharedContractPath", ROOT / "scripts/backup/signoz_metadata_contract.py"),
|
||||
("ExporterPath", ROOT / "scripts/backup/signoz-metadata-export.py"),
|
||||
("VerifierPath", ROOT / "scripts/backup/verify-signoz-metadata-export.py"),
|
||||
("RestoreDriverPath", ROOT / "scripts/backup/signoz-metadata-restore-drill.py"),
|
||||
("IsolatedRestorePath", ROOT / "scripts/backup/signoz-metadata-isolated-restore.py"),
|
||||
(
|
||||
"IsolatedClusterConfigPath",
|
||||
ROOT
|
||||
/ "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
),
|
||||
)
|
||||
hashes = [hashlib.sha256(path.read_bytes()).hexdigest() for _, path in rows]
|
||||
bundle_id = hashlib.sha256(
|
||||
"".join(f"{value}\n" for value in hashes).encode("ascii")
|
||||
).hexdigest()
|
||||
|
||||
assert f'$BundleId = "{bundle_id}"' in source
|
||||
for variable, expected_hash in zip(
|
||||
(variable for variable, _path in rows), hashes, strict=True
|
||||
):
|
||||
assert f'${variable} = "{expected_hash}"' in source
|
||||
assert "PENDING_ISOLATED_RESTORE_SHA256" not in source
|
||||
|
||||
|
||||
def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
lock_helper = source[
|
||||
@@ -184,6 +219,16 @@ def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> No
|
||||
assert "outputPresent = if ($outputStateReadbackVerified)" in preflight
|
||||
assert "receiptPresent = if ($receiptStateReadbackVerified)" in preflight
|
||||
assert "credentialReferencePresent = if ($credentialReadbackVerified)" in preflight
|
||||
assert "agent99_restore_startup:" in preflight
|
||||
assert "restoreStartupContractReadbackVerified" in preflight
|
||||
assert "restoreStartupContractReady" in preflight
|
||||
assert "restoreStartupEnvironmentValuesOutput = $false" in preflight
|
||||
assert "restoreStartupFullEnvironmentOutput = $false" in preflight
|
||||
assert "{{json .Config.Env}}" not in preflight
|
||||
assert "{{range .Config.Env}}" not in preflight
|
||||
assert "docker exec signoz /bin/sh -c" in preflight
|
||||
assert "tcp://clickhouse:9000*" in preflight
|
||||
assert "/var/lib/signoz/signoz.db" in preflight
|
||||
|
||||
credential_unverified = main.index("credential_reference_readback_unverified")
|
||||
credential_absent = main.index("credential_reference_absent")
|
||||
@@ -205,6 +250,9 @@ def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> No
|
||||
transport_reserve = 30
|
||||
assert guard + 90 + kill_after + transport_reserve < 300
|
||||
assert guard + 720 + kill_after + transport_reserve < 900
|
||||
restore_cleanup_kill_after = 600
|
||||
assert guard + 1800 + restore_cleanup_kill_after + transport_reserve < 2550
|
||||
assert guard + 300 + restore_cleanup_kill_after + transport_reserve < 1050
|
||||
|
||||
|
||||
def test_entrypoint_never_reads_or_persists_credential_value_in_controller() -> None:
|
||||
@@ -217,7 +265,9 @@ def test_entrypoint_never_reads_or_persists_credential_value_in_controller() ->
|
||||
assert 'secretValueTransmitted = $false' in source
|
||||
assert 'secretValuePersistedInEvidence = $false' in source
|
||||
assert "credentialReferenceDispatchAttempted" in source
|
||||
assert "$script:ApplyDispatchAttempted = $true" in source
|
||||
assert "$script:ExportCredentialDispatchAttempted = $true" in source
|
||||
assert "$script:IsolatedApplyDispatchAttempted = $true" in source
|
||||
assert "credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted" in source
|
||||
assert "credentialReferenceConsumedOnTarget = if" in source
|
||||
assert "elseif (Test-Agent99SignozReceiptIdentity $ApplyReceipt" in source
|
||||
assert 'rawSqliteRead = $false' in source
|
||||
@@ -248,8 +298,8 @@ def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal
|
||||
assert '--receipt-file $RemoteReceipt' in source
|
||||
assert 'pass_export_verified_restore_pending' in source
|
||||
assert 'export_verified_restore_pending' in source
|
||||
assert 'restoreTerminal = "pending_isolated_same_version_target"' in source
|
||||
assert 'zeroResidueTerminal = "pending"' in source
|
||||
assert 'else { "pending_isolated_same_version_target" }' in source
|
||||
assert 'else { "pending" }' in source
|
||||
assert 'completionClaim = $false' in source
|
||||
assert "controlledApplyPhases = [pscustomobject]$controlledApplyPhases" in source
|
||||
assert 'credential_reference_absent' in source
|
||||
@@ -258,6 +308,40 @@ def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal
|
||||
assert 'post_verifier_runtime_or_artifact_drift' in source
|
||||
|
||||
|
||||
def test_restore_modes_are_bounded_distinct_and_cleanup_remains_available() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
main_start = source.index("foreach ($identity")
|
||||
cleanup_start = source.index(
|
||||
'if ($Mode -eq "RestoreVerifyCleanup")', main_start
|
||||
)
|
||||
restore_apply_start = source.index(
|
||||
'if ($Mode -in @("RestoreCheck", "RestoreApply"))', cleanup_start
|
||||
)
|
||||
export_verify_start = source.index('if ($Mode -eq "Verify")', restore_apply_start)
|
||||
cleanup_helper = source[
|
||||
source.index("function New-Agent99SignozCleanupLockedRemoteCommand") :
|
||||
source.index("function Get-Agent99RuntimeSourceBinding")
|
||||
]
|
||||
cleanup_branch = source[cleanup_start:restore_apply_start]
|
||||
restore_apply_branch = source[restore_apply_start:export_verify_start]
|
||||
|
||||
assert "Get-Agent99SignozCleanupPreflight" in cleanup_branch
|
||||
assert "New-Agent99SignozCleanupLockedRemoteCommand" in cleanup_branch
|
||||
assert "Get-Agent99SignozPreflight" not in cleanup_branch
|
||||
assert "$OutputDir" not in cleanup_branch
|
||||
assert "$BaseUrl/api/v1/health" not in cleanup_helper
|
||||
assert "ExpectedArtifactState" not in cleanup_helper
|
||||
assert "/usr/bin/docker info" in cleanup_helper
|
||||
assert "--apply-receipt-file $RestoreRemoteReceipt" in cleanup_branch
|
||||
assert '"independent_restore_verifier"' in cleanup_branch
|
||||
assert '"portable_metadata_restore_independently_verified_zero_residue"' in cleanup_branch
|
||||
assert '"portable_metadata_restore_verified_zero_residue_pending_independent_verifier"' in restore_apply_branch
|
||||
assert cleanup_start < restore_apply_start < export_verify_start
|
||||
assert "$RestoreCleanupKillAfterSeconds = 600" in source
|
||||
assert "$RestoreApplyControllerWaitSeconds = 2550" in source
|
||||
assert "$RestoreCleanupControllerWaitSeconds = 1050" in source
|
||||
|
||||
|
||||
def test_entrypoint_reserves_durable_evidence_before_any_remote_dispatch() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
main = source[source.index("foreach ($identity") :]
|
||||
|
||||
@@ -143,14 +143,14 @@ def test_check_mode_fails_closed_on_remote_drift(tmp_path: Path) -> None:
|
||||
assert rows[-1]["terminal"] == "failed_no_write"
|
||||
|
||||
|
||||
def test_apply_uses_five_transfers_and_exact_postcheck(tmp_path: Path) -> None:
|
||||
def test_apply_uses_seven_transfers_and_exact_postcheck(tmp_path: Path) -> None:
|
||||
result = run_deployer(tmp_path, mode="apply", apply=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
rows = receipt_rows(result.stdout)
|
||||
assert rows[-1]["terminal"] == "pass_source_deployed_inactive"
|
||||
assert rows[-1]["detail"] == "runtime_export_not_executed"
|
||||
events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines()
|
||||
assert len([line for line in events if line.startswith("scp ")]) == 5
|
||||
assert len([line for line in events if line.startswith("scp ")]) == 7
|
||||
assert len([line for line in events if line.startswith("ssh ")]) == 4
|
||||
|
||||
|
||||
@@ -170,6 +170,10 @@ def test_deployer_has_no_active_pointer_or_destructive_fallback() -> None:
|
||||
assert "curl -fsSL" not in source
|
||||
assert "\nrm " not in source
|
||||
assert "ln -s" not in source
|
||||
assert "signoz-metadata-isolated-restore.py" in source
|
||||
assert "isolated-cluster.xml" in source
|
||||
assert "local_exact_sources_7" in source
|
||||
assert "[ \"${#EXPECTED_HASHES[@]}\" -eq 7 ]" in source
|
||||
|
||||
|
||||
def test_transport_propagates_strict_fixed_paths_with_spaces(tmp_path: Path) -> None:
|
||||
@@ -247,7 +251,7 @@ def test_remote_check_uses_privileged_read_only_visibility() -> None:
|
||||
assert "REMOTE_CHECK contains only stat/hash/process/API" in remote_check
|
||||
|
||||
|
||||
def test_exact_five_file_supply_chain_and_modes_are_fixed() -> None:
|
||||
def test_exact_seven_file_supply_chain_and_modes_are_fixed() -> None:
|
||||
source = DEPLOYER.read_text(encoding="utf-8")
|
||||
for path in (
|
||||
"config/signoz/metadata-export-policy.json",
|
||||
@@ -255,7 +259,9 @@ def test_exact_five_file_supply_chain_and_modes_are_fixed() -> None:
|
||||
"scripts/backup/signoz-metadata-export.py",
|
||||
"scripts/backup/verify-signoz-metadata-export.py",
|
||||
"scripts/backup/signoz-metadata-restore-drill.py",
|
||||
"scripts/backup/signoz-metadata-isolated-restore.py",
|
||||
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
):
|
||||
assert path in source
|
||||
assert "MODES=(0644 0644 0755 0755 0755)" in source
|
||||
assert '[ "${#EXPECTED_HASHES[@]}" -eq 5 ]' in source
|
||||
assert "MODES=(0644 0644 0755 0755 0755 0755 0644)" in source
|
||||
assert '[ "${#EXPECTED_HASHES[@]}" -eq 7 ]' in source
|
||||
|
||||
Reference in New Issue
Block a user