feat(signoz): add recoverable Agent99 metadata toolchain route
This commit is contained in:
Your Name
@@ -149,7 +149,7 @@ def test_agent99_entrypoint_is_fixed_and_never_transports_secret() -> None:
|
||||
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
|
||||
assert '$TargetHost = "192.168.0.110"' in source
|
||||
assert '$ExpectedWorkItemId = "P0-OBS-002"' in source
|
||||
assert '$ExpectedRuntimeFileCount = 20' in source
|
||||
assert '$ExpectedRuntimeFileCount = 21' in source
|
||||
assert '"sudo -n /usr/bin/python3 - --mode $($Mode.ToLowerInvariant())"' in source
|
||||
assert "$process.Refresh()" in source
|
||||
assert 'if (-not $process.HasExited) { throw "target_exit_state_unavailable" }' in source
|
||||
|
||||
@@ -4,7 +4,6 @@ import hashlib
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
ENTRYPOINT = ROOT / "agent99-signoz-metadata-executor.ps1"
|
||||
|
||||
@@ -20,7 +19,8 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
|
||||
|
||||
assert (
|
||||
'[ValidateSet("Check", "Apply", "Verify", "RestoreCheck", '
|
||||
'"RestoreApply", "RestoreVerifyCleanup")]' in source
|
||||
'"RestoreApply", "RestoreVerifyCleanup", "ToolchainCheck", '
|
||||
'"ToolchainApply", "ToolchainVerify", "ToolchainReconcile")]' in source
|
||||
)
|
||||
assert '$CanonicalAsset = "signoz-110-control-plane-metadata"' in source
|
||||
assert '$TargetHost = "192.168.0.110"' in source
|
||||
@@ -28,24 +28,26 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
|
||||
assert '$WorkItemId -ne "P0-OBS-002"' in source
|
||||
assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source
|
||||
assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source
|
||||
assert '$EvidenceRunId = if ($RunId -match $SafeIdPattern)' in source
|
||||
assert "$EvidenceRunId = if ($RunId -match $SafeIdPattern)" in source
|
||||
assert "Get-Agent99RuntimeSourceBinding" in source
|
||||
assert "$result.fileCount -eq 20" in source
|
||||
assert "$result.fileCount -eq 21" in source
|
||||
assert "$result.sourceRevision -eq $SourceRevision" in source
|
||||
assert "$result.entrypointHashMatched" in source
|
||||
assert '$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"' in source
|
||||
assert (
|
||||
'$BundleId = "ed4e5fbfc5f732b64d6a78f6ab26b90118e1d42a4f38b40d583876445c32e3aa"'
|
||||
'$BundleId = "b83bfc898cd43cf8ec5688d75ec20231c2faa32de5be3fa886db33038d620dff"'
|
||||
in source
|
||||
)
|
||||
assert re.search(r'\$BundleId = "[0-9a-f]{64}"', source)
|
||||
assert '$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"' in source
|
||||
assert '$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"' in source
|
||||
assert '$BaseUrl = "http://127.0.0.1:8080"' in source
|
||||
assert (
|
||||
'$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"'
|
||||
'$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"'
|
||||
in source
|
||||
)
|
||||
assert (
|
||||
'$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"' in source
|
||||
)
|
||||
assert '$BaseUrl = "http://127.0.0.1:8080"' in source
|
||||
assert '$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"' in source
|
||||
assert '$OutputDir = "/backup/signoz-metadata-export-$RunId"' not in source
|
||||
assert '$SignozImage = "signoz/signoz:v0.113.0"' in source
|
||||
assert '$SignozContainer = "signoz"' in source
|
||||
@@ -63,11 +65,13 @@ def test_entrypoint_pins_current_seven_file_toolchain_bundle() -> None:
|
||||
("ExporterPath", ROOT / "scripts/backup/signoz-metadata-export.py"),
|
||||
("VerifierPath", ROOT / "scripts/backup/verify-signoz-metadata-export.py"),
|
||||
("RestoreDriverPath", ROOT / "scripts/backup/signoz-metadata-restore-drill.py"),
|
||||
("IsolatedRestorePath", ROOT / "scripts/backup/signoz-metadata-isolated-restore.py"),
|
||||
(
|
||||
"IsolatedRestorePath",
|
||||
ROOT / "scripts/backup/signoz-metadata-isolated-restore.py",
|
||||
),
|
||||
(
|
||||
"IsolatedClusterConfigPath",
|
||||
ROOT
|
||||
/ "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
),
|
||||
)
|
||||
hashes = [hashlib.sha256(path.read_bytes()).hexdigest() for _, path in rows]
|
||||
@@ -86,8 +90,9 @@ def test_entrypoint_pins_current_seven_file_toolchain_bundle() -> None:
|
||||
def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
lock_helper = source[
|
||||
source.index("function New-Agent99SignozLockedRemoteCommand") :
|
||||
source.index("function Get-Agent99RuntimeSourceBinding")
|
||||
source.index("function New-Agent99SignozLockedRemoteCommand") : source.index(
|
||||
"function Get-Agent99RuntimeSourceBinding"
|
||||
)
|
||||
]
|
||||
|
||||
assert 'Start-Process -FilePath "ssh.exe"' in source
|
||||
@@ -95,9 +100,9 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() ->
|
||||
assert '"-o", "IdentitiesOnly=yes"' in source
|
||||
assert '"-o", "StrictHostKeyChecking=yes"' in source
|
||||
assert '"-o", "NumberOfPasswordPrompts=0"' in source
|
||||
assert 'windows99_agent99_to_host110_fixed_metadata_toolchain' in source
|
||||
assert 'arbitraryCommandAllowed = $false' in source
|
||||
assert 'crossDomainFallbackAllowed = $false' in source
|
||||
assert "windows99_agent99_to_host110_fixed_metadata_toolchain" in source
|
||||
assert "arbitraryCommandAllowed = $false" in source
|
||||
assert "crossDomainFallbackAllowed = $false" in source
|
||||
assert "Invoke-Expression" not in source
|
||||
assert "-EncodedCommand" not in source
|
||||
assert "cmd.exe" not in source
|
||||
@@ -128,9 +133,9 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() ->
|
||||
)
|
||||
assert "targetTimeoutFinishesBeforeControllerWait" in source
|
||||
assert "target_timeout_not_bounded_before_controller_wait" in source
|
||||
assert '--kill-after=5s 15s /usr/bin/docker inspect' in source
|
||||
assert '--kill-after=5s 15s /usr/bin/curl' in source
|
||||
assert '--kill-after=5s 10s /usr/bin/pgrep' in lock_helper
|
||||
assert "--kill-after=5s 15s /usr/bin/docker inspect" in source
|
||||
assert "--kill-after=5s 15s /usr/bin/curl" in source
|
||||
assert "--kill-after=5s 10s /usr/bin/pgrep" in lock_helper
|
||||
assert "active_rc=`$?; set -e" in lock_helper
|
||||
assert '[ `"`$active_rc`" -eq 1 ] && [ -z `"`$active_output`" ]' in lock_helper
|
||||
assert "$bashProgram = (" in lock_helper
|
||||
@@ -142,49 +147,52 @@ def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() ->
|
||||
assert "sudo -n /bin/bash -c '" not in lock_helper
|
||||
assert "pgrep -af" in lock_helper
|
||||
assert "|| true" not in lock_helper
|
||||
assert '--signal=TERM --kill-after=$($TargetKillAfterSeconds)s' in source
|
||||
assert "--signal=TERM --kill-after=$($TargetKillAfterSeconds)s" in source
|
||||
|
||||
|
||||
def test_entrypoint_uses_typed_stdout_without_windows_process_exit_code() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
transport = source[
|
||||
source.index("function Invoke-Agent99SignozFixedRemote") :
|
||||
source.index("function New-Agent99SignozLockedRemoteCommand")
|
||||
source.index("function Invoke-Agent99SignozFixedRemote") : source.index(
|
||||
"function Test-Agent99SignozTransportOutcome"
|
||||
)
|
||||
]
|
||||
|
||||
assert "$process.ExitCode" not in transport
|
||||
assert "exitCodeObserved" not in transport
|
||||
assert "AllowedExitCodes" not in transport
|
||||
assert "function Test-Agent99SignozTransportOutcome" in transport
|
||||
assert "$Result.completed -eq $true" in transport
|
||||
assert "-not $Result.timedOut" in transport
|
||||
assert '$process.WaitForExit($TimeoutSeconds * 1000)' in transport
|
||||
assert "function Test-Agent99SignozTransportOutcome" in source
|
||||
assert "$Result.completed -eq $true" in source
|
||||
assert "-not $Result.timedOut" in source
|
||||
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in transport
|
||||
assert "taskkill.exe /PID $process.Id /T /F" in transport
|
||||
|
||||
|
||||
def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contracts() -> None:
|
||||
def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contracts() -> (
|
||||
None
|
||||
):
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'function New-Agent99SignozPathStateCommand' in source
|
||||
assert 'agent99_credential:symlink' in source
|
||||
assert 'agent99_credential:absent' in source
|
||||
assert 'agent99_credential:present:%u:%a:%s' in source
|
||||
assert 'agent99_path_state:$ExpectedKind' in source
|
||||
assert 'agent99_path_state:absent' in source
|
||||
assert 'agent99_hashes_begin' in source
|
||||
assert 'agent99_hashes_end' in source
|
||||
assert '/usr/bin/echo agent99_hashes_begin' in source
|
||||
assert '/usr/bin/echo agent99_hashes_end' in source
|
||||
assert 'agent99_hashes_begin\\n' not in source
|
||||
assert 'agent99_image:' in source
|
||||
assert 'agent99_runtime:' in source
|
||||
assert 'agent99_health:' in source
|
||||
assert 'agent99_active:' in source
|
||||
assert "function New-Agent99SignozPathStateCommand" in source
|
||||
assert "agent99_credential:symlink" in source
|
||||
assert "agent99_credential:absent" in source
|
||||
assert "agent99_credential:present:%u:%a:%s" in source
|
||||
assert "agent99_path_state:$ExpectedKind" in source
|
||||
assert "agent99_path_state:absent" in source
|
||||
assert "agent99_hashes_begin" in source
|
||||
assert "agent99_hashes_end" in source
|
||||
assert "/usr/bin/echo agent99_hashes_begin" in source
|
||||
assert "/usr/bin/echo agent99_hashes_end" in source
|
||||
assert "agent99_hashes_begin\\n" not in source
|
||||
assert "agent99_image:" in source
|
||||
assert "agent99_runtime:" in source
|
||||
assert "agent99_health:" in source
|
||||
assert "agent99_active:" in source
|
||||
assert '$credentialState -in @("absent", "symlink", "other")' in source
|
||||
assert 'credentialReadbackVerified = $credentialReadbackVerified' in source
|
||||
assert 'transportReadbackVerified = $transportReadbackVerified' in source
|
||||
assert 'outputStateReadbackVerified = $outputStateReadbackVerified' in source
|
||||
assert 'receiptStateReadbackVerified = $receiptStateReadbackVerified' in source
|
||||
assert "credentialReadbackVerified = $credentialReadbackVerified" in source
|
||||
assert "transportReadbackVerified = $transportReadbackVerified" in source
|
||||
assert "outputStateReadbackVerified = $outputStateReadbackVerified" in source
|
||||
assert "receiptStateReadbackVerified = $receiptStateReadbackVerified" in source
|
||||
assert '"credential_reference_readback_unverified"' in source
|
||||
assert '"preflight_transport_unverified"' in source
|
||||
assert source.count("Test-Agent99SignozTransportOutcome $verify") == 2
|
||||
@@ -195,24 +203,29 @@ def test_entrypoint_preflight_has_explicit_credential_and_artifact_state_contrac
|
||||
def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
preflight = source[
|
||||
source.index("function Get-Agent99SignozPreflight") :
|
||||
source.index("function Write-Agent99SignozEvidenceCollision")
|
||||
source.index("function Get-Agent99SignozPreflight") : source.index(
|
||||
"function Write-Agent99SignozEvidenceCollision"
|
||||
)
|
||||
]
|
||||
main = source[
|
||||
source.index(
|
||||
'$preflight = Get-Agent99SignozPreflight '
|
||||
'-ExpectedArtifactState "absent"'
|
||||
"$preflight = Get-Agent99SignozPreflight " '-ExpectedArtifactState "absent"'
|
||||
) :
|
||||
]
|
||||
|
||||
assert "$hashLines.Count -eq $ToolchainFiles.Count + 2" in preflight
|
||||
assert "$hashLines[0] -eq \"agent99_hashes_begin\"" in preflight
|
||||
assert '$hashLines[0] -eq "agent99_hashes_begin"' in preflight
|
||||
assert '$hashLines[$hashLines.Count - 1] -eq "agent99_hashes_end"' in preflight
|
||||
assert "$actualHashes.ContainsKey($path)" in preflight
|
||||
assert "-not $ToolchainFiles.Contains($path)" in preflight
|
||||
assert '"^agent99_image:(sha256:[0-9a-f]{64})$"' in preflight
|
||||
assert '--format={{.Id}},{{.Image}},{{.Config.Image}},{{.State.Running}}' in preflight
|
||||
assert '--format=`"{{.Id}}|{{.Image}}|{{.Config.Image}}|{{.State.Running}}`"' not in preflight
|
||||
assert (
|
||||
"--format={{.Id}},{{.Image}},{{.Config.Image}},{{.State.Running}}" in preflight
|
||||
)
|
||||
assert (
|
||||
'--format=`"{{.Id}}|{{.Image}}|{{.Config.Image}}|{{.State.Running}}`"'
|
||||
not in preflight
|
||||
)
|
||||
assert '"^agent99_runtime:([0-9a-f]{64}),(sha256:' in preflight
|
||||
assert '"^agent99_active:([0-9]+)$"' in preflight
|
||||
assert '[ `"`$active_rc`" -eq 0 ] || [ `"`$active_rc`" -eq 1 ]' in preflight
|
||||
@@ -258,69 +271,72 @@ def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> No
|
||||
def test_entrypoint_never_reads_or_persists_credential_value_in_controller() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
|
||||
assert '/usr/bin/stat -c `"agent99_credential:present:%u:%a:%s`" `"$CredentialFile`"' in source
|
||||
assert 'credentialValueRead = $false' in source
|
||||
assert 'credentialValueOutput = $false' in source
|
||||
assert 'secretValueReadByController = $false' in source
|
||||
assert 'secretValueTransmitted = $false' in source
|
||||
assert 'secretValuePersistedInEvidence = $false' in source
|
||||
assert (
|
||||
'/usr/bin/stat -c `"agent99_credential:present:%u:%a:%s`" `"$CredentialFile`"'
|
||||
in source
|
||||
)
|
||||
assert "credentialValueRead = $false" in source
|
||||
assert "credentialValueOutput = $false" in source
|
||||
assert "secretValueReadByController = $false" in source
|
||||
assert "secretValueTransmitted = $false" in source
|
||||
assert "secretValuePersistedInEvidence = $false" in source
|
||||
assert "credentialReferenceDispatchAttempted" in source
|
||||
assert "$script:ExportCredentialDispatchAttempted = $true" in source
|
||||
assert "$script:IsolatedApplyDispatchAttempted = $true" in source
|
||||
assert "credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted" in source
|
||||
assert (
|
||||
"credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted"
|
||||
in source
|
||||
)
|
||||
assert "credentialReferenceConsumedOnTarget = if" in source
|
||||
assert "elseif (Test-Agent99SignozReceiptIdentity $ApplyReceipt" in source
|
||||
assert 'rawSqliteRead = $false' in source
|
||||
assert 'rawVolumeRead = $false' in source
|
||||
assert "rawSqliteRead = $false" in source
|
||||
assert "rawVolumeRead = $false" in source
|
||||
assert "Get-Content -LiteralPath $CredentialFile" not in source
|
||||
assert "ReadAllText($CredentialFile" not in source
|
||||
assert "ReadAllBytes($CredentialFile" not in source
|
||||
|
||||
|
||||
def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal() -> None:
|
||||
def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal() -> (
|
||||
None
|
||||
):
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'Receipt.schema -eq "awoooi_signoz_metadata_receipt_v1"' in source
|
||||
assert "awoooi_controlled_apply_receipt_v1" not in source
|
||||
assert '[string]$Receipt.phase -eq $ExpectedPhase' in source
|
||||
assert "[string]$Receipt.phase -eq $ExpectedPhase" in source
|
||||
assert (
|
||||
'Test-Agent99SignozReceiptIdentity $verifyReceipt '
|
||||
"Test-Agent99SignozReceiptIdentity $verifyReceipt "
|
||||
'"independent_export_verifier"' in source
|
||||
)
|
||||
assert (
|
||||
'Test-Agent99SignozReceiptIdentity $checkReceipt "terminal"' in source
|
||||
)
|
||||
assert (
|
||||
'Test-Agent99SignozReceiptIdentity $applyReceipt "terminal"' in source
|
||||
)
|
||||
assert 'Test-Agent99SignozReceiptIdentity $checkReceipt "terminal"' in source
|
||||
assert 'Test-Agent99SignozReceiptIdentity $applyReceipt "terminal"' in source
|
||||
assert "$ExporterPath --check" in source
|
||||
assert "$ExporterPath --apply" in source
|
||||
assert '--receipt-file $RemoteReceipt' in source
|
||||
assert 'pass_export_verified_restore_pending' in source
|
||||
assert 'export_verified_restore_pending' in source
|
||||
assert "--receipt-file $RemoteReceipt" in source
|
||||
assert "pass_export_verified_restore_pending" in source
|
||||
assert "export_verified_restore_pending" in source
|
||||
assert 'else { "pending_isolated_same_version_target" }' in source
|
||||
assert 'else { "pending" }' in source
|
||||
assert 'completionClaim = $false' in source
|
||||
assert "completionClaim = $false" in source
|
||||
assert "controlledApplyPhases = [pscustomobject]$controlledApplyPhases" in source
|
||||
assert 'credential_reference_absent' in source
|
||||
assert 'blocked_with_safe_next_action' in source
|
||||
assert 'apply_failed_artifact_preserved_for_readback' in source
|
||||
assert 'post_verifier_runtime_or_artifact_drift' in source
|
||||
assert "credential_reference_absent" in source
|
||||
assert "blocked_with_safe_next_action" in source
|
||||
assert "apply_failed_artifact_preserved_for_readback" in source
|
||||
assert "post_verifier_runtime_or_artifact_drift" in source
|
||||
|
||||
|
||||
def test_restore_modes_are_bounded_distinct_and_cleanup_remains_available() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
main_start = source.index("foreach ($identity")
|
||||
cleanup_start = source.index(
|
||||
'if ($Mode -eq "RestoreVerifyCleanup")', main_start
|
||||
)
|
||||
cleanup_start = source.index('if ($Mode -eq "RestoreVerifyCleanup")', main_start)
|
||||
restore_apply_start = source.index(
|
||||
'if ($Mode -in @("RestoreCheck", "RestoreApply"))', cleanup_start
|
||||
)
|
||||
export_verify_start = source.index('if ($Mode -eq "Verify")', restore_apply_start)
|
||||
cleanup_helper = source[
|
||||
source.index("function New-Agent99SignozCleanupLockedRemoteCommand") :
|
||||
source.index("function Get-Agent99RuntimeSourceBinding")
|
||||
source.index(
|
||||
"function New-Agent99SignozCleanupLockedRemoteCommand"
|
||||
) : source.index("function Get-Agent99RuntimeSourceBinding")
|
||||
]
|
||||
cleanup_branch = source[cleanup_start:restore_apply_start]
|
||||
restore_apply_branch = source[restore_apply_start:export_verify_start]
|
||||
@@ -328,18 +344,82 @@ def test_restore_modes_are_bounded_distinct_and_cleanup_remains_available() -> N
|
||||
assert "Get-Agent99SignozCleanupPreflight" in cleanup_branch
|
||||
assert "New-Agent99SignozCleanupLockedRemoteCommand" in cleanup_branch
|
||||
assert "Get-Agent99SignozPreflight" not in cleanup_branch
|
||||
assert "$OutputDir" not in cleanup_branch
|
||||
assert "--bundle-dir $OutputDir" in cleanup_branch
|
||||
assert "$BaseUrl/api/v1/health" not in cleanup_helper
|
||||
assert "ExpectedArtifactState" not in cleanup_helper
|
||||
assert "/usr/bin/docker info" in cleanup_helper
|
||||
assert "--apply-receipt-file $RestoreRemoteReceipt" in cleanup_branch
|
||||
assert '"independent_restore_verifier"' in cleanup_branch
|
||||
assert '"portable_metadata_restore_independently_verified_zero_residue"' in cleanup_branch
|
||||
assert '"portable_metadata_restore_verified_zero_residue_pending_independent_verifier"' in restore_apply_branch
|
||||
assert (
|
||||
'"portable_metadata_restore_independently_verified_zero_residue"'
|
||||
in cleanup_branch
|
||||
)
|
||||
assert (
|
||||
'"portable_metadata_restore_verified_zero_residue_pending_independent_verifier"'
|
||||
in restore_apply_branch
|
||||
)
|
||||
assert cleanup_start < restore_apply_start < export_verify_start
|
||||
assert "$RestoreCleanupKillAfterSeconds = 600" in source
|
||||
assert "$RestoreApplyControllerWaitSeconds = 2550" in source
|
||||
assert "$RestoreCleanupControllerWaitSeconds = 1050" in source
|
||||
assert '[string]$AttemptId = ""' in source
|
||||
assert '"$EvidenceRunId-attempt-$EvidenceAttemptId"' in source
|
||||
assert 'cleanupAttemptId = if ($Mode -eq "RestoreVerifyCleanup")' in source
|
||||
assert "cleanup_attempt_id_missing_or_invalid" in source
|
||||
assert "attempt_id_not_allowed_for_mode" in source
|
||||
|
||||
|
||||
def test_toolchain_modes_use_separate_reconcile_attempt_and_never_fall_through() -> (
|
||||
None
|
||||
):
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
main_start = source.index("foreach ($identity")
|
||||
reconcile_start = source.index('if ($Mode -eq "ToolchainReconcile")', main_start)
|
||||
toolchain_start = source.index(
|
||||
'if ($Mode -in @("ToolchainCheck", "ToolchainApply", "ToolchainVerify"))',
|
||||
reconcile_start,
|
||||
)
|
||||
restore_start = source.index(
|
||||
'if ($Mode -eq "RestoreVerifyCleanup")', toolchain_start
|
||||
)
|
||||
reconcile_branch = source[reconcile_start:toolchain_start]
|
||||
toolchain_branch = source[toolchain_start:restore_start]
|
||||
|
||||
assert '"$EvidenceRunId-attempt-$EvidenceAttemptId"' in source
|
||||
assert 'toolchainReconcileAttemptId = if ($Mode -eq "ToolchainReconcile")' in source
|
||||
assert 'Invoke-Agent99SignozToolchainTarget "reconcile"' in reconcile_branch
|
||||
assert 'Invoke-Agent99SignozToolchainTarget "reconcile"' not in toolchain_branch
|
||||
assert '"toolchain_reconcile_exact_deployment_verified"' in reconcile_branch
|
||||
assert '"toolchain_reconcile_zero_owned_residue"' in reconcile_branch
|
||||
assert "_toolchain_reconcile_required" in toolchain_branch
|
||||
assert "Get-Agent99SignozPreflight" not in reconcile_branch
|
||||
assert "New-Agent99SignozLockedRemoteCommand" not in reconcile_branch
|
||||
assert "Get-Agent99SignozPreflight" not in toolchain_branch
|
||||
assert "$ToolchainEnvelopeDeletedBeforeExecutor" in source
|
||||
assert "toolchainEnvelopeTransportPayloadDeletedBeforeExecutor" in source
|
||||
assert "productionSourceMutationState" in source
|
||||
|
||||
|
||||
def test_restore_check_failure_terminals_preserve_no_write_semantics() -> None:
|
||||
source = ENTRYPOINT.read_text(encoding="utf-8")
|
||||
cleanup_start = source.index(
|
||||
'if ($Mode -eq "RestoreVerifyCleanup")', source.index("foreach ($identity")
|
||||
)
|
||||
restore_start = source.index(
|
||||
'if ($Mode -in @("RestoreCheck", "RestoreApply"))', cleanup_start
|
||||
)
|
||||
restore_end = source.index('if ($Mode -eq "Verify")', restore_start)
|
||||
restore_branch = source[restore_start:restore_end]
|
||||
|
||||
assert restore_branch.count('"restore_check_failed_no_write"') >= 4
|
||||
assert (
|
||||
'"restore_check_failed_unexpected_write_requires_reconcile"' in restore_branch
|
||||
)
|
||||
assert '"restore_check_wrote_receipt"' in restore_branch
|
||||
assert (
|
||||
'if ($Terminal -eq "restore_check_failed_unexpected_write_requires_reconcile") '
|
||||
'{ "unexpected_receipt_write_requires_reconcile" }' in source
|
||||
)
|
||||
|
||||
|
||||
def test_entrypoint_reserves_durable_evidence_before_any_remote_dispatch() -> None:
|
||||
@@ -349,7 +429,7 @@ def test_entrypoint_reserves_durable_evidence_before_any_remote_dispatch() -> No
|
||||
reservation = main.index("New-Agent99SignozEvidenceReservation")
|
||||
first_remote = main.index("Get-Agent99RuntimeSourceBinding")
|
||||
assert reservation < first_remote
|
||||
assert '[IO.FileMode]::CreateNew' in source
|
||||
assert "[IO.FileMode]::CreateNew" in source
|
||||
assert '$EvidenceReservationPath = "$EvidencePath.reserve"' in source
|
||||
assert "blocked_evidence_identity_collision" in source
|
||||
assert "evidence_path_or_reservation_already_exists" in source
|
||||
@@ -379,8 +459,7 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
|
||||
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
|
||||
).read_text(encoding="utf-8")
|
||||
receiver = (
|
||||
ROOT
|
||||
/ "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
|
||||
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert expected in _quoted_array(bootstrap, "$agentFiles = @(", ")")
|
||||
@@ -388,8 +467,8 @@ def test_entrypoint_is_in_exact_agent99_runtime_bundle() -> None:
|
||||
assert expected in _quoted_array(contract, "$requiredFiles = @(", ")")
|
||||
assert expected in _quoted_array(sender, "RUNTIME_FILES=(", ")")
|
||||
assert expected in _quoted_array(receiver, "$FixedRuntimeFiles = @(", ")")
|
||||
assert 'expectedRuntimeFileCount": 20' in sender
|
||||
assert '$runtimeManifest.fileCount -eq 20' in receiver
|
||||
assert 'expectedRuntimeFileCount": 21' in sender
|
||||
assert "$runtimeManifest.fileCount -eq 21" in receiver
|
||||
|
||||
|
||||
def test_runtime_contract_check_requires_typed_signoz_transport_readback() -> None:
|
||||
|
||||
357
scripts/ops/tests/test_agent99_signoz_toolchain_target.py
Normal file
357
scripts/ops/tests/test_agent99_signoz_toolchain_target.py
Normal file
@@ -0,0 +1,357 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
TARGET = ROOT / "agent99-signoz-toolchain-target.py"
|
||||
|
||||
|
||||
def _load_target() -> ModuleType:
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"agent99_signoz_toolchain_target", TARGET
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def _payloads() -> dict[str, bytes]:
|
||||
policy = {
|
||||
"work_item_id": "P0-OBS-002",
|
||||
"source_runtime": {
|
||||
"raw_database_access_allowed": False,
|
||||
"raw_volume_access_allowed": False,
|
||||
},
|
||||
"completion_contract": {"full_sqlite_completion_claim_allowed": False},
|
||||
}
|
||||
values: dict[str, bytes] = {
|
||||
"metadata-export-policy.json": (
|
||||
json.dumps(policy, sort_keys=True, separators=(",", ":")) + "\n"
|
||||
).encode(),
|
||||
"isolated-cluster.xml": b"<clickhouse><remote_servers/></clickhouse>\n",
|
||||
}
|
||||
for name in (
|
||||
"signoz_metadata_contract.py",
|
||||
"signoz-metadata-export.py",
|
||||
"verify-signoz-metadata-export.py",
|
||||
"signoz-metadata-restore-drill.py",
|
||||
"signoz-metadata-isolated-restore.py",
|
||||
):
|
||||
values[name] = b"VALUE = 'bounded-toolchain-test'\n"
|
||||
return values
|
||||
|
||||
|
||||
def _manifest(module: ModuleType, values: dict[str, bytes]) -> list[dict[str, str]]:
|
||||
return [
|
||||
{
|
||||
"name": name,
|
||||
"sha256": hashlib.sha256(values[name]).hexdigest(),
|
||||
"mode": f"{mode:04o}",
|
||||
}
|
||||
for name, mode in zip(module.LABELS, module.MODES, strict=True)
|
||||
]
|
||||
|
||||
|
||||
def _args(
|
||||
module: ModuleType,
|
||||
manifest: list[dict[str, str]],
|
||||
*,
|
||||
mode: str,
|
||||
run_id: str = "run-001",
|
||||
attempt_id: str = "",
|
||||
stage: Path | None = None,
|
||||
) -> SimpleNamespace:
|
||||
bundle_id = module.sha256_bytes(
|
||||
"".join(f"{row['sha256']}\n" for row in manifest).encode("ascii")
|
||||
)
|
||||
return SimpleNamespace(
|
||||
mode=mode,
|
||||
trace_id="trace-001",
|
||||
run_id=run_id,
|
||||
work_item_id="P0-OBS-002",
|
||||
source_revision="1" * 40,
|
||||
bundle_id=bundle_id,
|
||||
stage_dir=str(stage) if stage else None,
|
||||
attempt_id=attempt_id,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def target_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
|
||||
module = _load_target()
|
||||
uid = os.getuid()
|
||||
gid = os.getgid()
|
||||
remote_parent = tmp_path / "backup" / "toolchains"
|
||||
receipt_parent = tmp_path / "backup" / "deploy-receipts"
|
||||
remote_parent.mkdir(parents=True)
|
||||
receipt_parent.mkdir(parents=True)
|
||||
lock = tmp_path / "operation.lock"
|
||||
lock.write_bytes(b"")
|
||||
os.chmod(lock, 0o600)
|
||||
snapshot = {
|
||||
"id": "a" * 64,
|
||||
"image_id": module.SIGNOZ_IMAGE_ID,
|
||||
"image_ref": module.SIGNOZ_IMAGE_REF,
|
||||
"running": True,
|
||||
"started_at": "2026-07-22T00:00:00Z",
|
||||
"restart_count": 0,
|
||||
}
|
||||
monkeypatch.setattr(module, "ROOT_UID", uid)
|
||||
monkeypatch.setattr(module, "REMOTE_ROOT", remote_parent / "signoz-metadata")
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"RECEIPT_ROOT",
|
||||
receipt_parent / "signoz-metadata-toolchain",
|
||||
)
|
||||
module.REMOTE_ROOT.mkdir(mode=0o700)
|
||||
module.RECEIPT_ROOT.mkdir(mode=0o700)
|
||||
monkeypatch.setattr(module, "OPERATION_LOCK", lock)
|
||||
monkeypatch.setattr(module, "production_snapshot", lambda: dict(snapshot))
|
||||
monkeypatch.setattr(module, "require_health", lambda: None)
|
||||
monkeypatch.setattr(module, "require_no_active_operations", lambda: None)
|
||||
monkeypatch.setenv("SUDO_UID", str(uid))
|
||||
monkeypatch.setenv("SUDO_GID", str(gid))
|
||||
values = _payloads()
|
||||
manifest = _manifest(module, values)
|
||||
return module, values, manifest, snapshot
|
||||
|
||||
|
||||
def _write_tree(
|
||||
root: Path,
|
||||
manifest: list[dict[str, str]],
|
||||
values: dict[str, bytes],
|
||||
*,
|
||||
directory_mode: int,
|
||||
) -> None:
|
||||
root.mkdir(mode=directory_mode, parents=True)
|
||||
for row in manifest:
|
||||
path = root / row["name"]
|
||||
path.write_bytes(values[row["name"]])
|
||||
os.chmod(path, int(row["mode"], 8))
|
||||
|
||||
|
||||
def test_target_state_distinguishes_absent_exact_drift_and_active_pointer(
|
||||
target_env,
|
||||
) -> None:
|
||||
module, values, manifest, _snapshot = target_env
|
||||
args = _args(module, manifest, mode="check")
|
||||
assert module.target_state(args.bundle_id, manifest) == "absent"
|
||||
|
||||
target = module.REMOTE_ROOT / args.bundle_id
|
||||
_write_tree(target, manifest, values, directory_mode=0o750)
|
||||
assert module.target_state(args.bundle_id, manifest) == "exact"
|
||||
|
||||
(target / manifest[0]["name"]).write_bytes(b"drift")
|
||||
with pytest.raises(module.TargetError, match="target_file_drift"):
|
||||
module.target_state(args.bundle_id, manifest)
|
||||
|
||||
(target / manifest[0]["name"]).write_bytes(values[manifest[0]["name"]])
|
||||
os.chmod(target / manifest[0]["name"], int(manifest[0]["mode"], 8))
|
||||
(module.REMOTE_ROOT / "current").symlink_to(target)
|
||||
with pytest.raises(module.TargetError, match="active_pointer_must_remain_absent"):
|
||||
module.target_state(args.bundle_id, manifest)
|
||||
|
||||
|
||||
def test_prepare_creates_private_transport_owned_stage(
|
||||
target_env, tmp_path: Path
|
||||
) -> None:
|
||||
module, _values, manifest, _snapshot = target_env
|
||||
stage = tmp_path / "transport-stage"
|
||||
args = _args(module, manifest, mode="prepare", stage=stage)
|
||||
|
||||
module.reset_mutation_state()
|
||||
receipt = module.prepare_stage(args, manifest)
|
||||
|
||||
metadata = stage.lstat()
|
||||
assert receipt["terminal"] == "stage_prepared"
|
||||
assert metadata.st_uid == os.getuid()
|
||||
assert stat.S_IMODE(metadata.st_mode) == 0o700
|
||||
assert receipt["source_write_attempted"] is True
|
||||
assert receipt["source_write_performed"] is True
|
||||
|
||||
|
||||
def test_apply_writes_receipt_only_after_exact_target_and_live_stage_cleanup(
|
||||
target_env, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
module, values, manifest, _snapshot = target_env
|
||||
stage = tmp_path / "transport-stage"
|
||||
_write_tree(stage, manifest, values, directory_mode=0o700)
|
||||
args = _args(module, manifest, mode="apply", stage=stage)
|
||||
original_write = module.write_deployment_receipt
|
||||
observed: list[tuple[bool, str]] = []
|
||||
|
||||
def guarded_write(*call_args, **call_kwargs):
|
||||
observed.append(
|
||||
(
|
||||
os.path.lexists(stage),
|
||||
module.target_state(args.bundle_id, manifest),
|
||||
)
|
||||
)
|
||||
return original_write(*call_args, **call_kwargs)
|
||||
|
||||
monkeypatch.setattr(module, "write_deployment_receipt", guarded_write)
|
||||
module.reset_mutation_state()
|
||||
receipt = module.apply_bundle(args, manifest)
|
||||
|
||||
assert receipt["terminal"] == "apply_pass_deployed_inactive"
|
||||
assert observed == [(False, "exact")]
|
||||
assert not os.path.lexists(stage)
|
||||
assert not os.path.lexists(
|
||||
module.REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}"
|
||||
)
|
||||
durable = module.load_deploy_receipt(args)
|
||||
assert durable is not None
|
||||
assert durable["deployment_method"] == "created"
|
||||
assert (
|
||||
durable["production_continuity_sha256"]
|
||||
== receipt["production_continuity_sha256"]
|
||||
)
|
||||
assert not os.path.lexists(module.REMOTE_ROOT / "current")
|
||||
|
||||
|
||||
def test_reconcile_retries_half_quarantine_and_preserves_shared_target(
|
||||
target_env, tmp_path: Path
|
||||
) -> None:
|
||||
module, values, manifest, _snapshot = target_env
|
||||
stage = tmp_path / "transport-stage"
|
||||
args = _args(
|
||||
module,
|
||||
manifest,
|
||||
mode="reconcile",
|
||||
run_id="run-half",
|
||||
attempt_id="attempt-002",
|
||||
stage=stage,
|
||||
)
|
||||
receipt_dir = module.RECEIPT_ROOT / args.run_id
|
||||
receipt_dir.mkdir(mode=0o700, parents=True)
|
||||
# Model a signal after the stage was moved but before the candidate move.
|
||||
(receipt_dir / "transport-stage").mkdir(mode=0o700)
|
||||
candidate = module.REMOTE_ROOT / f".{args.bundle_id}.candidate.{args.run_id}"
|
||||
candidate.mkdir(mode=0o700, parents=True)
|
||||
target = module.REMOTE_ROOT / args.bundle_id
|
||||
_write_tree(target, manifest, values, directory_mode=0o750)
|
||||
|
||||
module.reset_mutation_state()
|
||||
receipt = module.reconcile_apply(args, manifest)
|
||||
|
||||
assert receipt["terminal"] == "reconcile_exact_target_preserved_unverified"
|
||||
assert receipt["owned_live_residue_verified_absent"] is True
|
||||
assert target.is_dir()
|
||||
assert (receipt_dir / "transport-stage").is_dir()
|
||||
assert (receipt_dir / "quarantine-candidate").is_dir()
|
||||
assert not candidate.exists()
|
||||
|
||||
|
||||
def test_reconcile_finalizes_interrupted_exact_apply_from_durable_intent(
|
||||
target_env, tmp_path: Path
|
||||
) -> None:
|
||||
module, values, manifest, snapshot = target_env
|
||||
stage = tmp_path / "transport-stage"
|
||||
args = _args(
|
||||
module,
|
||||
manifest,
|
||||
mode="reconcile",
|
||||
run_id="run-interrupted",
|
||||
attempt_id="attempt-003",
|
||||
stage=stage,
|
||||
)
|
||||
receipt_dir = module.RECEIPT_ROOT / args.run_id
|
||||
receipt_dir.mkdir(mode=0o700, parents=True)
|
||||
continuity = module.sha256_bytes(module.canonical_bytes(snapshot))
|
||||
intent = module.apply_intent(args, "absent", continuity)
|
||||
module.write_private_new(
|
||||
receipt_dir / "apply-intent.json", module.canonical_bytes(intent), 0o600
|
||||
)
|
||||
target = module.REMOTE_ROOT / args.bundle_id
|
||||
_write_tree(target, manifest, values, directory_mode=0o750)
|
||||
|
||||
module.reset_mutation_state()
|
||||
receipt = module.reconcile_apply(args, manifest)
|
||||
|
||||
assert receipt["terminal"] == "reconcile_exact_deployment_verified"
|
||||
assert receipt["durable_deploy_receipt_valid"] is True
|
||||
durable = module.load_deploy_receipt(args)
|
||||
assert durable is not None
|
||||
assert durable["deployment_method"] == "reconciled_exact"
|
||||
assert durable["reconcile_attempt_id"] == "attempt-003"
|
||||
assert target.is_dir()
|
||||
|
||||
|
||||
def test_reconcile_zero_live_residue_is_repeatable_with_new_attempt(
|
||||
target_env, tmp_path: Path
|
||||
) -> None:
|
||||
module, _values, manifest, _snapshot = target_env
|
||||
stage = tmp_path / "transport-stage"
|
||||
first = _args(
|
||||
module,
|
||||
manifest,
|
||||
mode="reconcile",
|
||||
run_id="run-cleanup",
|
||||
attempt_id="attempt-004",
|
||||
stage=stage,
|
||||
)
|
||||
stage.mkdir(mode=0o700)
|
||||
|
||||
module.reset_mutation_state()
|
||||
first_receipt = module.reconcile_apply(first, manifest)
|
||||
second = _args(
|
||||
module,
|
||||
manifest,
|
||||
mode="reconcile",
|
||||
run_id=first.run_id,
|
||||
attempt_id="attempt-005",
|
||||
stage=stage,
|
||||
)
|
||||
module.reset_mutation_state()
|
||||
second_receipt = module.reconcile_apply(second, manifest)
|
||||
|
||||
assert first_receipt["terminal"] == "reconcile_zero_owned_residue"
|
||||
assert second_receipt["terminal"] == "reconcile_zero_owned_residue"
|
||||
assert not stage.exists()
|
||||
assert (module.RECEIPT_ROOT / first.run_id / "transport-stage").is_dir()
|
||||
|
||||
|
||||
def test_reconcile_fails_closed_on_production_continuity_change(
|
||||
target_env, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
module, values, manifest, snapshot = target_env
|
||||
stage = tmp_path / "transport-stage"
|
||||
args = _args(
|
||||
module,
|
||||
manifest,
|
||||
mode="reconcile",
|
||||
run_id="run-continuity",
|
||||
attempt_id="attempt-006",
|
||||
stage=stage,
|
||||
)
|
||||
receipt_dir = module.RECEIPT_ROOT / args.run_id
|
||||
receipt_dir.mkdir(mode=0o700, parents=True)
|
||||
intent = module.apply_intent(
|
||||
args,
|
||||
"absent",
|
||||
module.sha256_bytes(module.canonical_bytes(snapshot)),
|
||||
)
|
||||
module.write_private_new(
|
||||
receipt_dir / "apply-intent.json", module.canonical_bytes(intent), 0o600
|
||||
)
|
||||
_write_tree(
|
||||
module.REMOTE_ROOT / args.bundle_id, manifest, values, directory_mode=0o750
|
||||
)
|
||||
changed = dict(snapshot)
|
||||
changed["restart_count"] = 1
|
||||
monkeypatch.setattr(module, "production_snapshot", lambda: dict(changed))
|
||||
|
||||
module.reset_mutation_state()
|
||||
with pytest.raises(
|
||||
module.TargetError, match="apply_intent_production_continuity_mismatch"
|
||||
):
|
||||
module.reconcile_apply(args, manifest)
|
||||
@@ -122,7 +122,7 @@ def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
|
||||
assert "$dbExecutorRecoveryContract" in contract
|
||||
assert "awoooi_db_bounded_executor_receipt_v2" in contract
|
||||
assert "Receipt.database_sqlstate" in contract
|
||||
assert 'expectedRuntimeFileCount": 20' in transport
|
||||
assert "expectedRuntimeFileCount=20" in transport
|
||||
assert "expectedRuntimeFileCount -ne 20" in receiver
|
||||
assert "fileCount = 20" in receiver
|
||||
assert 'expectedRuntimeFileCount": 21' in transport
|
||||
assert "expectedRuntimeFileCount=21" in transport
|
||||
assert "expectedRuntimeFileCount -ne 21" in receiver
|
||||
assert "fileCount = 21" in receiver
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SENDER = ROOT / "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh"
|
||||
|
||||
|
||||
def _array(source: str, name: str) -> tuple[str, ...]:
|
||||
match = re.search(rf"{re.escape(name)}=\(\n(?P<body>.*?)\n\)", source, re.DOTALL)
|
||||
assert match is not None
|
||||
return tuple(re.findall(r'"([^"\n]+)"', match.group("body")))
|
||||
|
||||
|
||||
def test_sender_is_syntax_valid_and_routes_only_through_windows99_agent99() -> None:
|
||||
source = SENDER.read_text(encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
["bash", "-n", str(SENDER)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert 'readonly TARGET="Administrator@192.168.0.99"' in source
|
||||
assert "wooo@192.168.0.110" not in source
|
||||
assert (
|
||||
'dispatchPath": "windows99_agent99_to_host110_fixed_metadata_toolchain"'
|
||||
in source
|
||||
)
|
||||
assert "C:\\Wooo\\Agent99\\bin\\agent99-signoz-metadata-executor.ps1" in source
|
||||
assert "Invoke-Expression" not in source
|
||||
assert "github.com" not in source
|
||||
assert "ghcr.io" not in source
|
||||
|
||||
|
||||
def test_sender_binds_exact_fresh_gitea_main_and_all_ten_transport_sources() -> None:
|
||||
source = SENDER.read_text(encoding="utf-8")
|
||||
toolchain_sources = _array(source, "TOOLCHAIN_SOURCES")
|
||||
|
||||
assert len(toolchain_sources) == 7
|
||||
assert len(set(toolchain_sources)) == 7
|
||||
assert 'readonly GITEA_REMOTE="gitea"' in source
|
||||
assert "+refs/heads/main:refs/remotes/gitea/main" in source
|
||||
assert 'SOURCE_REVISION}" == "${GITEA_MAIN_REVISION}' in source
|
||||
assert 'git -C "${SOURCE_ROOT}" diff --quiet "${SOURCE_REVISION}"' in source
|
||||
assert '"${TOOLCHAIN_SOURCES[@]}"' in source
|
||||
assert '"${EXECUTOR_RELATIVE}"' in source
|
||||
assert '"${TARGET_RELATIVE}"' in source
|
||||
assert '"${WRAPPER_RELATIVE}"' in source
|
||||
assert "source_bound_file_untracked" in source
|
||||
assert "source_bundle_differs_from_revision" in source
|
||||
|
||||
|
||||
def test_sender_envelope_is_exact_seven_file_secret_free_and_deleted_before_executor() -> (
|
||||
None
|
||||
):
|
||||
source = SENDER.read_text(encoding="utf-8")
|
||||
|
||||
assert '"schemaVersion": "awoooi_signoz_metadata_toolchain_envelope_v1"' in source
|
||||
assert '"fileCount": 7' in source
|
||||
assert '"transportPayloadContainsSecrets": False' in source
|
||||
assert '"contentBase64": base64.b64encode(content).decode("ascii")' in source
|
||||
assert "fixed_toolchain_file_contract_failed" in source
|
||||
assert "toolchain_envelope_transport_file_invalid" in source
|
||||
assert "toolchain_envelope_transport_delete_failed" in source
|
||||
delete = source.index("Remove-Item -LiteralPath $payloadPath")
|
||||
invoke = source.index("& $executor @parameters")
|
||||
assert delete < invoke
|
||||
assert "$parameters.ToolchainEnvelopeDeletedBeforeExecutor = $true" in source
|
||||
assert "REMOTE_PAYLOAD_PREPARED" in source
|
||||
assert "transport_payload_cleanup_failed" in source
|
||||
|
||||
|
||||
def test_sender_has_bounded_check_apply_verify_and_attempt_scoped_reconcile() -> None:
|
||||
source = SENDER.read_text(encoding="utf-8")
|
||||
apply_branch = source[source.index(" apply)") :]
|
||||
|
||||
for mode in (
|
||||
"ToolchainCheck",
|
||||
"ToolchainApply",
|
||||
"ToolchainVerify",
|
||||
"ToolchainReconcile",
|
||||
):
|
||||
assert mode in source
|
||||
assert "reconcile_attempt_id_missing_or_invalid" in source
|
||||
assert "reconcile-$(date -u +%Y%m%dT%H%M%SZ)-$$" in source
|
||||
assert apply_branch.index('run_checked_mode "ToolchainCheck"') < apply_branch.index(
|
||||
'invoke_remote "ToolchainApply"'
|
||||
)
|
||||
assert apply_branch.index('invoke_remote "ToolchainApply"') < apply_branch.index(
|
||||
'run_checked_mode "ToolchainVerify"'
|
||||
)
|
||||
assert 'run_checked_mode "ToolchainReconcile"' in apply_branch
|
||||
assert "toolchain_reconcile_zero_owned_residue" in source
|
||||
assert "exit 75" in source
|
||||
assert 'timeout --kill-after=20 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}"' in source
|
||||
@@ -1,267 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
DEPLOYER = ROOT / "scripts" / "ops" / "deploy-signoz-metadata-toolchain.sh"
|
||||
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-metadata-toolchain.sh"
|
||||
|
||||
|
||||
def _write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def fake_environment(tmp_path: Path, *, mode: str) -> dict[str, str]:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
event_log = tmp_path / "events.log"
|
||||
ssh_count = tmp_path / "ssh.count"
|
||||
event_log.touch()
|
||||
ssh_count.write_text("0\n", encoding="utf-8")
|
||||
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""#!/bin/bash
|
||||
set -eu
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do shift; done
|
||||
shift
|
||||
exec "$@"
|
||||
""",
|
||||
def test_historical_direct_host110_deployer_is_syntax_valid_and_fail_closed() -> None:
|
||||
syntax = subprocess.run(
|
||||
["bash", "-n", str(DEPLOYER)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "ssh",
|
||||
"""#!/bin/bash
|
||||
set -eu
|
||||
count="$(cat "${TEST_SSH_COUNT:?}")"
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "$count" > "${TEST_SSH_COUNT:?}"
|
||||
printf 'ssh %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
cat >/dev/null || true
|
||||
case "${FAKE_SSH_MODE:?}" in
|
||||
check_absent)
|
||||
printf 'REMOTE_CHECK terminal=check_pass_no_write state=absent drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
|
||||
;;
|
||||
check_drift)
|
||||
exit 3
|
||||
;;
|
||||
apply)
|
||||
case "$count" in
|
||||
1)
|
||||
printf 'REMOTE_CHECK terminal=check_pass_no_write state=absent drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
|
||||
;;
|
||||
2) ;;
|
||||
3)
|
||||
printf '{"phase":"terminal","terminal":"pass_source_deployed_inactive"}\n'
|
||||
;;
|
||||
4)
|
||||
printf 'REMOTE_CHECK terminal=check_pass_no_write state=exact drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
|
||||
;;
|
||||
*) exit 90 ;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--apply"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "scp",
|
||||
"""#!/bin/bash
|
||||
set -eu
|
||||
printf 'scp %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
""",
|
||||
)
|
||||
return {
|
||||
**os.environ,
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TEST_EVENT_LOG": str(event_log),
|
||||
"TEST_SSH_COUNT": str(ssh_count),
|
||||
"FAKE_SSH_MODE": mode,
|
||||
"TRACE_ID": "trace-P0-OBS-002-metadata-deploy",
|
||||
"RUN_ID": f"run-{mode}",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
|
||||
assert syntax.returncode == 0, syntax.stderr
|
||||
assert result.returncode == 78
|
||||
receipt = json.loads(result.stdout)
|
||||
assert receipt == {
|
||||
"schema": "awoooi_signoz_metadata_toolchain_route_retirement_v1",
|
||||
"terminal": "blocked_direct_host110_route_retired",
|
||||
"replacement": "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh",
|
||||
"production_mutation_performed": False,
|
||||
"completion_claim": False,
|
||||
}
|
||||
|
||||
|
||||
def run_deployer(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
mode: str,
|
||||
apply: bool,
|
||||
env_overrides: dict[str, str] | None = None,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
env = fake_environment(tmp_path, mode=mode)
|
||||
if env_overrides:
|
||||
env.update(env_overrides)
|
||||
return subprocess.run(
|
||||
["bash", str(DEPLOYER), "--apply" if apply else "--check"],
|
||||
env=env,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def receipt_rows(stdout: str) -> list[dict[str, object]]:
|
||||
return [json.loads(line) for line in stdout.splitlines() if line.startswith("{")]
|
||||
|
||||
|
||||
def test_help_documents_inactive_immutable_contract() -> None:
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--help"],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert "immutable exact-hash directory" in result.stdout
|
||||
assert "does not create or" in result.stdout
|
||||
assert "active pointer" in result.stdout
|
||||
|
||||
|
||||
def test_check_mode_is_no_write_and_ready_when_bundle_absent(tmp_path: Path) -> None:
|
||||
result = run_deployer(tmp_path, mode="check_absent", apply=False)
|
||||
assert result.returncode == 0, result.stderr
|
||||
rows = receipt_rows(result.stdout)
|
||||
assert rows[-1]["terminal"] == "check_pass_no_write"
|
||||
assert rows[-1]["detail"].endswith("remote_state_absent")
|
||||
events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines()
|
||||
assert len([line for line in events if line.startswith("ssh ")]) == 1
|
||||
assert not [line for line in events if line.startswith("scp ")]
|
||||
|
||||
|
||||
def test_check_mode_fails_closed_on_remote_drift(tmp_path: Path) -> None:
|
||||
result = run_deployer(tmp_path, mode="check_drift", apply=False)
|
||||
assert result.returncode == 1
|
||||
rows = receipt_rows(result.stdout)
|
||||
assert rows[-1]["terminal"] == "failed_no_write"
|
||||
|
||||
|
||||
def test_apply_uses_seven_transfers_and_exact_postcheck(tmp_path: Path) -> None:
|
||||
result = run_deployer(tmp_path, mode="apply", apply=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
rows = receipt_rows(result.stdout)
|
||||
assert rows[-1]["terminal"] == "pass_source_deployed_inactive"
|
||||
assert rows[-1]["detail"] == "runtime_export_not_executed"
|
||||
events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines()
|
||||
assert len([line for line in events if line.startswith("scp ")]) == 7
|
||||
assert len([line for line in events if line.startswith("ssh ")]) == 4
|
||||
|
||||
|
||||
def test_deployer_has_no_active_pointer_or_destructive_fallback() -> None:
|
||||
def test_historical_entrypoint_contains_no_transport_or_mutation_primitive() -> None:
|
||||
source = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert '"${REMOTE_ROOT}/current"' in source
|
||||
assert "active_pointer_0" in source
|
||||
assert "pass_source_deployed_inactive" in source
|
||||
assert "quarantine-candidate" in source
|
||||
assert "quarantine-target" in source
|
||||
assert "docker stop" not in source
|
||||
assert "docker start" not in source
|
||||
assert "docker restart" not in source
|
||||
assert "sqlite3" not in source
|
||||
assert "github.com" not in source.casefold()
|
||||
assert "curl |" not in source
|
||||
assert "curl -fsSL" not in source
|
||||
assert "\nrm " not in source
|
||||
assert "ln -s" not in source
|
||||
assert "signoz-metadata-isolated-restore.py" in source
|
||||
assert "isolated-cluster.xml" in source
|
||||
assert "local_exact_sources_7" in source
|
||||
assert "[ \"${#EXPECTED_HASHES[@]}\" -eq 7 ]" in source
|
||||
|
||||
|
||||
def test_transport_propagates_strict_fixed_paths_with_spaces(tmp_path: Path) -> None:
|
||||
transport_dir = tmp_path / "transport paths"
|
||||
transport_dir.mkdir()
|
||||
identity = transport_dir / "agent99 identity"
|
||||
known_hosts = transport_dir / "known hosts"
|
||||
identity.write_text("test-only-identity\n", encoding="utf-8")
|
||||
known_hosts.write_text("test-only-known-host\n", encoding="utf-8")
|
||||
|
||||
result = run_deployer(
|
||||
tmp_path,
|
||||
mode="check_absent",
|
||||
apply=False,
|
||||
env_overrides={
|
||||
"SIGNOZ_METADATA_SSH_IDENTITY_FILE": str(identity),
|
||||
"SIGNOZ_METADATA_KNOWN_HOSTS_FILE": str(known_hosts),
|
||||
},
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
event = (tmp_path / "events.log").read_text(encoding="utf-8")
|
||||
assert "IdentitiesOnly=yes" in event
|
||||
assert "StrictHostKeyChecking=yes" in event
|
||||
assert str(identity) in event
|
||||
assert f"UserKnownHostsFile={known_hosts}" in event
|
||||
|
||||
|
||||
def test_transport_rejects_missing_fixed_path_before_remote_use(tmp_path: Path) -> None:
|
||||
result = run_deployer(
|
||||
tmp_path,
|
||||
mode="check_absent",
|
||||
apply=False,
|
||||
env_overrides={
|
||||
"SIGNOZ_METADATA_SSH_IDENTITY_FILE": str(tmp_path / "missing identity")
|
||||
},
|
||||
)
|
||||
|
||||
assert result.returncode == 64
|
||||
assert "invalid SSH identity file" in result.stderr
|
||||
assert not (tmp_path / "events.log").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_transport_rejects_symlink_fixed_path_before_remote_use(tmp_path: Path) -> None:
|
||||
target = tmp_path / "known-hosts-target"
|
||||
target.write_text("test-only-known-host\n", encoding="utf-8")
|
||||
symlink = tmp_path / "known hosts link"
|
||||
symlink.symlink_to(target)
|
||||
|
||||
result = run_deployer(
|
||||
tmp_path,
|
||||
mode="check_absent",
|
||||
apply=False,
|
||||
env_overrides={"SIGNOZ_METADATA_KNOWN_HOSTS_FILE": str(symlink)},
|
||||
)
|
||||
|
||||
assert result.returncode == 64
|
||||
assert "invalid SSH known_hosts file" in result.stderr
|
||||
assert not (tmp_path / "events.log").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_shared_operation_lock_is_existing_read_only_and_fail_closed() -> None:
|
||||
source = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock" in source
|
||||
assert '[ -f "${OPERATION_LOCK}" ] && [ ! -L "${OPERATION_LOCK}" ]' in source
|
||||
assert 'exec 8<"${OPERATION_LOCK}"' in source
|
||||
assert "flock -n 8" in source
|
||||
assert "exec 8>>/tmp/awoooi-signoz-backup-operation.lock" not in source
|
||||
|
||||
|
||||
def test_remote_check_uses_privileged_read_only_visibility() -> None:
|
||||
source = DEPLOYER.read_text(encoding="utf-8")
|
||||
remote_check = source.split("remote_check() {", 1)[1].split("\n}", 1)[0]
|
||||
assert '"${TARGET_HOST}" sudo -n bash -s --' in remote_check
|
||||
assert "REMOTE_CHECK contains only stat/hash/process/API" in remote_check
|
||||
|
||||
|
||||
def test_exact_seven_file_supply_chain_and_modes_are_fixed() -> None:
|
||||
source = DEPLOYER.read_text(encoding="utf-8")
|
||||
for path in (
|
||||
"config/signoz/metadata-export-policy.json",
|
||||
"scripts/backup/signoz_metadata_contract.py",
|
||||
"scripts/backup/signoz-metadata-export.py",
|
||||
"scripts/backup/verify-signoz-metadata-export.py",
|
||||
"scripts/backup/signoz-metadata-restore-drill.py",
|
||||
"scripts/backup/signoz-metadata-isolated-restore.py",
|
||||
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
for forbidden in (
|
||||
"192.168.0.110",
|
||||
"ssh ",
|
||||
"scp ",
|
||||
"docker ",
|
||||
"sudo ",
|
||||
"curl ",
|
||||
"rm ",
|
||||
"mv ",
|
||||
"ln ",
|
||||
):
|
||||
assert path in source
|
||||
assert "MODES=(0644 0644 0755 0755 0755 0755 0644)" in source
|
||||
assert '[ "${#EXPECTED_HASHES[@]}" -eq 7 ]' in source
|
||||
assert forbidden not in source
|
||||
assert "blocked_direct_host110_route_retired" in source
|
||||
assert "production_mutation_performed" in source
|
||||
assert "completion_claim" in source
|
||||
|
||||
|
||||
def test_historical_entrypoint_points_only_to_agent99_replacement() -> None:
|
||||
source = DEPLOYER.read_text(encoding="utf-8")
|
||||
|
||||
assert "scripts/ops/deploy-signoz-metadata-toolchain-via-agent99.sh" in source
|
||||
assert "github.com" not in source.casefold()
|
||||
assert "ghcr.io" not in source.casefold()
|
||||
|
||||
Reference in New Issue
Block a user