wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

feat(signoz): add isolated metadata restore controller

Browse Source
This commit is contained in:
Your Name
2026-07-22 19:39:51 +08:00
parent 15bebe3edc
commit 5a51fa3dd3
9 changed files with 3311 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
scripts/ops/deploy-signoz-metadata-toolchain.sh
View File

@@ -26,6 +26,8 @@ LABELS=(
signoz-metadata-export.py
verify-signoz-metadata-export.py
signoz-metadata-restore-drill.py
signoz-metadata-isolated-restore.py
isolated-cluster.xml
)
LOCAL_SOURCES=(
"${ROOT_DIR}/config/signoz/metadata-export-policy.json"
@@ -33,8 +35,10 @@ LOCAL_SOURCES=(
"${ROOT_DIR}/scripts/backup/signoz-metadata-export.py"
"${ROOT_DIR}/scripts/backup/verify-signoz-metadata-export.py"
"${ROOT_DIR}/scripts/backup/signoz-metadata-restore-drill.py"
"${ROOT_DIR}/scripts/backup/signoz-metadata-isolated-restore.py"
"${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
)
MODES=(0644 0644 0755 0755 0755)
MODES=(0644 0644 0755 0755 0755 0755 0644)
SSH_OPTIONS=(
-o BatchMode=yes
-o IdentitiesOnly=yes
@@ -72,7 +76,7 @@ Optional fixed transport paths:
SIGNOZ_METADATA_SSH_IDENTITY_FILE
SIGNOZ_METADATA_KNOWN_HOSTS_FILE
--check validates the five local sources and performs a no-write host110 diff.
--check validates the seven local sources and performs a no-write host110 diff.
--apply creates one immutable exact-hash directory. It does not create or
change an active pointer and does not run an authenticated metadata export.
USAGE
@@ -153,7 +157,8 @@ import sys
for value in sys.argv[1:]:
path = Path(value)
compile(path.read_text(encoding="utf-8"), str(path), "exec")
if path.suffix == ".py":
compile(path.read_text(encoding="utf-8"), str(path), "exec")
PY
PYTHONPATH="${ROOT_DIR}/scripts/backup" "${LOCAL_PYTHON_BIN}" - \
"${LOCAL_SOURCES[0]}" <<'PY'
@@ -182,7 +187,7 @@ BUNDLE_ID="$(printf '%s\n' "${SOURCE_HASHES[@]}" | sha256sum | awk '{print $1}')
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-metadata-toolchain.${RUN_ID}"
emit sensor_source pass "local_exact_sources_5"
emit sensor_source pass "local_exact_sources_7"
emit normalized_asset_identity pass "bundle_${BUNDLE_ID}"
emit ai_decision pass "candidate_additive_immutable_inactive_source_deploy"
emit risk_policy_decision pass "medium_no_active_pointer_no_secret_no_raw_sqlite"
@@ -209,11 +214,13 @@ LABELS=(
signoz-metadata-export.py
verify-signoz-metadata-export.py
signoz-metadata-restore-drill.py
signoz-metadata-isolated-restore.py
isolated-cluster.xml
)
MODES=(0644 0644 0755 0755 0755)
MODES=(0644 0644 0755 0755 0755 0755 0644)
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
[ "${#EXPECTED_HASHES[@]}" -eq 5 ]
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
[ -d /backup ] && [ ! -L /backup ]
[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ]
for command_name in curl docker pgrep python3 sha256sum stat timeout; do
@@ -230,7 +237,7 @@ api_status="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
[ "${api_status}" = "200" ]
active_operations="$(
(pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \
(pgrep -af '([s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$))' || true) \
| wc -l | tr -d ' '
)"
[ "${active_operations}" = "0" ]
@@ -369,8 +376,10 @@ LABELS=(
signoz-metadata-export.py
verify-signoz-metadata-export.py
signoz-metadata-restore-drill.py
signoz-metadata-isolated-restore.py
isolated-cluster.xml
)
MODES=(0644 0644 0755 0755 0755)
MODES=(0644 0644 0755 0755 0755 0755 0644)
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
CANDIDATE_DIR="${REMOTE_ROOT}/.${BUNDLE_ID}.candidate.${RUN_ID}"
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
@@ -408,7 +417,7 @@ finalize() {
}
trap finalize EXIT HUP INT TERM
[ "${#EXPECTED_HASHES[@]}" -eq 5 ]
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
[ ! -e "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]
[ ! -e "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]
@@ -431,7 +440,7 @@ OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock
exec 8<"${OPERATION_LOCK}"
flock -n 8
active_operations="$(
(pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \
(pgrep -af '([s]ignoz-metadata-isolated-restore[.]py|(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$))' || true) \
| wc -l | tr -d ' '
)"
[ "${active_operations}" = "0" ]
@@ -443,7 +452,7 @@ api_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
http://127.0.0.1:8080/api/v1/health)"
[ "${api_before}" = "200" ]
emit_remote sensor_source pass "staged_sources_5_runtime_api_200"
emit_remote sensor_source pass "staged_sources_7_runtime_api_200"
for index in "${!LABELS[@]}"; do
staged="${STAGE_DIR}/${LABELS[index]}"
@@ -463,6 +472,7 @@ for name in (
"signoz-metadata-export.py",
"verify-signoz-metadata-export.py",
"signoz-metadata-restore-drill.py",
"signoz-metadata-isolated-restore.py",
):
path = root / name
compile(path.read_text(encoding="utf-8"), str(path), "exec")

96
scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py
View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import hashlib
import re
from pathlib import Path
@@ -17,13 +18,16 @@ def _quoted_array(source: str, start: str, end: str) -> tuple[str, ...]:
def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
assert (
'[ValidateSet("Check", "Apply", "Verify", "RestoreCheck", '
'"RestoreApply", "RestoreVerifyCleanup")]' in source
)
assert '$CanonicalAsset = "signoz-110-control-plane-metadata"' in source
assert '$TargetHost = "192.168.0.110"' in source
assert '$RemoteUser = "wooo"' in source
assert '$WorkItemId -ne "P0-OBS-002"' in source
assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source
assert "._+" not in source
assert '$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$"' in source
assert '$EvidenceRunId = if ($RunId -match $SafeIdPattern)' in source
assert "Get-Agent99RuntimeSourceBinding" in source
assert "$result.fileCount -eq 20" in source
@@ -31,9 +35,12 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
assert "$result.entrypointHashMatched" in source
assert '$CredentialFile = "/etc/awoooi/secrets/signoz-metadata-api-key"' in source
assert (
'$BundleId = "d1318c412aa303c5d8a5795f22ee62dce16ca79354c5a15e95bfad41a5e35533"'
'$BundleId = "ed4e5fbfc5f732b64d6a78f6ab26b90118e1d42a4f38b40d583876445c32e3aa"'
in source
)
assert re.search(r'\$BundleId = "[0-9a-f]{64}"', source)
assert '$IsolatedRestorePath = "$ToolchainRoot/signoz-metadata-isolated-restore.py"' in source
assert '$IsolatedClusterConfigPath = "$ToolchainRoot/isolated-cluster.xml"' in source
assert '$BaseUrl = "http://127.0.0.1:8080"' in source
assert (
'$OutputDir = "/var/backups/awoooi/signoz-metadata-export-$RunId"'
@@ -48,6 +55,34 @@ def test_entrypoint_is_fixed_to_windows99_host110_and_p0_obs_002() -> None:
)
def test_entrypoint_pins_current_seven_file_toolchain_bundle() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
rows = (
("PolicyPath", ROOT / "config/signoz/metadata-export-policy.json"),
("SharedContractPath", ROOT / "scripts/backup/signoz_metadata_contract.py"),
("ExporterPath", ROOT / "scripts/backup/signoz-metadata-export.py"),
("VerifierPath", ROOT / "scripts/backup/verify-signoz-metadata-export.py"),
("RestoreDriverPath", ROOT / "scripts/backup/signoz-metadata-restore-drill.py"),
("IsolatedRestorePath", ROOT / "scripts/backup/signoz-metadata-isolated-restore.py"),
(
"IsolatedClusterConfigPath",
ROOT
/ "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
),
)
hashes = [hashlib.sha256(path.read_bytes()).hexdigest() for _, path in rows]
bundle_id = hashlib.sha256(
"".join(f"{value}\n" for value in hashes).encode("ascii")
).hexdigest()
assert f'$BundleId = "{bundle_id}"' in source
for variable, expected_hash in zip(
(variable for variable, _path in rows), hashes, strict=True
):
assert f'${variable} = "{expected_hash}"' in source
assert "PENDING_ISOLATED_RESTORE_SHA256" not in source
def test_entrypoint_uses_fixed_public_key_transport_and_no_arbitrary_shell() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
lock_helper = source[
@@ -184,6 +219,16 @@ def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> No
assert "outputPresent = if ($outputStateReadbackVerified)" in preflight
assert "receiptPresent = if ($receiptStateReadbackVerified)" in preflight
assert "credentialReferencePresent = if ($credentialReadbackVerified)" in preflight
assert "agent99_restore_startup:" in preflight
assert "restoreStartupContractReadbackVerified" in preflight
assert "restoreStartupContractReady" in preflight
assert "restoreStartupEnvironmentValuesOutput = $false" in preflight
assert "restoreStartupFullEnvironmentOutput = $false" in preflight
assert "{{json .Config.Env}}" not in preflight
assert "{{range .Config.Env}}" not in preflight
assert "docker exec signoz /bin/sh -c" in preflight
assert "tcp://clickhouse:9000*" in preflight
assert "/var/lib/signoz/signoz.db" in preflight
credential_unverified = main.index("credential_reference_readback_unverified")
credential_absent = main.index("credential_reference_absent")
@@ -205,6 +250,9 @@ def test_entrypoint_typed_readbacks_are_exact_and_preserve_unknown_state() -> No
transport_reserve = 30
assert guard + 90 + kill_after + transport_reserve < 300
assert guard + 720 + kill_after + transport_reserve < 900
restore_cleanup_kill_after = 600
assert guard + 1800 + restore_cleanup_kill_after + transport_reserve < 2550
assert guard + 300 + restore_cleanup_kill_after + transport_reserve < 1050
def test_entrypoint_never_reads_or_persists_credential_value_in_controller() -> None:
@@ -217,7 +265,9 @@ def test_entrypoint_never_reads_or_persists_credential_value_in_controller() ->
assert 'secretValueTransmitted = $false' in source
assert 'secretValuePersistedInEvidence = $false' in source
assert "credentialReferenceDispatchAttempted" in source
assert "$script:ApplyDispatchAttempted = $true" in source
assert "$script:ExportCredentialDispatchAttempted = $true" in source
assert "$script:IsolatedApplyDispatchAttempted = $true" in source
assert "credentialReferenceDispatchAttempted = $script:ExportCredentialDispatchAttempted" in source
assert "credentialReferenceConsumedOnTarget = if" in source
assert "elseif (Test-Agent99SignozReceiptIdentity $ApplyReceipt" in source
assert 'rawSqliteRead = $false' in source
@@ -248,8 +298,8 @@ def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal
assert '--receipt-file $RemoteReceipt' in source
assert 'pass_export_verified_restore_pending' in source
assert 'export_verified_restore_pending' in source
assert 'restoreTerminal = "pending_isolated_same_version_target"' in source
assert 'zeroResidueTerminal = "pending"' in source
assert 'else { "pending_isolated_same_version_target" }' in source
assert 'else { "pending" }' in source
assert 'completionClaim = $false' in source
assert "controlledApplyPhases = [pscustomobject]$controlledApplyPhases" in source
assert 'credential_reference_absent' in source
@@ -258,6 +308,40 @@ def test_entrypoint_requires_check_apply_independent_verify_and_partial_terminal
assert 'post_verifier_runtime_or_artifact_drift' in source
def test_restore_modes_are_bounded_distinct_and_cleanup_remains_available() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
main_start = source.index("foreach ($identity")
cleanup_start = source.index(
'if ($Mode -eq "RestoreVerifyCleanup")', main_start
)
restore_apply_start = source.index(
'if ($Mode -in @("RestoreCheck", "RestoreApply"))', cleanup_start
)
export_verify_start = source.index('if ($Mode -eq "Verify")', restore_apply_start)
cleanup_helper = source[
source.index("function New-Agent99SignozCleanupLockedRemoteCommand") :
source.index("function Get-Agent99RuntimeSourceBinding")
]
cleanup_branch = source[cleanup_start:restore_apply_start]
restore_apply_branch = source[restore_apply_start:export_verify_start]
assert "Get-Agent99SignozCleanupPreflight" in cleanup_branch
assert "New-Agent99SignozCleanupLockedRemoteCommand" in cleanup_branch
assert "Get-Agent99SignozPreflight" not in cleanup_branch
assert "$OutputDir" not in cleanup_branch
assert "$BaseUrl/api/v1/health" not in cleanup_helper
assert "ExpectedArtifactState" not in cleanup_helper
assert "/usr/bin/docker info" in cleanup_helper
assert "--apply-receipt-file $RestoreRemoteReceipt" in cleanup_branch
assert '"independent_restore_verifier"' in cleanup_branch
assert '"portable_metadata_restore_independently_verified_zero_residue"' in cleanup_branch
assert '"portable_metadata_restore_verified_zero_residue_pending_independent_verifier"' in restore_apply_branch
assert cleanup_start < restore_apply_start < export_verify_start
assert "$RestoreCleanupKillAfterSeconds = 600" in source
assert "$RestoreApplyControllerWaitSeconds = 2550" in source
assert "$RestoreCleanupControllerWaitSeconds = 1050" in source
def test_entrypoint_reserves_durable_evidence_before_any_remote_dispatch() -> None:
source = ENTRYPOINT.read_text(encoding="utf-8")
main = source[source.index("foreach ($identity") :]

16
scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py
View File

@@ -143,14 +143,14 @@ def test_check_mode_fails_closed_on_remote_drift(tmp_path: Path) -> None:
assert rows[-1]["terminal"] == "failed_no_write"
def test_apply_uses_five_transfers_and_exact_postcheck(tmp_path: Path) -> None:
def test_apply_uses_seven_transfers_and_exact_postcheck(tmp_path: Path) -> None:
result = run_deployer(tmp_path, mode="apply", apply=True)
assert result.returncode == 0, result.stderr
rows = receipt_rows(result.stdout)
assert rows[-1]["terminal"] == "pass_source_deployed_inactive"
assert rows[-1]["detail"] == "runtime_export_not_executed"
events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines()
assert len([line for line in events if line.startswith("scp ")]) == 5
assert len([line for line in events if line.startswith("scp ")]) == 7
assert len([line for line in events if line.startswith("ssh ")]) == 4
@@ -170,6 +170,10 @@ def test_deployer_has_no_active_pointer_or_destructive_fallback() -> None:
assert "curl -fsSL" not in source
assert "\nrm " not in source
assert "ln -s" not in source
assert "signoz-metadata-isolated-restore.py" in source
assert "isolated-cluster.xml" in source
assert "local_exact_sources_7" in source
assert "[ \"${#EXPECTED_HASHES[@]}\" -eq 7 ]" in source
def test_transport_propagates_strict_fixed_paths_with_spaces(tmp_path: Path) -> None:
@@ -247,7 +251,7 @@ def test_remote_check_uses_privileged_read_only_visibility() -> None:
assert "REMOTE_CHECK contains only stat/hash/process/API" in remote_check
def test_exact_five_file_supply_chain_and_modes_are_fixed() -> None:
def test_exact_seven_file_supply_chain_and_modes_are_fixed() -> None:
source = DEPLOYER.read_text(encoding="utf-8")
for path in (
"config/signoz/metadata-export-policy.json",
@@ -255,7 +259,9 @@ def test_exact_five_file_supply_chain_and_modes_are_fixed() -> None:
"scripts/backup/signoz-metadata-export.py",
"scripts/backup/verify-signoz-metadata-export.py",
"scripts/backup/signoz-metadata-restore-drill.py",
"scripts/backup/signoz-metadata-isolated-restore.py",
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
):
assert path in source
assert "MODES=(0644 0644 0755 0755 0755)" in source
assert '[ "${#EXPECTED_HASHES[@]}" -eq 5 ]' in source
assert "MODES=(0644 0644 0755 0755 0755 0755 0644)" in source
assert '[ "${#EXPECTED_HASHES[@]}" -eq 7 ]' in source