feat(observability): close native SigNoz backup restore canary

scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import subprocess
+import xml.etree.ElementTree as ET
+
+
+ROOT = Path(__file__).resolve().parents[3]
+CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
+OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
+DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
+
+
+def executable_text() -> str:
+    return "\n".join(
+        line
+        for line in DEPLOYER.read_text(encoding="utf-8").splitlines()
+        if not line.lstrip().startswith("#")
+    )
+
+
+def test_clickhouse_config_declares_one_allowed_local_backup_disk() -> None:
+    root = ET.parse(CONFIG).getroot()
+    assert root.tag == "clickhouse"
+
+    disks = root.findall("./storage_configuration/disks/*")
+    assert [disk.tag for disk in disks] == ["backups"]
+    assert disks[0].findtext("type") == "local"
+    assert disks[0].findtext("path") == "/backups/"
+
+    backups = root.find("./backups")
+    assert backups is not None
+    assert backups.findtext("allowed_disk") == "backups"
+    assert backups.findtext("allowed_path") == "/backups/"
+    assert backups.findtext("allow_concurrent_backups") == "false"
+    assert backups.findtext("allow_concurrent_restores") == "false"
+    assert CONFIG.read_text(encoding="utf-8").count("<allowed_disk>") == 1
+
+
+def test_compose_override_is_additive_clickhouse_only_and_uses_dedicated_mounts() -> (
+    None
+):
+    text = OVERRIDE.read_text(encoding="utf-8")
+    assert text.count("  clickhouse:") == 1
+    assert "otel-collector:" not in text
+    assert "signoz:" not in text
+    assert "zookeeper:" not in text
+    assert "source: /backup/staging/signoz-clickhouse" in text
+    assert "target: /backups" in text
+    assert (
+        "source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml"
+    ) in text
+    assert ("target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml") in text
+    assert text.count("read_only: true") == 1
+    assert text.count("read_only: false") == 1
+    for forbidden in ("image:", "build:", "command:", "environment:"):
+        assert forbidden not in text
+
+
+def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
+    syntax = subprocess.run(
+        ["bash", "-n", str(DEPLOYER)],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert syntax.returncode == 0, syntax.stderr
+
+    help_result = subprocess.run(
+        ["bash", str(DEPLOYER), "--help"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env={
+            key: value
+            for key, value in os.environ.items()
+            if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+        },
+    )
+    assert help_result.returncode == 0
+    assert "--check" in help_result.stdout
+    assert "--apply" in help_result.stdout
+
+
+def test_deployer_requires_all_three_controlled_apply_identifiers_before_ssh() -> None:
+    env = {
+        key: value
+        for key, value in os.environ.items()
+        if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+    }
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "must be path-safe" in result.stderr
+    assert "ssh:" not in result.stderr
+
+
+def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> None:
+    text = DEPLOYER.read_text(encoding="utf-8")
+    check_terminal = text.index('if [ "${MODE}" = "--check" ]')
+    stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
+    first_scp = text.index("scp -q")
+
+    assert check_terminal < stage_assignment < first_scp
+    assert "-f '${REMOTE_BASE}' -f - config -q" in text
+    assert '< "${LOCAL_OVERRIDE}"' in text
+    assert "check_pass_no_write" in text
+
+
+def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
+    text = DEPLOYER.read_text(encoding="utf-8")
+    executable = executable_text()
+
+    assert 'REMOTE_SERVICE="clickhouse"' in text
+    assert "--env-file /dev/null" in text
+    assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
+    assert "timeout --signal=TERM --kill-after=30 300" in text
+    assert "up -d --no-deps --pull never --force-recreate" in text
+    assert '"${REMOTE_SERVICE}"' in text
+    assert "docker compose down" not in executable
+    assert "docker pull" not in executable
+    assert "docker volume" not in executable
+    assert "docker system prune" not in executable
+    assert "github.com" not in text.lower()
+    assert "ghcr.io" not in text.lower()
+
+
+def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
+    text = DEPLOYER.read_text(encoding="utf-8")
+    assert "active_signoz_backup_detected" in text
+    assert "active_native_backup_or_restore" in text
+    assert "partial_managed_file_drift" in text
+    assert "another_deploy_holds_lock" in text
+    assert "backup-signoz[.]sh" in text
+    assert "CREATING_BACKUP" in text
+    assert "RESTORING" in text
+
+
+def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
+    text = DEPLOYER.read_text(encoding="utf-8")
+    assert "BEFORE_IMAGE_ID" in text
+    assert "AFTER_IMAGE_ID" in text
+    assert "BEFORE_DATA_VOLUME" in text
+    assert "AFTER_DATA_VOLUME" in text
+    assert 'eq .Destination "/var/lib/clickhouse"' in text
+    for prefix in ("COLLECTOR", "SIGNOZ", "ZOOKEEPER"):
+        assert f"BEFORE_{prefix}_ID" in text
+        assert f"BEFORE_{prefix}_RESTARTS" in text
+    assert "image_and_data_volume_unchanged" in text
+
+
+def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
+    text = DEPLOYER.read_text(encoding="utf-8")
+    assert "system.disks WHERE name='backups'" in text
+    assert "Local|0|0|0|1" in text
+    assert "EXISTS TABLE system.backups" in text
+    assert "SHOW GRANTS" in text
+    assert "BACKUP([,[:space:]]|$)" in text
+    assert 'eq .Destination "/backups"' in text
+    assert "101:101:750" in text
+    assert "server_uid" in text
+    assert "server_gid" in text
+    assert "docker exec -u 101:101" in text
+    assert "test -w /backups" in text
+    assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
+    assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
+    identity_start = text.index('server_uid="')
+    identity_block = text[
+        identity_start : text.index("docker exec -u 101:101", identity_start)
+    ]
+    assert "sh -c" not in identity_block
+    assert "concurrency_disabled" in text
+    for port in (4317, 4318, 8080):
+        assert f"listener_up {port}" in text
+    assert "api_code" in text
+    assert "disk_backups_local_rw_healthy" in text
+
+
+def test_failed_apply_restores_prior_managed_compose_state() -> None:
+    text = DEPLOYER.read_text(encoding="utf-8")
+    assert "restore_managed_files" in text
+    assert "rollback_deploy" in text
+    assert "PRIOR_OVERRIDE_PRESENT" in text
+    assert "PRIOR_CONFIG_PRESENT" in text
+    assert "prior-override.yaml" in text
+    assert "prior-backup-disk.xml" in text
+    assert "prior_compose_state_restored" in text
+    assert "retained_nonempty_no_delete" in text
+    assert "rc=91" in text
+    assert "trap on_exit EXIT" in text
+
+
+def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:
+    executable = executable_text()
+    lowered = executable.lower()
+    assert ".config.env" not in lowered
+    assert "config.env" not in lowered
+    assert ".env" not in lowered
+    assert "docker inspect --format '{{json .config.env}}'" not in lowered
+    assert " backup all" not in lowered
+    assert " restore all" not in lowered
+    assert "clickhouse-client --query 'backup" not in lowered
+    assert "clickhouse-client --query 'restore" not in lowered
+    assert "no_backup_or_restore_executed" in executable
+
+
+def test_receipts_cover_controlled_apply_and_durable_terminal() -> None:
+    text = DEPLOYER.read_text(encoding="utf-8")
+    for phase in (
+        "sensor_source",
+        "normalized_asset_identity",
+        "source_of_truth_diff",
+        "ai_decision",
+        "risk_policy",
+        "check",
+        "execution",
+        "post_verifier",
+        "rollback",
+        "closure_writeback",
+        "terminal",
+    ):
+        assert phase in text
+    assert "receipts.jsonl" in text
+    assert "override_sha_" in text
+    assert "config_sha_" in text

scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py

@@ -0,0 +1,339 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import subprocess
+
+
+ROOT = Path(__file__).resolve().parents[3]
+DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
+
+
+def deployer_text() -> str:
+    return DEPLOYER.read_text(encoding="utf-8")
+
+
+def executable_text() -> str:
+    return "\n".join(
+        line
+        for line in deployer_text().splitlines()
+        if not line.lstrip().startswith("#")
+    )
+
+
+def base_env() -> dict[str, str]:
+    env = os.environ.copy()
+    env.update(
+        {
+            "TRACE_ID": "P0-OBS-002",
+            "RUN_ID": "toolchain-test-run",
+            "WORK_ITEM_ID": "P0-OBS-002",
+        }
+    )
+    return env
+
+
+def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
+    syntax = subprocess.run(
+        ["bash", "-n", str(DEPLOYER)],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert syntax.returncode == 0, syntax.stderr
+
+    help_result = subprocess.run(
+        ["bash", str(DEPLOYER), "--help"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env={
+            key: value
+            for key, value in os.environ.items()
+            if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+        },
+    )
+    assert help_result.returncode == 0
+    assert "--check" in help_result.stdout
+    assert "--apply" in help_result.stdout
+    assert "CLICKHOUSE_RESTORE_INVENTORY_HELPER" in help_result.stdout
+    assert "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" in help_result.stdout
+    assert "CLICKHOUSE_RESTORE_CLUSTER_CONFIG" in help_result.stdout
+
+
+def test_deployer_requires_controlled_apply_ids_before_network_access() -> None:
+    env = {
+        key: value
+        for key, value in os.environ.items()
+        if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+    }
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "must be path-safe" in result.stderr
+    assert "ssh:" not in result.stderr
+
+
+def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> None:
+    env = base_env()
+    env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/scripts/../escape"
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "safe absolute paths" in result.stderr
+    assert "ssh:" not in result.stderr
+
+    env = base_env()
+    env["SIGNOZ_TOOLCHAIN_RECEIPT_ROOT"] = "/var/tmp/receipts"
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "receipt root must stay under /backup" in result.stderr
+
+    env = base_env()
+    env["SIGNOZ_TOOLCHAIN_STAGE_ROOT"] = "/backup/deploy-staging"
+    env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/deploy-staging/scripts"
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "must not overlap" in result.stderr
+
+
+def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
+    text = deployer_text()
+    assert 'TARGET_HOST="wooo@192.168.0.110"' in text
+    assert "TARGET_HOST:-" not in text
+    assert "TARGET_HOST=" not in text.replace('TARGET_HOST="wooo@192.168.0.110"', "", 1)
+
+
+def test_exact_seven_artifacts_and_target_modes_are_declared() -> None:
+    text = deployer_text()
+    for source in (
+        "scripts/backup/backup-signoz.sh",
+        "scripts/backup/clickhouse-native-backup.sh",
+        "scripts/backup/clickhouse-native-restore-drill.sh",
+        "scripts/backup/clickhouse-restore-inventory.py",
+        "scripts/backup/run-signoz-backup-canary.sh",
+        "ops/signoz/clickhouse/config.d/backup_disk.xml",
+        "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
+    ):
+        assert source in text
+
+    for target in (
+        "${REMOTE_SCRIPT_DIR}/backup-signoz.sh",
+        "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh",
+        "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh",
+        "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py",
+        "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh",
+        "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml",
+        "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml",
+    ):
+        assert target in text
+    assert "MODES=(0755 0755 0755 0755 0755 0644 0644)" in text
+    assert 'REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"' in text
+    assert (
+        'REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-'
+        '/backup/config/signoz/clickhouse}"'
+    ) in text
+
+
+def test_local_and_remote_stage_validation_cover_hash_syntax_python_and_xml() -> None:
+    text = deployer_text()
+    assert "validate_local_sources" in text
+    assert "SOURCE_HASHES" in text
+    assert "bash -n" in text
+    assert "python3 -m py_compile" in text
+    assert "validate_xml_pair" in text
+    for expected in (
+        '"/backups/"',
+        '"restore-zookeeper"',
+        '"restore-clickhouse"',
+        '"CLICKHOUSE_RESTORE_REPLICA"',
+    ):
+        assert expected in text
+
+
+def test_check_mode_exits_before_remote_stage_or_scp() -> None:
+    text = deployer_text()
+    check_exit = text.index('if [ "${MODE}" = --check ]')
+    stage_create = text.index("STAGE_CREATED=0")
+    first_scp = text.index("scp -q")
+    assert check_exit < stage_create < first_scp
+    assert "check_pass_no_write" in text
+    assert "remote_target_unchanged_no_stage_no_candidate" in text
+
+    remote_check = text[
+        text.index("remote_read_only_check() {") : text.index(
+            "local_receipt sensor_source", text.index("remote_read_only_check() {")
+        )
+    ]
+    for forbidden in ("mkdir ", "install ", "mv ", "rm ", "scp "):
+        assert forbidden not in remote_check
+
+
+def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    log = tmp_path / "calls.log"
+    ssh = fake_bin / "ssh"
+    ssh.write_text(
+        "#!/bin/sh\n"
+        'printf "ssh %s\\n" "$*" >> "$FAKE_CALL_LOG"\n'
+        "cat >/dev/null\n"
+        'printf \'{"phase":"source_of_truth_diff","result":"pass"}\\n\'\n',
+        encoding="utf-8",
+    )
+    scp = fake_bin / "scp"
+    scp.write_text(
+        '#!/bin/sh\nprintf "scp %s\\n" "$*" >> "$FAKE_CALL_LOG"\nexit 99\n',
+        encoding="utf-8",
+    )
+    ssh.chmod(0o755)
+    scp.chmod(0o755)
+
+    env = base_env()
+    env["PATH"] = f"{fake_bin}:{env['PATH']}"
+    env["FAKE_CALL_LOG"] = str(log)
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 0, result.stderr
+    calls = log.read_text(encoding="utf-8").splitlines()
+    assert len(calls) == 1
+    assert calls[0].startswith("ssh ")
+    assert "wooo@192.168.0.110" in calls[0]
+    assert "check_pass_no_write" in result.stdout
+
+
+def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
+    text = deployer_text()
+    for token in (
+        "prior-manifest.tsv",
+        "PRIOR_PRESENT",
+        "PRIOR_MODES",
+        "PRIOR_UIDS",
+        "PRIOR_GIDS",
+        "PRIOR_HASHES",
+        "cp -p",
+        "rollback_targets",
+        "all_prior_files_modes_hashes_owners_and_runtime_restored",
+    ):
+        assert token in text
+    assert "trap on_exit EXIT" in text
+    assert "rc=91" in text
+    assert 'run_root rm -f "${target}"' in text
+    assert 'run_root rmdir "${CREATED_DIRS[index]}"' in text
+    assert "bounded_execution_not_started_targets_unchanged" in text
+
+
+def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
+    text = deployer_text()
+    assert '"${target}.candidate.${RUN_ID}"' in text
+    assert (
+        'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
+    ) in text
+    assert "deployed-manifest.tsv" in text
+    assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
+    assert "stage_candidate_residue_0" in text
+    assert "verify_residue_zero" in text
+
+
+def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
+    text = deployer_text()
+    for container in (
+        "signoz-clickhouse",
+        "signoz-otel-collector",
+        "signoz",
+        "signoz-zookeeper-1",
+    ):
+        assert container in text
+    for field in (
+        ".Id",
+        ".State.StartedAt",
+        ".RestartCount",
+        ".State.Running",
+        'index .State "Health"',
+    ):
+        assert field in text
+    assert "runtime-before.tsv" in text
+    assert "runtime-after.tsv" in text
+    assert 'cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"' in text
+    for port in (4317, 4318, 8080):
+        assert "for port in 4317 4318 8080" in text
+        assert str(port) in text
+    assert "api_200_unchanged" in text
+
+
+def test_apply_fails_closed_while_backup_or_restore_is_active() -> None:
+    text = deployer_text()
+    assert "active_signoz_backup_toolchain_process" in text
+    assert "active_native_backup_or_restore" in text
+    assert "CREATING_BACKUP" in text
+    assert "RESTORING" in text
+    assert "another_toolchain_deploy_holds_lock" in text
+
+
+def test_deployer_never_executes_runtime_work_or_uses_forbidden_supply_chain() -> None:
+    executable = executable_text().lower()
+    for forbidden in (
+        "docker restart",
+        "docker compose",
+        "docker pull",
+        "docker run",
+        "backup all",
+        "restore all",
+        "ansible-playbook",
+        "github.com",
+        "api.github.com",
+        "raw.githubusercontent.com",
+        "codeload.github.com",
+        "ghcr.io",
+        " gh ",
+    ):
+        assert forbidden not in executable
+    assert "no_backup_restore_restart_or_pull" in executable
+
+
+def test_receipts_cover_complete_controlled_apply_contract() -> None:
+    text = deployer_text()
+    for phase in (
+        "sensor_source",
+        "normalized_asset_identity",
+        "source_of_truth_diff",
+        "ai_decision",
+        "risk_policy",
+        "check",
+        "execution",
+        "post_verifier",
+        "rollback",
+        "closure_writeback",
+        "terminal",
+    ):
+        assert phase in text
+    assert "receipts.jsonl" in text
+    assert '"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"' in text
+    assert "durable_manifests_and_receipts_ack" in text