feat(observability): close native SigNoz backup restore canary

scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py

@@ -0,0 +1,339 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import subprocess
+
+
+ROOT = Path(__file__).resolve().parents[3]
+DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
+
+
+def deployer_text() -> str:
+    return DEPLOYER.read_text(encoding="utf-8")
+
+
+def executable_text() -> str:
+    return "\n".join(
+        line
+        for line in deployer_text().splitlines()
+        if not line.lstrip().startswith("#")
+    )
+
+
+def base_env() -> dict[str, str]:
+    env = os.environ.copy()
+    env.update(
+        {
+            "TRACE_ID": "P0-OBS-002",
+            "RUN_ID": "toolchain-test-run",
+            "WORK_ITEM_ID": "P0-OBS-002",
+        }
+    )
+    return env
+
+
+def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
+    syntax = subprocess.run(
+        ["bash", "-n", str(DEPLOYER)],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert syntax.returncode == 0, syntax.stderr
+
+    help_result = subprocess.run(
+        ["bash", str(DEPLOYER), "--help"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env={
+            key: value
+            for key, value in os.environ.items()
+            if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+        },
+    )
+    assert help_result.returncode == 0
+    assert "--check" in help_result.stdout
+    assert "--apply" in help_result.stdout
+    assert "CLICKHOUSE_RESTORE_INVENTORY_HELPER" in help_result.stdout
+    assert "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" in help_result.stdout
+    assert "CLICKHOUSE_RESTORE_CLUSTER_CONFIG" in help_result.stdout
+
+
+def test_deployer_requires_controlled_apply_ids_before_network_access() -> None:
+    env = {
+        key: value
+        for key, value in os.environ.items()
+        if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+    }
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "must be path-safe" in result.stderr
+    assert "ssh:" not in result.stderr
+
+
+def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> None:
+    env = base_env()
+    env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/scripts/../escape"
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "safe absolute paths" in result.stderr
+    assert "ssh:" not in result.stderr
+
+    env = base_env()
+    env["SIGNOZ_TOOLCHAIN_RECEIPT_ROOT"] = "/var/tmp/receipts"
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "receipt root must stay under /backup" in result.stderr
+
+    env = base_env()
+    env["SIGNOZ_TOOLCHAIN_STAGE_ROOT"] = "/backup/deploy-staging"
+    env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/deploy-staging/scripts"
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 64
+    assert "must not overlap" in result.stderr
+
+
+def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
+    text = deployer_text()
+    assert 'TARGET_HOST="wooo@192.168.0.110"' in text
+    assert "TARGET_HOST:-" not in text
+    assert "TARGET_HOST=" not in text.replace('TARGET_HOST="wooo@192.168.0.110"', "", 1)
+
+
+def test_exact_seven_artifacts_and_target_modes_are_declared() -> None:
+    text = deployer_text()
+    for source in (
+        "scripts/backup/backup-signoz.sh",
+        "scripts/backup/clickhouse-native-backup.sh",
+        "scripts/backup/clickhouse-native-restore-drill.sh",
+        "scripts/backup/clickhouse-restore-inventory.py",
+        "scripts/backup/run-signoz-backup-canary.sh",
+        "ops/signoz/clickhouse/config.d/backup_disk.xml",
+        "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
+    ):
+        assert source in text
+
+    for target in (
+        "${REMOTE_SCRIPT_DIR}/backup-signoz.sh",
+        "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh",
+        "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh",
+        "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py",
+        "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh",
+        "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml",
+        "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml",
+    ):
+        assert target in text
+    assert "MODES=(0755 0755 0755 0755 0755 0644 0644)" in text
+    assert 'REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"' in text
+    assert (
+        'REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-'
+        '/backup/config/signoz/clickhouse}"'
+    ) in text
+
+
+def test_local_and_remote_stage_validation_cover_hash_syntax_python_and_xml() -> None:
+    text = deployer_text()
+    assert "validate_local_sources" in text
+    assert "SOURCE_HASHES" in text
+    assert "bash -n" in text
+    assert "python3 -m py_compile" in text
+    assert "validate_xml_pair" in text
+    for expected in (
+        '"/backups/"',
+        '"restore-zookeeper"',
+        '"restore-clickhouse"',
+        '"CLICKHOUSE_RESTORE_REPLICA"',
+    ):
+        assert expected in text
+
+
+def test_check_mode_exits_before_remote_stage_or_scp() -> None:
+    text = deployer_text()
+    check_exit = text.index('if [ "${MODE}" = --check ]')
+    stage_create = text.index("STAGE_CREATED=0")
+    first_scp = text.index("scp -q")
+    assert check_exit < stage_create < first_scp
+    assert "check_pass_no_write" in text
+    assert "remote_target_unchanged_no_stage_no_candidate" in text
+
+    remote_check = text[
+        text.index("remote_read_only_check() {") : text.index(
+            "local_receipt sensor_source", text.index("remote_read_only_check() {")
+        )
+    ]
+    for forbidden in ("mkdir ", "install ", "mv ", "rm ", "scp "):
+        assert forbidden not in remote_check
+
+
+def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    log = tmp_path / "calls.log"
+    ssh = fake_bin / "ssh"
+    ssh.write_text(
+        "#!/bin/sh\n"
+        'printf "ssh %s\\n" "$*" >> "$FAKE_CALL_LOG"\n'
+        "cat >/dev/null\n"
+        'printf \'{"phase":"source_of_truth_diff","result":"pass"}\\n\'\n',
+        encoding="utf-8",
+    )
+    scp = fake_bin / "scp"
+    scp.write_text(
+        '#!/bin/sh\nprintf "scp %s\\n" "$*" >> "$FAKE_CALL_LOG"\nexit 99\n',
+        encoding="utf-8",
+    )
+    ssh.chmod(0o755)
+    scp.chmod(0o755)
+
+    env = base_env()
+    env["PATH"] = f"{fake_bin}:{env['PATH']}"
+    env["FAKE_CALL_LOG"] = str(log)
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+    assert result.returncode == 0, result.stderr
+    calls = log.read_text(encoding="utf-8").splitlines()
+    assert len(calls) == 1
+    assert calls[0].startswith("ssh ")
+    assert "wooo@192.168.0.110" in calls[0]
+    assert "check_pass_no_write" in result.stdout
+
+
+def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
+    text = deployer_text()
+    for token in (
+        "prior-manifest.tsv",
+        "PRIOR_PRESENT",
+        "PRIOR_MODES",
+        "PRIOR_UIDS",
+        "PRIOR_GIDS",
+        "PRIOR_HASHES",
+        "cp -p",
+        "rollback_targets",
+        "all_prior_files_modes_hashes_owners_and_runtime_restored",
+    ):
+        assert token in text
+    assert "trap on_exit EXIT" in text
+    assert "rc=91" in text
+    assert 'run_root rm -f "${target}"' in text
+    assert 'run_root rmdir "${CREATED_DIRS[index]}"' in text
+    assert "bounded_execution_not_started_targets_unchanged" in text
+
+
+def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
+    text = deployer_text()
+    assert '"${target}.candidate.${RUN_ID}"' in text
+    assert (
+        'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
+    ) in text
+    assert "deployed-manifest.tsv" in text
+    assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
+    assert "stage_candidate_residue_0" in text
+    assert "verify_residue_zero" in text
+
+
+def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
+    text = deployer_text()
+    for container in (
+        "signoz-clickhouse",
+        "signoz-otel-collector",
+        "signoz",
+        "signoz-zookeeper-1",
+    ):
+        assert container in text
+    for field in (
+        ".Id",
+        ".State.StartedAt",
+        ".RestartCount",
+        ".State.Running",
+        'index .State "Health"',
+    ):
+        assert field in text
+    assert "runtime-before.tsv" in text
+    assert "runtime-after.tsv" in text
+    assert 'cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"' in text
+    for port in (4317, 4318, 8080):
+        assert "for port in 4317 4318 8080" in text
+        assert str(port) in text
+    assert "api_200_unchanged" in text
+
+
+def test_apply_fails_closed_while_backup_or_restore_is_active() -> None:
+    text = deployer_text()
+    assert "active_signoz_backup_toolchain_process" in text
+    assert "active_native_backup_or_restore" in text
+    assert "CREATING_BACKUP" in text
+    assert "RESTORING" in text
+    assert "another_toolchain_deploy_holds_lock" in text
+
+
+def test_deployer_never_executes_runtime_work_or_uses_forbidden_supply_chain() -> None:
+    executable = executable_text().lower()
+    for forbidden in (
+        "docker restart",
+        "docker compose",
+        "docker pull",
+        "docker run",
+        "backup all",
+        "restore all",
+        "ansible-playbook",
+        "github.com",
+        "api.github.com",
+        "raw.githubusercontent.com",
+        "codeload.github.com",
+        "ghcr.io",
+        " gh ",
+    ):
+        assert forbidden not in executable
+    assert "no_backup_restore_restart_or_pull" in executable
+
+
+def test_receipts_cover_complete_controlled_apply_contract() -> None:
+    text = deployer_text()
+    for phase in (
+        "sensor_source",
+        "normalized_asset_identity",
+        "source_of_truth_diff",
+        "ai_decision",
+        "risk_policy",
+        "check",
+        "execution",
+        "post_verifier",
+        "rollback",
+        "closure_writeback",
+        "terminal",
+    ):
+        assert phase in text
+    assert "receipts.jsonl" in text
+    assert '"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"' in text
+    assert "durable_manifests_and_receipts_ack" in text