340 lines
10 KiB
Python
340 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
|
|
|
|
|
|
def deployer_text() -> str:
|
|
return DEPLOYER.read_text(encoding="utf-8")
|
|
|
|
|
|
def executable_text() -> str:
|
|
return "\n".join(
|
|
line
|
|
for line in deployer_text().splitlines()
|
|
if not line.lstrip().startswith("#")
|
|
)
|
|
|
|
|
|
def base_env() -> dict[str, str]:
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"TRACE_ID": "P0-OBS-002",
|
|
"RUN_ID": "toolchain-test-run",
|
|
"WORK_ITEM_ID": "P0-OBS-002",
|
|
}
|
|
)
|
|
return env
|
|
|
|
|
|
def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
|
|
syntax = subprocess.run(
|
|
["bash", "-n", str(DEPLOYER)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert syntax.returncode == 0, syntax.stderr
|
|
|
|
help_result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--help"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env={
|
|
key: value
|
|
for key, value in os.environ.items()
|
|
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
|
},
|
|
)
|
|
assert help_result.returncode == 0
|
|
assert "--check" in help_result.stdout
|
|
assert "--apply" in help_result.stdout
|
|
assert "CLICKHOUSE_RESTORE_INVENTORY_HELPER" in help_result.stdout
|
|
assert "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" in help_result.stdout
|
|
assert "CLICKHOUSE_RESTORE_CLUSTER_CONFIG" in help_result.stdout
|
|
|
|
|
|
def test_deployer_requires_controlled_apply_ids_before_network_access() -> None:
|
|
env = {
|
|
key: value
|
|
for key, value in os.environ.items()
|
|
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
|
}
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--check"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert result.returncode == 64
|
|
assert "must be path-safe" in result.stderr
|
|
assert "ssh:" not in result.stderr
|
|
|
|
|
|
def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> None:
|
|
env = base_env()
|
|
env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/scripts/../escape"
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--check"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert result.returncode == 64
|
|
assert "safe absolute paths" in result.stderr
|
|
assert "ssh:" not in result.stderr
|
|
|
|
env = base_env()
|
|
env["SIGNOZ_TOOLCHAIN_RECEIPT_ROOT"] = "/var/tmp/receipts"
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--check"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert result.returncode == 64
|
|
assert "receipt root must stay under /backup" in result.stderr
|
|
|
|
env = base_env()
|
|
env["SIGNOZ_TOOLCHAIN_STAGE_ROOT"] = "/backup/deploy-staging"
|
|
env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/deploy-staging/scripts"
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--check"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert result.returncode == 64
|
|
assert "must not overlap" in result.stderr
|
|
|
|
|
|
def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
|
|
text = deployer_text()
|
|
assert 'TARGET_HOST="wooo@192.168.0.110"' in text
|
|
assert "TARGET_HOST:-" not in text
|
|
assert "TARGET_HOST=" not in text.replace('TARGET_HOST="wooo@192.168.0.110"', "", 1)
|
|
|
|
|
|
def test_exact_seven_artifacts_and_target_modes_are_declared() -> None:
|
|
text = deployer_text()
|
|
for source in (
|
|
"scripts/backup/backup-signoz.sh",
|
|
"scripts/backup/clickhouse-native-backup.sh",
|
|
"scripts/backup/clickhouse-native-restore-drill.sh",
|
|
"scripts/backup/clickhouse-restore-inventory.py",
|
|
"scripts/backup/run-signoz-backup-canary.sh",
|
|
"ops/signoz/clickhouse/config.d/backup_disk.xml",
|
|
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
|
):
|
|
assert source in text
|
|
|
|
for target in (
|
|
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh",
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh",
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh",
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py",
|
|
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh",
|
|
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml",
|
|
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml",
|
|
):
|
|
assert target in text
|
|
assert "MODES=(0755 0755 0755 0755 0755 0644 0644)" in text
|
|
assert 'REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"' in text
|
|
assert (
|
|
'REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-'
|
|
'/backup/config/signoz/clickhouse}"'
|
|
) in text
|
|
|
|
|
|
def test_local_and_remote_stage_validation_cover_hash_syntax_python_and_xml() -> None:
|
|
text = deployer_text()
|
|
assert "validate_local_sources" in text
|
|
assert "SOURCE_HASHES" in text
|
|
assert "bash -n" in text
|
|
assert "python3 -m py_compile" in text
|
|
assert "validate_xml_pair" in text
|
|
for expected in (
|
|
'"/backups/"',
|
|
'"restore-zookeeper"',
|
|
'"restore-clickhouse"',
|
|
'"CLICKHOUSE_RESTORE_REPLICA"',
|
|
):
|
|
assert expected in text
|
|
|
|
|
|
def test_check_mode_exits_before_remote_stage_or_scp() -> None:
|
|
text = deployer_text()
|
|
check_exit = text.index('if [ "${MODE}" = --check ]')
|
|
stage_create = text.index("STAGE_CREATED=0")
|
|
first_scp = text.index("scp -q")
|
|
assert check_exit < stage_create < first_scp
|
|
assert "check_pass_no_write" in text
|
|
assert "remote_target_unchanged_no_stage_no_candidate" in text
|
|
|
|
remote_check = text[
|
|
text.index("remote_read_only_check() {") : text.index(
|
|
"local_receipt sensor_source", text.index("remote_read_only_check() {")
|
|
)
|
|
]
|
|
for forbidden in ("mkdir ", "install ", "mv ", "rm ", "scp "):
|
|
assert forbidden not in remote_check
|
|
|
|
|
|
def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
|
|
fake_bin = tmp_path / "bin"
|
|
fake_bin.mkdir()
|
|
log = tmp_path / "calls.log"
|
|
ssh = fake_bin / "ssh"
|
|
ssh.write_text(
|
|
"#!/bin/sh\n"
|
|
'printf "ssh %s\\n" "$*" >> "$FAKE_CALL_LOG"\n'
|
|
"cat >/dev/null\n"
|
|
'printf \'{"phase":"source_of_truth_diff","result":"pass"}\\n\'\n',
|
|
encoding="utf-8",
|
|
)
|
|
scp = fake_bin / "scp"
|
|
scp.write_text(
|
|
'#!/bin/sh\nprintf "scp %s\\n" "$*" >> "$FAKE_CALL_LOG"\nexit 99\n',
|
|
encoding="utf-8",
|
|
)
|
|
ssh.chmod(0o755)
|
|
scp.chmod(0o755)
|
|
|
|
env = base_env()
|
|
env["PATH"] = f"{fake_bin}:{env['PATH']}"
|
|
env["FAKE_CALL_LOG"] = str(log)
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--check"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
calls = log.read_text(encoding="utf-8").splitlines()
|
|
assert len(calls) == 1
|
|
assert calls[0].startswith("ssh ")
|
|
assert "wooo@192.168.0.110" in calls[0]
|
|
assert "check_pass_no_write" in result.stdout
|
|
|
|
|
|
def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
|
|
text = deployer_text()
|
|
for token in (
|
|
"prior-manifest.tsv",
|
|
"PRIOR_PRESENT",
|
|
"PRIOR_MODES",
|
|
"PRIOR_UIDS",
|
|
"PRIOR_GIDS",
|
|
"PRIOR_HASHES",
|
|
"cp -p",
|
|
"rollback_targets",
|
|
"all_prior_files_modes_hashes_owners_and_runtime_restored",
|
|
):
|
|
assert token in text
|
|
assert "trap on_exit EXIT" in text
|
|
assert "rc=91" in text
|
|
assert 'run_root rm -f "${target}"' in text
|
|
assert 'run_root rmdir "${CREATED_DIRS[index]}"' in text
|
|
assert "bounded_execution_not_started_targets_unchanged" in text
|
|
|
|
|
|
def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
|
|
text = deployer_text()
|
|
assert '"${target}.candidate.${RUN_ID}"' in text
|
|
assert (
|
|
'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
|
|
) in text
|
|
assert "deployed-manifest.tsv" in text
|
|
assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
|
|
assert "stage_candidate_residue_0" in text
|
|
assert "verify_residue_zero" in text
|
|
|
|
|
|
def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
|
|
text = deployer_text()
|
|
for container in (
|
|
"signoz-clickhouse",
|
|
"signoz-otel-collector",
|
|
"signoz",
|
|
"signoz-zookeeper-1",
|
|
):
|
|
assert container in text
|
|
for field in (
|
|
".Id",
|
|
".State.StartedAt",
|
|
".RestartCount",
|
|
".State.Running",
|
|
'index .State "Health"',
|
|
):
|
|
assert field in text
|
|
assert "runtime-before.tsv" in text
|
|
assert "runtime-after.tsv" in text
|
|
assert 'cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"' in text
|
|
for port in (4317, 4318, 8080):
|
|
assert "for port in 4317 4318 8080" in text
|
|
assert str(port) in text
|
|
assert "api_200_unchanged" in text
|
|
|
|
|
|
def test_apply_fails_closed_while_backup_or_restore_is_active() -> None:
|
|
text = deployer_text()
|
|
assert "active_signoz_backup_toolchain_process" in text
|
|
assert "active_native_backup_or_restore" in text
|
|
assert "CREATING_BACKUP" in text
|
|
assert "RESTORING" in text
|
|
assert "another_toolchain_deploy_holds_lock" in text
|
|
|
|
|
|
def test_deployer_never_executes_runtime_work_or_uses_forbidden_supply_chain() -> None:
|
|
executable = executable_text().lower()
|
|
for forbidden in (
|
|
"docker restart",
|
|
"docker compose",
|
|
"docker pull",
|
|
"docker run",
|
|
"backup all",
|
|
"restore all",
|
|
"ansible-playbook",
|
|
"github.com",
|
|
"api.github.com",
|
|
"raw.githubusercontent.com",
|
|
"codeload.github.com",
|
|
"ghcr.io",
|
|
" gh ",
|
|
):
|
|
assert forbidden not in executable
|
|
assert "no_backup_restore_restart_or_pull" in executable
|
|
|
|
|
|
def test_receipts_cover_complete_controlled_apply_contract() -> None:
|
|
text = deployer_text()
|
|
for phase in (
|
|
"sensor_source",
|
|
"normalized_asset_identity",
|
|
"source_of_truth_diff",
|
|
"ai_decision",
|
|
"risk_policy",
|
|
"check",
|
|
"execution",
|
|
"post_verifier",
|
|
"rollback",
|
|
"closure_writeback",
|
|
"terminal",
|
|
):
|
|
assert phase in text
|
|
assert "receipts.jsonl" in text
|
|
assert '"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"' in text
|
|
assert "durable_manifests_and_receipts_ack" in text
|