406 lines
14 KiB
Python
406 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
import xml.etree.ElementTree as ET
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
|
|
OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
|
|
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
|
|
BACKUP_SCRIPT = ROOT / "scripts/backup/backup-signoz.sh"
|
|
|
|
|
|
def write_executable(path: Path, text: str) -> None:
|
|
path.write_text(text, encoding="utf-8")
|
|
path.chmod(0o755)
|
|
|
|
|
|
def executable_text() -> str:
|
|
return "\n".join(
|
|
line
|
|
for line in DEPLOYER.read_text(encoding="utf-8").splitlines()
|
|
if not line.lstrip().startswith("#")
|
|
)
|
|
|
|
|
|
def test_clickhouse_config_declares_one_allowed_local_backup_disk() -> None:
|
|
root = ET.parse(CONFIG).getroot()
|
|
assert root.tag == "clickhouse"
|
|
|
|
disks = root.findall("./storage_configuration/disks/*")
|
|
assert [disk.tag for disk in disks] == ["backups"]
|
|
assert disks[0].findtext("type") == "local"
|
|
assert disks[0].findtext("path") == "/backups/"
|
|
|
|
backups = root.find("./backups")
|
|
assert backups is not None
|
|
assert backups.findtext("allowed_disk") == "backups"
|
|
assert backups.findtext("allowed_path") == "/backups/"
|
|
assert backups.findtext("allow_concurrent_backups") == "false"
|
|
assert backups.findtext("allow_concurrent_restores") == "false"
|
|
assert CONFIG.read_text(encoding="utf-8").count("<allowed_disk>") == 1
|
|
|
|
|
|
def test_compose_override_is_additive_clickhouse_only_and_uses_dedicated_mounts() -> (
|
|
None
|
|
):
|
|
text = OVERRIDE.read_text(encoding="utf-8")
|
|
assert text.count(" clickhouse:") == 1
|
|
assert "otel-collector:" not in text
|
|
assert "signoz:" not in text
|
|
assert "zookeeper:" not in text
|
|
assert "source: /backup/staging/signoz-clickhouse" in text
|
|
assert "target: /backups" in text
|
|
assert (
|
|
"source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml"
|
|
) in text
|
|
assert ("target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml") in text
|
|
assert text.count("read_only: true") == 1
|
|
assert text.count("read_only: false") == 1
|
|
for forbidden in ("image:", "build:", "command:", "environment:"):
|
|
assert forbidden not in text
|
|
|
|
|
|
def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
|
|
syntax = subprocess.run(
|
|
["bash", "-n", str(DEPLOYER)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert syntax.returncode == 0, syntax.stderr
|
|
|
|
help_result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--help"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env={
|
|
key: value
|
|
for key, value in os.environ.items()
|
|
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
|
},
|
|
)
|
|
assert help_result.returncode == 0
|
|
assert "--check" in help_result.stdout
|
|
assert "--apply" in help_result.stdout
|
|
|
|
|
|
def test_deployer_requires_all_three_controlled_apply_identifiers_before_ssh() -> None:
|
|
env = {
|
|
key: value
|
|
for key, value in os.environ.items()
|
|
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
|
}
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--check"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert result.returncode == 64
|
|
assert "must be path-safe" in result.stderr
|
|
assert "ssh:" not in result.stderr
|
|
|
|
|
|
def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
check_terminal = text.index('if [ "${MODE}" = "--check" ]')
|
|
stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
|
|
first_transfer = text.index('bounded_scp "${LOCAL_OVERRIDE}"')
|
|
|
|
assert check_terminal < stage_assignment < first_transfer
|
|
assert "-f '${REMOTE_BASE}' -f - config -q" in text
|
|
assert '< "${LOCAL_OVERRIDE}"' in text
|
|
assert "check_pass_no_write" in text
|
|
|
|
|
|
def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
executable = executable_text()
|
|
|
|
assert 'REMOTE_SERVICE="clickhouse"' in text
|
|
assert "--env-file /dev/null" in text
|
|
assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
|
|
assert "bounded_ssh" in text
|
|
assert "bounded_scp" in text
|
|
assert "docker_cmd" in text
|
|
assert "compose_config_cmd" in text
|
|
assert "compose_apply_cmd" in text
|
|
assert "REMOTE_APPLY_TIMEOUT_SECONDS=3600" in text
|
|
assert "up -d --no-deps --pull never --force-recreate" in text
|
|
assert '"${REMOTE_SERVICE}"' in text
|
|
assert "docker compose down" not in executable
|
|
assert "docker pull" not in executable
|
|
assert "docker volume" not in executable
|
|
assert "docker system prune" not in executable
|
|
assert "github.com" not in text.lower()
|
|
assert "ghcr.io" not in text.lower()
|
|
|
|
|
|
def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
assert "active_signoz_backup_or_restore_process_detected" in text
|
|
assert "active_native_backup_or_restore" in text
|
|
assert "signoz_backup_process_probe_failed" in text
|
|
assert "partial_managed_file_drift" in text
|
|
assert "another_deploy_holds_lock" in text
|
|
for process in (
|
|
"backup-signoz",
|
|
"clickhouse-native-backup",
|
|
"clickhouse-native-restore-drill",
|
|
"run-signoz-backup-canary",
|
|
):
|
|
assert process in text
|
|
probe_block = text[
|
|
text.index("require_backup_idle()") : text.index(
|
|
"require_backup_idle preflight"
|
|
)
|
|
]
|
|
assert "process_rc=$?" in probe_block
|
|
assert 'case "${process_rc}" in' in probe_block
|
|
assert "1) ;;" in probe_block
|
|
assert "return 69" in probe_block
|
|
assert "CREATING_BACKUP" in text
|
|
assert "RESTORING" in text
|
|
|
|
|
|
def test_target_host_is_fixed_and_ignores_ambient_override(tmp_path: Path) -> None:
|
|
fake_bin = tmp_path / "bin"
|
|
fake_bin.mkdir()
|
|
ssh_log = tmp_path / "ssh.log"
|
|
write_executable(
|
|
fake_bin / "ssh",
|
|
"""#!/bin/bash
|
|
printf '%s\\n' "$*" >> "${FAKE_SSH_LOG:?}"
|
|
cat >/dev/null
|
|
""",
|
|
)
|
|
write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"PATH": f"{fake_bin}:{env['PATH']}",
|
|
"TRACE_ID": "P0-OBS-002-test",
|
|
"RUN_ID": "fixed-host-test",
|
|
"WORK_ITEM_ID": "P0-OBS-002",
|
|
"TARGET_HOST": "attacker@example.invalid",
|
|
"FAKE_SSH_LOG": str(ssh_log),
|
|
}
|
|
)
|
|
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--check"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
invocations = ssh_log.read_text(encoding="utf-8").splitlines()
|
|
assert len(invocations) == 2
|
|
assert all("wooo@192.168.0.110" in line for line in invocations)
|
|
assert all("attacker@example.invalid" not in line for line in invocations)
|
|
|
|
|
|
def test_all_transport_and_remote_docker_operations_are_bounded() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
executable = executable_text()
|
|
|
|
assert executable.count(' ssh "${SSH_OPTS[@]}"') == 1
|
|
assert executable.count(' scp -q "${SSH_OPTS[@]}"') == 1
|
|
assert not re.search(r"\bdocker (inspect|exec)\b", executable)
|
|
assert 'timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}"' in text
|
|
assert 'docker_cmd exec "${CLICKHOUSE_CONTAINER}"' in text
|
|
assert "ss_cmd -H -lnt" in text
|
|
for mutation in (
|
|
"bounded_cmd cp -p",
|
|
"bounded_cmd install -m",
|
|
"bounded_cmd mv -f",
|
|
"bounded_cmd sudo -n install",
|
|
"bounded_cmd rm -f",
|
|
):
|
|
assert mutation in text
|
|
|
|
|
|
def test_stage_cleanup_failure_changes_apply_terminal_to_failed(tmp_path: Path) -> None:
|
|
fake_bin = tmp_path / "bin"
|
|
fake_bin.mkdir()
|
|
ssh_count = tmp_path / "ssh-count"
|
|
write_executable(
|
|
fake_bin / "ssh",
|
|
"""#!/bin/bash
|
|
count=0
|
|
if [ -f "${FAKE_SSH_COUNT:?}" ]; then
|
|
count="$(cat "${FAKE_SSH_COUNT}")"
|
|
fi
|
|
count=$((count + 1))
|
|
printf '%s\\n' "${count}" > "${FAKE_SSH_COUNT}"
|
|
cat >/dev/null
|
|
if [ "${count}" -eq 5 ]; then
|
|
exit 1
|
|
fi
|
|
""",
|
|
)
|
|
write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"PATH": f"{fake_bin}:{env['PATH']}",
|
|
"TRACE_ID": "P0-OBS-002-test",
|
|
"RUN_ID": "cleanup-failure-test",
|
|
"WORK_ITEM_ID": "P0-OBS-002",
|
|
"FAKE_SSH_COUNT": str(ssh_count),
|
|
}
|
|
)
|
|
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--apply"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
|
|
assert result.returncode == 92
|
|
assert '"phase":"cleanup","result":"failed"' in result.stdout
|
|
assert '"phase":"terminal","result":"failed"' in result.stdout
|
|
assert "failed_or_residue_present" in result.stdout
|
|
|
|
|
|
def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
assert "BEFORE_IMAGE_ID" in text
|
|
assert "AFTER_IMAGE_ID" in text
|
|
assert "BEFORE_DATA_VOLUME" in text
|
|
assert "AFTER_DATA_VOLUME" in text
|
|
assert 'eq .Destination "/var/lib/clickhouse"' in text
|
|
for prefix in ("COLLECTOR", "SIGNOZ", "ZOOKEEPER"):
|
|
assert f"BEFORE_{prefix}_ID" in text
|
|
assert f"BEFORE_{prefix}_RESTARTS" in text
|
|
assert "image_and_data_volume_unchanged" in text
|
|
|
|
|
|
def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
assert "system.disks WHERE name='backups'" in text
|
|
assert "Local|0|0|0|1" in text
|
|
assert "EXISTS TABLE system.backups" in text
|
|
assert "SHOW GRANTS" in text
|
|
assert "BACKUP([,[:space:]]|$)" in text
|
|
assert 'eq .Destination "/backups"' in text
|
|
assert "101:101:750" in text
|
|
assert "server_uid" in text
|
|
assert "server_gid" in text
|
|
assert "docker_cmd exec -u 101:101" in text
|
|
assert "test -w /backups" in text
|
|
assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
|
|
assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
|
|
identity_start = text.index('server_uid="')
|
|
identity_block = text[
|
|
identity_start : text.index("docker_cmd exec -u 101:101", identity_start)
|
|
]
|
|
assert "sh -c" not in identity_block
|
|
assert "concurrency_disabled" in text
|
|
for port in (4317, 4318, 8080):
|
|
assert f"listener_up {port}" in text
|
|
assert "api_code" in text
|
|
assert "disk_backups_local_rw_healthy" in text
|
|
|
|
|
|
def test_failed_apply_restores_prior_managed_compose_state() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
assert "restore_managed_files" in text
|
|
assert "rollback_deploy" in text
|
|
assert "PRIOR_OVERRIDE_PRESENT" in text
|
|
assert "PRIOR_CONFIG_PRESENT" in text
|
|
assert "prior-override.yaml" in text
|
|
assert "prior-backup-disk.xml" in text
|
|
assert "prior_compose_state_restored" in text
|
|
assert "retained_nonempty_no_delete" in text
|
|
assert "rc=91" in text
|
|
assert "trap on_exit EXIT" in text
|
|
for token in (
|
|
"PRIOR_OVERRIDE_HASH",
|
|
"PRIOR_OVERRIDE_MODE",
|
|
"PRIOR_OVERRIDE_UID",
|
|
"PRIOR_OVERRIDE_GID",
|
|
"PRIOR_CONFIG_HASH",
|
|
"PRIOR_CONFIG_MODE",
|
|
"PRIOR_CONFIG_UID",
|
|
"PRIOR_CONFIG_GID",
|
|
"file_matches_metadata",
|
|
"restore_exact_file",
|
|
"prior_compose_state_restored_exact_hash_mode_uid_gid",
|
|
):
|
|
assert token in text
|
|
assert "exact_stage_candidate_and_rollback_residue_absence_verified" in text
|
|
|
|
|
|
def test_deployer_and_backup_execution_share_operation_lock_contract() -> None:
|
|
deployer = DEPLOYER.read_text(encoding="utf-8")
|
|
backup = BACKUP_SCRIPT.read_text(encoding="utf-8")
|
|
operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
|
|
canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
|
|
|
|
assert operation_lock in deployer
|
|
assert operation_lock in backup
|
|
assert canary_lock in deployer
|
|
assert 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' in deployer
|
|
assert 'exec 7>>"${CANARY_LOCK_FILE}"' in deployer
|
|
assert "flock -n 8 || { receipt check failed backup_operation_lock_held" in deployer
|
|
assert "flock -n 7 || { receipt check failed backup_canary_lock_held" in deployer
|
|
assert 'exec 8>>"${OPERATION_LOCK_FILE}"' in backup
|
|
assert "require_backup_idle pre_mutation" in deployer
|
|
assert deployer.index('exec 7>>"${CANARY_LOCK_FILE}"') < deployer.index(
|
|
'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"'
|
|
)
|
|
assert deployer.index('exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"') < deployer.index(
|
|
"require_backup_idle pre_mutation"
|
|
)
|
|
remote_cleanup_call = deployer.rindex(" cleanup_owned_residue")
|
|
assert remote_cleanup_call < deployer.index(
|
|
'receipt terminal "${terminal}"', remote_cleanup_call
|
|
)
|
|
|
|
|
|
def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:
|
|
executable = executable_text()
|
|
lowered = executable.lower()
|
|
assert ".config.env" not in lowered
|
|
assert "config.env" not in lowered
|
|
assert ".env" not in lowered
|
|
assert "docker inspect --format '{{json .config.env}}'" not in lowered
|
|
assert " backup all" not in lowered
|
|
assert " restore all" not in lowered
|
|
assert "clickhouse-client --query 'backup" not in lowered
|
|
assert "clickhouse-client --query 'restore" not in lowered
|
|
assert "no_backup_or_restore_executed" in executable
|
|
|
|
|
|
def test_receipts_cover_controlled_apply_and_durable_terminal() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
for phase in (
|
|
"sensor_source",
|
|
"normalized_asset_identity",
|
|
"source_of_truth_diff",
|
|
"ai_decision",
|
|
"risk_policy",
|
|
"check",
|
|
"execution",
|
|
"post_verifier",
|
|
"rollback",
|
|
"closure_writeback",
|
|
"terminal",
|
|
):
|
|
assert phase in text
|
|
assert "receipts.jsonl" in text
|
|
assert "override_sha_" in text
|
|
assert "config_sha_" in text
|