232 lines
8.2 KiB
Python
232 lines
8.2 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import xml.etree.ElementTree as ET
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
|
|
OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
|
|
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
|
|
|
|
|
|
def executable_text() -> str:
|
|
return "\n".join(
|
|
line
|
|
for line in DEPLOYER.read_text(encoding="utf-8").splitlines()
|
|
if not line.lstrip().startswith("#")
|
|
)
|
|
|
|
|
|
def test_clickhouse_config_declares_one_allowed_local_backup_disk() -> None:
|
|
root = ET.parse(CONFIG).getroot()
|
|
assert root.tag == "clickhouse"
|
|
|
|
disks = root.findall("./storage_configuration/disks/*")
|
|
assert [disk.tag for disk in disks] == ["backups"]
|
|
assert disks[0].findtext("type") == "local"
|
|
assert disks[0].findtext("path") == "/backups/"
|
|
|
|
backups = root.find("./backups")
|
|
assert backups is not None
|
|
assert backups.findtext("allowed_disk") == "backups"
|
|
assert backups.findtext("allowed_path") == "/backups/"
|
|
assert backups.findtext("allow_concurrent_backups") == "false"
|
|
assert backups.findtext("allow_concurrent_restores") == "false"
|
|
assert CONFIG.read_text(encoding="utf-8").count("<allowed_disk>") == 1
|
|
|
|
|
|
def test_compose_override_is_additive_clickhouse_only_and_uses_dedicated_mounts() -> (
|
|
None
|
|
):
|
|
text = OVERRIDE.read_text(encoding="utf-8")
|
|
assert text.count(" clickhouse:") == 1
|
|
assert "otel-collector:" not in text
|
|
assert "signoz:" not in text
|
|
assert "zookeeper:" not in text
|
|
assert "source: /backup/staging/signoz-clickhouse" in text
|
|
assert "target: /backups" in text
|
|
assert (
|
|
"source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml"
|
|
) in text
|
|
assert ("target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml") in text
|
|
assert text.count("read_only: true") == 1
|
|
assert text.count("read_only: false") == 1
|
|
for forbidden in ("image:", "build:", "command:", "environment:"):
|
|
assert forbidden not in text
|
|
|
|
|
|
def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
|
|
syntax = subprocess.run(
|
|
["bash", "-n", str(DEPLOYER)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert syntax.returncode == 0, syntax.stderr
|
|
|
|
help_result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--help"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env={
|
|
key: value
|
|
for key, value in os.environ.items()
|
|
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
|
},
|
|
)
|
|
assert help_result.returncode == 0
|
|
assert "--check" in help_result.stdout
|
|
assert "--apply" in help_result.stdout
|
|
|
|
|
|
def test_deployer_requires_all_three_controlled_apply_identifiers_before_ssh() -> None:
|
|
env = {
|
|
key: value
|
|
for key, value in os.environ.items()
|
|
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
|
}
|
|
result = subprocess.run(
|
|
["bash", str(DEPLOYER), "--check"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=env,
|
|
)
|
|
assert result.returncode == 64
|
|
assert "must be path-safe" in result.stderr
|
|
assert "ssh:" not in result.stderr
|
|
|
|
|
|
def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
check_terminal = text.index('if [ "${MODE}" = "--check" ]')
|
|
stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
|
|
first_scp = text.index("scp -q")
|
|
|
|
assert check_terminal < stage_assignment < first_scp
|
|
assert "-f '${REMOTE_BASE}' -f - config -q" in text
|
|
assert '< "${LOCAL_OVERRIDE}"' in text
|
|
assert "check_pass_no_write" in text
|
|
|
|
|
|
def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
executable = executable_text()
|
|
|
|
assert 'REMOTE_SERVICE="clickhouse"' in text
|
|
assert "--env-file /dev/null" in text
|
|
assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
|
|
assert "timeout --signal=TERM --kill-after=30 300" in text
|
|
assert "up -d --no-deps --pull never --force-recreate" in text
|
|
assert '"${REMOTE_SERVICE}"' in text
|
|
assert "docker compose down" not in executable
|
|
assert "docker pull" not in executable
|
|
assert "docker volume" not in executable
|
|
assert "docker system prune" not in executable
|
|
assert "github.com" not in text.lower()
|
|
assert "ghcr.io" not in text.lower()
|
|
|
|
|
|
def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
assert "active_signoz_backup_detected" in text
|
|
assert "active_native_backup_or_restore" in text
|
|
assert "partial_managed_file_drift" in text
|
|
assert "another_deploy_holds_lock" in text
|
|
assert "backup-signoz[.]sh" in text
|
|
assert "CREATING_BACKUP" in text
|
|
assert "RESTORING" in text
|
|
|
|
|
|
def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
assert "BEFORE_IMAGE_ID" in text
|
|
assert "AFTER_IMAGE_ID" in text
|
|
assert "BEFORE_DATA_VOLUME" in text
|
|
assert "AFTER_DATA_VOLUME" in text
|
|
assert 'eq .Destination "/var/lib/clickhouse"' in text
|
|
for prefix in ("COLLECTOR", "SIGNOZ", "ZOOKEEPER"):
|
|
assert f"BEFORE_{prefix}_ID" in text
|
|
assert f"BEFORE_{prefix}_RESTARTS" in text
|
|
assert "image_and_data_volume_unchanged" in text
|
|
|
|
|
|
def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
assert "system.disks WHERE name='backups'" in text
|
|
assert "Local|0|0|0|1" in text
|
|
assert "EXISTS TABLE system.backups" in text
|
|
assert "SHOW GRANTS" in text
|
|
assert "BACKUP([,[:space:]]|$)" in text
|
|
assert 'eq .Destination "/backups"' in text
|
|
assert "101:101:750" in text
|
|
assert "server_uid" in text
|
|
assert "server_gid" in text
|
|
assert "docker exec -u 101:101" in text
|
|
assert "test -w /backups" in text
|
|
assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
|
|
assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
|
|
identity_start = text.index('server_uid="')
|
|
identity_block = text[
|
|
identity_start : text.index("docker exec -u 101:101", identity_start)
|
|
]
|
|
assert "sh -c" not in identity_block
|
|
assert "concurrency_disabled" in text
|
|
for port in (4317, 4318, 8080):
|
|
assert f"listener_up {port}" in text
|
|
assert "api_code" in text
|
|
assert "disk_backups_local_rw_healthy" in text
|
|
|
|
|
|
def test_failed_apply_restores_prior_managed_compose_state() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
assert "restore_managed_files" in text
|
|
assert "rollback_deploy" in text
|
|
assert "PRIOR_OVERRIDE_PRESENT" in text
|
|
assert "PRIOR_CONFIG_PRESENT" in text
|
|
assert "prior-override.yaml" in text
|
|
assert "prior-backup-disk.xml" in text
|
|
assert "prior_compose_state_restored" in text
|
|
assert "retained_nonempty_no_delete" in text
|
|
assert "rc=91" in text
|
|
assert "trap on_exit EXIT" in text
|
|
|
|
|
|
def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:
|
|
executable = executable_text()
|
|
lowered = executable.lower()
|
|
assert ".config.env" not in lowered
|
|
assert "config.env" not in lowered
|
|
assert ".env" not in lowered
|
|
assert "docker inspect --format '{{json .config.env}}'" not in lowered
|
|
assert " backup all" not in lowered
|
|
assert " restore all" not in lowered
|
|
assert "clickhouse-client --query 'backup" not in lowered
|
|
assert "clickhouse-client --query 'restore" not in lowered
|
|
assert "no_backup_or_restore_executed" in executable
|
|
|
|
|
|
def test_receipts_cover_controlled_apply_and_durable_terminal() -> None:
|
|
text = DEPLOYER.read_text(encoding="utf-8")
|
|
for phase in (
|
|
"sensor_source",
|
|
"normalized_asset_identity",
|
|
"source_of_truth_diff",
|
|
"ai_decision",
|
|
"risk_policy",
|
|
"check",
|
|
"execution",
|
|
"post_verifier",
|
|
"rollback",
|
|
"closure_writeback",
|
|
"terminal",
|
|
):
|
|
assert phase in text
|
|
assert "receipts.jsonl" in text
|
|
assert "override_sha_" in text
|
|
assert "config_sha_" in text
|