wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(observability): harden native backup controlled apply

Browse Source
This commit is contained in:
ogt
2026-07-15 09:43:40 +08:00
parent 322e812d93
commit 032be5f400
12 changed files with 1849 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import os
from pathlib import Path
import re
import subprocess
import xml.etree.ElementTree as ET
@@ -10,6 +11,12 @@ ROOT = Path(__file__).resolve().parents[3]
CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
BACKUP_SCRIPT = ROOT / "scripts/backup/backup-signoz.sh"
def write_executable(path: Path, text: str) -> None:
path.write_text(text, encoding="utf-8")
path.chmod(0o755)
def executable_text() -> str:
@@ -105,9 +112,9 @@ def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> No
text = DEPLOYER.read_text(encoding="utf-8")
check_terminal = text.index('if [ "${MODE}" = "--check" ]')
stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
first_scp = text.index("scp -q")
first_transfer = text.index('bounded_scp "${LOCAL_OVERRIDE}"')
assert check_terminal < stage_assignment < first_scp
assert check_terminal < stage_assignment < first_transfer
assert "-f '${REMOTE_BASE}' -f - config -q" in text
assert '< "${LOCAL_OVERRIDE}"' in text
assert "check_pass_no_write" in text
@@ -120,7 +127,12 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
assert 'REMOTE_SERVICE="clickhouse"' in text
assert "--env-file /dev/null" in text
assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
assert "timeout --signal=TERM --kill-after=30 300" in text
assert "bounded_ssh" in text
assert "bounded_scp" in text
assert "docker_cmd" in text
assert "compose_config_cmd" in text
assert "compose_apply_cmd" in text
assert "REMOTE_APPLY_TIMEOUT_SECONDS=3600" in text
assert "up -d --no-deps --pull never --force-recreate" in text
assert '"${REMOTE_SERVICE}"' in text
assert "docker compose down" not in executable
@@ -133,15 +145,135 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
text = DEPLOYER.read_text(encoding="utf-8")
assert "active_signoz_backup_detected" in text
assert "active_signoz_backup_or_restore_process_detected" in text
assert "active_native_backup_or_restore" in text
assert "signoz_backup_process_probe_failed" in text
assert "partial_managed_file_drift" in text
assert "another_deploy_holds_lock" in text
assert "backup-signoz[.]sh" in text
for process in (
"backup-signoz",
"clickhouse-native-backup",
"clickhouse-native-restore-drill",
"run-signoz-backup-canary",
):
assert process in text
probe_block = text[
text.index("require_backup_idle()") : text.index(
"require_backup_idle preflight"
)
]
assert "process_rc=$?" in probe_block
assert 'case "${process_rc}" in' in probe_block
assert "1) ;;" in probe_block
assert "return 69" in probe_block
assert "CREATING_BACKUP" in text
assert "RESTORING" in text
def test_target_host_is_fixed_and_ignores_ambient_override(tmp_path: Path) -> None:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
ssh_log = tmp_path / "ssh.log"
write_executable(
fake_bin / "ssh",
"""#!/bin/bash
printf '%s\\n' "$*" >> "${FAKE_SSH_LOG:?}"
cat >/dev/null
""",
)
write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
env = os.environ.copy()
env.update(
{
"PATH": f"{fake_bin}:{env['PATH']}",
"TRACE_ID": "P0-OBS-002-test",
"RUN_ID": "fixed-host-test",
"WORK_ITEM_ID": "P0-OBS-002",
"TARGET_HOST": "attacker@example.invalid",
"FAKE_SSH_LOG": str(ssh_log),
}
)
result = subprocess.run(
["bash", str(DEPLOYER), "--check"],
check=False,
capture_output=True,
text=True,
env=env,
)
assert result.returncode == 0, result.stderr
invocations = ssh_log.read_text(encoding="utf-8").splitlines()
assert len(invocations) == 2
assert all("wooo@192.168.0.110" in line for line in invocations)
assert all("attacker@example.invalid" not in line for line in invocations)
def test_all_transport_and_remote_docker_operations_are_bounded() -> None:
text = DEPLOYER.read_text(encoding="utf-8")
executable = executable_text()
assert executable.count(' ssh "${SSH_OPTS[@]}"') == 1
assert executable.count(' scp -q "${SSH_OPTS[@]}"') == 1
assert not re.search(r"\bdocker (inspect|exec)\b", executable)
assert 'timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}"' in text
assert 'docker_cmd exec "${CLICKHOUSE_CONTAINER}"' in text
assert "ss_cmd -H -lnt" in text
for mutation in (
"bounded_cmd cp -p",
"bounded_cmd install -m",
"bounded_cmd mv -f",
"bounded_cmd sudo -n install",
"bounded_cmd rm -f",
):
assert mutation in text
def test_stage_cleanup_failure_changes_apply_terminal_to_failed(tmp_path: Path) -> None:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
ssh_count = tmp_path / "ssh-count"
write_executable(
fake_bin / "ssh",
"""#!/bin/bash
count=0
if [ -f "${FAKE_SSH_COUNT:?}" ]; then
count="$(cat "${FAKE_SSH_COUNT}")"
fi
count=$((count + 1))
printf '%s\\n' "${count}" > "${FAKE_SSH_COUNT}"
cat >/dev/null
if [ "${count}" -eq 5 ]; then
exit 1
fi
""",
)
write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
env = os.environ.copy()
env.update(
{
"PATH": f"{fake_bin}:{env['PATH']}",
"TRACE_ID": "P0-OBS-002-test",
"RUN_ID": "cleanup-failure-test",
"WORK_ITEM_ID": "P0-OBS-002",
"FAKE_SSH_COUNT": str(ssh_count),
}
)
result = subprocess.run(
["bash", str(DEPLOYER), "--apply"],
check=False,
capture_output=True,
text=True,
env=env,
)
assert result.returncode == 92
assert '"phase":"cleanup","result":"failed"' in result.stdout
assert '"phase":"terminal","result":"failed"' in result.stdout
assert "failed_or_residue_present" in result.stdout
def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
text = DEPLOYER.read_text(encoding="utf-8")
assert "BEFORE_IMAGE_ID" in text
@@ -166,13 +298,13 @@ def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
assert "101:101:750" in text
assert "server_uid" in text
assert "server_gid" in text
assert "docker exec -u 101:101" in text
assert "docker_cmd exec -u 101:101" in text
assert "test -w /backups" in text
assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
identity_start = text.index('server_uid="')
identity_block = text[
identity_start : text.index("docker exec -u 101:101", identity_start)
identity_start : text.index("docker_cmd exec -u 101:101", identity_start)
]
assert "sh -c" not in identity_block
assert "concurrency_disabled" in text
@@ -194,6 +326,48 @@ def test_failed_apply_restores_prior_managed_compose_state() -> None:
assert "retained_nonempty_no_delete" in text
assert "rc=91" in text
assert "trap on_exit EXIT" in text
for token in (
"PRIOR_OVERRIDE_HASH",
"PRIOR_OVERRIDE_MODE",
"PRIOR_OVERRIDE_UID",
"PRIOR_OVERRIDE_GID",
"PRIOR_CONFIG_HASH",
"PRIOR_CONFIG_MODE",
"PRIOR_CONFIG_UID",
"PRIOR_CONFIG_GID",
"file_matches_metadata",
"restore_exact_file",
"prior_compose_state_restored_exact_hash_mode_uid_gid",
):
assert token in text
assert "exact_stage_candidate_and_rollback_residue_absence_verified" in text
def test_deployer_and_backup_execution_share_operation_lock_contract() -> None:
deployer = DEPLOYER.read_text(encoding="utf-8")
backup = BACKUP_SCRIPT.read_text(encoding="utf-8")
operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
assert operation_lock in deployer
assert operation_lock in backup
assert canary_lock in deployer
assert 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' in deployer
assert 'exec 7>>"${CANARY_LOCK_FILE}"' in deployer
assert "flock -n 8 || { receipt check failed backup_operation_lock_held" in deployer
assert "flock -n 7 || { receipt check failed backup_canary_lock_held" in deployer
assert 'exec 8>>"${OPERATION_LOCK_FILE}"' in backup
assert "require_backup_idle pre_mutation" in deployer
assert deployer.index('exec 7>>"${CANARY_LOCK_FILE}"') < deployer.index(
'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"'
)
assert deployer.index('exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"') < deployer.index(
"require_backup_idle pre_mutation"
)
remote_cleanup_call = deployer.rindex(" cleanup_owned_residue")
assert remote_cleanup_call < deployer.index(
'receipt terminal "${terminal}"', remote_cleanup_call
)
def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:

140
scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
View File

@@ -3,10 +3,13 @@ from __future__ import annotations
import os
from pathlib import Path
import subprocess
import time
ROOT = Path(__file__).resolve().parents[3]
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
BACKUP = ROOT / "scripts/backup/backup-signoz.sh"
CANARY = ROOT / "scripts/backup/run-signoz-backup-canary.sh"
def deployer_text() -> str:
@@ -119,6 +122,33 @@ def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> Non
assert "must not overlap" in result.stderr
def test_timeout_overrides_fail_closed_before_network_access() -> None:
for key, value, expected in (
(
"SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS",
"0",
"positive integer seconds",
),
(
"SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS",
"301",
"exceeds bounded maximum",
),
):
env = base_env()
env[key] = value
result = subprocess.run(
["bash", str(DEPLOYER), "--check"],
check=False,
capture_output=True,
text=True,
env=env,
)
assert result.returncode == 64
assert expected in result.stderr
assert "ssh:" not in result.stderr
def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
text = deployer_text()
assert 'TARGET_HOST="wooo@192.168.0.110"' in text
@@ -229,6 +259,70 @@ def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
assert "check_pass_no_write" in result.stdout
def test_hung_ssh_is_bounded_and_cannot_claim_check_pass(tmp_path: Path) -> None:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
ssh = fake_bin / "ssh"
ssh.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8")
scp = fake_bin / "scp"
scp.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8")
ssh.chmod(0o755)
scp.chmod(0o755)
env = base_env()
env["PATH"] = f"{fake_bin}:{env['PATH']}"
env["SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS"] = "1"
env["SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS"] = "1"
started = time.monotonic()
result = subprocess.run(
["bash", str(DEPLOYER), "--check"],
check=False,
capture_output=True,
text=True,
env=env,
timeout=5,
)
elapsed = time.monotonic() - started
assert result.returncode == 124
assert elapsed < 4
assert "check_pass_no_write" not in result.stdout
def test_all_ssh_scp_and_docker_calls_use_bounded_wrappers() -> None:
text = deployer_text()
executable = executable_text()
assert (
'SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"'
in text
)
assert (
'SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"'
in text
)
assert (
'DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"'
in text
)
assert (
'TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"'
in text
)
assert "bounded_ssh()" in text
assert "bounded_scp()" in text
assert text.count('ssh "$@"') == 1
assert text.count('scp "$@"') == 1
assert executable.count("bounded_docker()") == 2
assert "$(docker inspect" not in executable
assert "$(docker exec" not in executable
assert "\n docker inspect" not in executable
assert "\n docker exec" not in executable
assert executable.count("bounded_docker inspect") == 2
assert executable.count("bounded_docker exec") == 2
assert "timeout --signal=TERM" in executable
def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
text = deployer_text()
for token in (
@@ -250,18 +344,60 @@ def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
assert "bounded_execution_not_started_targets_unchanged" in text
def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
def test_apply_uses_same_filesystem_candidates_and_accurate_replace_receipt() -> None:
text = deployer_text()
assert '"${target}.candidate.${RUN_ID}"' in text
assert (
'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
) in text
assert "deployed-manifest.tsv" in text
assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
assert (
"seven_same_filesystem_candidates_replaced_with_full_rollback_"
"expected_hashes_and_modes"
) in text
assert "seven_candidates_atomically_replaced" not in text
assert "atomically replaces exact targets" not in text
assert "stage_candidate_residue_0" in text
assert "verify_residue_zero" in text
def test_apply_holds_dual_backup_locks_through_postverify_or_rollback() -> None:
text = deployer_text()
backup_text = BACKUP.read_text(encoding="utf-8")
canary_text = CANARY.read_text(encoding="utf-8")
canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
assert f'CANARY_LOCK="{canary_lock}"' in text
assert f'OPERATION_LOCK="{operation_lock}"' in text
assert canary_lock in canary_text
assert operation_lock in backup_text
canary_acquire = text.index('exec 8>>"${CANARY_LOCK}"')
operation_acquire = text.index('exec 7>>"${OPERATION_LOCK}"')
active_check = text.index("if pgrep -af", operation_acquire)
apply_started = text.index("APPLY_STARTED=1", operation_acquire)
target_replace = text.index(
'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}"', apply_started
)
assert canary_acquire < operation_acquire < active_check < apply_started
assert apply_started < target_replace
on_exit = text[text.index("on_exit() {") : text.index("trap on_exit EXIT")]
rollback = on_exit.index("rollback_targets")
release_operation = on_exit.index("flock -u 7")
release_canary = on_exit.index("flock -u 8")
assert rollback < release_operation < release_canary
assert 'rm -f "${OPERATION_LOCK}"' not in text
assert 'rm -f "${CANARY_LOCK}"' not in text
assert "another_signoz_backup_holds_operation_lock" in text
assert "another_signoz_canary_holds_canary_lock" in text
assert (
"risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_"
"backup_operation_locks_"
"no_runtime_execution_no_container_change_full_rollback"
) in text
def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
text = deployer_text()
for container in (