feat(observability): close native SigNoz backup restore canary
This commit is contained in:
ogt
@@ -5,6 +5,7 @@ import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
|
||||
BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh"
|
||||
@@ -20,27 +21,24 @@ def _run_backup(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
initial_running: bool = True,
|
||||
docker_run_mode: str = "success",
|
||||
docker_run_target: str = "signoz-clickhouse",
|
||||
archive_timeout_target: str = "",
|
||||
tar_verify_mode: str = "success",
|
||||
tar_verify_target: str = "clickhouse_",
|
||||
tar_timeout_target: str = "",
|
||||
docker_start_failures: int = 0,
|
||||
native_check_status: int = 0,
|
||||
native_apply_status: int = 0,
|
||||
continuity_mode: str = "stable",
|
||||
restic_backup_status: int = 0,
|
||||
restore_max_attempts: int = 3,
|
||||
skip_retention: bool = False,
|
||||
) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
|
||||
harness = tmp_path / "backup"
|
||||
fake_bin = tmp_path / "bin"
|
||||
backup_base = tmp_path / "repository"
|
||||
event_log = tmp_path / "events.log"
|
||||
collector_id = tmp_path / "collector.id"
|
||||
collector_state = tmp_path / "collector.state"
|
||||
start_count = tmp_path / "start.count"
|
||||
harness.mkdir()
|
||||
fake_bin.mkdir()
|
||||
(backup_base / "signoz" / "data").mkdir(parents=True)
|
||||
shutil.copy2(SCRIPT, harness / SCRIPT.name)
|
||||
event_log.touch()
|
||||
collector_id.write_text("sha256:collector-stable\n", encoding="utf-8")
|
||||
collector_state.write_text(
|
||||
"true\n" if initial_running else "false\n",
|
||||
encoding="utf-8",
|
||||
@@ -50,37 +48,62 @@ def _run_backup(
|
||||
"""\
|
||||
BACKUP_BASE="${TEST_BACKUP_BASE:?}"
|
||||
RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password"
|
||||
log_info() { :; }
|
||||
log_warn() { :; }
|
||||
log_error() { :; }
|
||||
log_success() { :; }
|
||||
log_info() { printf 'info %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
log_warn() { printf 'warn %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
log_error() { printf 'error %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
log_success() { printf 'success %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
build_tags() { :; }
|
||||
cleanup_old_backups() { :; }
|
||||
build_tags() { printf '%s' '--tag signoz'; }
|
||||
cleanup_old_backups() { printf 'retention %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_write_executable(
|
||||
harness / "clickhouse-native-backup.sh",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'native %s trace=%s run=%s work=%s\\n' \
|
||||
"${1:-}" "${TRACE_ID:-}" "${RUN_ID:-}" "${WORK_ITEM_ID:-}" \
|
||||
>> "${TEST_EVENT_LOG:?}"
|
||||
case "${1:-}" in
|
||||
--check)
|
||||
exit "${FAKE_NATIVE_CHECK_STATUS:-0}"
|
||||
;;
|
||||
--apply)
|
||||
status="${FAKE_NATIVE_APPLY_STATUS:-0}"
|
||||
[ "$status" -eq 0 ] || exit "$status"
|
||||
case "${FAKE_CONTINUITY_MODE:-stable}" in
|
||||
stopped)
|
||||
printf 'false\\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
sleep 0.08
|
||||
printf 'true\\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
;;
|
||||
identity)
|
||||
printf 'sha256:collector-replaced\\n' > "${TEST_COLLECTOR_ID:?}"
|
||||
sleep 0.08
|
||||
;;
|
||||
*) sleep 0.08 ;;
|
||||
esac
|
||||
printf 'native-zip\\n' \
|
||||
> "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip"
|
||||
printf '{}\\n' \
|
||||
> "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip.manifest.json"
|
||||
printf 'db\\ttable\\tengine\\trows\\tbytes\\t%s\\n' \
|
||||
"$(printf 'A%.0s' {1..64})" \
|
||||
> "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.source-inventory.tsv"
|
||||
;;
|
||||
*) exit 64 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do
|
||||
shift
|
||||
done
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do shift; done
|
||||
shift
|
||||
if [ -n "${FAKE_ARCHIVE_TIMEOUT_TARGET:-}" ] \
|
||||
&& [ "${1:-}" = "docker" ] \
|
||||
&& [[ " $* " == *"${FAKE_ARCHIVE_TIMEOUT_TARGET}"* ]]; then
|
||||
exit 124
|
||||
fi
|
||||
if [ -n "${FAKE_TAR_TIMEOUT_TARGET:-}" ] \
|
||||
&& [ "${1:-}" = "tar" ] \
|
||||
&& [[ " $* " == *"${FAKE_TAR_TIMEOUT_TARGET}"* ]]; then
|
||||
exit 124
|
||||
fi
|
||||
exec "$@"
|
||||
""",
|
||||
)
|
||||
@@ -89,61 +112,12 @@ exec "$@"
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'docker %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
case "${1:-}" in
|
||||
inspect)
|
||||
cat "${TEST_COLLECTOR_STATE:?}"
|
||||
;;
|
||||
stop)
|
||||
if [ "${2:-}" = "signoz-otel-collector" ]; then
|
||||
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
fi
|
||||
;;
|
||||
start)
|
||||
count=0
|
||||
if [ -e "${TEST_START_COUNT:?}" ]; then
|
||||
count=$(cat "${TEST_START_COUNT}")
|
||||
fi
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "$count" > "${TEST_START_COUNT}"
|
||||
if [ "$count" -le "${FAKE_DOCKER_START_FAILURES:-0}" ]; then
|
||||
exit 1
|
||||
fi
|
||||
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
;;
|
||||
run)
|
||||
mode=success
|
||||
if [[ " $* " == *"${FAKE_DOCKER_RUN_TARGET:-signoz-clickhouse}"* ]]; then
|
||||
mode="${FAKE_DOCKER_RUN_MODE:-success}"
|
||||
fi
|
||||
if [ "$mode" = "signal" ]; then
|
||||
kill -TERM "${PPID}"
|
||||
sleep 0.1
|
||||
exit 0
|
||||
fi
|
||||
if [ "$mode" = "empty" ]; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$mode" = "partial_nonzero" ]; then
|
||||
printf 'partial-archive-data\n'
|
||||
exit 23
|
||||
fi
|
||||
printf 'archive-data\n'
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "tar",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'tar %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
if [[ " $* " == *"${FAKE_TAR_VERIFY_TARGET:-signoz-clickhouse}"* ]] \
|
||||
&& [ "${FAKE_TAR_VERIFY_MODE:-success}" = "corrupt" ]; then
|
||||
exit 2
|
||||
fi
|
||||
exit 0
|
||||
printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
[ "${1:-}" = "inspect" ] || exit 70
|
||||
printf '%s\\t%s\\t%s\\t%s\\t%s\\n' \
|
||||
"$(tr -d '\\r\\n' < "${TEST_COLLECTOR_ID:?}")" \
|
||||
"$(tr -d '\\r\\n' < "${TEST_COLLECTOR_STATE:?}")" \
|
||||
'2026-07-15T00:00:00Z' '0' 'healthy'
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
@@ -151,18 +125,20 @@ exit 0
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'restic %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
printf 'restic %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
case " $* " in
|
||||
*" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
|
||||
*" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
|
||||
*" backup "*)
|
||||
dump_dir="${4:?}"
|
||||
if [ -f "$dump_dir/signoz-metadata-gap.json" ]; then
|
||||
printf 'gap %s\\n' "$(cat "$dump_dir/signoz-metadata-gap.json")" \
|
||||
>> "${TEST_EVENT_LOG:?}"
|
||||
fi
|
||||
exit "${FAKE_RESTIC_BACKUP_STATUS:-0}"
|
||||
;;
|
||||
*" snapshots "*)
|
||||
printf '[{"short_id":"native123"}]\\n'
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "du",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
printf '4K\t%s\n' "${2:-unknown}"
|
||||
""",
|
||||
)
|
||||
|
||||
@@ -171,22 +147,20 @@ printf '4K\t%s\n' "${2:-unknown}"
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TEST_BACKUP_BASE": str(backup_base),
|
||||
"TEST_EVENT_LOG": str(event_log),
|
||||
"TEST_COLLECTOR_ID": str(collector_id),
|
||||
"TEST_COLLECTOR_STATE": str(collector_state),
|
||||
"TEST_START_COUNT": str(start_count),
|
||||
"FAKE_DOCKER_RUN_MODE": docker_run_mode,
|
||||
"FAKE_DOCKER_RUN_TARGET": docker_run_target,
|
||||
"FAKE_ARCHIVE_TIMEOUT_TARGET": archive_timeout_target,
|
||||
"FAKE_TAR_VERIFY_MODE": tar_verify_mode,
|
||||
"FAKE_TAR_VERIFY_TARGET": tar_verify_target,
|
||||
"FAKE_TAR_TIMEOUT_TARGET": tar_timeout_target,
|
||||
"FAKE_DOCKER_START_FAILURES": str(docker_start_failures),
|
||||
"FAKE_NATIVE_CHECK_STATUS": str(native_check_status),
|
||||
"FAKE_NATIVE_APPLY_STATUS": str(native_apply_status),
|
||||
"FAKE_CONTINUITY_MODE": continuity_mode,
|
||||
"FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
|
||||
"COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts),
|
||||
"COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0",
|
||||
"TRACE_ID": "trace-P0-OBS-002",
|
||||
"RUN_ID": f"run-{tmp_path.name}",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"SIGNOZ_BACKUP_RECEIPT_ROOT": str(tmp_path / "receipts"),
|
||||
"COLLECTOR_CONTINUITY_POLL_SECONDS": "0.01",
|
||||
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
|
||||
"ARCHIVE_CREATE_TIMEOUT_SECONDS": "1",
|
||||
"ARCHIVE_VERIFY_TIMEOUT_SECONDS": "1",
|
||||
"TIMEOUT_KILL_AFTER_SECONDS": "1",
|
||||
"BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0",
|
||||
}
|
||||
process = subprocess.Popen(
|
||||
["bash", str(harness / SCRIPT.name)],
|
||||
@@ -210,274 +184,155 @@ def _events(events: list[str], prefix: str) -> list[str]:
|
||||
return [event for event in events if event.startswith(prefix)]
|
||||
|
||||
|
||||
def _run_backup_all_with_signoz_failure(
|
||||
def test_source_sunsets_raw_volumes_and_collector_mutation() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "signoz-clickhouse" not in source
|
||||
assert "signoz-sqlite" not in source
|
||||
assert "docker run" not in source
|
||||
assert "docker stop" not in source
|
||||
assert "docker start" not in source
|
||||
assert "create_volume_archive" not in source
|
||||
assert "pending_application_consistent_export" in source
|
||||
assert '"raw_volume_read":false' in source
|
||||
assert "{{.State.StartedAt}}" in source
|
||||
assert "{{.RestartCount}}" in source
|
||||
assert '{{if (index .State "Health")}}' in source
|
||||
assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
|
||||
assert 'notify_clawbot "success"' not in source
|
||||
assert "no-downtime continuity verifier" in source
|
||||
|
||||
|
||||
def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
|
||||
tmp_path: Path,
|
||||
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
|
||||
harness = tmp_path / "backup-all"
|
||||
fake_scripts = tmp_path / "deployed-scripts"
|
||||
event_log = tmp_path / "backup-all-events.log"
|
||||
harness.mkdir()
|
||||
fake_scripts.mkdir()
|
||||
event_log.touch()
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path)
|
||||
|
||||
source = BACKUP_ALL.read_text(encoding="utf-8").replace(
|
||||
"/backup/scripts",
|
||||
str(fake_scripts),
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert len(_events(events, "native --check")) == 1
|
||||
assert len(_events(events, "native --apply")) == 1
|
||||
check_identity = _events(events, "native --check")[0].split(" trace=", 1)[1]
|
||||
apply_identity = _events(events, "native --apply")[0].split(" trace=", 1)[1]
|
||||
assert check_identity == apply_identity
|
||||
assert not _events(events, "docker stop")
|
||||
assert not _events(events, "docker start")
|
||||
assert not _events(events, "docker run")
|
||||
assert any("pending_application_consistent_export" in event for event in events)
|
||||
assert len(_events(events, "notify warning signoz ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert any("SigNoz ClickHouse native 備份完成" in event for event in events)
|
||||
assert any(
|
||||
"--tag run:run-" in event for event in events if event.startswith("restic ")
|
||||
)
|
||||
backup_all = harness / "backup-all.sh"
|
||||
_write_executable(backup_all, source)
|
||||
(harness / "common.sh").write_text(
|
||||
"""\
|
||||
log_info() { :; }
|
||||
log_warn() { :; }
|
||||
log_error() { :; }
|
||||
log_success() { :; }
|
||||
notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
for name in (
|
||||
"backup-gitea.sh",
|
||||
"backup-momo.sh",
|
||||
"backup-harbor.sh",
|
||||
"backup-awoooi.sh",
|
||||
"backup-langfuse.sh",
|
||||
"backup-monitoring.sh",
|
||||
"backup-signoz.sh",
|
||||
"backup-open-webui.sh",
|
||||
"backup-clawbot.sh",
|
||||
):
|
||||
status = 17 if name == "backup-signoz.sh" else 0
|
||||
_write_executable(
|
||||
fake_scripts / name,
|
||||
f"#!/bin/bash\nprintf 'run {name}\\n' >> \"${{TEST_EVENT_LOG:?}}\"\nexit {status}\n",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(backup_all)],
|
||||
env={**os.environ, "TEST_EVENT_LOG": str(event_log)},
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
return result, event_log.read_text(encoding="utf-8").splitlines()
|
||||
assert any("restic -r" in event and " check " in event for event in events)
|
||||
receipts = list((tmp_path / "receipts").glob("*/restic-snapshot.json"))
|
||||
assert len(receipts) == 1
|
||||
receipt = receipts[0].read_text(encoding="utf-8")
|
||||
assert '"snapshot_id":"native123"' in receipt
|
||||
assert '"offsite_verified":false' in receipt
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_backup_jobs_deploys_the_guarded_signoz_script() -> None:
|
||||
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
|
||||
|
||||
assert 'dest: "/backup/scripts/{{ item }}"' in playbook
|
||||
assert "- backup-signoz.sh" in playbook
|
||||
assert "tags: backup_jobs" in playbook
|
||||
|
||||
|
||||
def test_restore_policy_is_bounded_timed_and_independently_verified() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source
|
||||
assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source
|
||||
assert 'ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"' in source
|
||||
assert 'ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"' in source
|
||||
assert 'TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"' in source
|
||||
assert 'timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}"' in source
|
||||
assert 'tar -tzf "${output_file}"' in source
|
||||
assert "docker run" in source
|
||||
assert "|| true" not in source.split("create_volume_archive()", 1)[1].split(
|
||||
"verify_volume_archive()",
|
||||
1,
|
||||
)[0]
|
||||
assert "verify_collector_final_state" in source
|
||||
assert source.index("verify_collector_final_state") < source.rindex(
|
||||
'notify_clawbot "success"'
|
||||
)
|
||||
|
||||
|
||||
def test_canary_retention_skip_is_explicit_and_defaults_off() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"' in source
|
||||
assert 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' in source
|
||||
assert source.index('restic -r "${LOCAL_REPO}" backup') < source.index(
|
||||
'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]'
|
||||
)
|
||||
assert source.index('if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]') < source.index(
|
||||
"if ! verify_collector_final_state"
|
||||
)
|
||||
|
||||
|
||||
def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal")
|
||||
def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, native_check_status=19)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert _events(events, "docker stop signoz-otel-collector")
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert _events(events, "native --check")
|
||||
assert not _events(events, "native --apply")
|
||||
assert not _events(events, "restic ")
|
||||
assert len(_events(events, "notify failed signoz ")) == 1
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_collector_is_restored_when_clickhouse_backup_fails(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="empty")
|
||||
def test_native_apply_failure_never_reports_scoped_success(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, native_apply_status=23)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert result.returncode != 0
|
||||
assert _events(events, "native --apply")
|
||||
assert not _events(events, "restic ")
|
||||
assert not _events(events, "notify warning ")
|
||||
assert not _events(events, "notify success ")
|
||||
assert len(_events(events, "notify failed signoz ")) == 1
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_exit_cleanup_does_not_restart_an_already_restored_collector(
|
||||
def test_initially_stopped_collector_fails_without_apply(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert _events(events, "native --check")
|
||||
assert not _events(events, "native --apply")
|
||||
assert not _events(events, "restic ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_continuity_observer_rejects_stopped_sample(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, continuity_mode="stopped")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert any("continuity verifier 失敗" in event for event in events)
|
||||
assert not _events(events, "restic ")
|
||||
assert not _events(events, "notify warning ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_continuity_observer_rejects_container_identity_drift(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, continuity_mode="identity")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert any("continuity verifier 失敗" in event for event in events)
|
||||
assert not _events(events, "restic ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_restic_failure_is_nonzero_and_no_completion_notification(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, restic_backup_status=17)
|
||||
|
||||
assert result.returncode == 17
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert _events(events, "restic ")
|
||||
assert not _events(events, "notify warning ")
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_fail_once_restores_within_budget_and_succeeds(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=1)
|
||||
def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None:
|
||||
result, events, _dump_dir = _run_backup(tmp_path, skip_retention=True)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 2
|
||||
assert len(_events(events, "notify success ")) == 1
|
||||
assert not _events(events, "notify failed ")
|
||||
assert not dump_dir.exists()
|
||||
assert any("BACKUP_SKIP_RETENTION_CLEANUP=1" in event for event in events)
|
||||
assert not _events(events, "retention ")
|
||||
|
||||
|
||||
def test_permanent_restore_failure_is_nonzero_and_never_notifies_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=99)
|
||||
def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None:
|
||||
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 3
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_initially_stopped_fails_closed_without_stop_start_or_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert not _events(events, "docker stop signoz-otel-collector")
|
||||
assert not _events(events, "docker start signoz-otel-collector")
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_success_notification_follows_independent_final_running_readback(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, _dump_dir = _run_backup(tmp_path)
|
||||
|
||||
assert result.returncode == 0
|
||||
inspect_positions = [
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.startswith("docker inspect --format")
|
||||
]
|
||||
success_position = next(
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.startswith("notify success ")
|
||||
assert 'dest: "/backup/scripts/{{ item }}"' in playbook
|
||||
assert "- backup-signoz.sh" in playbook
|
||||
assert "- clickhouse-native-backup.sh" in playbook
|
||||
assert "- clickhouse-native-restore-drill.sh" in playbook
|
||||
assert "- clickhouse-restore-inventory.py" in playbook
|
||||
assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in playbook
|
||||
assert (
|
||||
"/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
in playbook
|
||||
)
|
||||
assert len(inspect_positions) >= 3
|
||||
assert inspect_positions[-1] < success_position
|
||||
assert "tags: backup_jobs" in playbook
|
||||
|
||||
start_position = next(
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.startswith("docker start signoz-otel-collector")
|
||||
|
||||
def test_scripts_are_valid_bash() -> None:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
"-n",
|
||||
str(SCRIPT),
|
||||
str(ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
tar_positions = [
|
||||
index for index, event in enumerate(events) if event.startswith("tar -tzf")
|
||||
]
|
||||
assert len(tar_positions) == 2
|
||||
assert start_position < tar_positions[0]
|
||||
|
||||
|
||||
def test_archive_create_timeout_is_bounded_restores_and_notifies_once(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
archive_timeout_target="signoz-clickhouse",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_partial_sqlite_archive_nonzero_fails_overall_and_removes_output(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
docker_run_mode="partial_nonzero",
|
||||
docker_run_target="signoz-sqlite",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_corrupt_archive_fails_after_collector_restore_and_notifies_once(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
tar_verify_mode="corrupt",
|
||||
tar_verify_target="clickhouse_",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
start_position = next(
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.startswith("docker start signoz-otel-collector")
|
||||
)
|
||||
tar_position = next(
|
||||
index for index, event in enumerate(events) if event.startswith("tar -tzf")
|
||||
)
|
||||
assert start_position < tar_position
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_integrity_verifier_timeout_is_bounded_and_never_reports_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
tar_timeout_target="clickhouse_",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_backup_all_propagates_signoz_failure_to_nonzero_terminal(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events = _run_backup_all_with_signoz_failure(tmp_path)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "run backup-signoz.sh" in events
|
||||
assert any(event.startswith("notify warning all ") for event in events)
|
||||
assert not any(event.startswith("notify success all ") for event in events)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
478
scripts/backup/tests/test_clickhouse_native_backup.py
Normal file
478
scripts/backup/tests/test_clickhouse_native_backup.py
Normal file
@@ -0,0 +1,478 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"
|
||||
PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
|
||||
|
||||
|
||||
def _write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _install_fakes(fake_bin: Path) -> None:
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do shift; done
|
||||
shift
|
||||
exec "$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(fake_bin / "flock", "#!/bin/bash\nexit 0\n")
|
||||
_write_executable(
|
||||
fake_bin / "sha256sum",
|
||||
"""\
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import sys
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
payload = sys.stdin.buffer.read()
|
||||
print(f"{hashlib.sha256(payload).hexdigest()} -")
|
||||
else:
|
||||
with open(sys.argv[1], "rb") as source:
|
||||
payload = source.read()
|
||||
print(f"{hashlib.sha256(payload).hexdigest()} {sys.argv[1]}")
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "unzip",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'unzip %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
exit "${FAKE_UNZIP_STATUS:-0}"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "docker",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
|
||||
find_arg() {
|
||||
local wanted="$1"
|
||||
shift
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = "$wanted" ]; then
|
||||
printf '%s\\n' "${2:-}"
|
||||
return 0
|
||||
fi
|
||||
shift
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
inspect)
|
||||
printf 'sha256:clickhouse-test-id\\ttrue\\n'
|
||||
;;
|
||||
cp)
|
||||
[ -e "${TEST_SERVER_ARTIFACT:?}" ] || exit 44
|
||||
printf 'clickhouse-native-zip-bytes\\n' > "${3:?}"
|
||||
;;
|
||||
exec)
|
||||
shift
|
||||
container="${1:?}"
|
||||
shift
|
||||
case "${1:-}" in
|
||||
test)
|
||||
shift
|
||||
if [ "${1:-}" = "!" ]; then
|
||||
[ ! -e "${TEST_SERVER_ARTIFACT:?}" ]
|
||||
elif [ "${1:-}" = "-d" ] || [ "${1:-}" = "-w" ]; then
|
||||
[ "${FAKE_DISK_READY:-1}" = "1" ]
|
||||
[ "${2:-}" = "${FAKE_DISK_PATH:-/backups/}" ]
|
||||
else
|
||||
exit 45
|
||||
fi
|
||||
;;
|
||||
rm)
|
||||
rm -f "${TEST_SERVER_ARTIFACT:?}"
|
||||
;;
|
||||
clickhouse-client)
|
||||
query="$(find_arg --query "$@")"
|
||||
operation_id="$(find_arg --param_operation_id "$@" || true)"
|
||||
artifact="$(find_arg --param_artifact "$@" || true)"
|
||||
disk="$(find_arg --param_disk "$@" || true)"
|
||||
case "$query" in
|
||||
'SELECT version()')
|
||||
printf '25.5.1.1\\n'
|
||||
;;
|
||||
*"database='system' AND name='backups'"*)
|
||||
printf '%s\\n' "${FAKE_SYSTEM_BACKUPS_COUNT:-1}"
|
||||
;;
|
||||
'CHECK GRANT BACKUP ON *.*')
|
||||
printf '%s\\n' "${FAKE_BACKUP_GRANT:-1}"
|
||||
;;
|
||||
*"FROM system.disks"*)
|
||||
printf '%s\\n' "${FAKE_DISK_PATH:-/backups/}"
|
||||
;;
|
||||
*"FROM system.databases"*)
|
||||
printf '%s\\n' signoz_analytics signoz_logs signoz_metadata \
|
||||
signoz_meter signoz_metrics signoz_traces
|
||||
if [ "${FAKE_EXTRA_DATABASE:-0}" = "1" ]; then
|
||||
printf 'unexpected_db\\n'
|
||||
fi
|
||||
;;
|
||||
*"FROM system.tables WHERE database IN"*)
|
||||
schema_hash="$(printf 'A%.0s' {1..64})"
|
||||
printf 'signoz_analytics\\tanalytics\\tReplicatedMergeTree\\t10\\t1000\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_logs\\tlogs_v2\\tReplicatedMergeTree\\t20\\t2000\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_metadata\\tmetadata\\tMergeTree\\t3\\t300\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_meter\\tmeter\\tDistributed\\t0\\t0\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_metrics\\tsamples_v4\\tReplicatedMergeTree\\t40\\t4000\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_traces\\tsignoz_index_v3\\tReplicatedMergeTree\\t50\\t5000\\t%s\\n' "$schema_hash"
|
||||
;;
|
||||
BACKUP*)
|
||||
case "$query" in
|
||||
*" SETTINGS id='${EXPECTED_FAKE_OPERATION_ID:?}' ASYNC")
|
||||
operation_id="${EXPECTED_FAKE_OPERATION_ID}"
|
||||
;;
|
||||
*) exit 49 ;;
|
||||
esac
|
||||
printf '%s\\n' "$operation_id" > "${TEST_OPERATION_ID:?}"
|
||||
printf '%s\\n' "$artifact" > "${TEST_ARTIFACT_NAME:?}"
|
||||
printf '%s\\n' "$disk" > "${TEST_DISK_NAME:?}"
|
||||
: > "${TEST_SUBMITTED:?}"
|
||||
: > "${TEST_SERVER_ARTIFACT:?}"
|
||||
printf '%s\\tCREATING_BACKUP\\n' "$operation_id"
|
||||
;;
|
||||
*"FROM system.backups WHERE id="*)
|
||||
mode="${FAKE_STATUS_MODE:-success}"
|
||||
if [ "$mode" = "existing_success" ] || [ -e "${TEST_SUBMITTED:?}" ]; then
|
||||
artifact_name="$(cat "${TEST_ARTIFACT_NAME:?}" 2>/dev/null || true)"
|
||||
[ -n "$artifact_name" ] || artifact_name="${EXPECTED_FAKE_ARTIFACT:?}"
|
||||
backup_name="Disk('backups', '$artifact_name')"
|
||||
case "$mode" in
|
||||
failed)
|
||||
printf '%s\\tBACKUP_FAILED\\t0\\t0\\t0\\t0\\t434F44453A20353938\\t%s\\n' \
|
||||
"$backup_name" "$(printf '1%.0s' {1..64})"
|
||||
;;
|
||||
*)
|
||||
: > "${TEST_SERVER_ARTIFACT:?}"
|
||||
printf '%s\\tBACKUP_CREATED\\t12\\t1200\\t2400\\t1100\\tnone\\t%s\\n' \
|
||||
"$backup_name" "$(printf '0%.0s' {1..64})"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
;;
|
||||
*) exit 46 ;;
|
||||
esac
|
||||
;;
|
||||
*) exit 47 ;;
|
||||
esac
|
||||
;;
|
||||
*) exit 48 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
def _harness(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
status_mode: str = "success",
|
||||
extra_database: bool = False,
|
||||
disk_ready: bool = True,
|
||||
unzip_status: int = 0,
|
||||
with_hook: bool = False,
|
||||
) -> tuple[dict[str, str], dict[str, Path]]:
|
||||
fake_bin = tmp_path / "bin"
|
||||
output = tmp_path / "output"
|
||||
receipts = tmp_path / "receipts"
|
||||
fake_bin.mkdir()
|
||||
output.mkdir()
|
||||
receipts.mkdir()
|
||||
_install_fakes(fake_bin)
|
||||
|
||||
event_log = tmp_path / "events.log"
|
||||
event_log.touch()
|
||||
paths = {
|
||||
"output": output,
|
||||
"receipts": receipts,
|
||||
"events": event_log,
|
||||
"submitted": tmp_path / "submitted",
|
||||
"server_artifact": tmp_path / "server-artifact",
|
||||
"operation_id": tmp_path / "operation-id",
|
||||
"artifact_name": tmp_path / "artifact-name",
|
||||
"disk_name": tmp_path / "disk-name",
|
||||
"hook_events": tmp_path / "hook-events.log",
|
||||
}
|
||||
identity_hash = (
|
||||
subprocess.check_output(
|
||||
[str(fake_bin / "sha256sum")],
|
||||
input=b"trace-native\0run-native\0P0-OBS-002",
|
||||
)
|
||||
.decode()
|
||||
.split()[0]
|
||||
)
|
||||
expected_artifact = f"signoz-{identity_hash}.zip"
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TRACE_ID": "trace-native",
|
||||
"RUN_ID": "run-native",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"CLICKHOUSE_NATIVE_OUTPUT_DIR": str(output),
|
||||
"CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(receipts),
|
||||
"CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS": "2",
|
||||
"CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS": "2",
|
||||
"CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS": "1",
|
||||
"CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS": "1",
|
||||
"TEST_EVENT_LOG": str(event_log),
|
||||
"TEST_SUBMITTED": str(paths["submitted"]),
|
||||
"TEST_SERVER_ARTIFACT": str(paths["server_artifact"]),
|
||||
"TEST_OPERATION_ID": str(paths["operation_id"]),
|
||||
"TEST_ARTIFACT_NAME": str(paths["artifact_name"]),
|
||||
"TEST_DISK_NAME": str(paths["disk_name"]),
|
||||
"EXPECTED_FAKE_ARTIFACT": expected_artifact,
|
||||
"EXPECTED_FAKE_OPERATION_ID": f"awoooi-{identity_hash}",
|
||||
"FAKE_STATUS_MODE": status_mode,
|
||||
"FAKE_EXTRA_DATABASE": "1" if extra_database else "0",
|
||||
"FAKE_DISK_READY": "1" if disk_ready else "0",
|
||||
"FAKE_UNZIP_STATUS": str(unzip_status),
|
||||
}
|
||||
if with_hook:
|
||||
hook = tmp_path / "restore-verifier-hook.sh"
|
||||
_write_executable(
|
||||
hook,
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
if [ "$1" = "--apply" ]; then
|
||||
test -f "$CLICKHOUSE_NATIVE_ARTIFACT_PATH"
|
||||
test -f "$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH"
|
||||
test -f "$CLICKHOUSE_NATIVE_MANIFEST_PATH"
|
||||
grep -q '"status":"BACKUP_CREATED"' "$CLICKHOUSE_NATIVE_MANIFEST_PATH"
|
||||
fi
|
||||
printf '%s trace=%s run=%s work=%s dbs=%s inventory=%s manifest=%s\\n' \
|
||||
"$1" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
|
||||
"$CLICKHOUSE_NATIVE_EXPECTED_DATABASES" \
|
||||
"$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH" \
|
||||
"${CLICKHOUSE_NATIVE_MANIFEST_PATH:-none}" \
|
||||
>> "${TEST_HOOK_EVENTS:?}"
|
||||
""",
|
||||
)
|
||||
env["CLICKHOUSE_NATIVE_POST_VERIFY_HOOK"] = str(hook)
|
||||
env["TEST_HOOK_EVENTS"] = str(paths["hook_events"])
|
||||
return env, paths
|
||||
|
||||
|
||||
def _run(env: dict[str, str], mode: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["bash", str(SCRIPT), mode],
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _receipts(path: Path) -> list[dict[str, object]]:
|
||||
receipt_file = path / "run-native" / "receipts.jsonl"
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in receipt_file.read_text(encoding="utf-8").splitlines()
|
||||
]
|
||||
|
||||
|
||||
def test_check_validates_exact_scope_without_backup_or_artifact_write(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
result = _run(env, "--check")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
events = paths["events"].read_text(encoding="utf-8")
|
||||
assert "BACKUP DATABASE" not in events
|
||||
assert "docker cp" not in events
|
||||
assert "docker exec signoz-clickhouse rm" not in events
|
||||
assert not list(paths["output"].iterdir())
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert receipts[-1]["terminal"] == "check_pass_no_write"
|
||||
|
||||
|
||||
def test_apply_requires_same_run_check_receipt(tmp_path: Path) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "same_run_check_pass_receipt_required_before_apply" in result.stderr
|
||||
assert paths["events"].read_text(encoding="utf-8") == ""
|
||||
|
||||
|
||||
def test_apply_submits_exact_six_database_backup_and_verifies_zip(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
assert _run(env, "--check").returncode == 0
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
events = paths["events"].read_text(encoding="utf-8")
|
||||
assert "BACKUP DATABASE signoz_analytics,DATABASE signoz_logs" in events
|
||||
assert "DATABASE signoz_metadata,DATABASE signoz_meter" in events
|
||||
assert "DATABASE signoz_metrics,DATABASE signoz_traces" in events
|
||||
assert "BACKUP ALL" not in events
|
||||
backup_line = next(
|
||||
line for line in events.splitlines() if "--query BACKUP " in line
|
||||
)
|
||||
assert " SETTINGS id='awoooi-" in backup_line
|
||||
assert "id={operation_id:String}" not in backup_line
|
||||
assert "--param_operation_id" not in backup_line
|
||||
assert "docker cp signoz-clickhouse:/backups/signoz-" in events
|
||||
assert "unzip -tq" in events
|
||||
assert "docker exec signoz-clickhouse rm -f -- /backups/signoz-" in events
|
||||
assert not paths["server_artifact"].exists()
|
||||
outputs = list(paths["output"].glob("clickhouse-native-*.zip"))
|
||||
assert len(outputs) == 1
|
||||
manifest = json.loads(Path(str(outputs[0]) + ".manifest.json").read_text())
|
||||
assert manifest["databases"] == [
|
||||
"signoz_analytics",
|
||||
"signoz_logs",
|
||||
"signoz_metadata",
|
||||
"signoz_meter",
|
||||
"signoz_metrics",
|
||||
"signoz_traces",
|
||||
]
|
||||
inventory = list(paths["output"].glob("*.source-inventory.tsv"))
|
||||
assert len(inventory) == 1
|
||||
assert "ReplicatedMergeTree" in inventory[0].read_text(encoding="utf-8")
|
||||
assert "Distributed" in inventory[0].read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_database_allowlist_drift_fails_check(tmp_path: Path) -> None:
|
||||
env, _paths = _harness(tmp_path, extra_database=True)
|
||||
result = _run(env, "--check")
|
||||
assert result.returncode != 0
|
||||
assert "signoz_database_allowlist_drift" in result.stderr
|
||||
|
||||
|
||||
def test_missing_backup_grant_fails_check_before_submission(tmp_path: Path) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
env["FAKE_BACKUP_GRANT"] = "0"
|
||||
|
||||
result = _run(env, "--check")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "native_backup_grant_absent" in result.stderr
|
||||
assert not paths["submitted"].exists()
|
||||
|
||||
|
||||
def test_missing_runtime_backup_disk_fails_check(tmp_path: Path) -> None:
|
||||
env, _paths = _harness(tmp_path, disk_ready=False)
|
||||
result = _run(env, "--check")
|
||||
assert result.returncode != 0
|
||||
assert "backup_disk_directory_absent" in result.stderr
|
||||
|
||||
|
||||
def test_failed_async_terminal_has_exact_cleanup_and_no_local_artifact(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, paths = _harness(tmp_path, status_mode="failed")
|
||||
assert _run(env, "--check").returncode == 0
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "native_backup_terminal_BACKUP_FAILED" in result.stderr
|
||||
assert not paths["server_artifact"].exists()
|
||||
assert not list(paths["output"].iterdir())
|
||||
receipt_dir = paths["receipts"] / "run-native"
|
||||
assert list(receipt_dir.glob("*.stderr"))
|
||||
assert list(receipt_dir.glob("*.status"))
|
||||
bounded_errors = list(receipt_dir.glob("*-backup-error-bounded.hex"))
|
||||
assert len(bounded_errors) == 1
|
||||
assert bounded_errors[0].read_text(encoding="utf-8") == ("434F44453A20353938\n")
|
||||
|
||||
|
||||
def test_zip_failure_removes_local_partial_and_preserves_server_evidence(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, paths = _harness(tmp_path, unzip_status=7)
|
||||
assert _run(env, "--check").returncode == 0
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "native_backup_archive_integrity_failed" in result.stderr
|
||||
assert paths["server_artifact"].exists()
|
||||
assert not list(paths["output"].iterdir())
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert any(
|
||||
item["stage"] == "cleanup" and item["terminal"] == "preserved"
|
||||
for item in receipts
|
||||
)
|
||||
|
||||
|
||||
def test_restore_verifier_hook_uses_check_and_apply_contract(tmp_path: Path) -> None:
|
||||
env, paths = _harness(tmp_path, with_hook=True)
|
||||
assert _run(env, "--check").returncode == 0
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
hook_events = paths["hook_events"].read_text(encoding="utf-8").splitlines()
|
||||
assert sum(event.startswith("--check ") for event in hook_events) == 1
|
||||
assert sum(event.startswith("--apply ") for event in hook_events) == 1
|
||||
assert all("signoz_analytics,signoz_logs" in event for event in hook_events)
|
||||
assert any(
|
||||
"manifest=" in event and "manifest=none" not in event
|
||||
for event in hook_events
|
||||
if event.startswith("--apply ")
|
||||
)
|
||||
|
||||
|
||||
def test_verified_local_artifact_is_idempotently_reused(tmp_path: Path) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
assert _run(env, "--check").returncode == 0
|
||||
assert _run(env, "--apply").returncode == 0
|
||||
submissions_before = (
|
||||
paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE")
|
||||
)
|
||||
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
submissions_after = (
|
||||
paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE")
|
||||
)
|
||||
assert submissions_after == submissions_before
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert any(
|
||||
item["stage"] == "idempotency" and item["terminal"] == "reused"
|
||||
for item in receipts
|
||||
)
|
||||
|
||||
|
||||
def test_source_contract_has_no_raw_volume_fallback_and_is_deployed() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
playbook = PLAYBOOK.read_text(encoding="utf-8")
|
||||
|
||||
assert "BACKUP ${BACKUP_SCOPE_SQL}" in source
|
||||
assert "SETTINGS id='${OPERATION_ID}' ASYNC" in source
|
||||
assert "SETTINGS id={operation_id:String}" not in source
|
||||
assert "BACKUP ALL" not in source
|
||||
assert ".zip" in source
|
||||
assert "unzip -tq" in source
|
||||
assert "--format $'{{.Id}}\\t{{.State.Running}}'" in source
|
||||
assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
|
||||
assert "docker run" not in source
|
||||
assert "clickhouse-native-backup.sh" in playbook
|
||||
result = subprocess.run(
|
||||
["bash", "-n", str(SCRIPT)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
848
scripts/backup/tests/test_clickhouse_native_restore_drill.py
Normal file
848
scripts/backup/tests/test_clickhouse_native_restore_drill.py
Normal file
@@ -0,0 +1,848 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "clickhouse-native-restore-drill.sh"
|
||||
INVENTORY_HELPER = ROOT / "clickhouse-restore-inventory.py"
|
||||
REPO_ROOT = ROOT.parents[1]
|
||||
CLICKHOUSE_IMAGE_ID = (
|
||||
"sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6"
|
||||
)
|
||||
ZOOKEEPER_IMAGE_ID = (
|
||||
"sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0"
|
||||
)
|
||||
EMPTY_SHA256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"
|
||||
EXPECTED_DATABASES = [
|
||||
"signoz_analytics",
|
||||
"signoz_logs",
|
||||
"signoz_metadata",
|
||||
"signoz_meter",
|
||||
"signoz_metrics",
|
||||
"signoz_traces",
|
||||
]
|
||||
|
||||
|
||||
def _inventory(
|
||||
*,
|
||||
restored_delta: int = 0,
|
||||
engine_mismatch: bool = False,
|
||||
schema_mismatch: bool = False,
|
||||
) -> str:
|
||||
rows = [
|
||||
("signoz_analytics", "events", "ReplicatedMergeTree", 10, 100),
|
||||
("signoz_analytics", "events_all", "Distributed", 10, 100),
|
||||
("signoz_logs", "logs_v2", "ReplicatedMergeTree", 20, 200),
|
||||
("signoz_metadata", "resource_attrs", "ReplicatedMergeTree", 2, 20),
|
||||
("signoz_meter", "meter", "ReplicatedMergeTree", 3, 30),
|
||||
("signoz_metrics", "samples_v4", "ReplicatedMergeTree", 30, 300),
|
||||
("signoz_metrics", "time_series_v4", "ReplicatedMergeTree", 12, 120),
|
||||
("signoz_traces", "signoz_index_v3", "ReplicatedMergeTree", 40, 400),
|
||||
]
|
||||
rendered: list[str] = []
|
||||
for index, (database, table, engine, row_count, byte_count) in enumerate(rows):
|
||||
if engine_mismatch and table == "meter":
|
||||
engine = "MergeTree"
|
||||
schema_hash = (
|
||||
hashlib.sha256(f"{database}.{table}:{engine}".encode("utf-8"))
|
||||
.hexdigest()
|
||||
.upper()
|
||||
)
|
||||
if schema_mismatch and table == "meter":
|
||||
schema_hash = "F" * 64
|
||||
rendered.append(
|
||||
"\t".join(
|
||||
(
|
||||
database,
|
||||
table,
|
||||
engine,
|
||||
str(row_count + (restored_delta if index == 0 else 0)),
|
||||
str(byte_count + (restored_delta if index == 0 else 0)),
|
||||
schema_hash,
|
||||
)
|
||||
)
|
||||
)
|
||||
return "\n".join(rendered) + "\n"
|
||||
|
||||
|
||||
def _write_backup_fixture(tmp_path: Path) -> tuple[Path, Path, Path]:
|
||||
artifact = tmp_path / "clickhouse-native.zip.partial"
|
||||
with zipfile.ZipFile(artifact, "w") as archive:
|
||||
archive.writestr(".backup", "bounded native backup fixture\n")
|
||||
inventory = tmp_path / "source-inventory.tsv.partial"
|
||||
inventory.write_text(_inventory(), encoding="utf-8")
|
||||
artifact_sha = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
inventory_sha = hashlib.sha256(inventory.read_bytes()).hexdigest()
|
||||
manifest = tmp_path / "manifest.json.partial"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"trace_id": "backup-trace",
|
||||
"run_id": "backup-run",
|
||||
"work_item_id": "P0-OBS-002",
|
||||
"operation_id": "awoooi-backup-operation",
|
||||
"databases": EXPECTED_DATABASES,
|
||||
"source_inventory_sha256": inventory_sha,
|
||||
"sha256": artifact_sha,
|
||||
"size_bytes": artifact.stat().st_size,
|
||||
"status": "BACKUP_CREATED",
|
||||
},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return artifact, manifest, inventory
|
||||
|
||||
|
||||
def _write_fake_docker(tmp_path: Path) -> tuple[Path, Path, Path]:
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
state_dir = tmp_path / "docker-state"
|
||||
state_dir.mkdir()
|
||||
log_path = tmp_path / "docker.log"
|
||||
docker = bin_dir / "docker"
|
||||
docker.write_text(
|
||||
r"""#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
printf '%s\n' "$*" >> "${FAKE_DOCKER_LOG:?}"
|
||||
state="${FAKE_DOCKER_STATE:?}"
|
||||
cmd="${1:?}"
|
||||
shift
|
||||
last_arg() { eval "printf '%s' \"\${$#}\""; }
|
||||
|
||||
case "$cmd" in
|
||||
info)
|
||||
[ "${FAKE_DOCKER_INFO_FAIL:-0}" = 0 ] || exit 78
|
||||
printf '25.0.0\\n'
|
||||
;;
|
||||
image)
|
||||
sub="${1:?}"
|
||||
shift
|
||||
[ "$sub" = inspect ] || exit 90
|
||||
image="$(last_arg "$@")"
|
||||
case "$image" in
|
||||
clickhouse/clickhouse-server:25.5.6) printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" ;;
|
||||
signoz/zookeeper:3.7.1) printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
;;
|
||||
container)
|
||||
sub="${1:?}"
|
||||
shift
|
||||
case "$sub" in
|
||||
inspect)
|
||||
format=""
|
||||
if [ "${1:-}" = --format ]; then
|
||||
format="$2"
|
||||
shift 2
|
||||
fi
|
||||
name="${1:?}"
|
||||
[ -f "$state/container-$name" ] || exit 1
|
||||
if [[ "$format" == *Labels* ]]; then
|
||||
if [ "${FAKE_LABEL_INSPECT_HANG:-0}" = 1 ] && [[ "$name" == *ch-restore* ]]; then
|
||||
sleep 10
|
||||
fi
|
||||
cat "$state/container-label-$name"
|
||||
fi
|
||||
;;
|
||||
ls)
|
||||
for item in "$state"/container-*; do
|
||||
[ -e "$item" ] || continue
|
||||
case "$item" in
|
||||
*-label-*|*-network-*|*-image-*) continue ;;
|
||||
esac
|
||||
basename "$item" | sed 's/^container-//'
|
||||
done
|
||||
;;
|
||||
*) exit 90 ;;
|
||||
esac
|
||||
;;
|
||||
inspect)
|
||||
format=""
|
||||
if [ "${1:-}" = --format ]; then
|
||||
format="$2"
|
||||
shift 2
|
||||
fi
|
||||
name="${1:?}"
|
||||
[ -f "$state/container-$name" ] || exit 1
|
||||
case "$format" in
|
||||
*State.Running*) printf 'true\n' ;;
|
||||
*HostConfig.NetworkMode*) cat "$state/container-network-$name" ;;
|
||||
*.Image*) cat "$state/container-image-$name" ;;
|
||||
*) printf '%s\n' "fake-$name" ;;
|
||||
esac
|
||||
;;
|
||||
network)
|
||||
sub="${1:?}"
|
||||
shift
|
||||
case "$sub" in
|
||||
create)
|
||||
name="$(last_arg "$@")"
|
||||
label=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi
|
||||
done
|
||||
touch "$state/network-$name"
|
||||
printf '%s\n' "$label" > "$state/network-label-$name"
|
||||
printf '%s\n' "$name" > "$state/network-name"
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
inspect)
|
||||
format=""
|
||||
if [ "${1:-}" = --format ]; then
|
||||
format="$2"
|
||||
shift 2
|
||||
fi
|
||||
name="${1:?}"
|
||||
[ -f "$state/network-$name" ] || exit 1
|
||||
if [[ "$format" == *Internal* ]]; then printf 'true\n'; fi
|
||||
if [[ "$format" == *Labels* ]]; then cat "$state/network-label-$name"; fi
|
||||
;;
|
||||
ls)
|
||||
for item in "$state"/network-*; do
|
||||
[ -e "$item" ] || continue
|
||||
case "$item" in *-label-*|*/network-name) continue ;; esac
|
||||
basename "$item" | sed 's/^network-//'
|
||||
done
|
||||
;;
|
||||
rm)
|
||||
name="${1:?}"
|
||||
rm -f "$state/network-$name" "$state/network-label-$name" "$state/network-name"
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
*) exit 90 ;;
|
||||
esac
|
||||
;;
|
||||
volume)
|
||||
sub="${1:?}"
|
||||
shift
|
||||
case "$sub" in
|
||||
create)
|
||||
name="$(last_arg "$@")"
|
||||
label=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi
|
||||
done
|
||||
touch "$state/volume-$name"
|
||||
printf '%s\n' "$label" > "$state/volume-label-$name"
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
inspect)
|
||||
format=""
|
||||
if [ "${1:-}" = --format ]; then format="$2"; shift 2; fi
|
||||
name="${1:?}"
|
||||
[ -f "$state/volume-$name" ] || exit 1
|
||||
if [[ "$format" == *Labels* ]]; then cat "$state/volume-label-$name"; fi
|
||||
;;
|
||||
ls)
|
||||
for item in "$state"/volume-*; do
|
||||
[ -e "$item" ] || continue
|
||||
case "$item" in *-label-*) continue ;; esac
|
||||
basename "$item" | sed 's/^volume-//'
|
||||
done
|
||||
;;
|
||||
rm)
|
||||
name="${1:?}"
|
||||
rm -f "$state/volume-$name" "$state/volume-label-$name"
|
||||
if [[ "$name" == *-backup ]]; then
|
||||
rm -f "$state/staged-artifact-sha256"
|
||||
fi
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
*) exit 90 ;;
|
||||
esac
|
||||
;;
|
||||
run)
|
||||
name=""
|
||||
replica=""
|
||||
label=""
|
||||
network=""
|
||||
image=""
|
||||
expected_artifact_sha256=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--name)
|
||||
name="$2"
|
||||
shift 2
|
||||
;;
|
||||
--env)
|
||||
if [[ "$2" == CLICKHOUSE_RESTORE_REPLICA=* ]]; then
|
||||
replica="${2#*=}"
|
||||
fi
|
||||
if [[ "$2" == EXPECTED_ARTIFACT_SHA256=* ]]; then
|
||||
expected_artifact_sha256="${2#*=}"
|
||||
fi
|
||||
shift 2
|
||||
;;
|
||||
--label)
|
||||
label="${2#*=}"
|
||||
shift 2
|
||||
;;
|
||||
--network)
|
||||
network="$2"
|
||||
shift 2
|
||||
;;
|
||||
--network-alias|--restart|--mount|--user|--entrypoint)
|
||||
shift 2
|
||||
;;
|
||||
--detach|--pull=never)
|
||||
shift
|
||||
;;
|
||||
clickhouse/clickhouse-server:25.5.6|signoz/zookeeper:3.7.1)
|
||||
image="$1"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[ -n "$name" ] || exit 91
|
||||
touch "$state/container-$name"
|
||||
printf '%s\n' "$label" > "$state/container-label-$name"
|
||||
printf '%s\n' "$network" > "$state/container-network-$name"
|
||||
case "$image" in
|
||||
clickhouse/clickhouse-server:25.5.6)
|
||||
printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" > "$state/container-image-$name"
|
||||
;;
|
||||
signoz/zookeeper:3.7.1)
|
||||
printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" > "$state/container-image-$name"
|
||||
;;
|
||||
*) exit 96 ;;
|
||||
esac
|
||||
if [ -n "$replica" ]; then printf '%s\n' "$replica" > "$state/replica"; fi
|
||||
if [ "$network" = none ]; then
|
||||
if [ "${FAKE_STAGING_HANG:-0}" = 1 ]; then sleep 10; fi
|
||||
[ "${FAKE_STAGING_FAIL:-0}" = 0 ] || exit 94
|
||||
staged_hash="$expected_artifact_sha256"
|
||||
if [ "${FAKE_STAGING_HASH_MISMATCH:-0}" = 1 ]; then
|
||||
staged_hash="$(printf '0%.0s' {1..64})"
|
||||
fi
|
||||
printf '%s\n' "$staged_hash" > "$state/staged-artifact-sha256"
|
||||
printf '%s\n' "$staged_hash"
|
||||
else
|
||||
printf 'fake-container-%s\n' "$name"
|
||||
fi
|
||||
;;
|
||||
exec)
|
||||
container="${1:?}"
|
||||
shift
|
||||
[ -f "$state/container-$container" ] || exit 1
|
||||
if [ "${1:-}" = sha256sum ]; then
|
||||
[ -f "$state/staged-artifact-sha256" ] || exit 97
|
||||
artifact_hash="$(cat "$state/staged-artifact-sha256")"
|
||||
if [ "${FAKE_CLICKHOUSE_HASH_MISMATCH:-0}" = 1 ]; then
|
||||
artifact_hash="$(printf 'f%.0s' {1..64})"
|
||||
fi
|
||||
printf '%s /backups/native-backup.zip\n' "$artifact_hash"
|
||||
exit 0
|
||||
fi
|
||||
query=""
|
||||
operation=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--query)
|
||||
query="$2"
|
||||
shift 2
|
||||
;;
|
||||
--param_operation_id)
|
||||
operation="$2"
|
||||
shift 2
|
||||
;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
case "$query" in
|
||||
CHECK\ TABLE*)
|
||||
if [ "${FAKE_CHECK_TABLE_FAIL:-0}" = 1 ]; then printf '0\n'; else printf '1\n'; fi
|
||||
;;
|
||||
RESTORE\ *)
|
||||
case "$query" in
|
||||
*" SETTINGS id='${EXPECTED_FAKE_RESTORE_OPERATION_ID:?}' ASYNC")
|
||||
operation="${EXPECTED_FAKE_RESTORE_OPERATION_ID}"
|
||||
;;
|
||||
*) exit 93 ;;
|
||||
esac
|
||||
touch "$state/restored"
|
||||
printf '%s\tRESTORING\n' "$operation"
|
||||
;;
|
||||
*FROM\ system.backups*)
|
||||
printf "Disk('backups', 'native-backup.zip')\tRESTORED\t100\t1000\t%s\n" "${EMPTY_SHA256:?}"
|
||||
;;
|
||||
*FROM\ system.disks*) printf '/backups/\n' ;;
|
||||
*FROM\ system.macros*)
|
||||
printf 'replica\t%s\nshard\t01\n' "$(cat "$state/replica")"
|
||||
;;
|
||||
*FROM\ system.clusters*) printf 'cluster\trestore-clickhouse\t9000\n' ;;
|
||||
*FROM\ system.zookeeper*) printf '1\n' ;;
|
||||
*"SELECT count() FROM system.databases"*) printf '0\n' ;;
|
||||
*"SELECT count() FROM system.tables"*) printf '0\n' ;;
|
||||
*"SELECT database,name,engine"*) cat "${FAKE_RESTORED_INVENTORY:?}" ;;
|
||||
"SELECT 1") printf '1\n' ;;
|
||||
*) printf 'UNHANDLED QUERY: %s\n' "$query" >&2; exit 92 ;;
|
||||
esac
|
||||
;;
|
||||
logs)
|
||||
printf 'bounded fake container log\n'
|
||||
;;
|
||||
rm)
|
||||
if [ "${1:-}" = -f ]; then shift; fi
|
||||
name="${1:?}"
|
||||
rm -f "$state/container-$name" "$state/container-label-$name" \
|
||||
"$state/container-network-$name" "$state/container-image-$name"
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
*) exit 99 ;;
|
||||
esac
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
docker.chmod(0o755)
|
||||
return bin_dir, state_dir, log_path
|
||||
|
||||
|
||||
def _environment(
|
||||
tmp_path: Path,
|
||||
artifact: Path,
|
||||
manifest: Path,
|
||||
inventory: Path,
|
||||
restored_inventory: Path,
|
||||
) -> dict[str, str]:
|
||||
bin_dir, state_dir, log_path = _write_fake_docker(tmp_path)
|
||||
return os.environ | {
|
||||
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||
"TRACE_ID": "trace-restore-test",
|
||||
"RUN_ID": "run-restore-test",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"CLICKHOUSE_NATIVE_ARTIFACT_PATH": str(artifact),
|
||||
"CLICKHOUSE_NATIVE_MANIFEST_PATH": str(manifest),
|
||||
"CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH": str(inventory),
|
||||
"CLICKHOUSE_NATIVE_ARTIFACT_SHA256": hashlib.sha256(
|
||||
artifact.read_bytes()
|
||||
).hexdigest(),
|
||||
"CLICKHOUSE_NATIVE_OPERATION_ID": "awoooi-backup-operation",
|
||||
"CLICKHOUSE_NATIVE_EXPECTED_DATABASES": ",".join(EXPECTED_DATABASES),
|
||||
"CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(tmp_path / "receipts"),
|
||||
"CLICKHOUSE_RESTORE_POLL_INTERVAL_SECONDS": "1",
|
||||
"CLICKHOUSE_RESTORE_STARTUP_TIMEOUT_SECONDS": "5",
|
||||
"CLICKHOUSE_RESTORE_TIMEOUT_SECONDS": "5",
|
||||
"CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS": "5",
|
||||
"FAKE_DOCKER_LOG": str(log_path),
|
||||
"FAKE_DOCKER_STATE": str(state_dir),
|
||||
"FAKE_CLICKHOUSE_IMAGE_ID": CLICKHOUSE_IMAGE_ID,
|
||||
"FAKE_ZOOKEEPER_IMAGE_ID": ZOOKEEPER_IMAGE_ID,
|
||||
"FAKE_RESTORED_INVENTORY": str(restored_inventory),
|
||||
"EXPECTED_FAKE_RESTORE_OPERATION_ID": "restore-"
|
||||
+ hashlib.sha256(
|
||||
b"trace-restore-test\0run-restore-test\0P0-OBS-002"
|
||||
).hexdigest(),
|
||||
"EMPTY_SHA256": EMPTY_SHA256,
|
||||
}
|
||||
|
||||
|
||||
def _run(mode: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["bash", str(SCRIPT), mode],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
|
||||
def _status(tmp_path: Path) -> dict[str, object]:
|
||||
return json.loads(
|
||||
(tmp_path / "receipts" / "run-restore-test" / "status.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _assert_ephemeral_resources_absent(state_dir: Path) -> None:
|
||||
assert not list(state_dir.glob("container-*"))
|
||||
assert not list(state_dir.glob("volume-*"))
|
||||
assert not list(state_dir.glob("network-*"))
|
||||
assert not (state_dir / "staged-artifact-sha256").exists()
|
||||
|
||||
|
||||
def test_hook_check_uses_env_contract_without_runtime_creation(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(restored_delta=3), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
|
||||
result = _run("--check", env)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "terminal=check_pass_no_runtime_write" in result.stdout
|
||||
docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8")
|
||||
assert "image inspect" in docker_log
|
||||
assert "network create" not in docker_log
|
||||
assert "volume create" not in docker_log
|
||||
assert not any(line.startswith("run ") for line in docker_log.splitlines())
|
||||
status = _status(tmp_path)
|
||||
assert status["check_pass"] is True
|
||||
assert status["ephemeral_runtime_created"] is False
|
||||
assert status["production_network_attached"] is False
|
||||
assert status["production_restore_performed"] is False
|
||||
|
||||
|
||||
def test_apply_restores_on_internal_pair_and_cleans_every_resource(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(restored_delta=3), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
check = _run("--check", env)
|
||||
assert check.returncode == 0, check.stderr
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "terminal=verified" in result.stdout
|
||||
docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines()
|
||||
network_create = next(
|
||||
line for line in docker_lines if line.startswith("network create")
|
||||
)
|
||||
assert "--internal" in network_create
|
||||
run_lines = [line for line in docker_lines if line.startswith("run ")]
|
||||
assert len(run_lines) == 3
|
||||
assert all("--pull=never" in line for line in run_lines)
|
||||
assert all("--publish" not in line and " -p " not in line for line in run_lines)
|
||||
|
||||
staging_run = next(line for line in run_lines if "--network none" in line)
|
||||
zookeeper_run = next(line for line in run_lines if "signoz/zookeeper:3.7.1" in line)
|
||||
clickhouse_run = next(
|
||||
line for line in run_lines if "--network-alias restore-clickhouse" in line
|
||||
)
|
||||
assert "--user 0:0" in staging_run
|
||||
assert "--entrypoint /bin/sh" in staging_run
|
||||
assert (
|
||||
f"type=bind,src={artifact},dst=/tmp/awoooi-source-native-backup.zip,readonly"
|
||||
in staging_run
|
||||
)
|
||||
assert sum(f"src={artifact}" in line for line in run_lines) == 1
|
||||
assert "type=volume,src=awoooi-ch-restore-" in staging_run
|
||||
assert "-backup,dst=/backups" in staging_run
|
||||
assert "ALLOW_ANONYMOUS_LOGIN=yes" in zookeeper_run
|
||||
assert "--network awoooi-ch-restore-" in zookeeper_run
|
||||
assert "--network awoooi-ch-restore-" in clickhouse_run
|
||||
assert str(artifact) not in clickhouse_run
|
||||
assert "type=volume,src=awoooi-ch-restore-" in clickhouse_run
|
||||
assert "-backup,dst=/backups" in clickhouse_run
|
||||
assert "dst=/backups,readonly" not in clickhouse_run
|
||||
assert (
|
||||
"dst=/etc/clickhouse-server/config.d/awoooi-backup-disk.xml,readonly"
|
||||
in clickhouse_run
|
||||
)
|
||||
assert (
|
||||
"dst=/etc/clickhouse-server/config.d/awoooi-restore-isolation.xml,readonly"
|
||||
in clickhouse_run
|
||||
)
|
||||
hash_readback = next(
|
||||
line
|
||||
for line in docker_lines
|
||||
if line.startswith("exec ") and "sha256sum /backups/native-backup.zip" in line
|
||||
)
|
||||
assert "exec awoooi-ch-restore-" in hash_readback
|
||||
restore_line = next(line for line in docker_lines if "RESTORE DATABASE" in line)
|
||||
assert "allow_non_empty_tables" not in restore_line
|
||||
assert " SETTINGS id='restore-" in restore_line
|
||||
assert "id={operation_id:String}" not in restore_line
|
||||
macro_line = next(line for line in docker_lines if "FROM system.macros" in line)
|
||||
assert "SELECT macro,substitution" in macro_line
|
||||
assert "SELECT name,value" not in macro_line
|
||||
state_dir = Path(env["FAKE_DOCKER_STATE"])
|
||||
_assert_ephemeral_resources_absent(state_dir)
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "verified"
|
||||
assert status["check_pass"] is True
|
||||
assert status["restore_terminal"] == "RESTORED"
|
||||
assert status["manifest_parity"] == 1
|
||||
assert status["check_table_count"] == status["check_table_pass_count"] == 7
|
||||
assert status["critical_nonzero_count"] == 4
|
||||
assert status["row_delta"] == 3
|
||||
expected_hash = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
assert status["artifact_sha256"] == expected_hash
|
||||
assert status["staged_artifact_sha256"] == expected_hash
|
||||
assert status["clickhouse_artifact_sha256"] == expected_hash
|
||||
assert status["artifact_staging_verified"] is True
|
||||
assert status["clickhouse_artifact_readback_verified"] is True
|
||||
assert status["staging_network_mode"] == "none"
|
||||
assert status["artifact_source_mount"] == "read_only_staging_only"
|
||||
assert status["backup_volume_mount"] == "writable_ephemeral"
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_apply_failure_still_cleans_pair_network_and_volumes(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_CHECK_TABLE_FAIL"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "terminal=failed" in result.stdout
|
||||
state_dir = Path(env["FAKE_DOCKER_STATE"])
|
||||
_assert_ephemeral_resources_absent(state_dir)
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["cleanup_verified"] is True
|
||||
assert status["apply_pass"] is False
|
||||
|
||||
|
||||
def test_artifact_staging_failure_cleans_exact_resources(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_STAGING_FAIL"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
_assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["reason"] == "isolated_artifact_staging_failed"
|
||||
assert status["artifact_staging_verified"] is False
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_artifact_staging_hang_is_bounded_and_cleans_exact_resources(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1"
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_STAGING_HANG"] = "1"
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(SCRIPT), "--apply"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
timeout=12,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
_assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["reason"] == "isolated_artifact_staging_failed"
|
||||
assert status["artifact_staging_verified"] is False
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_staged_artifact_hash_mismatch_fails_before_isolated_services(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_STAGING_HASH_MISMATCH"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines()
|
||||
assert not any(
|
||||
"signoz/zookeeper:3.7.1" in line
|
||||
for line in docker_lines
|
||||
if line.startswith("run ")
|
||||
)
|
||||
_assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
|
||||
status = _status(tmp_path)
|
||||
assert status["reason"] == "staged_artifact_hash_mismatch"
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_clickhouse_artifact_hash_readback_mismatch_fails_and_cleans(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_CLICKHOUSE_HASH_MISMATCH"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
_assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
|
||||
status = _status(tmp_path)
|
||||
assert status["reason"] == "isolated_clickhouse_artifact_hash_mismatch"
|
||||
assert status["artifact_staging_verified"] is True
|
||||
assert status["clickhouse_artifact_readback_verified"] is False
|
||||
assert status["clickhouse_artifact_sha256"] == "f" * 64
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_docker_daemon_failure_cannot_claim_cleanup_verified(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_DOCKER_INFO_FAIL"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["cleanup_verified"] is False
|
||||
assert status["reason"] == "ephemeral_resource_cleanup_failed"
|
||||
|
||||
|
||||
def test_cleanup_label_inspect_hang_is_bounded_and_fails_closed(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1"
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_CHECK_TABLE_FAIL"] = "1"
|
||||
env["FAKE_LABEL_INSPECT_HANG"] = "1"
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(SCRIPT), "--apply"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
timeout=12,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["cleanup_verified"] is False
|
||||
assert status["reason"] == "ephemeral_resource_cleanup_failed"
|
||||
|
||||
|
||||
def test_check_fails_closed_on_local_image_id_drift(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
env["FAKE_CLICKHOUSE_IMAGE_ID"] = "sha256:" + "0" * 64
|
||||
|
||||
result = _run("--check", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "clickhouse_local_image_id_mismatch" in result.stderr
|
||||
docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8")
|
||||
assert "network create" not in docker_log
|
||||
assert "volume create" not in docker_log
|
||||
|
||||
|
||||
def test_flat_host_deploy_accepts_reviewed_absolute_helper_and_config_paths(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
host_root = tmp_path / "host-backup"
|
||||
host_root.mkdir()
|
||||
helper = host_root / "clickhouse-restore-inventory.py"
|
||||
backup_config = host_root / "backup_disk.xml"
|
||||
cluster_config = host_root / "isolated-cluster.xml"
|
||||
shutil.copyfile(INVENTORY_HELPER, helper)
|
||||
shutil.copyfile(
|
||||
REPO_ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml", backup_config
|
||||
)
|
||||
shutil.copyfile(
|
||||
REPO_ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
cluster_config,
|
||||
)
|
||||
env.update(
|
||||
{
|
||||
"CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(helper),
|
||||
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_config),
|
||||
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config),
|
||||
}
|
||||
)
|
||||
|
||||
result = _run("--check", env)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "terminal=check_pass_no_runtime_write" in result.stdout
|
||||
|
||||
|
||||
def test_inventory_compare_allows_online_row_delta_but_not_engine_drift(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source = tmp_path / "source.tsv"
|
||||
restored = tmp_path / "restored.tsv"
|
||||
source.write_text(_inventory(), encoding="utf-8")
|
||||
restored.write_text(_inventory(restored_delta=9), encoding="utf-8")
|
||||
|
||||
accepted = subprocess.run(
|
||||
[
|
||||
"python3",
|
||||
str(INVENTORY_HELPER),
|
||||
"compare",
|
||||
"--source-inventory",
|
||||
str(source),
|
||||
"--restored-inventory",
|
||||
str(restored),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
assert accepted.returncode == 0, accepted.stderr
|
||||
assert "manifest_parity=1" in accepted.stdout
|
||||
assert "row_delta=9" in accepted.stdout
|
||||
|
||||
restored.write_text(_inventory(engine_mismatch=True), encoding="utf-8")
|
||||
rejected = subprocess.run(
|
||||
[
|
||||
"python3",
|
||||
str(INVENTORY_HELPER),
|
||||
"compare",
|
||||
"--source-inventory",
|
||||
str(source),
|
||||
"--restored-inventory",
|
||||
str(restored),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
assert rejected.returncode == 1
|
||||
assert "table_engine_mismatch_signoz_meter_meter" in rejected.stderr
|
||||
|
||||
restored.write_text(_inventory(schema_mismatch=True), encoding="utf-8")
|
||||
schema_rejected = subprocess.run(
|
||||
[
|
||||
"python3",
|
||||
str(INVENTORY_HELPER),
|
||||
"compare",
|
||||
"--source-inventory",
|
||||
str(source),
|
||||
"--restored-inventory",
|
||||
str(restored),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
assert schema_rejected.returncode == 1
|
||||
assert "table_schema_mismatch_signoz_meter_meter" in schema_rejected.stderr
|
||||
528
scripts/backup/tests/test_signoz_backup_canary_wrapper.py
Normal file
528
scripts/backup/tests/test_signoz_backup_canary_wrapper.py
Normal file
@@ -0,0 +1,528 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh"
|
||||
PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
|
||||
ANSIBLE_VALIDATE = ROOT / "scripts" / "ops" / "ansible-validate.sh"
|
||||
READINESS_AUDIT = (
|
||||
ROOT / "scripts" / "reboot-recovery" / "reboot-recovery-readiness-audit.sh"
|
||||
)
|
||||
|
||||
|
||||
def _write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _monitor_source(cooldown_dir: Path, *, omit_contract: bool = False) -> str:
|
||||
cooldown_log = (
|
||||
'log "COOLDOWN: ${container} skipped"'
|
||||
if not omit_contract
|
||||
else 'log "maintenance skip for ${container}"'
|
||||
)
|
||||
return f"""\
|
||||
#!/bin/bash
|
||||
: "${{ACTION_COOLDOWN_SECONDS:=${{SEND_COOLDOWN_SECONDS}}}}"
|
||||
: "${{COOLDOWN_DIR:={cooldown_dir}}}"
|
||||
: "${{EXCLUDE_CONTAINERS:=signoz-clickhouse}}"
|
||||
is_in_cooldown() {{
|
||||
local container="$1"
|
||||
local cooldown_file="${{COOLDOWN_DIR}}/${{container}}.cooldown"
|
||||
local last_sent now elapsed
|
||||
last_sent=$(cat "$cooldown_file")
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - last_sent ))
|
||||
if (( elapsed < ACTION_COOLDOWN_SECONDS )); then
|
||||
{cooldown_log}
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}}
|
||||
set_cooldown() {{ :; }}
|
||||
# A second monitor path may use the same elapsed expression. The wrapper must
|
||||
# validate the expression inside is_in_cooldown, not require global uniqueness.
|
||||
# elapsed=$(( now - last_sent ))
|
||||
while read -r container_name; do
|
||||
is_in_cooldown "$container_name" && continue
|
||||
set_cooldown "$container_name"
|
||||
container="$container_name"
|
||||
log "AUTO_REPAIR: docker restart ${{container}}"
|
||||
if docker restart "$container"; then
|
||||
:
|
||||
fi
|
||||
done
|
||||
"""
|
||||
|
||||
|
||||
def _install_fake_commands(fake_bin: Path) -> None:
|
||||
_write_executable(
|
||||
fake_bin / "crontab",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
[ "${1:-}" = "-l" ]
|
||||
printf '*/5 * * * * %s >> %s 2>&1\n' \
|
||||
"${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
|
||||
if [ "${FAKE_DUPLICATE_CRON:-0}" = "1" ]; then
|
||||
printf '*/5 * * * * %s >> %s 2>&1\n' \
|
||||
"${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
|
||||
fi
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "docker",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
[ "${1:-}" = "inspect" ] || exit 70
|
||||
case " $* " in
|
||||
*"{{.State.StartedAt}}"*)
|
||||
printf 'sha256:collector-test-id\\t%s\\t2026-07-15T00:00:00Z\\t0\\thealthy\\n' \
|
||||
"$(cat "${TEST_COLLECTOR_STATE:?}")"
|
||||
;;
|
||||
*"{{.Id}}"*) printf 'sha256:collector-test-id\n' ;;
|
||||
*"{{.State.Running}}"*) cat "${TEST_COLLECTOR_STATE:?}" ;;
|
||||
*) exit 71 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "ss",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
while IFS= read -r port; do
|
||||
[ -n "$port" ] || continue
|
||||
printf 'LISTEN 0 4096 *:%s *:*\n' "$port"
|
||||
done < "${TEST_LISTENERS:?}"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do
|
||||
shift
|
||||
done
|
||||
[ "$#" -gt 1 ]
|
||||
shift
|
||||
exec "$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "flock",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
exit 0
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "pgrep",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
exit 1
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "stat",
|
||||
"""\
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 4 or sys.argv[1] != "-c":
|
||||
raise SystemExit(64)
|
||||
value = os.stat(sys.argv[3])
|
||||
formats = {
|
||||
"%a": format(stat.S_IMODE(value.st_mode), "o"),
|
||||
"%u": str(value.st_uid),
|
||||
"%g": str(value.st_gid),
|
||||
"%s": str(value.st_size),
|
||||
"%d:%i": f"{value.st_dev}:{value.st_ino}",
|
||||
}
|
||||
if sys.argv[2] not in formats:
|
||||
raise SystemExit(65)
|
||||
print(formats[sys.argv[2]])
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "sha256sum",
|
||||
"""\
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
raise SystemExit(64)
|
||||
with open(sys.argv[1], "rb") as source:
|
||||
digest = hashlib.sha256(source.read()).hexdigest()
|
||||
print(f"{digest} {sys.argv[1]}")
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
def _backup_source() -> str:
|
||||
return """\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80
|
||||
[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ]
|
||||
printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
|
||||
> "${TEST_BACKUP_IDENTITIES:?}"
|
||||
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
|
||||
cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}"
|
||||
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
case "${FAKE_MONITOR_MODE:-cooldown}" in
|
||||
cooldown)
|
||||
printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
;;
|
||||
auto_repair)
|
||||
printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
printf '[test] AUTO_REPAIR: docker restart signoz-otel-collector\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
;;
|
||||
none) : ;;
|
||||
*) exit 81 ;;
|
||||
esac
|
||||
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
|
||||
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
else
|
||||
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
|
||||
fi
|
||||
if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then
|
||||
printf '4317\n' > "${TEST_LISTENERS:?}"
|
||||
fi
|
||||
printf '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)\n'
|
||||
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
|
||||
printf '[SUCCESS] ========== SignOz 備份完成 (1s) ==========\n'
|
||||
else
|
||||
printf '[SUCCESS] ========== SigNoz ClickHouse native 備份完成 (1s) ==========\n'
|
||||
fi
|
||||
"""
|
||||
|
||||
|
||||
def _run_canary(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
mode: str = "apply",
|
||||
prior_lease: bool = True,
|
||||
monitor_mode: str = "cooldown",
|
||||
backup_sleep_seconds: float = 0.2,
|
||||
monitor_interval_seconds: int = 300,
|
||||
omit_monitor_contract: bool = False,
|
||||
duplicate_cron: bool = False,
|
||||
drop_listener: bool = False,
|
||||
expect_collector_stop: bool = True,
|
||||
) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]:
|
||||
fake_bin = tmp_path / "bin"
|
||||
cooldown_dir = tmp_path / "cooldown"
|
||||
receipt_root = tmp_path / "receipts"
|
||||
fake_bin.mkdir()
|
||||
cooldown_dir.mkdir()
|
||||
receipt_root.mkdir()
|
||||
|
||||
monitor_script = tmp_path / "docker-health-monitor.sh"
|
||||
monitor_log = tmp_path / "monitor.log"
|
||||
backup_script = tmp_path / "backup-signoz.sh"
|
||||
collector_state = tmp_path / "collector.state"
|
||||
listeners = tmp_path / "listeners"
|
||||
identities = tmp_path / "backup-identities.tsv"
|
||||
lease_observed = tmp_path / "lease-observed.txt"
|
||||
cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown"
|
||||
run_id = f"canary-{tmp_path.name}"
|
||||
|
||||
_write_executable(
|
||||
monitor_script,
|
||||
_monitor_source(cooldown_dir, omit_contract=omit_monitor_contract),
|
||||
)
|
||||
_write_executable(backup_script, _backup_source())
|
||||
_install_fake_commands(fake_bin)
|
||||
monitor_log.write_text("[test] monitor seed\n", encoding="utf-8")
|
||||
collector_state.write_text("true\n", encoding="utf-8")
|
||||
listeners.write_text("4317\n4318\n", encoding="utf-8")
|
||||
if prior_lease:
|
||||
cooldown_file.write_text("123\n", encoding="utf-8")
|
||||
cooldown_file.chmod(0o640)
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TRACE_ID": "trace-P0-OBS-002",
|
||||
"RUN_ID": run_id,
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"SIGNOZ_CANARY_BACKUP_SCRIPT": str(backup_script),
|
||||
"SIGNOZ_CANARY_MONITOR_SCRIPT": str(monitor_script),
|
||||
"SIGNOZ_CANARY_MONITOR_LOG": str(monitor_log),
|
||||
"SIGNOZ_CANARY_COOLDOWN_DIR": str(cooldown_dir),
|
||||
"SIGNOZ_CANARY_RECEIPT_ROOT": str(receipt_root),
|
||||
"SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"),
|
||||
"SIGNOZ_CANARY_TIMEOUT_SECONDS": "5",
|
||||
"SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2",
|
||||
"SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2",
|
||||
"SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds),
|
||||
"SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05",
|
||||
"SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"),
|
||||
"TEST_MONITOR_SCRIPT": str(monitor_script),
|
||||
"TEST_MONITOR_LOG": str(monitor_log),
|
||||
"TEST_COLLECTOR_STATE": str(collector_state),
|
||||
"TEST_LISTENERS": str(listeners),
|
||||
"TEST_BACKUP_IDENTITIES": str(identities),
|
||||
"TEST_COOLDOWN_FILE": str(cooldown_file),
|
||||
"TEST_LEASE_OBSERVED": str(lease_observed),
|
||||
"FAKE_MONITOR_MODE": monitor_mode,
|
||||
"FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds),
|
||||
"FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0",
|
||||
"FAKE_DROP_LISTENER": "1" if drop_listener else "0",
|
||||
"FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0",
|
||||
}
|
||||
started_at = int(time.time())
|
||||
result = subprocess.run(
|
||||
["bash", str(WRAPPER), f"--{mode}"],
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
paths = {
|
||||
"cooldown": cooldown_file,
|
||||
"receipts": receipt_root / run_id / "receipts.jsonl",
|
||||
"rollback_metadata": receipt_root / run_id / "cooldown.rollback.json",
|
||||
"identities": identities,
|
||||
"lease_observed": lease_observed,
|
||||
"collector_state": collector_state,
|
||||
"listeners": listeners,
|
||||
}
|
||||
return result, paths, str(started_at)
|
||||
|
||||
|
||||
def _receipts(path: Path) -> list[dict[str, object]]:
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
|
||||
def _terminal(receipts: list[dict[str, object]], stage: str) -> str:
|
||||
return str([item for item in receipts if item["stage"] == stage][-1]["terminal"])
|
||||
|
||||
|
||||
def test_existing_lease_is_atomically_replaced_and_exactly_restored(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, started_at = _run_canary(tmp_path)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
|
||||
assert int(paths["lease_observed"].read_text(encoding="utf-8")) >= (
|
||||
int(started_at) + 7
|
||||
)
|
||||
identities = paths["identities"].read_text(encoding="utf-8").split("\t")
|
||||
assert identities[0] == "trace-P0-OBS-002"
|
||||
assert identities[1].startswith("canary-")
|
||||
assert identities[2] == "P0-OBS-002\n"
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "restored"
|
||||
assert _terminal(receipts, "terminal") == "pass"
|
||||
metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8"))
|
||||
assert metadata["trace_id"] == "trace-P0-OBS-002"
|
||||
assert metadata["prior_present"] == 1
|
||||
|
||||
stages = [str(item["stage"]) for item in receipts]
|
||||
ordered = [
|
||||
"sensor_source",
|
||||
"normalized_asset_identity",
|
||||
"source_of_truth_diff",
|
||||
"ai_decision",
|
||||
"risk_policy",
|
||||
"check",
|
||||
"execution",
|
||||
"post_verifier",
|
||||
"rollback",
|
||||
"closure_writeback",
|
||||
"terminal",
|
||||
]
|
||||
assert [stages.index(stage) for stage in ordered] == sorted(
|
||||
stages.index(stage) for stage in ordered
|
||||
)
|
||||
|
||||
|
||||
def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, prior_lease=False)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert not paths["cooldown"].exists()
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "restored"
|
||||
assert _terminal(receipts, "terminal") == "pass"
|
||||
|
||||
|
||||
def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, expect_collector_stop=False)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert not paths["lease_observed"].exists()
|
||||
assert paths["collector_state"].read_text(encoding="utf-8") == "true\n"
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert any(
|
||||
item["stage"] == "lease" and item["terminal"] == "not_required"
|
||||
for item in receipts
|
||||
)
|
||||
assert _terminal(receipts, "rollback") == "no_write"
|
||||
assert _terminal(receipts, "terminal") == "pass"
|
||||
|
||||
|
||||
def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
expect_collector_stop=False,
|
||||
omit_monitor_contract=True,
|
||||
duplicate_cron=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert not paths["lease_observed"].exists()
|
||||
receipts = _receipts(paths["receipts"])
|
||||
source_diff = [
|
||||
item for item in receipts if item["stage"] == "source_of_truth_diff"
|
||||
][-1]
|
||||
assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"]
|
||||
assert _terminal(receipts, "terminal") == "pass"
|
||||
|
||||
|
||||
def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, monitor_mode="auto_repair")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "monitor_auto_repair_contaminated_canary" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "restored"
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
def test_cron_crossing_requires_monitor_cooldown_receipt(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
monitor_mode="none",
|
||||
backup_sleep_seconds=1.2,
|
||||
monitor_interval_seconds=1,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "cron_crossed_without_collector_cooldown_receipt" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
|
||||
|
||||
def test_cron_crossing_with_detected_and_cooldown_receipts_passes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
monitor_mode="cooldown",
|
||||
backup_sleep_seconds=1.2,
|
||||
monitor_interval_seconds=1,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
receipts = _receipts(paths["receipts"])
|
||||
monitor_receipt = [
|
||||
item for item in receipts if item["stage"] == "monitor_verifier"
|
||||
][-1]
|
||||
assert monitor_receipt["terminal"] == "pass"
|
||||
assert "cron_crossed_1" in str(monitor_receipt["detail"])
|
||||
|
||||
|
||||
def test_missing_monitor_contract_fails_before_lease_or_backup(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, omit_monitor_contract=True)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "monitor_contract_ambiguous_fragment_count_0" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert not paths["lease_observed"].exists()
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "not_armed"
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
def test_ambiguous_cron_contract_fails_before_lease_or_backup(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, duplicate_cron=True)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "monitor_cron_contract_ambiguous_count_2" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert not paths["lease_observed"].exists()
|
||||
|
||||
|
||||
def test_post_backup_listener_failure_fails_and_restores_lease(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, drop_listener=True)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "collector_post_backup_runtime_failed" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "restored"
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
def test_wrapper_deployment_and_static_contracts_are_owned() -> None:
|
||||
wrapper = WRAPPER.read_text(encoding="utf-8")
|
||||
assert "BACKUP_SKIP_RETENTION_CLEANUP=1" in wrapper
|
||||
assert 'TRACE_ID="${TRACE_ID:-}"' in wrapper
|
||||
assert 'RUN_ID="${RUN_ID:-}"' in wrapper
|
||||
assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper
|
||||
assert "monitor_auto_repair_contaminated_canary" in wrapper
|
||||
assert "collector_post_backup_runtime_failed" in wrapper
|
||||
assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8")
|
||||
assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert str(WRAPPER.relative_to(ROOT)) in READINESS_AUDIT.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_check_mode_proves_contract_without_lease_or_backup_write(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, mode="check")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
|
||||
assert not paths["identities"].exists()
|
||||
assert not paths["lease_observed"].exists()
|
||||
check_receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(check_receipts, "rollback") == "no_write"
|
||||
assert _terminal(check_receipts, "terminal") == "check_pass_no_write"
|
||||
|
||||
|
||||
def test_wrapper_is_valid_bash() -> None:
|
||||
result = subprocess.run(
|
||||
["bash", "-n", str(WRAPPER)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
Reference in New Issue
Block a user