wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(backup): validate bounded SigNoz archives

Browse Source
This commit is contained in:
ogt
2026-07-15 04:14:26 +08:00
parent dba193273f
commit 038743bf44
2 changed files with 307 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
scripts/backup/tests/test_backup_signoz_collector_restore.py
View File

@@ -7,6 +7,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh"
DEPLOYMENT_PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
@@ -20,6 +21,11 @@ def _run_backup(
*,
initial_running: bool = True,
docker_run_mode: str = "success",
docker_run_target: str = "signoz-clickhouse",
archive_timeout_target: str = "",
tar_verify_mode: str = "success",
tar_verify_target: str = "clickhouse_",
tar_timeout_target: str = "",
docker_start_failures: int = 0,
restic_backup_status: int = 0,
restore_max_attempts: int = 3,
@@ -61,7 +67,20 @@ cleanup_old_backups() { :; }
#!/bin/bash
set -eu
printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
while [[ "${1:-}" == --kill-after=* ]]; do
shift
done
shift
if [ -n "${FAKE_ARCHIVE_TIMEOUT_TARGET:-}" ] \
&& [ "${1:-}" = "docker" ] \
&& [[ " $* " == *"${FAKE_ARCHIVE_TIMEOUT_TARGET}"* ]]; then
exit 124
fi
if [ -n "${FAKE_TAR_TIMEOUT_TARGET:-}" ] \
&& [ "${1:-}" = "tar" ] \
&& [[ " $* " == *"${FAKE_TAR_TIMEOUT_TARGET}"* ]]; then
exit 124
fi
exec "$@"
""",
)
@@ -93,17 +112,38 @@ case "${1:-}" in
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
;;
run)
if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then
mode=success
if [[ " $* " == *"${FAKE_DOCKER_RUN_TARGET:-signoz-clickhouse}"* ]]; then
mode="${FAKE_DOCKER_RUN_MODE:-success}"
fi
if [ "$mode" = "signal" ]; then
kill -TERM "${PPID}"
sleep 0.1
exit 0
fi
if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "empty" ]; then
if [ "$mode" = "empty" ]; then
exit 0
fi
if [ "$mode" = "partial_nonzero" ]; then
printf 'partial-archive-data\n'
exit 23
fi
printf 'archive-data\n'
;;
esac
""",
)
_write_executable(
fake_bin / "tar",
"""\
#!/bin/bash
set -eu
printf 'tar %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
if [[ " $* " == *"${FAKE_TAR_VERIFY_TARGET:-signoz-clickhouse}"* ]] \
&& [ "${FAKE_TAR_VERIFY_MODE:-success}" = "corrupt" ]; then
exit 2
fi
exit 0
""",
)
_write_executable(
@@ -134,11 +174,19 @@ printf '4K\t%s\n' "${2:-unknown}"
"TEST_COLLECTOR_STATE": str(collector_state),
"TEST_START_COUNT": str(start_count),
"FAKE_DOCKER_RUN_MODE": docker_run_mode,
"FAKE_DOCKER_RUN_TARGET": docker_run_target,
"FAKE_ARCHIVE_TIMEOUT_TARGET": archive_timeout_target,
"FAKE_TAR_VERIFY_MODE": tar_verify_mode,
"FAKE_TAR_VERIFY_TARGET": tar_verify_target,
"FAKE_TAR_TIMEOUT_TARGET": tar_timeout_target,
"FAKE_DOCKER_START_FAILURES": str(docker_start_failures),
"FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
"COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts),
"COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0",
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
"ARCHIVE_CREATE_TIMEOUT_SECONDS": "1",
"ARCHIVE_VERIFY_TIMEOUT_SECONDS": "1",
"TIMEOUT_KILL_AFTER_SECONDS": "1",
}
process = subprocess.Popen(
["bash", str(harness / SCRIPT.name)],
@@ -162,6 +210,60 @@ def _events(events: list[str], prefix: str) -> list[str]:
return [event for event in events if event.startswith(prefix)]
def _run_backup_all_with_signoz_failure(
tmp_path: Path,
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
harness = tmp_path / "backup-all"
fake_scripts = tmp_path / "deployed-scripts"
event_log = tmp_path / "backup-all-events.log"
harness.mkdir()
fake_scripts.mkdir()
event_log.touch()
source = BACKUP_ALL.read_text(encoding="utf-8").replace(
"/backup/scripts",
str(fake_scripts),
)
backup_all = harness / "backup-all.sh"
_write_executable(backup_all, source)
(harness / "common.sh").write_text(
"""\
log_info() { :; }
log_warn() { :; }
log_error() { :; }
log_success() { :; }
notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
""",
encoding="utf-8",
)
for name in (
"backup-gitea.sh",
"backup-momo.sh",
"backup-harbor.sh",
"backup-awoooi.sh",
"backup-langfuse.sh",
"backup-monitoring.sh",
"backup-signoz.sh",
"backup-open-webui.sh",
"backup-clawbot.sh",
):
status = 17 if name == "backup-signoz.sh" else 0
_write_executable(
fake_scripts / name,
f"#!/bin/bash\nprintf 'run {name}\\n' >> \"${{TEST_EVENT_LOG:?}}\"\nexit {status}\n",
)
result = subprocess.run(
["bash", str(backup_all)],
env={**os.environ, "TEST_EVENT_LOG": str(event_log)},
text=True,
capture_output=True,
timeout=10,
)
return result, event_log.read_text(encoding="utf-8").splitlines()
def test_backup_jobs_deploys_the_guarded_signoz_script() -> None:
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
@@ -175,13 +277,35 @@ def test_restore_policy_is_bounded_timed_and_independently_verified() -> None:
assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source
assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source
assert 'timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"' in source
assert 'ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"' in source
assert 'ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"' in source
assert 'TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"' in source
assert 'timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}"' in source
assert 'tar -tzf "${output_file}"' in source
assert "docker run" in source
assert "|| true" not in source.split("create_volume_archive()", 1)[1].split(
"verify_volume_archive()",
1,
)[0]
assert "verify_collector_final_state" in source
assert source.index("verify_collector_final_state") < source.rindex(
'notify_clawbot "success"'
)
def test_canary_retention_skip_is_explicit_and_defaults_off() -> None:
source = SCRIPT.read_text(encoding="utf-8")
assert 'BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"' in source
assert 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' in source
assert source.index('restic -r "${LOCAL_REPO}" backup') < source.index(
'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]'
)
assert source.index('if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]') < source.index(
"if ! verify_collector_final_state"
)
def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None:
result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal")
@@ -265,3 +389,95 @@ def test_success_notification_follows_independent_final_running_readback(
)
assert len(inspect_positions) >= 3
assert inspect_positions[-1] < success_position
start_position = next(
index
for index, event in enumerate(events)
if event.startswith("docker start signoz-otel-collector")
)
tar_positions = [
index for index, event in enumerate(events) if event.startswith("tar -tzf")
]
assert len(tar_positions) == 2
assert start_position < tar_positions[0]
def test_archive_create_timeout_is_bounded_restores_and_notifies_once(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(
tmp_path,
archive_timeout_target="signoz-clickhouse",
)
assert result.returncode != 0
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_partial_sqlite_archive_nonzero_fails_overall_and_removes_output(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(
tmp_path,
docker_run_mode="partial_nonzero",
docker_run_target="signoz-sqlite",
)
assert result.returncode != 0
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_corrupt_archive_fails_after_collector_restore_and_notifies_once(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(
tmp_path,
tar_verify_mode="corrupt",
tar_verify_target="clickhouse_",
)
assert result.returncode != 0
start_position = next(
index
for index, event in enumerate(events)
if event.startswith("docker start signoz-otel-collector")
)
tar_position = next(
index for index, event in enumerate(events) if event.startswith("tar -tzf")
)
assert start_position < tar_position
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_integrity_verifier_timeout_is_bounded_and_never_reports_success(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(
tmp_path,
tar_timeout_target="clickhouse_",
)
assert result.returncode != 0
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_backup_all_propagates_signoz_failure_to_nonzero_terminal(
tmp_path: Path,
) -> None:
result, events = _run_backup_all_with_signoz_failure(tmp_path)
assert result.returncode == 1
assert "run backup-signoz.sh" in events
assert any(event.startswith("notify warning all ") for event in events)
assert not any(event.startswith("notify success all ") for event in events)