diff --git a/scripts/backup/backup-signoz.sh b/scripts/backup/backup-signoz.sh index a5c75fee0..a942e4957 100755 --- a/scripts/backup/backup-signoz.sh +++ b/scripts/backup/backup-signoz.sh @@ -20,14 +20,19 @@ COLLECTOR_NAME="signoz-otel-collector" COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}" COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}" COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}" +ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}" +ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}" +TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}" +BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}" COLLECTOR_WAS_RUNNING=0 COLLECTOR_RESTORE_PENDING=0 COLLECTOR_RESTORE_ATTEMPTS_USED=0 CLEANUP_COMPLETE=0 -RESTORE_FAILURE_NOTIFIED=0 +FAILURE_NOTIFIED=0 run_collector_docker() { - timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@" + timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@" } collector_running_state() { @@ -53,12 +58,12 @@ collector_is_running() { [ "$(collector_running_state)" = "running" ] } -notify_restore_failure_once() { +notify_failure_once() { local detail="$1" - if [ "${RESTORE_FAILURE_NOTIFIED}" -eq 1 ]; then + if [ "${FAILURE_NOTIFIED}" -eq 1 ]; then return 0 fi - RESTORE_FAILURE_NOTIFIED=1 + FAILURE_NOTIFIED=1 notify_clawbot "failed" "${SERVICE}" "${detail}" || true } @@ -119,7 +124,7 @@ cleanup() { # cleanup 期間忽略重複 signal,避免還原與刪除流程重入。 trap '' HUP INT TERM if ! restore_collector; then - notify_restore_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}" + notify_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}" fi rm -rf "${DUMP_DIR}" return "${exit_code}" @@ -130,25 +135,61 @@ on_signal() { exit "$((128 + signal_number))" } -backup_volume() { +create_volume_archive() { local volume_name="$1" local output_file="$2" local extra_exclude="${3:-}" + local partial_file="${output_file}.partial" + log_info "備份 volume: ${volume_name}" - # 使用 || true 處理 tar 備份運行中 volume 的 exit 1 警告 + rm -f "${partial_file}" "${output_file}" + if [ -n "${extra_exclude}" ]; then - docker run --rm -v "${volume_name}:/data" alpine tar czf - "${extra_exclude}" /data 2>/dev/null > "${output_file}" || true + if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \ + docker run --rm -v "${volume_name}:/data" alpine \ + tar czf - "${extra_exclude}" /data \ + > "${partial_file}" 2>/dev/null; then + log_error " Volume ${volume_name} archive 建立失敗" + rm -f "${partial_file}" "${output_file}" + return 1 + fi else - docker run --rm -v "${volume_name}:/data" alpine tar czf - /data 2>/dev/null > "${output_file}" || true + if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \ + docker run --rm -v "${volume_name}:/data" alpine \ + tar czf - /data \ + > "${partial_file}" 2>/dev/null; then + log_error " Volume ${volume_name} archive 建立失敗" + rm -f "${partial_file}" "${output_file}" + return 1 + fi fi - if [ -s "${output_file}" ]; then - local size=$(du -h "${output_file}" | cut -f1) - log_success " Volume ${volume_name} 備份完成 (${size})" - return 0 - else + + if [ ! -s "${partial_file}" ]; then log_error " Volume ${volume_name} 備份失敗 (空檔案)" + rm -f "${partial_file}" "${output_file}" return 1 fi + + mv "${partial_file}" "${output_file}" +} + +verify_volume_archive() { + local volume_name="$1" + local output_file="$2" + + if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${ARCHIVE_VERIFY_TIMEOUT_SECONDS}" \ + tar -tzf "${output_file}" >/dev/null 2>&1; then + log_error " Volume ${volume_name} archive integrity verifier 失敗" + rm -f "${output_file}" "${output_file}.partial" + return 1 + fi + + local size + size="$(du -h "${output_file}" | cut -f1)" + log_success " Volume ${volume_name} 備份與 integrity verifier 完成 (${size})" } main() { @@ -168,7 +209,7 @@ main() { local collector_initial_state if ! collector_initial_state="$(collector_running_state)"; then log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態" - notify_restore_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態" + notify_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態" exit 1 fi @@ -178,35 +219,49 @@ main() { log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..." if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止" - notify_restore_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}" + notify_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}" exit 1 fi else log_error "${COLLECTOR_NAME} 初始狀態為 stopped,停止備份且不自動啟動" - notify_restore_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止" + notify_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止" exit 1 fi docker stop signoz-telemetrystore-migrator 2>/dev/null || true # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積) - backup_volume "signoz-clickhouse" "${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" "--exclude=/data/tmp" || { + local clickhouse_archive="${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" + local sqlite_archive="${DUMP_DIR}/sqlite_${timestamp}.tar.gz" + create_volume_archive "signoz-clickhouse" "${clickhouse_archive}" "--exclude=/data/tmp" || { log_error "ClickHouse volume 備份失敗" - notify_clawbot "failed" "${SERVICE}" "SignOz ClickHouse 備份失敗" + notify_failure_once "SignOz ClickHouse 備份失敗" exit 1 } # Step 3: 備份 SQLite volume (SignOz metadata) - backup_volume "signoz-sqlite" "${DUMP_DIR}/sqlite_${timestamp}.tar.gz" || { - log_warn "SQLite volume 備份失敗,繼續..." + create_volume_archive "signoz-sqlite" "${sqlite_archive}" || { + log_error "SQLite volume 備份失敗" + notify_failure_once "SignOz SQLite 備份失敗" + exit 1 } # Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。 if ! restore_collector; then - notify_restore_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}" + notify_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}" exit 1 fi - # Step 5: 初始化 Restic 倉庫 + # Step 5: Collector 恢復後才在 host 做有界 archive integrity verifier。 + verify_volume_archive "signoz-clickhouse" "${clickhouse_archive}" || { + notify_failure_once "SignOz ClickHouse archive integrity verifier 失敗" + exit 1 + } + verify_volume_archive "signoz-sqlite" "${sqlite_archive}" || { + notify_failure_once "SignOz SQLite archive integrity verifier 失敗" + exit 1 + } + + # Step 6: 初始化 Restic 倉庫 if [ ! -d "${LOCAL_REPO}/data" ]; then log_info "初始化 Restic 倉庫: ${LOCAL_REPO}" restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || { @@ -215,7 +270,7 @@ main() { } fi - # Step 6: Restic 備份 + # Step 7: Restic 備份 log_info "建立 Restic 備份..." local tags=$(build_tags "${SERVICE}") restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" --password-file "${RESTIC_PASSWORD_FILE}" ${tags} 2>&1 @@ -223,12 +278,16 @@ main() { local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1) log_success "Restic 備份完成: ${snapshot_id}" - # Step 7: GFS 清理 - cleanup_old_backups "${LOCAL_REPO}" + # Step 8: GFS 清理;real canary 可明確跳過 destructive retention。 + if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then + log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)" + else + cleanup_old_backups "${LOCAL_REPO}" + fi - # Step 8: success 前獨立檢查 runtime state;失敗時不得發送 success。 + # Step 9: success 前獨立檢查 runtime state;失敗時不得發送 success。 if ! verify_collector_final_state; then - notify_restore_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗" + notify_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗" exit 1 fi diff --git a/scripts/backup/tests/test_backup_signoz_collector_restore.py b/scripts/backup/tests/test_backup_signoz_collector_restore.py index ab717c345..01d9635e8 100644 --- a/scripts/backup/tests/test_backup_signoz_collector_restore.py +++ b/scripts/backup/tests/test_backup_signoz_collector_restore.py @@ -7,6 +7,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[3] SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh" +BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh" DEPLOYMENT_PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml" @@ -20,6 +21,11 @@ def _run_backup( *, initial_running: bool = True, docker_run_mode: str = "success", + docker_run_target: str = "signoz-clickhouse", + archive_timeout_target: str = "", + tar_verify_mode: str = "success", + tar_verify_target: str = "clickhouse_", + tar_timeout_target: str = "", docker_start_failures: int = 0, restic_backup_status: int = 0, restore_max_attempts: int = 3, @@ -61,7 +67,20 @@ cleanup_old_backups() { :; } #!/bin/bash set -eu printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}" +while [[ "${1:-}" == --kill-after=* ]]; do + shift +done shift +if [ -n "${FAKE_ARCHIVE_TIMEOUT_TARGET:-}" ] \ + && [ "${1:-}" = "docker" ] \ + && [[ " $* " == *"${FAKE_ARCHIVE_TIMEOUT_TARGET}"* ]]; then + exit 124 +fi +if [ -n "${FAKE_TAR_TIMEOUT_TARGET:-}" ] \ + && [ "${1:-}" = "tar" ] \ + && [[ " $* " == *"${FAKE_TAR_TIMEOUT_TARGET}"* ]]; then + exit 124 +fi exec "$@" """, ) @@ -93,17 +112,38 @@ case "${1:-}" in printf 'true\n' > "${TEST_COLLECTOR_STATE:?}" ;; run) - if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then + mode=success + if [[ " $* " == *"${FAKE_DOCKER_RUN_TARGET:-signoz-clickhouse}"* ]]; then + mode="${FAKE_DOCKER_RUN_MODE:-success}" + fi + if [ "$mode" = "signal" ]; then kill -TERM "${PPID}" sleep 0.1 exit 0 fi - if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "empty" ]; then + if [ "$mode" = "empty" ]; then exit 0 fi + if [ "$mode" = "partial_nonzero" ]; then + printf 'partial-archive-data\n' + exit 23 + fi printf 'archive-data\n' ;; esac +""", + ) + _write_executable( + fake_bin / "tar", + """\ +#!/bin/bash +set -eu +printf 'tar %s\n' "$*" >> "${TEST_EVENT_LOG:?}" +if [[ " $* " == *"${FAKE_TAR_VERIFY_TARGET:-signoz-clickhouse}"* ]] \ + && [ "${FAKE_TAR_VERIFY_MODE:-success}" = "corrupt" ]; then + exit 2 +fi +exit 0 """, ) _write_executable( @@ -134,11 +174,19 @@ printf '4K\t%s\n' "${2:-unknown}" "TEST_COLLECTOR_STATE": str(collector_state), "TEST_START_COUNT": str(start_count), "FAKE_DOCKER_RUN_MODE": docker_run_mode, + "FAKE_DOCKER_RUN_TARGET": docker_run_target, + "FAKE_ARCHIVE_TIMEOUT_TARGET": archive_timeout_target, + "FAKE_TAR_VERIFY_MODE": tar_verify_mode, + "FAKE_TAR_VERIFY_TARGET": tar_verify_target, + "FAKE_TAR_TIMEOUT_TARGET": tar_timeout_target, "FAKE_DOCKER_START_FAILURES": str(docker_start_failures), "FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status), "COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts), "COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0", "COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1", + "ARCHIVE_CREATE_TIMEOUT_SECONDS": "1", + "ARCHIVE_VERIFY_TIMEOUT_SECONDS": "1", + "TIMEOUT_KILL_AFTER_SECONDS": "1", } process = subprocess.Popen( ["bash", str(harness / SCRIPT.name)], @@ -162,6 +210,60 @@ def _events(events: list[str], prefix: str) -> list[str]: return [event for event in events if event.startswith(prefix)] +def _run_backup_all_with_signoz_failure( + tmp_path: Path, +) -> tuple[subprocess.CompletedProcess[str], list[str]]: + harness = tmp_path / "backup-all" + fake_scripts = tmp_path / "deployed-scripts" + event_log = tmp_path / "backup-all-events.log" + harness.mkdir() + fake_scripts.mkdir() + event_log.touch() + + source = BACKUP_ALL.read_text(encoding="utf-8").replace( + "/backup/scripts", + str(fake_scripts), + ) + backup_all = harness / "backup-all.sh" + _write_executable(backup_all, source) + (harness / "common.sh").write_text( + """\ +log_info() { :; } +log_warn() { :; } +log_error() { :; } +log_success() { :; } +notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } +""", + encoding="utf-8", + ) + + for name in ( + "backup-gitea.sh", + "backup-momo.sh", + "backup-harbor.sh", + "backup-awoooi.sh", + "backup-langfuse.sh", + "backup-monitoring.sh", + "backup-signoz.sh", + "backup-open-webui.sh", + "backup-clawbot.sh", + ): + status = 17 if name == "backup-signoz.sh" else 0 + _write_executable( + fake_scripts / name, + f"#!/bin/bash\nprintf 'run {name}\\n' >> \"${{TEST_EVENT_LOG:?}}\"\nexit {status}\n", + ) + + result = subprocess.run( + ["bash", str(backup_all)], + env={**os.environ, "TEST_EVENT_LOG": str(event_log)}, + text=True, + capture_output=True, + timeout=10, + ) + return result, event_log.read_text(encoding="utf-8").splitlines() + + def test_backup_jobs_deploys_the_guarded_signoz_script() -> None: playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8") @@ -175,13 +277,35 @@ def test_restore_policy_is_bounded_timed_and_independently_verified() -> None: assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source - assert 'timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"' in source + assert 'ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"' in source + assert 'ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"' in source + assert 'TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"' in source + assert 'timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}"' in source + assert 'tar -tzf "${output_file}"' in source + assert "docker run" in source + assert "|| true" not in source.split("create_volume_archive()", 1)[1].split( + "verify_volume_archive()", + 1, + )[0] assert "verify_collector_final_state" in source assert source.index("verify_collector_final_state") < source.rindex( 'notify_clawbot "success"' ) +def test_canary_retention_skip_is_explicit_and_defaults_off() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert 'BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"' in source + assert 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' in source + assert source.index('restic -r "${LOCAL_REPO}" backup') < source.index( + 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' + ) + assert source.index('if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]') < source.index( + "if ! verify_collector_final_state" + ) + + def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None: result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal") @@ -265,3 +389,95 @@ def test_success_notification_follows_independent_final_running_readback( ) assert len(inspect_positions) >= 3 assert inspect_positions[-1] < success_position + + start_position = next( + index + for index, event in enumerate(events) + if event.startswith("docker start signoz-otel-collector") + ) + tar_positions = [ + index for index, event in enumerate(events) if event.startswith("tar -tzf") + ] + assert len(tar_positions) == 2 + assert start_position < tar_positions[0] + + +def test_archive_create_timeout_is_bounded_restores_and_notifies_once( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup( + tmp_path, + archive_timeout_target="signoz-clickhouse", + ) + + assert result.returncode != 0 + assert len(_events(events, "docker start signoz-otel-collector")) == 1 + assert len(_events(events, "notify failed ")) == 1 + assert not _events(events, "notify success ") + assert not dump_dir.exists() + + +def test_partial_sqlite_archive_nonzero_fails_overall_and_removes_output( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup( + tmp_path, + docker_run_mode="partial_nonzero", + docker_run_target="signoz-sqlite", + ) + + assert result.returncode != 0 + assert len(_events(events, "docker start signoz-otel-collector")) == 1 + assert len(_events(events, "notify failed ")) == 1 + assert not _events(events, "notify success ") + assert not dump_dir.exists() + + +def test_corrupt_archive_fails_after_collector_restore_and_notifies_once( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup( + tmp_path, + tar_verify_mode="corrupt", + tar_verify_target="clickhouse_", + ) + + assert result.returncode != 0 + start_position = next( + index + for index, event in enumerate(events) + if event.startswith("docker start signoz-otel-collector") + ) + tar_position = next( + index for index, event in enumerate(events) if event.startswith("tar -tzf") + ) + assert start_position < tar_position + assert len(_events(events, "notify failed ")) == 1 + assert not _events(events, "notify success ") + assert not dump_dir.exists() + + +def test_integrity_verifier_timeout_is_bounded_and_never_reports_success( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup( + tmp_path, + tar_timeout_target="clickhouse_", + ) + + assert result.returncode != 0 + assert len(_events(events, "docker start signoz-otel-collector")) == 1 + assert len(_events(events, "notify failed ")) == 1 + assert not _events(events, "notify success ") + assert not dump_dir.exists() + + +def test_backup_all_propagates_signoz_failure_to_nonzero_terminal( + tmp_path: Path, +) -> None: + result, events = _run_backup_all_with_signoz_failure(tmp_path) + + assert result.returncode == 1 + assert "run backup-signoz.sh" in events + assert any(event.startswith("notify warning all ") for event in events) + assert not any(event.startswith("notify success all ") for event in events)