diff --git a/scripts/backup/backup-signoz.sh b/scripts/backup/backup-signoz.sh index b37c6c246..a5c75fee0 100755 --- a/scripts/backup/backup-signoz.sh +++ b/scripts/backup/backup-signoz.sh @@ -1,11 +1,12 @@ #!/bin/bash # ============================================================================= # WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite) -# 版本: 1.2.0 +# 版本: 1.3.0 # 建立日期: 2026-04-05 # 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計 # 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true # 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector +# 2026-07-15 Codex: v1.3 加入初始狀態、全 run 有界還原與 success 前獨立 verifier # ============================================================================= set -euo pipefail @@ -15,22 +16,96 @@ source "$(dirname "$0")/common.sh" SERVICE="signoz" LOCAL_REPO="${BACKUP_BASE}/signoz" DUMP_DIR="/tmp/signoz-backup-$$" +COLLECTOR_NAME="signoz-otel-collector" +COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}" +COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}" +COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}" +COLLECTOR_WAS_RUNNING=0 COLLECTOR_RESTORE_PENDING=0 +COLLECTOR_RESTORE_ATTEMPTS_USED=0 CLEANUP_COMPLETE=0 +RESTORE_FAILURE_NOTIFIED=0 + +run_collector_docker() { + timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@" +} + +collector_running_state() { + local running + if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then + return 1 + fi + + case "${running}" in + true) + printf '%s\n' "running" + ;; + false) + printf '%s\n' "stopped" + ;; + *) + return 1 + ;; + esac +} + +collector_is_running() { + [ "$(collector_running_state)" = "running" ] +} + +notify_restore_failure_once() { + local detail="$1" + if [ "${RESTORE_FAILURE_NOTIFIED}" -eq 1 ]; then + return 0 + fi + RESTORE_FAILURE_NOTIFIED=1 + notify_clawbot "failed" "${SERVICE}" "${detail}" || true +} restore_collector() { if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then return 0 fi - log_info "重啟 signoz-otel-collector..." - if docker start signoz-otel-collector 2>/dev/null; then - COLLECTOR_RESTORE_PENDING=0 + while [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; do + COLLECTOR_RESTORE_ATTEMPTS_USED=$((COLLECTOR_RESTORE_ATTEMPTS_USED + 1)) + log_info "重啟 ${COLLECTOR_NAME} (${COLLECTOR_RESTORE_ATTEMPTS_USED}/${COLLECTOR_RESTORE_MAX_ATTEMPTS})..." + + if run_collector_docker start "${COLLECTOR_NAME}" >/dev/null 2>&1 \ + && collector_is_running; then + COLLECTOR_RESTORE_PENDING=0 + return 0 + fi + + log_warn "${COLLECTOR_NAME} 重啟或 running readback 失敗" + if [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; then + sleep "${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS}" + fi + done + + log_error "${COLLECTOR_NAME} 在有界重試後仍無法恢復" + return 1 +} + +verify_collector_final_state() { + if [ "${COLLECTOR_WAS_RUNNING}" -ne 1 ]; then + log_info "${COLLECTOR_NAME} 原本即為 stopped，不執行自動啟動" return 0 fi - log_warn "signoz-otel-collector 重啟失敗" - return 1 + if collector_is_running; then + log_success "${COLLECTOR_NAME} final running verifier 通過" + return 0 + fi + + log_warn "${COLLECTOR_NAME} final running verifier 首次失敗，使用剩餘有界還原額度" + COLLECTOR_RESTORE_PENDING=1 + if ! restore_collector; then + return 1 + fi + + # 與 docker start 的回傳值與 restore 內部 readback 分離，作為 success 前獨立 verifier。 + collector_is_running } cleanup() { @@ -43,7 +118,9 @@ cleanup() { # cleanup 期間忽略重複 signal，避免還原與刪除流程重入。 trap '' HUP INT TERM - restore_collector || true + if ! restore_collector; then + notify_restore_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}" + fi rm -rf "${DUMP_DIR}" return "${exit_code}" } @@ -87,11 +164,28 @@ main() { local timestamp=$(date "+%Y%m%d_%H%M%S") - # Step 1: 停止 OTEL Collector 確保數據一致性 - log_info "暫停 signoz-otel-collector 以確保數據一致性..." - # 先 arm 還原再停止，消除 signal 發生在 stop 與狀態記錄之間的空窗。 - COLLECTOR_RESTORE_PENDING=1 - docker stop signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 未在運行，繼續" + # Step 1: 只在 Collector 原本 running 時暫停，並在 stop 前 arm 還原。 + local collector_initial_state + if ! collector_initial_state="$(collector_running_state)"; then + log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態" + notify_restore_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態" + exit 1 + fi + + if [ "${collector_initial_state}" = "running" ]; then + COLLECTOR_WAS_RUNNING=1 + COLLECTOR_RESTORE_PENDING=1 + log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..." + if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then + log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止" + notify_restore_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}" + exit 1 + fi + else + log_error "${COLLECTOR_NAME} 初始狀態為 stopped，停止備份且不自動啟動" + notify_restore_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止" + exit 1 + fi docker stop signoz-telemetrystore-migrator 2>/dev/null || true # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積) @@ -106,9 +200,11 @@ main() { log_warn "SQLite volume 備份失敗，繼續..." } - # Step 4: 重啟 Collector - # 若此處失敗，保留 pending 讓 EXIT cleanup 再做一次有界還原。 - restore_collector || true + # Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。 + if ! restore_collector; then + notify_restore_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}" + exit 1 + fi # Step 5: 初始化 Restic 倉庫 if [ ! -d "${LOCAL_REPO}/data" ]; then @@ -130,6 +226,12 @@ main() { # Step 7: GFS 清理 cleanup_old_backups "${LOCAL_REPO}" + # Step 8: success 前獨立檢查 runtime state；失敗時不得發送 success。 + if ! verify_collector_final_state; then + notify_restore_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗" + exit 1 + fi + local end_time=$(date +%s) local duration=$((end_time - start_time)) log_success "========== SignOz 備份完成 (${duration}s) ==========" diff --git a/scripts/backup/tests/test_backup_signoz_collector_restore.py b/scripts/backup/tests/test_backup_signoz_collector_restore.py index 2e0337dfa..ab717c345 100644 --- a/scripts/backup/tests/test_backup_signoz_collector_restore.py +++ b/scripts/backup/tests/test_backup_signoz_collector_restore.py @@ -7,9 +7,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[3] SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh" -DEPLOYMENT_PLAYBOOK = ( - ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml" -) +DEPLOYMENT_PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml" def _write_executable(path: Path, text: str) -> None: @@ -20,18 +18,27 @@ def _write_executable(path: Path, text: str) -> None: def _run_backup( tmp_path: Path, *, + initial_running: bool = True, docker_run_mode: str = "success", - docker_start_fail_once: bool = False, + docker_start_failures: int = 0, restic_backup_status: int = 0, + restore_max_attempts: int = 3, ) -> tuple[subprocess.CompletedProcess[str], list[str], Path]: harness = tmp_path / "backup" fake_bin = tmp_path / "bin" backup_base = tmp_path / "repository" - docker_log = tmp_path / "docker.log" + event_log = tmp_path / "events.log" + collector_state = tmp_path / "collector.state" + start_count = tmp_path / "start.count" harness.mkdir() fake_bin.mkdir() (backup_base / "signoz" / "data").mkdir(parents=True) shutil.copy2(SCRIPT, harness / SCRIPT.name) + event_log.touch() + collector_state.write_text( + "true\n" if initial_running else "false\n", + encoding="utf-8", + ) (harness / "common.sh").write_text( """\ @@ -41,30 +48,49 @@ log_info() { :; } log_warn() { :; } log_error() { :; } log_success() { :; } -notify_clawbot() { :; } +notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } build_tags() { :; } cleanup_old_backups() { :; } """, encoding="utf-8", ) + _write_executable( + fake_bin / "timeout", + """\ +#!/bin/bash +set -eu +printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}" +shift +exec "$@" +""", + ) _write_executable( fake_bin / "docker", """\ #!/bin/bash set -eu -printf '%s\n' "$*" >> "${TEST_DOCKER_LOG:?}" +printf 'docker %s\n' "$*" >> "${TEST_EVENT_LOG:?}" case "${1:-}" in + inspect) + cat "${TEST_COLLECTOR_STATE:?}" + ;; stop) - exit 0 + if [ "${2:-}" = "signoz-otel-collector" ]; then + printf 'false\n' > "${TEST_COLLECTOR_STATE:?}" + fi ;; start) - if [ "${FAKE_DOCKER_START_FAIL_ONCE:-0}" = "1" ] && \ - [ ! -e "${TEST_DOCKER_START_MARKER:?}" ]; then - : > "${TEST_DOCKER_START_MARKER}" + count=0 + if [ -e "${TEST_START_COUNT:?}" ]; then + count=$(cat "${TEST_START_COUNT}") + fi + count=$((count + 1)) + printf '%s\n' "$count" > "${TEST_START_COUNT}" + if [ "$count" -le "${FAKE_DOCKER_START_FAILURES:-0}" ]; then exit 1 fi - exit 0 + printf 'true\n' > "${TEST_COLLECTOR_STATE:?}" ;; run) if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then @@ -85,6 +111,7 @@ esac """\ #!/bin/bash set -eu +printf 'restic %s\n' "$*" >> "${TEST_EVENT_LOG:?}" case " $* " in *" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;; *" snapshots "*) printf '[{"short_id":"test123"}]\n' ;; @@ -103,11 +130,15 @@ printf '4K\t%s\n' "${2:-unknown}" **os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}", "TEST_BACKUP_BASE": str(backup_base), - "TEST_DOCKER_LOG": str(docker_log), - "TEST_DOCKER_START_MARKER": str(tmp_path / "docker-start-failed-once"), + "TEST_EVENT_LOG": str(event_log), + "TEST_COLLECTOR_STATE": str(collector_state), + "TEST_START_COUNT": str(start_count), "FAKE_DOCKER_RUN_MODE": docker_run_mode, - "FAKE_DOCKER_START_FAIL_ONCE": "1" if docker_start_fail_once else "0", + "FAKE_DOCKER_START_FAILURES": str(docker_start_failures), "FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status), + "COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts), + "COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0", + "COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1", } process = subprocess.Popen( ["bash", str(harness / SCRIPT.name)], @@ -123,12 +154,12 @@ printf '4K\t%s\n' "${2:-unknown}" stdout, stderr, ) - docker_calls = docker_log.read_text(encoding="utf-8").splitlines() - return result, docker_calls, Path(f"/tmp/signoz-backup-{process.pid}") + events = event_log.read_text(encoding="utf-8").splitlines() + return result, events, Path(f"/tmp/signoz-backup-{process.pid}") -def _collector_start_count(docker_calls: list[str]) -> int: - return docker_calls.count("start signoz-otel-collector") +def _events(events: list[str], prefix: str) -> list[str]: + return [event for event in events if event.startswith(prefix)] def test_backup_jobs_deploys_the_guarded_signoz_script() -> None: @@ -139,51 +170,98 @@ def test_backup_jobs_deploys_the_guarded_signoz_script() -> None: assert "tags: backup_jobs" in playbook -def test_collector_is_restored_once_when_backup_receives_term(tmp_path: Path) -> None: - result, docker_calls, dump_dir = _run_backup( - tmp_path, - docker_run_mode="signal", +def test_restore_policy_is_bounded_timed_and_independently_verified() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source + assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source + assert 'timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"' in source + assert "verify_collector_final_state" in source + assert source.index("verify_collector_final_state") < source.rindex( + 'notify_clawbot "success"' ) + +def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None: + result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal") + assert result.returncode != 0 - assert "stop signoz-otel-collector" in docker_calls - assert _collector_start_count(docker_calls) == 1 + assert _events(events, "docker stop signoz-otel-collector") + assert len(_events(events, "docker start signoz-otel-collector")) == 1 + assert not _events(events, "notify success ") assert not dump_dir.exists() -def test_collector_is_restored_once_when_clickhouse_backup_fails(tmp_path: Path) -> None: - result, docker_calls, dump_dir = _run_backup( - tmp_path, - docker_run_mode="empty", - ) +def test_collector_is_restored_when_clickhouse_backup_fails(tmp_path: Path) -> None: + result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="empty") assert result.returncode == 1 - assert _collector_start_count(docker_calls) == 1 + assert len(_events(events, "docker start signoz-otel-collector")) == 1 + assert not _events(events, "notify success ") assert not dump_dir.exists() def test_exit_cleanup_does_not_restart_an_already_restored_collector( tmp_path: Path, ) -> None: - result, docker_calls, dump_dir = _run_backup( - tmp_path, - restic_backup_status=17, - ) + result, events, dump_dir = _run_backup(tmp_path, restic_backup_status=17) assert result.returncode == 17 - assert _collector_start_count(docker_calls) == 1 + assert len(_events(events, "docker start signoz-otel-collector")) == 1 + assert not _events(events, "notify success ") assert not dump_dir.exists() -def test_exit_cleanup_retries_only_after_explicit_restore_fails( +def test_fail_once_restores_within_budget_and_succeeds(tmp_path: Path) -> None: + result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=1) + + assert result.returncode == 0 + assert len(_events(events, "docker start signoz-otel-collector")) == 2 + assert len(_events(events, "notify success ")) == 1 + assert not _events(events, "notify failed ") + assert not dump_dir.exists() + + +def test_permanent_restore_failure_is_nonzero_and_never_notifies_success( tmp_path: Path, ) -> None: - result, docker_calls, dump_dir = _run_backup( - tmp_path, - docker_start_fail_once=True, - restic_backup_status=17, - ) + result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=99) - assert result.returncode == 17 - assert _collector_start_count(docker_calls) == 2 + assert result.returncode != 0 + assert len(_events(events, "docker start signoz-otel-collector")) == 3 + assert len(_events(events, "notify failed ")) == 1 + assert not _events(events, "notify success ") assert not dump_dir.exists() + + +def test_initially_stopped_fails_closed_without_stop_start_or_success( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup(tmp_path, initial_running=False) + + assert result.returncode != 0 + assert not _events(events, "docker stop signoz-otel-collector") + assert not _events(events, "docker start signoz-otel-collector") + assert len(_events(events, "notify failed ")) == 1 + assert not _events(events, "notify success ") + assert not dump_dir.exists() + + +def test_success_notification_follows_independent_final_running_readback( + tmp_path: Path, +) -> None: + result, events, _dump_dir = _run_backup(tmp_path) + + assert result.returncode == 0 + inspect_positions = [ + index + for index, event in enumerate(events) + if event.startswith("docker inspect --format") + ] + success_position = next( + index + for index, event in enumerate(events) + if event.startswith("notify success ") + ) + assert len(inspect_positions) >= 3 + assert inspect_positions[-1] < success_position