wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(backup): verify SigNoz collector restoration

Browse Source
This commit is contained in:
ogt
2026-07-15 03:54:12 +08:00
parent 5c96f2fb1b
commit 03d3236271
2 changed files with 239 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
scripts/backup/backup-signoz.sh
View File

@@ -1,11 +1,12 @@
#!/bin/bash
# =============================================================================
# WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
# 版本: 1.2.0
# 版本: 1.3.0
# 建立日期: 2026-04-05
# 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
# 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
# 2026-07-15 Codex: v1.3 加入初始狀態、全 run 有界還原與 success 前獨立 verifier
# =============================================================================
set -euo pipefail
@@ -15,22 +16,96 @@ source "$(dirname "$0")/common.sh"
SERVICE="signoz"
LOCAL_REPO="${BACKUP_BASE}/signoz"
DUMP_DIR="/tmp/signoz-backup-$$"
COLLECTOR_NAME="signoz-otel-collector"
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"
COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}"
COLLECTOR_WAS_RUNNING=0
COLLECTOR_RESTORE_PENDING=0
COLLECTOR_RESTORE_ATTEMPTS_USED=0
CLEANUP_COMPLETE=0
RESTORE_FAILURE_NOTIFIED=0
run_collector_docker() {
timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
}
collector_running_state() {
local running
if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
return 1
fi
case "${running}" in
true)
printf '%s\n' "running"
;;
false)
printf '%s\n' "stopped"
;;
*)
return 1
;;
esac
}
collector_is_running() {
[ "$(collector_running_state)" = "running" ]
}
notify_restore_failure_once() {
local detail="$1"
if [ "${RESTORE_FAILURE_NOTIFIED}" -eq 1 ]; then
return 0
fi
RESTORE_FAILURE_NOTIFIED=1
notify_clawbot "failed" "${SERVICE}" "${detail}" || true
}
restore_collector() {
if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
return 0
fi
log_info "重啟 signoz-otel-collector..."
if docker start signoz-otel-collector 2>/dev/null; then
COLLECTOR_RESTORE_PENDING=0
while [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; do
COLLECTOR_RESTORE_ATTEMPTS_USED=$((COLLECTOR_RESTORE_ATTEMPTS_USED + 1))
log_info "重啟 ${COLLECTOR_NAME} (${COLLECTOR_RESTORE_ATTEMPTS_USED}/${COLLECTOR_RESTORE_MAX_ATTEMPTS})..."
if run_collector_docker start "${COLLECTOR_NAME}" >/dev/null 2>&1 \
&& collector_is_running; then
COLLECTOR_RESTORE_PENDING=0
return 0
fi
log_warn "${COLLECTOR_NAME} 重啟或 running readback 失敗"
if [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; then
sleep "${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS}"
fi
done
log_error "${COLLECTOR_NAME} 在有界重試後仍無法恢復"
return 1
}
verify_collector_final_state() {
if [ "${COLLECTOR_WAS_RUNNING}" -ne 1 ]; then
log_info "${COLLECTOR_NAME} 原本即為 stopped，不執行自動啟動"
return 0
fi
log_warn "signoz-otel-collector 重啟失敗"
return 1
if collector_is_running; then
log_success "${COLLECTOR_NAME} final running verifier 通過"
return 0
fi
log_warn "${COLLECTOR_NAME} final running verifier 首次失敗，使用剩餘有界還原額度"
COLLECTOR_RESTORE_PENDING=1
if ! restore_collector; then
return 1
fi
# 與 docker start 的回傳值與 restore 內部 readback 分離，作為 success 前獨立 verifier。
collector_is_running
}
cleanup() {
@@ -43,7 +118,9 @@ cleanup() {
# cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
trap '' HUP INT TERM
restore_collector || true
if ! restore_collector; then
notify_restore_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}"
fi
rm -rf "${DUMP_DIR}"
return "${exit_code}"
}
@@ -87,11 +164,28 @@ main() {
local timestamp=$(date "+%Y%m%d_%H%M%S")
# Step 1: 停止 OTEL Collector 確保數據一致性
log_info "暫停 signoz-otel-collector 以確保數據一致性..."
# 先 arm 還原再停止，消除 signal 發生在 stop 與狀態記錄之間的空窗。
COLLECTOR_RESTORE_PENDING=1
docker stop signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 未在運行，繼續"
# Step 1: 只在 Collector 原本 running 時暫停，並在 stop 前 arm 還原。
local collector_initial_state
if ! collector_initial_state="$(collector_running_state)"; then
log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態"
notify_restore_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態"
exit 1
fi
if [ "${collector_initial_state}" = "running" ]; then
COLLECTOR_WAS_RUNNING=1
COLLECTOR_RESTORE_PENDING=1
log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..."
if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then
log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止"
notify_restore_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}"
exit 1
fi
else
log_error "${COLLECTOR_NAME} 初始狀態為 stopped，停止備份且不自動啟動"
notify_restore_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止"
exit 1
fi
docker stop signoz-telemetrystore-migrator 2>/dev/null || true
# Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
@@ -106,9 +200,11 @@ main() {
log_warn "SQLite volume 備份失敗，繼續..."
}
# Step 4: 重啟 Collector
# 若此處失敗，保留 pending 讓 EXIT cleanup 再做一次有界還原。
restore_collector || true
# Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。
if ! restore_collector; then
notify_restore_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}"
exit 1
fi
# Step 5: 初始化 Restic 倉庫
if [ ! -d "${LOCAL_REPO}/data" ]; then
@@ -130,6 +226,12 @@ main() {
# Step 7: GFS 清理
cleanup_old_backups "${LOCAL_REPO}"
# Step 8: success 前獨立檢查 runtime state；失敗時不得發送 success。
if ! verify_collector_final_state; then
notify_restore_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗"
exit 1
fi
local end_time=$(date +%s)
local duration=$((end_time - start_time))
log_success "========== SignOz 備份完成 (${duration}s) =========="

166
scripts/backup/tests/test_backup_signoz_collector_restore.py
View File

@@ -7,9 +7,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
DEPLOYMENT_PLAYBOOK = (
ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
)
DEPLOYMENT_PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
def _write_executable(path: Path, text: str) -> None:
@@ -20,18 +18,27 @@ def _write_executable(path: Path, text: str) -> None:
def _run_backup(
tmp_path: Path,
*,
initial_running: bool = True,
docker_run_mode: str = "success",
docker_start_fail_once: bool = False,
docker_start_failures: int = 0,
restic_backup_status: int = 0,
restore_max_attempts: int = 3,
) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
harness = tmp_path / "backup"
fake_bin = tmp_path / "bin"
backup_base = tmp_path / "repository"
docker_log = tmp_path / "docker.log"
event_log = tmp_path / "events.log"
collector_state = tmp_path / "collector.state"
start_count = tmp_path / "start.count"
harness.mkdir()
fake_bin.mkdir()
(backup_base / "signoz" / "data").mkdir(parents=True)
shutil.copy2(SCRIPT, harness / SCRIPT.name)
event_log.touch()
collector_state.write_text(
"true\n" if initial_running else "false\n",
encoding="utf-8",
)
(harness / "common.sh").write_text(
"""\
@@ -41,30 +48,49 @@ log_info() { :; }
log_warn() { :; }
log_error() { :; }
log_success() { :; }
notify_clawbot() { :; }
notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
build_tags() { :; }
cleanup_old_backups() { :; }
""",
encoding="utf-8",
)
_write_executable(
fake_bin / "timeout",
"""\
#!/bin/bash
set -eu
printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
shift
exec "$@"
""",
)
_write_executable(
fake_bin / "docker",
"""\
#!/bin/bash
set -eu
printf '%s\n' "$*" >> "${TEST_DOCKER_LOG:?}"
printf 'docker %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
case "${1:-}" in
inspect)
cat "${TEST_COLLECTOR_STATE:?}"
;;
stop)
exit 0
if [ "${2:-}" = "signoz-otel-collector" ]; then
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
fi
;;
start)
if [ "${FAKE_DOCKER_START_FAIL_ONCE:-0}" = "1" ] && \
[ ! -e "${TEST_DOCKER_START_MARKER:?}" ]; then
: > "${TEST_DOCKER_START_MARKER}"
count=0
if [ -e "${TEST_START_COUNT:?}" ]; then
count=$(cat "${TEST_START_COUNT}")
fi
count=$((count + 1))
printf '%s\n' "$count" > "${TEST_START_COUNT}"
if [ "$count" -le "${FAKE_DOCKER_START_FAILURES:-0}" ]; then
exit 1
fi
exit 0
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
;;
run)
if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then
@@ -85,6 +111,7 @@ esac
"""\
#!/bin/bash
set -eu
printf 'restic %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
case " $* " in
*" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
*" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
@@ -103,11 +130,15 @@ printf '4K\t%s\n' "${2:-unknown}"
**os.environ,
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"TEST_BACKUP_BASE": str(backup_base),
"TEST_DOCKER_LOG": str(docker_log),
"TEST_DOCKER_START_MARKER": str(tmp_path / "docker-start-failed-once"),
"TEST_EVENT_LOG": str(event_log),
"TEST_COLLECTOR_STATE": str(collector_state),
"TEST_START_COUNT": str(start_count),
"FAKE_DOCKER_RUN_MODE": docker_run_mode,
"FAKE_DOCKER_START_FAIL_ONCE": "1" if docker_start_fail_once else "0",
"FAKE_DOCKER_START_FAILURES": str(docker_start_failures),
"FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
"COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts),
"COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0",
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
}
process = subprocess.Popen(
["bash", str(harness / SCRIPT.name)],
@@ -123,12 +154,12 @@ printf '4K\t%s\n' "${2:-unknown}"
stdout,
stderr,
)
docker_calls = docker_log.read_text(encoding="utf-8").splitlines()
return result, docker_calls, Path(f"/tmp/signoz-backup-{process.pid}")
events = event_log.read_text(encoding="utf-8").splitlines()
return result, events, Path(f"/tmp/signoz-backup-{process.pid}")
def _collector_start_count(docker_calls: list[str]) -> int:
return docker_calls.count("start signoz-otel-collector")
def _events(events: list[str], prefix: str) -> list[str]:
return [event for event in events if event.startswith(prefix)]
def test_backup_jobs_deploys_the_guarded_signoz_script() -> None:
@@ -139,51 +170,98 @@ def test_backup_jobs_deploys_the_guarded_signoz_script() -> None:
assert "tags: backup_jobs" in playbook
def test_collector_is_restored_once_when_backup_receives_term(tmp_path: Path) -> None:
result, docker_calls, dump_dir = _run_backup(
tmp_path,
docker_run_mode="signal",
def test_restore_policy_is_bounded_timed_and_independently_verified() -> None:
source = SCRIPT.read_text(encoding="utf-8")
assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source
assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source
assert 'timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"' in source
assert "verify_collector_final_state" in source
assert source.index("verify_collector_final_state") < source.rindex(
'notify_clawbot "success"'
)
def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None:
result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal")
assert result.returncode != 0
assert "stop signoz-otel-collector" in docker_calls
assert _collector_start_count(docker_calls) == 1
assert _events(events, "docker stop signoz-otel-collector")
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_collector_is_restored_once_when_clickhouse_backup_fails(tmp_path: Path) -> None:
result, docker_calls, dump_dir = _run_backup(
tmp_path,
docker_run_mode="empty",
)
def test_collector_is_restored_when_clickhouse_backup_fails(tmp_path: Path) -> None:
result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="empty")
assert result.returncode == 1
assert _collector_start_count(docker_calls) == 1
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_exit_cleanup_does_not_restart_an_already_restored_collector(
tmp_path: Path,
) -> None:
result, docker_calls, dump_dir = _run_backup(
tmp_path,
restic_backup_status=17,
)
result, events, dump_dir = _run_backup(tmp_path, restic_backup_status=17)
assert result.returncode == 17
assert _collector_start_count(docker_calls) == 1
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_exit_cleanup_retries_only_after_explicit_restore_fails(
def test_fail_once_restores_within_budget_and_succeeds(tmp_path: Path) -> None:
result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=1)
assert result.returncode == 0
assert len(_events(events, "docker start signoz-otel-collector")) == 2
assert len(_events(events, "notify success ")) == 1
assert not _events(events, "notify failed ")
assert not dump_dir.exists()
def test_permanent_restore_failure_is_nonzero_and_never_notifies_success(
tmp_path: Path,
) -> None:
result, docker_calls, dump_dir = _run_backup(
tmp_path,
docker_start_fail_once=True,
restic_backup_status=17,
)
result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=99)
assert result.returncode == 17
assert _collector_start_count(docker_calls) == 2
assert result.returncode != 0
assert len(_events(events, "docker start signoz-otel-collector")) == 3
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_initially_stopped_fails_closed_without_stop_start_or_success(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
assert result.returncode != 0
assert not _events(events, "docker stop signoz-otel-collector")
assert not _events(events, "docker start signoz-otel-collector")
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_success_notification_follows_independent_final_running_readback(
tmp_path: Path,
) -> None:
result, events, _dump_dir = _run_backup(tmp_path)
assert result.returncode == 0
inspect_positions = [
index
for index, event in enumerate(events)
if event.startswith("docker inspect --format")
]
success_position = next(
index
for index, event in enumerate(events)
if event.startswith("notify success ")
)
assert len(inspect_positions) >= 3
assert inspect_positions[-1] < success_position