wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(backup): validate bounded SigNoz archives

Browse Source
This commit is contained in:
ogt
2026-07-15 04:14:26 +08:00
parent dba193273f
commit 038743bf44
2 changed files with 307 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
scripts/backup/backup-signoz.sh
View File

@@ -20,14 +20,19 @@ COLLECTOR_NAME="signoz-otel-collector"
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"
COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}"
ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"
ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"
COLLECTOR_WAS_RUNNING=0
COLLECTOR_RESTORE_PENDING=0
COLLECTOR_RESTORE_ATTEMPTS_USED=0
CLEANUP_COMPLETE=0
RESTORE_FAILURE_NOTIFIED=0
FAILURE_NOTIFIED=0
run_collector_docker() {
timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
}
collector_running_state() {
@@ -53,12 +58,12 @@ collector_is_running() {
[ "$(collector_running_state)" = "running" ]
}
notify_restore_failure_once() {
notify_failure_once() {
local detail="$1"
if [ "${RESTORE_FAILURE_NOTIFIED}" -eq 1 ]; then
if [ "${FAILURE_NOTIFIED}" -eq 1 ]; then
return 0
fi
RESTORE_FAILURE_NOTIFIED=1
FAILURE_NOTIFIED=1
notify_clawbot "failed" "${SERVICE}" "${detail}" || true
}
@@ -119,7 +124,7 @@ cleanup() {
# cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
trap '' HUP INT TERM
if ! restore_collector; then
notify_restore_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}"
notify_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}"
fi
rm -rf "${DUMP_DIR}"
return "${exit_code}"
@@ -130,25 +135,61 @@ on_signal() {
exit "$((128 + signal_number))"
}
backup_volume() {
create_volume_archive() {
local volume_name="$1"
local output_file="$2"
local extra_exclude="${3:-}"
local partial_file="${output_file}.partial"
log_info "備份 volume: ${volume_name}"
# 使用 || true 處理 tar 備份運行中 volume 的 exit 1 警告
rm -f "${partial_file}" "${output_file}"
if [ -n "${extra_exclude}" ]; then
docker run --rm -v "${volume_name}:/data" alpine tar czf - "${extra_exclude}" /data 2>/dev/null > "${output_file}" || true
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
docker run --rm -v "${volume_name}:/data" alpine \
tar czf - "${extra_exclude}" /data \
> "${partial_file}" 2>/dev/null; then
log_error " Volume ${volume_name} archive 建立失敗"
rm -f "${partial_file}" "${output_file}"
return 1
fi
else
docker run --rm -v "${volume_name}:/data" alpine tar czf - /data 2>/dev/null > "${output_file}" || true
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
docker run --rm -v "${volume_name}:/data" alpine \
tar czf - /data \
> "${partial_file}" 2>/dev/null; then
log_error " Volume ${volume_name} archive 建立失敗"
rm -f "${partial_file}" "${output_file}"
return 1
fi
fi
if [ -s "${output_file}" ]; then
local size=$(du -h "${output_file}" | cut -f1)
log_success " Volume ${volume_name} 備份完成 (${size})"
return 0
else
if [ ! -s "${partial_file}" ]; then
log_error " Volume ${volume_name} 備份失敗 (空檔案)"
rm -f "${partial_file}" "${output_file}"
return 1
fi
mv "${partial_file}" "${output_file}"
}
verify_volume_archive() {
local volume_name="$1"
local output_file="$2"
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${ARCHIVE_VERIFY_TIMEOUT_SECONDS}" \
tar -tzf "${output_file}" >/dev/null 2>&1; then
log_error " Volume ${volume_name} archive integrity verifier 失敗"
rm -f "${output_file}" "${output_file}.partial"
return 1
fi
local size
size="$(du -h "${output_file}" | cut -f1)"
log_success " Volume ${volume_name} 備份與 integrity verifier 完成 (${size})"
}
main() {
@@ -168,7 +209,7 @@ main() {
local collector_initial_state
if ! collector_initial_state="$(collector_running_state)"; then
log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態"
notify_restore_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態"
notify_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態"
exit 1
fi
@@ -178,35 +219,49 @@ main() {
log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..."
if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then
log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止"
notify_restore_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}"
notify_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}"
exit 1
fi
else
log_error "${COLLECTOR_NAME} 初始狀態為 stopped，停止備份且不自動啟動"
notify_restore_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止"
notify_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止"
exit 1
fi
docker stop signoz-telemetrystore-migrator 2>/dev/null || true
# Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
backup_volume "signoz-clickhouse" "${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" "--exclude=/data/tmp" || {
local clickhouse_archive="${DUMP_DIR}/clickhouse_${timestamp}.tar.gz"
local sqlite_archive="${DUMP_DIR}/sqlite_${timestamp}.tar.gz"
create_volume_archive "signoz-clickhouse" "${clickhouse_archive}" "--exclude=/data/tmp" || {
log_error "ClickHouse volume 備份失敗"
notify_clawbot "failed" "${SERVICE}" "SignOz ClickHouse 備份失敗"
notify_failure_once "SignOz ClickHouse 備份失敗"
exit 1
}
# Step 3: 備份 SQLite volume (SignOz metadata)
backup_volume "signoz-sqlite" "${DUMP_DIR}/sqlite_${timestamp}.tar.gz" || {
log_warn "SQLite volume 備份失敗，繼續..."
create_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
log_error "SQLite volume 備份失敗"
notify_failure_once "SignOz SQLite 備份失敗"
exit 1
}
# Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。
if ! restore_collector; then
notify_restore_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}"
notify_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}"
exit 1
fi
# Step 5: 初始化 Restic 倉庫
# Step 5: Collector 恢復後才在 host 做有界 archive integrity verifier。
verify_volume_archive "signoz-clickhouse" "${clickhouse_archive}" || {
notify_failure_once "SignOz ClickHouse archive integrity verifier 失敗"
exit 1
}
verify_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
notify_failure_once "SignOz SQLite archive integrity verifier 失敗"
exit 1
}
# Step 6: 初始化 Restic 倉庫
if [ ! -d "${LOCAL_REPO}/data" ]; then
log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
@@ -215,7 +270,7 @@ main() {
}
fi
# Step 6: Restic 備份
# Step 7: Restic 備份
log_info "建立 Restic 備份..."
local tags=$(build_tags "${SERVICE}")
restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" --password-file "${RESTIC_PASSWORD_FILE}" ${tags} 2>&1
@@ -223,12 +278,16 @@ main() {
local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1)
log_success "Restic 備份完成: ${snapshot_id}"
# Step 7: GFS 清理
cleanup_old_backups "${LOCAL_REPO}"
# Step 8: GFS 清理；real canary 可明確跳過 destructive retention。
if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
else
cleanup_old_backups "${LOCAL_REPO}"
fi
# Step 8: success 前獨立檢查 runtime state；失敗時不得發送 success。
# Step 9: success 前獨立檢查 runtime state；失敗時不得發送 success。
if ! verify_collector_final_state; then
notify_restore_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗"
notify_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗"
exit 1
fi

222
scripts/backup/tests/test_backup_signoz_collector_restore.py
View File

@@ -7,6 +7,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh"
DEPLOYMENT_PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
@@ -20,6 +21,11 @@ def _run_backup(
*,
initial_running: bool = True,
docker_run_mode: str = "success",
docker_run_target: str = "signoz-clickhouse",
archive_timeout_target: str = "",
tar_verify_mode: str = "success",
tar_verify_target: str = "clickhouse_",
tar_timeout_target: str = "",
docker_start_failures: int = 0,
restic_backup_status: int = 0,
restore_max_attempts: int = 3,
@@ -61,7 +67,20 @@ cleanup_old_backups() { :; }
#!/bin/bash
set -eu
printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
while [[ "${1:-}" == --kill-after=* ]]; do
shift
done
shift
if [ -n "${FAKE_ARCHIVE_TIMEOUT_TARGET:-}" ] \
&& [ "${1:-}" = "docker" ] \
&& [[ " $* " == *"${FAKE_ARCHIVE_TIMEOUT_TARGET}"* ]]; then
exit 124
fi
if [ -n "${FAKE_TAR_TIMEOUT_TARGET:-}" ] \
&& [ "${1:-}" = "tar" ] \
&& [[ " $* " == *"${FAKE_TAR_TIMEOUT_TARGET}"* ]]; then
exit 124
fi
exec "$@"
""",
)
@@ -93,17 +112,38 @@ case "${1:-}" in
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
;;
run)
if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then
mode=success
if [[ " $* " == *"${FAKE_DOCKER_RUN_TARGET:-signoz-clickhouse}"* ]]; then
mode="${FAKE_DOCKER_RUN_MODE:-success}"
fi
if [ "$mode" = "signal" ]; then
kill -TERM "${PPID}"
sleep 0.1
exit 0
fi
if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "empty" ]; then
if [ "$mode" = "empty" ]; then
exit 0
fi
if [ "$mode" = "partial_nonzero" ]; then
printf 'partial-archive-data\n'
exit 23
fi
printf 'archive-data\n'
;;
esac
""",
)
_write_executable(
fake_bin / "tar",
"""\
#!/bin/bash
set -eu
printf 'tar %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
if [[ " $* " == *"${FAKE_TAR_VERIFY_TARGET:-signoz-clickhouse}"* ]] \
&& [ "${FAKE_TAR_VERIFY_MODE:-success}" = "corrupt" ]; then
exit 2
fi
exit 0
""",
)
_write_executable(
@@ -134,11 +174,19 @@ printf '4K\t%s\n' "${2:-unknown}"
"TEST_COLLECTOR_STATE": str(collector_state),
"TEST_START_COUNT": str(start_count),
"FAKE_DOCKER_RUN_MODE": docker_run_mode,
"FAKE_DOCKER_RUN_TARGET": docker_run_target,
"FAKE_ARCHIVE_TIMEOUT_TARGET": archive_timeout_target,
"FAKE_TAR_VERIFY_MODE": tar_verify_mode,
"FAKE_TAR_VERIFY_TARGET": tar_verify_target,
"FAKE_TAR_TIMEOUT_TARGET": tar_timeout_target,
"FAKE_DOCKER_START_FAILURES": str(docker_start_failures),
"FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
"COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts),
"COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0",
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
"ARCHIVE_CREATE_TIMEOUT_SECONDS": "1",
"ARCHIVE_VERIFY_TIMEOUT_SECONDS": "1",
"TIMEOUT_KILL_AFTER_SECONDS": "1",
}
process = subprocess.Popen(
["bash", str(harness / SCRIPT.name)],
@@ -162,6 +210,60 @@ def _events(events: list[str], prefix: str) -> list[str]:
return [event for event in events if event.startswith(prefix)]
def _run_backup_all_with_signoz_failure(
tmp_path: Path,
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
harness = tmp_path / "backup-all"
fake_scripts = tmp_path / "deployed-scripts"
event_log = tmp_path / "backup-all-events.log"
harness.mkdir()
fake_scripts.mkdir()
event_log.touch()
source = BACKUP_ALL.read_text(encoding="utf-8").replace(
"/backup/scripts",
str(fake_scripts),
)
backup_all = harness / "backup-all.sh"
_write_executable(backup_all, source)
(harness / "common.sh").write_text(
"""\
log_info() { :; }
log_warn() { :; }
log_error() { :; }
log_success() { :; }
notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
""",
encoding="utf-8",
)
for name in (
"backup-gitea.sh",
"backup-momo.sh",
"backup-harbor.sh",
"backup-awoooi.sh",
"backup-langfuse.sh",
"backup-monitoring.sh",
"backup-signoz.sh",
"backup-open-webui.sh",
"backup-clawbot.sh",
):
status = 17 if name == "backup-signoz.sh" else 0
_write_executable(
fake_scripts / name,
f"#!/bin/bash\nprintf 'run {name}\\n' >> \"${{TEST_EVENT_LOG:?}}\"\nexit {status}\n",
)
result = subprocess.run(
["bash", str(backup_all)],
env={**os.environ, "TEST_EVENT_LOG": str(event_log)},
text=True,
capture_output=True,
timeout=10,
)
return result, event_log.read_text(encoding="utf-8").splitlines()
def test_backup_jobs_deploys_the_guarded_signoz_script() -> None:
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
@@ -175,13 +277,35 @@ def test_restore_policy_is_bounded_timed_and_independently_verified() -> None:
assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source
assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source
assert 'timeout "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"' in source
assert 'ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"' in source
assert 'ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"' in source
assert 'TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"' in source
assert 'timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}"' in source
assert 'tar -tzf "${output_file}"' in source
assert "docker run" in source
assert "|| true" not in source.split("create_volume_archive()", 1)[1].split(
"verify_volume_archive()",
1,
)[0]
assert "verify_collector_final_state" in source
assert source.index("verify_collector_final_state") < source.rindex(
'notify_clawbot "success"'
)
def test_canary_retention_skip_is_explicit_and_defaults_off() -> None:
source = SCRIPT.read_text(encoding="utf-8")
assert 'BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"' in source
assert 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' in source
assert source.index('restic -r "${LOCAL_REPO}" backup') < source.index(
'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]'
)
assert source.index('if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]') < source.index(
"if ! verify_collector_final_state"
)
def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None:
result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal")
@@ -265,3 +389,95 @@ def test_success_notification_follows_independent_final_running_readback(
)
assert len(inspect_positions) >= 3
assert inspect_positions[-1] < success_position
start_position = next(
index
for index, event in enumerate(events)
if event.startswith("docker start signoz-otel-collector")
)
tar_positions = [
index for index, event in enumerate(events) if event.startswith("tar -tzf")
]
assert len(tar_positions) == 2
assert start_position < tar_positions[0]
def test_archive_create_timeout_is_bounded_restores_and_notifies_once(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(
tmp_path,
archive_timeout_target="signoz-clickhouse",
)
assert result.returncode != 0
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_partial_sqlite_archive_nonzero_fails_overall_and_removes_output(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(
tmp_path,
docker_run_mode="partial_nonzero",
docker_run_target="signoz-sqlite",
)
assert result.returncode != 0
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_corrupt_archive_fails_after_collector_restore_and_notifies_once(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(
tmp_path,
tar_verify_mode="corrupt",
tar_verify_target="clickhouse_",
)
assert result.returncode != 0
start_position = next(
index
for index, event in enumerate(events)
if event.startswith("docker start signoz-otel-collector")
)
tar_position = next(
index for index, event in enumerate(events) if event.startswith("tar -tzf")
)
assert start_position < tar_position
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_integrity_verifier_timeout_is_bounded_and_never_reports_success(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(
tmp_path,
tar_timeout_target="clickhouse_",
)
assert result.returncode != 0
assert len(_events(events, "docker start signoz-otel-collector")) == 1
assert len(_events(events, "notify failed ")) == 1
assert not _events(events, "notify success ")
assert not dump_dir.exists()
def test_backup_all_propagates_signoz_failure_to_nonzero_terminal(
tmp_path: Path,
) -> None:
result, events = _run_backup_all_with_signoz_failure(tmp_path)
assert result.returncode == 1
assert "run backup-signoz.sh" in events
assert any(event.startswith("notify warning all ") for event in events)
assert not any(event.startswith("notify success all ") for event in events)