fix(ops): bound CD DB probes and restore SigNoz collector
This commit is contained in:
ogt
@@ -1,10 +1,11 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
|
||||
# 版本: 1.1.0
|
||||
# 版本: 1.2.0
|
||||
# 建立日期: 2026-04-05
|
||||
# 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
|
||||
# 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
|
||||
# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
@@ -14,11 +15,42 @@ source "$(dirname "$0")/common.sh"
|
||||
SERVICE="signoz"
|
||||
LOCAL_REPO="${BACKUP_BASE}/signoz"
|
||||
DUMP_DIR="/tmp/signoz-backup-$$"
|
||||
COLLECTOR_RESTORE_PENDING=0
|
||||
CLEANUP_COMPLETE=0
|
||||
|
||||
restore_collector() {
|
||||
if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "重啟 signoz-otel-collector..."
|
||||
if docker start signoz-otel-collector 2>/dev/null; then
|
||||
COLLECTOR_RESTORE_PENDING=0
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_warn "signoz-otel-collector 重啟失敗"
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
# 確保 collector 已重啟
|
||||
docker start signoz-otel-collector 2>/dev/null || true
|
||||
local exit_code=$?
|
||||
|
||||
if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
|
||||
return "${exit_code}"
|
||||
fi
|
||||
CLEANUP_COMPLETE=1
|
||||
|
||||
# cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
|
||||
trap '' HUP INT TERM
|
||||
restore_collector || true
|
||||
rm -rf "${DUMP_DIR}"
|
||||
return "${exit_code}"
|
||||
}
|
||||
|
||||
on_signal() {
|
||||
local signal_number="$1"
|
||||
exit "$((128 + signal_number))"
|
||||
}
|
||||
|
||||
backup_volume() {
|
||||
@@ -44,6 +76,12 @@ backup_volume() {
|
||||
|
||||
main() {
|
||||
local start_time=$(date +%s)
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'on_signal 1' HUP
|
||||
trap 'on_signal 2' INT
|
||||
trap 'on_signal 15' TERM
|
||||
|
||||
log_info "========== 開始 SignOz 備份 =========="
|
||||
mkdir -p "${DUMP_DIR}"
|
||||
|
||||
@@ -51,13 +89,14 @@ main() {
|
||||
|
||||
# Step 1: 停止 OTEL Collector 確保數據一致性
|
||||
log_info "暫停 signoz-otel-collector 以確保數據一致性..."
|
||||
# 先 arm 還原再停止，消除 signal 發生在 stop 與狀態記錄之間的空窗。
|
||||
COLLECTOR_RESTORE_PENDING=1
|
||||
docker stop signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 未在運行，繼續"
|
||||
docker stop signoz-telemetrystore-migrator 2>/dev/null || true
|
||||
|
||||
# Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
|
||||
backup_volume "signoz-clickhouse" "${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" "--exclude=/data/tmp" || {
|
||||
log_error "ClickHouse volume 備份失敗"
|
||||
cleanup
|
||||
notify_clawbot "failed" "${SERVICE}" "SignOz ClickHouse 備份失敗"
|
||||
exit 1
|
||||
}
|
||||
@@ -68,15 +107,14 @@ main() {
|
||||
}
|
||||
|
||||
# Step 4: 重啟 Collector
|
||||
log_info "重啟 signoz-otel-collector..."
|
||||
docker start signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 重啟失敗"
|
||||
# 若此處失敗，保留 pending 讓 EXIT cleanup 再做一次有界還原。
|
||||
restore_collector || true
|
||||
|
||||
# Step 5: 初始化 Restic 倉庫
|
||||
if [ ! -d "${LOCAL_REPO}/data" ]; then
|
||||
log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
|
||||
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||||
log_error "Restic 倉庫初始化失敗"
|
||||
rm -rf "${DUMP_DIR}"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
@@ -92,8 +130,6 @@ main() {
|
||||
# Step 7: GFS 清理
|
||||
cleanup_old_backups "${LOCAL_REPO}"
|
||||
|
||||
rm -rf "${DUMP_DIR}"
|
||||
|
||||
local end_time=$(date +%s)
|
||||
local duration=$((end_time - start_time))
|
||||
log_success "========== SignOz 備份完成 (${duration}s) =========="
|
||||
|
||||
179
scripts/backup/tests/test_backup_signoz_collector_restore.py
Normal file
179
scripts/backup/tests/test_backup_signoz_collector_restore.py
Normal file
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
|
||||
|
||||
|
||||
def _write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _run_backup(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
docker_run_mode: str = "success",
|
||||
docker_start_fail_once: bool = False,
|
||||
restic_backup_status: int = 0,
|
||||
) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
|
||||
harness = tmp_path / "backup"
|
||||
fake_bin = tmp_path / "bin"
|
||||
backup_base = tmp_path / "repository"
|
||||
docker_log = tmp_path / "docker.log"
|
||||
harness.mkdir()
|
||||
fake_bin.mkdir()
|
||||
(backup_base / "signoz" / "data").mkdir(parents=True)
|
||||
shutil.copy2(SCRIPT, harness / SCRIPT.name)
|
||||
|
||||
(harness / "common.sh").write_text(
|
||||
"""\
|
||||
BACKUP_BASE="${TEST_BACKUP_BASE:?}"
|
||||
RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password"
|
||||
log_info() { :; }
|
||||
log_warn() { :; }
|
||||
log_error() { :; }
|
||||
log_success() { :; }
|
||||
notify_clawbot() { :; }
|
||||
build_tags() { :; }
|
||||
cleanup_old_backups() { :; }
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_write_executable(
|
||||
fake_bin / "docker",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf '%s\n' "$*" >> "${TEST_DOCKER_LOG:?}"
|
||||
case "${1:-}" in
|
||||
stop)
|
||||
exit 0
|
||||
;;
|
||||
start)
|
||||
if [ "${FAKE_DOCKER_START_FAIL_ONCE:-0}" = "1" ] && \
|
||||
[ ! -e "${TEST_DOCKER_START_MARKER:?}" ]; then
|
||||
: > "${TEST_DOCKER_START_MARKER}"
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
;;
|
||||
run)
|
||||
if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then
|
||||
kill -TERM "${PPID}"
|
||||
sleep 0.1
|
||||
exit 0
|
||||
fi
|
||||
if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "empty" ]; then
|
||||
exit 0
|
||||
fi
|
||||
printf 'archive-data\n'
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "restic",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
case " $* " in
|
||||
*" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
|
||||
*" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "du",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
printf '4K\t%s\n' "${2:-unknown}"
|
||||
""",
|
||||
)
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TEST_BACKUP_BASE": str(backup_base),
|
||||
"TEST_DOCKER_LOG": str(docker_log),
|
||||
"TEST_DOCKER_START_MARKER": str(tmp_path / "docker-start-failed-once"),
|
||||
"FAKE_DOCKER_RUN_MODE": docker_run_mode,
|
||||
"FAKE_DOCKER_START_FAIL_ONCE": "1" if docker_start_fail_once else "0",
|
||||
"FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
|
||||
}
|
||||
process = subprocess.Popen(
|
||||
["bash", str(harness / SCRIPT.name)],
|
||||
env=env,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = process.communicate(timeout=10)
|
||||
result = subprocess.CompletedProcess(
|
||||
process.args,
|
||||
process.returncode,
|
||||
stdout,
|
||||
stderr,
|
||||
)
|
||||
docker_calls = docker_log.read_text(encoding="utf-8").splitlines()
|
||||
return result, docker_calls, Path(f"/tmp/signoz-backup-{process.pid}")
|
||||
|
||||
|
||||
def _collector_start_count(docker_calls: list[str]) -> int:
|
||||
return docker_calls.count("start signoz-otel-collector")
|
||||
|
||||
|
||||
def test_collector_is_restored_once_when_backup_receives_term(tmp_path: Path) -> None:
|
||||
result, docker_calls, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
docker_run_mode="signal",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "stop signoz-otel-collector" in docker_calls
|
||||
assert _collector_start_count(docker_calls) == 1
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_collector_is_restored_once_when_clickhouse_backup_fails(tmp_path: Path) -> None:
|
||||
result, docker_calls, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
docker_run_mode="empty",
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert _collector_start_count(docker_calls) == 1
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_exit_cleanup_does_not_restart_an_already_restored_collector(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, docker_calls, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
restic_backup_status=17,
|
||||
)
|
||||
|
||||
assert result.returncode == 17
|
||||
assert _collector_start_count(docker_calls) == 1
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_exit_cleanup_retries_only_after_explicit_restore_fails(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, docker_calls, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
docker_start_fail_once=True,
|
||||
restic_backup_status=17,
|
||||
)
|
||||
|
||||
assert result.returncode == 17
|
||||
assert _collector_start_count(docker_calls) == 2
|
||||
assert not dump_dir.exists()
|
||||
Reference in New Issue
Block a user