301 lines
10 KiB
Bash
Executable File
301 lines
10 KiB
Bash
Executable File
#!/bin/bash
|
||
# =============================================================================
|
||
# WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
|
||
# 版本: 1.3.0
|
||
# 建立日期: 2026-04-05
|
||
# 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
|
||
# 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
|
||
# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
|
||
# 2026-07-15 Codex: v1.3 加入初始狀態、全 run 有界還原與 success 前獨立 verifier
|
||
# =============================================================================
|
||
|
||
set -euo pipefail
|
||
|
||
source "$(dirname "$0")/common.sh"
|
||
|
||
SERVICE="signoz"
|
||
LOCAL_REPO="${BACKUP_BASE}/signoz"
|
||
DUMP_DIR="/tmp/signoz-backup-$$"
|
||
COLLECTOR_NAME="signoz-otel-collector"
|
||
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
|
||
COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"
|
||
COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}"
|
||
ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"
|
||
ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"
|
||
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
|
||
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"
|
||
COLLECTOR_WAS_RUNNING=0
|
||
COLLECTOR_RESTORE_PENDING=0
|
||
COLLECTOR_RESTORE_ATTEMPTS_USED=0
|
||
CLEANUP_COMPLETE=0
|
||
FAILURE_NOTIFIED=0
|
||
|
||
run_collector_docker() {
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||
"${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
|
||
}
|
||
|
||
collector_running_state() {
|
||
local running
|
||
if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
|
||
return 1
|
||
fi
|
||
|
||
case "${running}" in
|
||
true)
|
||
printf '%s\n' "running"
|
||
;;
|
||
false)
|
||
printf '%s\n' "stopped"
|
||
;;
|
||
*)
|
||
return 1
|
||
;;
|
||
esac
|
||
}
|
||
|
||
collector_is_running() {
|
||
[ "$(collector_running_state)" = "running" ]
|
||
}
|
||
|
||
notify_failure_once() {
|
||
local detail="$1"
|
||
if [ "${FAILURE_NOTIFIED}" -eq 1 ]; then
|
||
return 0
|
||
fi
|
||
FAILURE_NOTIFIED=1
|
||
notify_clawbot "failed" "${SERVICE}" "${detail}" || true
|
||
}
|
||
|
||
restore_collector() {
|
||
if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
|
||
return 0
|
||
fi
|
||
|
||
while [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; do
|
||
COLLECTOR_RESTORE_ATTEMPTS_USED=$((COLLECTOR_RESTORE_ATTEMPTS_USED + 1))
|
||
log_info "重啟 ${COLLECTOR_NAME} (${COLLECTOR_RESTORE_ATTEMPTS_USED}/${COLLECTOR_RESTORE_MAX_ATTEMPTS})..."
|
||
|
||
if run_collector_docker start "${COLLECTOR_NAME}" >/dev/null 2>&1 \
|
||
&& collector_is_running; then
|
||
COLLECTOR_RESTORE_PENDING=0
|
||
return 0
|
||
fi
|
||
|
||
log_warn "${COLLECTOR_NAME} 重啟或 running readback 失敗"
|
||
if [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; then
|
||
sleep "${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS}"
|
||
fi
|
||
done
|
||
|
||
log_error "${COLLECTOR_NAME} 在有界重試後仍無法恢復"
|
||
return 1
|
||
}
|
||
|
||
verify_collector_final_state() {
|
||
if [ "${COLLECTOR_WAS_RUNNING}" -ne 1 ]; then
|
||
log_info "${COLLECTOR_NAME} 原本即為 stopped，不執行自動啟動"
|
||
return 0
|
||
fi
|
||
|
||
if collector_is_running; then
|
||
log_success "${COLLECTOR_NAME} final running verifier 通過"
|
||
return 0
|
||
fi
|
||
|
||
log_warn "${COLLECTOR_NAME} final running verifier 首次失敗，使用剩餘有界還原額度"
|
||
COLLECTOR_RESTORE_PENDING=1
|
||
if ! restore_collector; then
|
||
return 1
|
||
fi
|
||
|
||
# 與 docker start 的回傳值與 restore 內部 readback 分離，作為 success 前獨立 verifier。
|
||
collector_is_running
|
||
}
|
||
|
||
cleanup() {
|
||
local exit_code=$?
|
||
|
||
if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
|
||
return "${exit_code}"
|
||
fi
|
||
CLEANUP_COMPLETE=1
|
||
|
||
# cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
|
||
trap '' HUP INT TERM
|
||
if ! restore_collector; then
|
||
notify_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}"
|
||
fi
|
||
rm -rf "${DUMP_DIR}"
|
||
return "${exit_code}"
|
||
}
|
||
|
||
on_signal() {
|
||
local signal_number="$1"
|
||
exit "$((128 + signal_number))"
|
||
}
|
||
|
||
create_volume_archive() {
|
||
local volume_name="$1"
|
||
local output_file="$2"
|
||
local extra_exclude="${3:-}"
|
||
local partial_file="${output_file}.partial"
|
||
|
||
log_info "備份 volume: ${volume_name}"
|
||
rm -f "${partial_file}" "${output_file}"
|
||
|
||
if [ -n "${extra_exclude}" ]; then
|
||
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||
"${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
|
||
docker run --rm -v "${volume_name}:/data" alpine \
|
||
tar czf - "${extra_exclude}" /data \
|
||
> "${partial_file}" 2>/dev/null; then
|
||
log_error " Volume ${volume_name} archive 建立失敗"
|
||
rm -f "${partial_file}" "${output_file}"
|
||
return 1
|
||
fi
|
||
else
|
||
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||
"${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
|
||
docker run --rm -v "${volume_name}:/data" alpine \
|
||
tar czf - /data \
|
||
> "${partial_file}" 2>/dev/null; then
|
||
log_error " Volume ${volume_name} archive 建立失敗"
|
||
rm -f "${partial_file}" "${output_file}"
|
||
return 1
|
||
fi
|
||
fi
|
||
|
||
if [ ! -s "${partial_file}" ]; then
|
||
log_error " Volume ${volume_name} 備份失敗 (空檔案)"
|
||
rm -f "${partial_file}" "${output_file}"
|
||
return 1
|
||
fi
|
||
|
||
mv "${partial_file}" "${output_file}"
|
||
}
|
||
|
||
verify_volume_archive() {
|
||
local volume_name="$1"
|
||
local output_file="$2"
|
||
|
||
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||
"${ARCHIVE_VERIFY_TIMEOUT_SECONDS}" \
|
||
tar -tzf "${output_file}" >/dev/null 2>&1; then
|
||
log_error " Volume ${volume_name} archive integrity verifier 失敗"
|
||
rm -f "${output_file}" "${output_file}.partial"
|
||
return 1
|
||
fi
|
||
|
||
local size
|
||
size="$(du -h "${output_file}" | cut -f1)"
|
||
log_success " Volume ${volume_name} 備份與 integrity verifier 完成 (${size})"
|
||
}
|
||
|
||
main() {
|
||
local start_time=$(date +%s)
|
||
|
||
trap cleanup EXIT
|
||
trap 'on_signal 1' HUP
|
||
trap 'on_signal 2' INT
|
||
trap 'on_signal 15' TERM
|
||
|
||
log_info "========== 開始 SignOz 備份 =========="
|
||
mkdir -p "${DUMP_DIR}"
|
||
|
||
local timestamp=$(date "+%Y%m%d_%H%M%S")
|
||
|
||
# Step 1: 只在 Collector 原本 running 時暫停，並在 stop 前 arm 還原。
|
||
local collector_initial_state
|
||
if ! collector_initial_state="$(collector_running_state)"; then
|
||
log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態"
|
||
notify_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態"
|
||
exit 1
|
||
fi
|
||
|
||
if [ "${collector_initial_state}" = "running" ]; then
|
||
COLLECTOR_WAS_RUNNING=1
|
||
COLLECTOR_RESTORE_PENDING=1
|
||
log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..."
|
||
if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then
|
||
log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止"
|
||
notify_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}"
|
||
exit 1
|
||
fi
|
||
else
|
||
log_error "${COLLECTOR_NAME} 初始狀態為 stopped，停止備份且不自動啟動"
|
||
notify_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止"
|
||
exit 1
|
||
fi
|
||
docker stop signoz-telemetrystore-migrator 2>/dev/null || true
|
||
|
||
# Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
|
||
local clickhouse_archive="${DUMP_DIR}/clickhouse_${timestamp}.tar.gz"
|
||
local sqlite_archive="${DUMP_DIR}/sqlite_${timestamp}.tar.gz"
|
||
create_volume_archive "signoz-clickhouse" "${clickhouse_archive}" "--exclude=/data/tmp" || {
|
||
log_error "ClickHouse volume 備份失敗"
|
||
notify_failure_once "SignOz ClickHouse 備份失敗"
|
||
exit 1
|
||
}
|
||
|
||
# Step 3: 備份 SQLite volume (SignOz metadata)
|
||
create_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
|
||
log_error "SQLite volume 備份失敗"
|
||
notify_failure_once "SignOz SQLite 備份失敗"
|
||
exit 1
|
||
}
|
||
|
||
# Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。
|
||
if ! restore_collector; then
|
||
notify_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}"
|
||
exit 1
|
||
fi
|
||
|
||
# Step 5: Collector 恢復後才在 host 做有界 archive integrity verifier。
|
||
verify_volume_archive "signoz-clickhouse" "${clickhouse_archive}" || {
|
||
notify_failure_once "SignOz ClickHouse archive integrity verifier 失敗"
|
||
exit 1
|
||
}
|
||
verify_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
|
||
notify_failure_once "SignOz SQLite archive integrity verifier 失敗"
|
||
exit 1
|
||
}
|
||
|
||
# Step 6: 初始化 Restic 倉庫
|
||
if [ ! -d "${LOCAL_REPO}/data" ]; then
|
||
log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
|
||
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||
log_error "Restic 倉庫初始化失敗"
|
||
exit 1
|
||
}
|
||
fi
|
||
|
||
# Step 7: Restic 備份
|
||
log_info "建立 Restic 備份..."
|
||
local tags=$(build_tags "${SERVICE}")
|
||
restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" --password-file "${RESTIC_PASSWORD_FILE}" ${tags} 2>&1
|
||
|
||
local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1)
|
||
log_success "Restic 備份完成: ${snapshot_id}"
|
||
|
||
# Step 8: GFS 清理；real canary 可明確跳過 destructive retention。
|
||
if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then
|
||
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
|
||
else
|
||
cleanup_old_backups "${LOCAL_REPO}"
|
||
fi
|
||
|
||
# Step 9: success 前獨立檢查 runtime state；失敗時不得發送 success。
|
||
if ! verify_collector_final_state; then
|
||
notify_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗"
|
||
exit 1
|
||
fi
|
||
|
||
local end_time=$(date +%s)
|
||
local duration=$((end_time - start_time))
|
||
log_success "========== SignOz 備份完成 (${duration}s) =========="
|
||
notify_clawbot "success" "${SERVICE}" "SignOz 備份完成 (ClickHouse+SQLite)" "${duration}"
|
||
}
|
||
|
||
main "$@"
|