338 lines
14 KiB
Bash
Executable File
338 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
||
# =============================================================================
|
||
# WOOO AIOps - SigNoz ClickHouse native backup orchestrator
|
||
# Version: 2.0.0
|
||
#
|
||
# ClickHouse is backed up online through its native BACKUP engine. This script
|
||
# never reads or archives the live ClickHouse/SQLite Docker volumes. SigNoz
|
||
# metadata/SQLite remains an explicit application-consistent export gap and is
|
||
# recorded in the Restic payload without being misreported as completed.
|
||
# =============================================================================
|
||
|
||
set -euo pipefail
|
||
|
||
source "$(dirname "$0")/common.sh"
|
||
|
||
SERVICE="signoz"
|
||
LOCAL_REPO="${BACKUP_BASE}/signoz"
|
||
DUMP_DIR="/tmp/signoz-backup-$$"
|
||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}"
|
||
COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}"
|
||
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
|
||
COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}"
|
||
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
|
||
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"
|
||
RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}"
|
||
RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}"
|
||
RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}"
|
||
RESTIC_READBACK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_READBACK_TIMEOUT_SECONDS:-120}"
|
||
RESTIC_CHECK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_CHECK_TIMEOUT_SECONDS:-1800}"
|
||
|
||
BACKUP_ID_SEED="$(date -u '+%Y%m%dT%H%M%SZ')-$$"
|
||
TRACE_ID="${TRACE_ID:-signoz-backup-${BACKUP_ID_SEED}}"
|
||
RUN_ID="${RUN_ID:-signoz-${BACKUP_ID_SEED}}"
|
||
WORK_ITEM_ID="${WORK_ITEM_ID:-SIGNOZ-BACKUP}"
|
||
export TRACE_ID RUN_ID WORK_ITEM_ID
|
||
|
||
COLLECTOR_ID_BEFORE=""
|
||
COLLECTOR_STARTED_AT_BEFORE=""
|
||
COLLECTOR_RESTARTS_BEFORE=""
|
||
COLLECTOR_HEALTH_BEFORE=""
|
||
COLLECTOR_OBSERVER_PID=""
|
||
COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
|
||
CLEANUP_COMPLETE=0
|
||
FAILURE_NOTIFIED=0
|
||
|
||
run_collector_docker() {
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||
"${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
|
||
}
|
||
|
||
collector_identity_state() {
|
||
run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
|
||
"${COLLECTOR_NAME}" 2>/dev/null
|
||
}
|
||
|
||
notify_failure_once() {
|
||
local detail="$1"
|
||
if [ "${FAILURE_NOTIFIED}" -eq 1 ]; then
|
||
return 0
|
||
fi
|
||
FAILURE_NOTIFIED=1
|
||
notify_clawbot "failed" "${SERVICE}" "${detail}" || true
|
||
}
|
||
|
||
observe_collector_continuity() {
|
||
local epoch observation
|
||
|
||
while true; do
|
||
epoch="$(date +%s)"
|
||
if observation="$(collector_identity_state)"; then
|
||
printf '%s\t%s\n' "${epoch}" "${observation}" >> "${COLLECTOR_OBSERVATIONS}"
|
||
else
|
||
printf '%s\tunknown\tunknown\tunknown\tunknown\tunknown\n' "${epoch}" >> "${COLLECTOR_OBSERVATIONS}"
|
||
fi
|
||
sleep "${COLLECTOR_CONTINUITY_POLL_SECONDS}"
|
||
done
|
||
}
|
||
|
||
start_collector_observer() {
|
||
: > "${COLLECTOR_OBSERVATIONS}"
|
||
observe_collector_continuity &
|
||
COLLECTOR_OBSERVER_PID=$!
|
||
}
|
||
|
||
stop_collector_observer() {
|
||
if [ -n "${COLLECTOR_OBSERVER_PID}" ] \
|
||
&& kill -0 "${COLLECTOR_OBSERVER_PID}" 2>/dev/null; then
|
||
kill -TERM "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
|
||
wait "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
|
||
fi
|
||
COLLECTOR_OBSERVER_PID=""
|
||
}
|
||
|
||
verify_collector_continuity() {
|
||
local observation final_id final_running final_started_at final_restarts final_health
|
||
|
||
stop_collector_observer
|
||
[ -s "${COLLECTOR_OBSERVATIONS}" ] || {
|
||
log_error "${COLLECTOR_NAME} continuity receipt 為空"
|
||
return 1
|
||
}
|
||
if awk -F '\t' -v expected_id="${COLLECTOR_ID_BEFORE}" \
|
||
-v expected_started="${COLLECTOR_STARTED_AT_BEFORE}" \
|
||
-v expected_restarts="${COLLECTOR_RESTARTS_BEFORE}" \
|
||
-v expected_health="${COLLECTOR_HEALTH_BEFORE}" \
|
||
'$2 == "unknown" || $3 != "true" || $2 != expected_id || \
|
||
$4 != expected_started || $5 != expected_restarts || $6 != expected_health { exit 1 }' \
|
||
"${COLLECTOR_OBSERVATIONS}"; then
|
||
:
|
||
else
|
||
log_error "${COLLECTOR_NAME} 在 native backup window 發生停止或 identity drift"
|
||
return 1
|
||
fi
|
||
|
||
observation="$(collector_identity_state)" || return 1
|
||
IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${observation}"
|
||
[ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
|
||
&& [ "${final_running}" = "true" ] \
|
||
&& [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||
&& [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
|
||
&& [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ]
|
||
}
|
||
|
||
cleanup() {
|
||
local exit_code=$?
|
||
|
||
if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
|
||
return "${exit_code}"
|
||
fi
|
||
CLEANUP_COMPLETE=1
|
||
trap '' HUP INT TERM
|
||
stop_collector_observer
|
||
rm -rf -- "${DUMP_DIR}"
|
||
return "${exit_code}"
|
||
}
|
||
|
||
on_signal() {
|
||
local signal_number="$1"
|
||
exit "$((128 + signal_number))"
|
||
}
|
||
|
||
run_native_clickhouse_backup() {
|
||
local mode="$1"
|
||
|
||
[ -f "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||
&& [ -x "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||
&& [ ! -L "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||
|| {
|
||
log_error "ClickHouse native backup adapter 缺失、不可執行或為 symlink"
|
||
return 1
|
||
}
|
||
|
||
CLICKHOUSE_NATIVE_OUTPUT_DIR="${DUMP_DIR}" \
|
||
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
||
"${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" "${mode}"
|
||
}
|
||
|
||
write_metadata_gap_receipt() {
|
||
cat > "${DUMP_DIR}/signoz-metadata-gap.json" <<EOF
|
||
{"trace_id":"${TRACE_ID}","run_id":"${RUN_ID}","work_item_id":"${WORK_ITEM_ID}","asset":"signoz-metadata-sqlite","terminal":"pending_application_consistent_export","raw_volume_read":false,"raw_volume_archive":false,"completion_claim":false}
|
||
EOF
|
||
log_warn "SigNoz metadata/SQLite 尚缺 application-consistent export；本 run 未讀取 live volume"
|
||
}
|
||
|
||
write_restic_receipt() {
|
||
local snapshot_id="$1"
|
||
local artifact_hash="$2"
|
||
local manifest_hash="$3"
|
||
local inventory_hash="$4"
|
||
local receipt_dir receipt_tmp receipt_file
|
||
|
||
[[ "${RESTIC_RECEIPT_ROOT}" == /* ]] && [ ! -L "${RESTIC_RECEIPT_ROOT}" ] || return 1
|
||
mkdir -p "${RESTIC_RECEIPT_ROOT}"
|
||
receipt_dir="${RESTIC_RECEIPT_ROOT}/${RUN_ID}"
|
||
[ ! -L "${receipt_dir}" ] || return 1
|
||
mkdir -m 700 -p "${receipt_dir}"
|
||
receipt_file="${receipt_dir}/restic-snapshot.json"
|
||
receipt_tmp="${receipt_file}.partial.$$"
|
||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","service":"signoz","snapshot_id":"%s","artifact_sha256":"%s","manifest_sha256":"%s","source_inventory_sha256":"%s","repository_scope":"local_same_failure_domain","restic_check_read_data_subset":"1%%","offsite_verified":false,"metadata_sqlite_terminal":"pending_application_consistent_export","completion_claim":"clickhouse_native_only"}\n' \
|
||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${snapshot_id}" \
|
||
"${artifact_hash}" "${manifest_hash}" "${inventory_hash}" > "${receipt_tmp}"
|
||
chmod 600 "${receipt_tmp}"
|
||
mv "${receipt_tmp}" "${receipt_file}"
|
||
}
|
||
|
||
main() {
|
||
local start_time end_time duration snapshot_id snapshot_json tags
|
||
local initial_observation initial_running final_id final_running
|
||
local final_started_at final_restarts final_health
|
||
local artifact_file manifest_file inventory_file artifact_hash manifest_hash inventory_hash
|
||
|
||
start_time="$(date +%s)"
|
||
trap cleanup EXIT
|
||
trap 'on_signal 1' HUP
|
||
trap 'on_signal 2' INT
|
||
trap 'on_signal 15' TERM
|
||
|
||
log_info "========== 開始 SigNoz ClickHouse native 備份 =========="
|
||
[[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]] \
|
||
&& [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0" ]] \
|
||
&& [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0.0" ]] || {
|
||
notify_failure_once "COLLECTOR_CONTINUITY_POLL_SECONDS 無效"
|
||
exit 1
|
||
}
|
||
for timeout_value in "${RESTIC_INIT_TIMEOUT_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
|
||
"${RESTIC_READBACK_TIMEOUT_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}"; do
|
||
[[ "${timeout_value}" =~ ^[0-9]+$ ]] && [ "${timeout_value}" -gt 0 ] || {
|
||
notify_failure_once "Restic timeout contract 無效"
|
||
exit 1
|
||
}
|
||
done
|
||
mkdir -p "${DUMP_DIR}"
|
||
|
||
# Check-mode must pass before the backup window. It writes only durable
|
||
# receipts and does not submit BACKUP or create a server artifact.
|
||
if ! run_native_clickhouse_backup --check; then
|
||
log_error "ClickHouse native BACKUP check-mode 失敗；沒有 volume tar fallback"
|
||
notify_failure_once "SigNoz ClickHouse native BACKUP runtime contract 未就緒"
|
||
exit 1
|
||
fi
|
||
|
||
initial_observation="$(collector_identity_state)" || {
|
||
notify_failure_once "無法讀取 ${COLLECTOR_NAME} 初始 identity/state"
|
||
exit 1
|
||
}
|
||
IFS=$'\t' read -r COLLECTOR_ID_BEFORE initial_running COLLECTOR_STARTED_AT_BEFORE \
|
||
COLLECTOR_RESTARTS_BEFORE COLLECTOR_HEALTH_BEFORE <<< "${initial_observation}"
|
||
[ -n "${COLLECTOR_ID_BEFORE}" ] && [ "${initial_running}" = "true" ] || {
|
||
notify_failure_once "${COLLECTOR_NAME} 初始狀態不是 running"
|
||
exit 1
|
||
}
|
||
[ -n "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||
&& [[ "${COLLECTOR_RESTARTS_BEFORE}" =~ ^[0-9]+$ ]] \
|
||
&& [[ "${COLLECTOR_HEALTH_BEFORE}" =~ ^(healthy|none)$ ]] || {
|
||
notify_failure_once "${COLLECTOR_NAME} 初始 runtime metadata 不完整"
|
||
exit 1
|
||
}
|
||
start_collector_observer
|
||
|
||
# Online native backup: no collector stop and no live-volume read.
|
||
if ! run_native_clickhouse_backup --apply; then
|
||
notify_failure_once "SigNoz ClickHouse native BACKUP 失敗"
|
||
exit 1
|
||
fi
|
||
|
||
if ! verify_collector_continuity; then
|
||
notify_failure_once "${COLLECTOR_NAME} no-downtime continuity verifier 失敗"
|
||
exit 1
|
||
fi
|
||
log_success "${COLLECTOR_NAME} no-downtime continuity verifier 通過"
|
||
|
||
write_metadata_gap_receipt
|
||
|
||
artifact_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip' -print)"
|
||
manifest_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip.manifest.json' -print)"
|
||
inventory_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.source-inventory.tsv' -print)"
|
||
[ "$(printf '%s\n' "${artifact_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
|
||
&& [ "$(printf '%s\n' "${manifest_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
|
||
&& [ "$(printf '%s\n' "${inventory_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] || {
|
||
log_error "ClickHouse native artifact set 不唯一或不完整"
|
||
exit 1
|
||
}
|
||
artifact_hash="$(sha256sum "${artifact_file}" | awk '{print $1}')"
|
||
manifest_hash="$(sha256sum "${manifest_file}" | awk '{print $1}')"
|
||
inventory_hash="$(sha256sum "${inventory_file}" | awk '{print $1}')"
|
||
|
||
if [ ! -d "${LOCAL_REPO}/data" ]; then
|
||
log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_INIT_TIMEOUT_SECONDS}" \
|
||
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||
log_error "Restic 倉庫初始化失敗"
|
||
exit 1
|
||
}
|
||
fi
|
||
|
||
log_info "建立 ClickHouse native artifact 的 Restic 備份..."
|
||
tags="$(build_tags "${SERVICE}")"
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
|
||
restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
|
||
--password-file "${RESTIC_PASSWORD_FILE}" ${tags} \
|
||
--tag "trace:${TRACE_ID}" --tag "run:${RUN_ID}" \
|
||
--tag "work-item:${WORK_ITEM_ID}" 2>&1
|
||
|
||
snapshot_json="$(timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||
"${RESTIC_READBACK_TIMEOUT_SECONDS}" restic -r "${LOCAL_REPO}" snapshots \
|
||
--tag "run:${RUN_ID}" --latest 1 --json \
|
||
--password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)"
|
||
snapshot_id="$(printf '%s\n' "${snapshot_json}" \
|
||
| sed -n 's/.*"short_id":"\([^"]*\)".*/\1/p' | head -1)"
|
||
[[ "${snapshot_id}" =~ ^[A-Za-z0-9]+$ ]] || {
|
||
log_error "Restic same-run snapshot durable readback 缺失"
|
||
exit 1
|
||
}
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}" \
|
||
restic -r "${LOCAL_REPO}" check --read-data-subset=1% \
|
||
--password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||
log_error "Restic repository/data-subset verifier 失敗"
|
||
exit 1
|
||
}
|
||
write_restic_receipt "${snapshot_id}" "${artifact_hash}" \
|
||
"${manifest_hash}" "${inventory_hash}" || {
|
||
log_error "Restic durable receipt writeback 失敗"
|
||
exit 1
|
||
}
|
||
log_success "Restic 備份完成: ${snapshot_id}"
|
||
|
||
if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then
|
||
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
|
||
else
|
||
cleanup_old_backups "${LOCAL_REPO}"
|
||
fi
|
||
|
||
# Final readback is separate from the continuity observer receipt.
|
||
initial_observation="$(collector_identity_state)" || {
|
||
notify_failure_once "${COLLECTOR_NAME} final readback 失敗"
|
||
exit 1
|
||
}
|
||
IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${initial_observation}"
|
||
[ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
|
||
&& [ "${final_running}" = "true" ] \
|
||
&& [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||
&& [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
|
||
&& [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ] || {
|
||
notify_failure_once "${COLLECTOR_NAME} final identity/start/restart/health verifier 失敗"
|
||
exit 1
|
||
}
|
||
|
||
end_time="$(date +%s)"
|
||
duration=$((end_time - start_time))
|
||
log_success "========== SigNoz ClickHouse native 備份完成 (${duration}s) =========="
|
||
notify_clawbot "warning" "${SERVICE}" \
|
||
"ClickHouse native backup 完成；metadata/SQLite application-consistent export 仍 pending" \
|
||
"${duration}"
|
||
}
|
||
|
||
main "$@"
|