410 lines
17 KiB
Bash
Executable File
410 lines
17 KiB
Bash
Executable File
#!/bin/bash
|
||
# =============================================================================
|
||
# WOOO AIOps - SigNoz ClickHouse native backup orchestrator
|
||
# Version: 2.0.0
|
||
#
|
||
# ClickHouse is backed up online through its native BACKUP engine. This script
|
||
# never reads or archives the live ClickHouse/SQLite Docker volumes. SigNoz
|
||
# metadata/SQLite remains an explicit application-consistent export gap and is
|
||
# recorded in the Restic payload without being misreported as completed.
|
||
# =============================================================================
|
||
|
||
set -euo pipefail
|
||
|
||
source "$(dirname "$0")/common.sh"
|
||
|
||
SERVICE="signoz"
|
||
LOCAL_REPO="${BACKUP_BASE}/signoz"
|
||
TMP_ROOT="${SIGNOZ_BACKUP_TMP_ROOT:-/tmp}"
|
||
DUMP_DIR=""
|
||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||
CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}"
|
||
COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}"
|
||
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
|
||
COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}"
|
||
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
|
||
# Destructive Restic forget/prune is a separate critical break-glass workflow.
|
||
# The regular and canary backup paths are backup-only and fail closed if a
|
||
# caller attempts to re-enable inline retention cleanup.
|
||
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-1}"
|
||
OPERATION_LOCK_FILE="${SIGNOZ_BACKUP_OPERATION_LOCK_FILE:-/tmp/awoooi-signoz-backup-operation.lock}"
|
||
RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}"
|
||
RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}"
|
||
RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}"
|
||
RESTIC_READBACK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_READBACK_TIMEOUT_SECONDS:-120}"
|
||
RESTIC_CHECK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_CHECK_TIMEOUT_SECONDS:-1800}"
|
||
|
||
BACKUP_ID_SEED="$(date -u '+%Y%m%dT%H%M%SZ')-$$"
|
||
TRACE_ID="${TRACE_ID:-signoz-backup-${BACKUP_ID_SEED}}"
|
||
RUN_ID="${RUN_ID:-signoz-${BACKUP_ID_SEED}}"
|
||
WORK_ITEM_ID="${WORK_ITEM_ID:-SIGNOZ-BACKUP}"
|
||
export TRACE_ID RUN_ID WORK_ITEM_ID
|
||
|
||
COLLECTOR_ID_BEFORE=""
|
||
COLLECTOR_STARTED_AT_BEFORE=""
|
||
COLLECTOR_RESTARTS_BEFORE=""
|
||
COLLECTOR_HEALTH_BEFORE=""
|
||
COLLECTOR_OBSERVER_PID=""
|
||
COLLECTOR_OBSERVATIONS=""
|
||
WORKSPACE_CREATED=0
|
||
CLEANUP_COMPLETE=0
|
||
FAILURE_NOTIFIED=0
|
||
|
||
run_collector_docker() {
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||
"${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
|
||
}
|
||
|
||
collector_identity_state() {
|
||
run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
|
||
"${COLLECTOR_NAME}" 2>/dev/null
|
||
}
|
||
|
||
notify_failure_once() {
|
||
local detail="$1"
|
||
if [ "${FAILURE_NOTIFIED}" -eq 1 ]; then
|
||
return 0
|
||
fi
|
||
FAILURE_NOTIFIED=1
|
||
notify_clawbot "failed" "${SERVICE}" "${detail}" || true
|
||
}
|
||
|
||
observe_collector_continuity() {
|
||
local epoch observation
|
||
|
||
while true; do
|
||
epoch="$(date +%s)"
|
||
if observation="$(collector_identity_state)"; then
|
||
printf '%s\t%s\n' "${epoch}" "${observation}" >> "${COLLECTOR_OBSERVATIONS}"
|
||
else
|
||
printf '%s\tunknown\tunknown\tunknown\tunknown\tunknown\n' "${epoch}" >> "${COLLECTOR_OBSERVATIONS}"
|
||
fi
|
||
sleep "${COLLECTOR_CONTINUITY_POLL_SECONDS}"
|
||
done
|
||
}
|
||
|
||
start_collector_observer() {
|
||
: > "${COLLECTOR_OBSERVATIONS}"
|
||
observe_collector_continuity &
|
||
COLLECTOR_OBSERVER_PID=$!
|
||
}
|
||
|
||
stop_collector_observer() {
|
||
if [ -n "${COLLECTOR_OBSERVER_PID}" ] \
|
||
&& kill -0 "${COLLECTOR_OBSERVER_PID}" 2>/dev/null; then
|
||
kill -TERM "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
|
||
wait "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
|
||
fi
|
||
COLLECTOR_OBSERVER_PID=""
|
||
}
|
||
|
||
verify_collector_continuity() {
|
||
local observation final_id final_running final_started_at final_restarts final_health
|
||
|
||
stop_collector_observer
|
||
[ -s "${COLLECTOR_OBSERVATIONS}" ] || {
|
||
log_error "${COLLECTOR_NAME} continuity receipt 為空"
|
||
return 1
|
||
}
|
||
if awk -F '\t' -v expected_id="${COLLECTOR_ID_BEFORE}" \
|
||
-v expected_started="${COLLECTOR_STARTED_AT_BEFORE}" \
|
||
-v expected_restarts="${COLLECTOR_RESTARTS_BEFORE}" \
|
||
-v expected_health="${COLLECTOR_HEALTH_BEFORE}" \
|
||
'$2 == "unknown" || $3 != "true" || $2 != expected_id || \
|
||
$4 != expected_started || $5 != expected_restarts || $6 != expected_health { exit 1 }' \
|
||
"${COLLECTOR_OBSERVATIONS}"; then
|
||
:
|
||
else
|
||
log_error "${COLLECTOR_NAME} 在 native backup window 發生停止或 identity drift"
|
||
return 1
|
||
fi
|
||
|
||
observation="$(collector_identity_state)" || return 1
|
||
IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${observation}"
|
||
[ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
|
||
&& [ "${final_running}" = "true" ] \
|
||
&& [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||
&& [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
|
||
&& [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ]
|
||
}
|
||
|
||
cleanup() {
|
||
local exit_code=$?
|
||
|
||
if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
|
||
exit "${exit_code}"
|
||
fi
|
||
CLEANUP_COMPLETE=1
|
||
trap - EXIT HUP INT TERM
|
||
set +e
|
||
stop_collector_observer
|
||
if [ "${WORKSPACE_CREATED}" -eq 1 ]; then
|
||
if [ -n "${DUMP_DIR}" ] && [ -d "${DUMP_DIR}" ] \
|
||
&& [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
|
||
&& [[ "${DUMP_DIR}" == "${TMP_ROOT%/}"/signoz-backup.* ]]; then
|
||
rm -rf -- "${DUMP_DIR}" || exit_code=90
|
||
[ ! -e "${DUMP_DIR}" ] || exit_code=90
|
||
else
|
||
log_error "拒絕清理不符合本 run identity 的 SigNoz workspace"
|
||
exit_code=90
|
||
fi
|
||
fi
|
||
exit "${exit_code}"
|
||
}
|
||
|
||
on_signal() {
|
||
local signal_number="$1"
|
||
exit "$((128 + signal_number))"
|
||
}
|
||
|
||
create_private_workspace() {
|
||
[[ "${TMP_ROOT}" == /* ]] \
|
||
&& [[ ! "${TMP_ROOT}" =~ (^|/)\.\.?(/|$) ]] \
|
||
&& [[ "${TMP_ROOT}" != *$'\n'* ]] \
|
||
&& [ -d "${TMP_ROOT}" ] && [ -w "${TMP_ROOT}" ] \
|
||
&& [ ! -L "${TMP_ROOT}" ] || {
|
||
log_error "SigNoz workspace root 必須是安全、可寫、非 symlink 的絕對目錄"
|
||
return 1
|
||
}
|
||
|
||
DUMP_DIR="$(mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX")" || return 1
|
||
WORKSPACE_CREATED=1
|
||
[ -d "${DUMP_DIR}" ] && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
|
||
&& [ -r "${DUMP_DIR}" ] && [ -w "${DUMP_DIR}" ] && [ -x "${DUMP_DIR}" ] || {
|
||
log_error "mktemp 未建立本 run 擁有的 private SigNoz workspace"
|
||
return 1
|
||
}
|
||
chmod 700 "${DUMP_DIR}" || return 1
|
||
COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
|
||
log_info "建立 SigNoz private workspace: ${DUMP_DIR}"
|
||
}
|
||
|
||
run_native_clickhouse_backup() {
|
||
local mode="$1"
|
||
|
||
[ -f "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||
&& [ -x "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||
&& [ ! -L "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||
|| {
|
||
log_error "ClickHouse native backup adapter 缺失、不可執行或為 symlink"
|
||
return 1
|
||
}
|
||
|
||
CLICKHOUSE_NATIVE_OUTPUT_DIR="${DUMP_DIR}" \
|
||
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
||
"${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" "${mode}"
|
||
}
|
||
|
||
write_metadata_gap_receipt() {
|
||
cat > "${DUMP_DIR}/signoz-metadata-gap.json" <<EOF
|
||
{"trace_id":"${TRACE_ID}","run_id":"${RUN_ID}","work_item_id":"${WORK_ITEM_ID}","asset":"signoz-metadata-sqlite","terminal":"pending_application_consistent_export","raw_volume_read":false,"raw_volume_archive":false,"completion_claim":false}
|
||
EOF
|
||
log_warn "SigNoz metadata/SQLite 尚缺 application-consistent export；本 run 未讀取 live volume"
|
||
}
|
||
|
||
write_restic_receipt() {
|
||
local snapshot_id="$1"
|
||
local artifact_hash="$2"
|
||
local manifest_hash="$3"
|
||
local inventory_hash="$4"
|
||
local receipt_dir receipt_tmp receipt_file
|
||
|
||
[[ "${RESTIC_RECEIPT_ROOT}" == /* ]] && [ ! -L "${RESTIC_RECEIPT_ROOT}" ] || return 1
|
||
mkdir -p "${RESTIC_RECEIPT_ROOT}"
|
||
receipt_dir="${RESTIC_RECEIPT_ROOT}/${RUN_ID}"
|
||
[ ! -L "${receipt_dir}" ] || return 1
|
||
mkdir -m 700 -p "${receipt_dir}"
|
||
receipt_file="${receipt_dir}/restic-snapshot.json"
|
||
receipt_tmp="${receipt_file}.partial.$$"
|
||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","service":"signoz","snapshot_id":"%s","artifact_sha256":"%s","manifest_sha256":"%s","source_inventory_sha256":"%s","repository_scope":"local_same_failure_domain","restic_check_read_data_subset":"1%%","offsite_verified":false,"metadata_sqlite_terminal":"pending_application_consistent_export","completion_claim":"clickhouse_native_only"}\n' \
|
||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${snapshot_id}" \
|
||
"${artifact_hash}" "${manifest_hash}" "${inventory_hash}" > "${receipt_tmp}"
|
||
chmod 600 "${receipt_tmp}"
|
||
mv "${receipt_tmp}" "${receipt_file}"
|
||
}
|
||
|
||
acquire_operation_lock() {
|
||
case "${OPERATION_LOCK_FILE}" in
|
||
/*) ;;
|
||
*)
|
||
log_error "SigNoz operation lock path 必須是絕對路徑"
|
||
return 1
|
||
;;
|
||
esac
|
||
[ ! -L "${OPERATION_LOCK_FILE}" ] || {
|
||
log_error "SigNoz operation lock 不得是 symlink"
|
||
return 1
|
||
}
|
||
if [ -e "${OPERATION_LOCK_FILE}" ]; then
|
||
[ -f "${OPERATION_LOCK_FILE}" ] || {
|
||
log_error "SigNoz operation lock 必須是 regular file"
|
||
return 1
|
||
}
|
||
fi
|
||
exec 8>>"${OPERATION_LOCK_FILE}"
|
||
flock -n 8 || {
|
||
log_error "另一個 SigNoz backup/deploy operation 正在執行"
|
||
return 1
|
||
}
|
||
log_info "取得 SigNoz backup/deploy 共用 operation lock"
|
||
}
|
||
|
||
main() {
|
||
local start_time end_time duration snapshot_id snapshot_json tags
|
||
local initial_observation initial_running final_id final_running
|
||
local final_started_at final_restarts final_health
|
||
local artifact_file manifest_file inventory_file artifact_hash manifest_hash inventory_hash
|
||
|
||
start_time="$(date +%s)"
|
||
trap cleanup EXIT
|
||
trap 'on_signal 1' HUP
|
||
trap 'on_signal 2' INT
|
||
trap 'on_signal 15' TERM
|
||
|
||
log_info "========== 開始 SigNoz ClickHouse native 備份 =========="
|
||
[[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]] \
|
||
&& [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0" ]] \
|
||
&& [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0.0" ]] || {
|
||
notify_failure_once "COLLECTOR_CONTINUITY_POLL_SECONDS 無效"
|
||
exit 1
|
||
}
|
||
for timeout_value in "${RESTIC_INIT_TIMEOUT_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
|
||
"${RESTIC_READBACK_TIMEOUT_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}"; do
|
||
[[ "${timeout_value}" =~ ^[0-9]+$ ]] && [ "${timeout_value}" -gt 0 ] || {
|
||
notify_failure_once "Restic timeout contract 無效"
|
||
exit 1
|
||
}
|
||
done
|
||
[ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ] || {
|
||
notify_failure_once "inline Restic forget/prune 已停用；僅允許獨立 critical break-glass workflow"
|
||
exit 1
|
||
}
|
||
acquire_operation_lock || {
|
||
notify_failure_once "SigNoz backup/deploy operation lock 取得失敗"
|
||
exit 1
|
||
}
|
||
create_private_workspace || {
|
||
notify_failure_once "建立 private SigNoz backup workspace 失敗"
|
||
exit 1
|
||
}
|
||
|
||
# Check-mode must pass before the backup window. It writes only durable
|
||
# receipts and does not submit BACKUP or create a server artifact.
|
||
if ! run_native_clickhouse_backup --check; then
|
||
log_error "ClickHouse native BACKUP check-mode 失敗；沒有 volume tar fallback"
|
||
notify_failure_once "SigNoz ClickHouse native BACKUP runtime contract 未就緒"
|
||
exit 1
|
||
fi
|
||
|
||
initial_observation="$(collector_identity_state)" || {
|
||
notify_failure_once "無法讀取 ${COLLECTOR_NAME} 初始 identity/state"
|
||
exit 1
|
||
}
|
||
IFS=$'\t' read -r COLLECTOR_ID_BEFORE initial_running COLLECTOR_STARTED_AT_BEFORE \
|
||
COLLECTOR_RESTARTS_BEFORE COLLECTOR_HEALTH_BEFORE <<< "${initial_observation}"
|
||
[ -n "${COLLECTOR_ID_BEFORE}" ] && [ "${initial_running}" = "true" ] || {
|
||
notify_failure_once "${COLLECTOR_NAME} 初始狀態不是 running"
|
||
exit 1
|
||
}
|
||
[ -n "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||
&& [[ "${COLLECTOR_RESTARTS_BEFORE}" =~ ^[0-9]+$ ]] \
|
||
&& [[ "${COLLECTOR_HEALTH_BEFORE}" =~ ^(healthy|none)$ ]] || {
|
||
notify_failure_once "${COLLECTOR_NAME} 初始 runtime metadata 不完整"
|
||
exit 1
|
||
}
|
||
start_collector_observer
|
||
|
||
# Online native backup: no collector stop and no live-volume read.
|
||
if ! run_native_clickhouse_backup --apply; then
|
||
notify_failure_once "SigNoz ClickHouse native BACKUP 失敗"
|
||
exit 1
|
||
fi
|
||
|
||
if ! verify_collector_continuity; then
|
||
notify_failure_once "${COLLECTOR_NAME} no-downtime continuity verifier 失敗"
|
||
exit 1
|
||
fi
|
||
log_success "${COLLECTOR_NAME} no-downtime continuity verifier 通過"
|
||
|
||
write_metadata_gap_receipt
|
||
|
||
artifact_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip' -print)"
|
||
manifest_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip.manifest.json' -print)"
|
||
inventory_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.source-inventory.tsv' -print)"
|
||
[ "$(printf '%s\n' "${artifact_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
|
||
&& [ "$(printf '%s\n' "${manifest_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
|
||
&& [ "$(printf '%s\n' "${inventory_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] || {
|
||
log_error "ClickHouse native artifact set 不唯一或不完整"
|
||
exit 1
|
||
}
|
||
artifact_hash="$(sha256sum "${artifact_file}" | awk '{print $1}')"
|
||
manifest_hash="$(sha256sum "${manifest_file}" | awk '{print $1}')"
|
||
inventory_hash="$(sha256sum "${inventory_file}" | awk '{print $1}')"
|
||
|
||
if [ ! -d "${LOCAL_REPO}/data" ]; then
|
||
log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_INIT_TIMEOUT_SECONDS}" \
|
||
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||
log_error "Restic 倉庫初始化失敗"
|
||
exit 1
|
||
}
|
||
fi
|
||
|
||
log_info "建立 ClickHouse native artifact 的 Restic 備份..."
|
||
tags="$(build_tags "${SERVICE}")"
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
|
||
restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
|
||
--password-file "${RESTIC_PASSWORD_FILE}" ${tags} \
|
||
--tag "trace:${TRACE_ID}" --tag "run:${RUN_ID}" \
|
||
--tag "work-item:${WORK_ITEM_ID}" 2>&1
|
||
|
||
snapshot_json="$(timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||
"${RESTIC_READBACK_TIMEOUT_SECONDS}" restic -r "${LOCAL_REPO}" snapshots \
|
||
--tag "run:${RUN_ID}" --latest 1 --json \
|
||
--password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)"
|
||
snapshot_id="$(printf '%s\n' "${snapshot_json}" \
|
||
| sed -n 's/.*"short_id":"\([^"]*\)".*/\1/p' | head -1)"
|
||
[[ "${snapshot_id}" =~ ^[A-Za-z0-9]+$ ]] || {
|
||
log_error "Restic same-run snapshot durable readback 缺失"
|
||
exit 1
|
||
}
|
||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}" \
|
||
restic -r "${LOCAL_REPO}" check --read-data-subset=1% \
|
||
--password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||
log_error "Restic repository/data-subset verifier 失敗"
|
||
exit 1
|
||
}
|
||
write_restic_receipt "${snapshot_id}" "${artifact_hash}" \
|
||
"${manifest_hash}" "${inventory_hash}" || {
|
||
log_error "Restic durable receipt writeback 失敗"
|
||
exit 1
|
||
}
|
||
log_success "Restic 備份完成: ${snapshot_id}"
|
||
|
||
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
|
||
|
||
# Final readback is separate from the continuity observer receipt.
|
||
initial_observation="$(collector_identity_state)" || {
|
||
notify_failure_once "${COLLECTOR_NAME} final readback 失敗"
|
||
exit 1
|
||
}
|
||
IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${initial_observation}"
|
||
[ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
|
||
&& [ "${final_running}" = "true" ] \
|
||
&& [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||
&& [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
|
||
&& [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ] || {
|
||
notify_failure_once "${COLLECTOR_NAME} final identity/start/restart/health verifier 失敗"
|
||
exit 1
|
||
}
|
||
|
||
end_time="$(date +%s)"
|
||
duration=$((end_time - start_time))
|
||
log_success "========== SigNoz ClickHouse native 備份完成 (${duration}s) =========="
|
||
notify_clawbot "warning" "${SERVICE}" \
|
||
"ClickHouse native backup 完成；metadata/SQLite application-consistent export 仍 pending" \
|
||
"${duration}"
|
||
}
|
||
|
||
main "$@"
|