629 lines
30 KiB
Bash
Executable File
629 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# P0-OBS-002 - ClickHouse native BACKUP adapter for SigNoz
|
|
#
|
|
# This adapter deliberately has no live-volume tar fallback. It requires a
|
|
# ClickHouse Disk configured and allowed for BACKUP, records every bounded
|
|
# command's stdout/stderr/exit status, and copies only the completed native
|
|
# backup artifact into the caller's staging directory.
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
CONTAINER_NAME="${CLICKHOUSE_NATIVE_CONTAINER_NAME:-signoz-clickhouse}"
|
|
BACKUP_DISK="${CLICKHOUSE_NATIVE_BACKUP_DISK:-backups}"
|
|
EXPECTED_DISK_PATH="${CLICKHOUSE_NATIVE_EXPECTED_DISK_PATH:-/backups/}"
|
|
OUTPUT_DIR="${CLICKHOUSE_NATIVE_OUTPUT_DIR:-}"
|
|
RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}"
|
|
COMMAND_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS:-30}"
|
|
BACKUP_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS:-1800}"
|
|
POLL_INTERVAL_SECONDS="${CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS:-5}"
|
|
KILL_AFTER_SECONDS="${CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS:-30}"
|
|
POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}"
|
|
EXPECTED_DATABASES_CSV="signoz_analytics,signoz_logs,signoz_metadata,signoz_meter,signoz_metrics,signoz_traces"
|
|
EXPECTED_DATABASES_JSON='["signoz_analytics","signoz_logs","signoz_metadata","signoz_meter","signoz_metrics","signoz_traces"]'
|
|
BACKUP_SCOPE_SQL="DATABASE signoz_analytics,DATABASE signoz_logs,DATABASE signoz_metadata,DATABASE signoz_meter,DATABASE signoz_metrics,DATABASE signoz_traces"
|
|
|
|
TRACE_ID="${TRACE_ID:-}"
|
|
RUN_ID="${RUN_ID:-}"
|
|
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
|
|
|
MODE=""
|
|
IDENTITY_HASH=""
|
|
OPERATION_ID=""
|
|
ARTIFACT_NAME=""
|
|
SERVER_ARTIFACT_PATH=""
|
|
OUTPUT_FILE=""
|
|
MANIFEST_FILE=""
|
|
HASH_FILE=""
|
|
INVENTORY_FILE=""
|
|
OUTPUT_PARTIAL=""
|
|
MANIFEST_PARTIAL=""
|
|
INVENTORY_PARTIAL=""
|
|
RECEIPT_DIR=""
|
|
RECEIPT_LOG=""
|
|
IDENTITY_FILE=""
|
|
INVOCATION_ID=""
|
|
|
|
LAST_STDOUT=""
|
|
LAST_STDERR=""
|
|
LAST_STATUS_FILE=""
|
|
LAST_EXIT_CODE=0
|
|
COMMAND_SEQUENCE=0
|
|
|
|
CONTAINER_ID=""
|
|
CLICKHOUSE_VERSION=""
|
|
DATABASES_HASH=""
|
|
SOURCE_INVENTORY_RECEIPT=""
|
|
SOURCE_INVENTORY_HASH=""
|
|
STATUS_PRESENT=0
|
|
BACKUP_NAME=""
|
|
BACKUP_STATUS=""
|
|
BACKUP_NUM_FILES="0"
|
|
BACKUP_TOTAL_SIZE="0"
|
|
BACKUP_UNCOMPRESSED_SIZE="0"
|
|
BACKUP_COMPRESSED_SIZE="0"
|
|
BACKUP_ERROR_HEX="none"
|
|
BACKUP_ERROR_HASH=""
|
|
|
|
RECEIPTS_READY=0
|
|
CHECK_PASS=0
|
|
APPLY_PASS=0
|
|
OUTPUT_CREATED=0
|
|
SERVER_ARTIFACT_ACTIVE=0
|
|
|
|
log_info() { printf '[INFO] %s\n' "$*"; }
|
|
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
|
|
|
valid_identity() {
|
|
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
|
}
|
|
|
|
valid_name() {
|
|
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]
|
|
}
|
|
|
|
valid_positive_integer() {
|
|
[[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
|
|
}
|
|
|
|
emit_receipt() {
|
|
local stage="$1"
|
|
local terminal="$2"
|
|
local detail="$3"
|
|
local observed_at
|
|
|
|
[ "${RECEIPTS_READY}" -eq 1 ] || return 0
|
|
observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
|
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${INVOCATION_ID}" \
|
|
"${observed_at}" "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}"
|
|
}
|
|
|
|
fail_closed() {
|
|
local detail="$1"
|
|
log_error "${detail}"
|
|
emit_receipt "failure" "failed" "${detail}"
|
|
exit 1
|
|
}
|
|
|
|
require_command() {
|
|
command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
|
|
}
|
|
|
|
cleanup() {
|
|
local exit_code=$?
|
|
local terminal="failed"
|
|
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
[ -z "${OUTPUT_PARTIAL}" ] || rm -f -- "${OUTPUT_PARTIAL}"
|
|
[ -z "${MANIFEST_PARTIAL}" ] || rm -f -- "${MANIFEST_PARTIAL}"
|
|
[ -z "${INVENTORY_PARTIAL}" ] || rm -f -- "${INVENTORY_PARTIAL}"
|
|
|
|
if [ "${OUTPUT_CREATED}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then
|
|
rm -f -- "${OUTPUT_FILE}" "${MANIFEST_FILE}" "${HASH_FILE}" "${INVENTORY_FILE}"
|
|
emit_receipt "cleanup" "removed" "unverified_local_artifact_removed"
|
|
fi
|
|
|
|
if [ "${SERVER_ARTIFACT_ACTIVE}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then
|
|
emit_receipt "cleanup" "preserved" \
|
|
"active_or_unverified_server_artifact_preserved_for_safe_followup"
|
|
fi
|
|
|
|
if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_PASS}" -eq 1 ]; then
|
|
terminal="check_pass_no_write"
|
|
elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "apply" ] && [ "${APPLY_PASS}" -eq 1 ]; then
|
|
terminal="pass"
|
|
fi
|
|
|
|
emit_receipt "terminal" "${terminal}" "clickhouse_native_backup_${terminal}"
|
|
printf 'CLICKHOUSE_NATIVE_BACKUP_TERMINAL trace_id=%s run_id=%s work_item_id=%s mode=%s terminal=%s exit_code=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" "${terminal}" "${exit_code}"
|
|
exit "${exit_code}"
|
|
}
|
|
|
|
on_signal() {
|
|
local signal_number="$1"
|
|
emit_receipt "signal" "failed" "signal_${signal_number}"
|
|
exit "$((128 + signal_number))"
|
|
}
|
|
|
|
setup() {
|
|
local identity_content identity_tmp
|
|
|
|
[ "$#" -eq 1 ] || { log_error "exactly one mode is required: --check or --apply"; exit 2; }
|
|
case "$1" in
|
|
--check) MODE="check" ;;
|
|
--apply) MODE="apply" ;;
|
|
*) log_error "unsupported mode: use --check or --apply"; exit 2 ;;
|
|
esac
|
|
|
|
valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and path-safe"; exit 2; }
|
|
valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and path-safe"; exit 2; }
|
|
valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and path-safe"; exit 2; }
|
|
valid_name "${CONTAINER_NAME}" || { log_error "invalid container name"; exit 2; }
|
|
valid_name "${BACKUP_DISK}" || { log_error "invalid backup disk name"; exit 2; }
|
|
valid_positive_integer "${COMMAND_TIMEOUT_SECONDS}" || { log_error "invalid command timeout"; exit 2; }
|
|
valid_positive_integer "${BACKUP_TIMEOUT_SECONDS}" || { log_error "invalid backup timeout"; exit 2; }
|
|
valid_positive_integer "${POLL_INTERVAL_SECONDS}" || { log_error "invalid poll interval"; exit 2; }
|
|
valid_positive_integer "${KILL_AFTER_SECONDS}" || { log_error "invalid kill-after timeout"; exit 2; }
|
|
[[ "${EXPECTED_DISK_PATH}" == /*/ ]] && [[ "${EXPECTED_DISK_PATH}" != *".."* ]] \
|
|
|| { log_error "expected disk path must be an absolute trailing-slash path without dot-dot"; exit 2; }
|
|
[ -n "${OUTPUT_DIR}" ] && [[ "${OUTPUT_DIR}" == /* ]] \
|
|
|| { log_error "CLICKHOUSE_NATIVE_OUTPUT_DIR must be absolute"; exit 2; }
|
|
[ -d "${OUTPUT_DIR}" ] && [ -w "${OUTPUT_DIR}" ] && [ ! -L "${OUTPUT_DIR}" ] \
|
|
|| { log_error "output directory missing, unwritable, or symlink"; exit 2; }
|
|
if [ -n "${POST_VERIFY_HOOK}" ]; then
|
|
[[ "${POST_VERIFY_HOOK}" == /* ]] && [ -f "${POST_VERIFY_HOOK}" ] \
|
|
&& [ -x "${POST_VERIFY_HOOK}" ] && [ ! -L "${POST_VERIFY_HOOK}" ] \
|
|
|| { log_error "post verifier hook must be an absolute executable regular file"; exit 2; }
|
|
fi
|
|
|
|
for command_name in awk basename cat chmod cmp cp date docker env flock grep mkdir mktemp mv rm sha256sum sleep timeout touch tr unzip wc; do
|
|
command -v "${command_name}" >/dev/null 2>&1 \
|
|
|| { log_error "required command missing: ${command_name}"; exit 2; }
|
|
done
|
|
|
|
IDENTITY_HASH="$(printf '%s\000%s\000%s' "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" | sha256sum | awk '{print $1}')"
|
|
OPERATION_ID="awoooi-${IDENTITY_HASH}"
|
|
ARTIFACT_NAME="signoz-${IDENTITY_HASH}.zip"
|
|
SERVER_ARTIFACT_PATH="${EXPECTED_DISK_PATH%/}/${ARTIFACT_NAME}"
|
|
OUTPUT_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.zip"
|
|
MANIFEST_FILE="${OUTPUT_FILE}.manifest.json"
|
|
HASH_FILE="${OUTPUT_FILE}.sha256"
|
|
INVENTORY_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.source-inventory.tsv"
|
|
OUTPUT_PARTIAL="${OUTPUT_FILE}.partial"
|
|
MANIFEST_PARTIAL="${MANIFEST_FILE}.partial"
|
|
INVENTORY_PARTIAL="${INVENTORY_FILE}.partial"
|
|
|
|
[ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root symlink is unsafe"; exit 2; }
|
|
mkdir -p "${RECEIPT_ROOT}"
|
|
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
|
[ ! -L "${RECEIPT_DIR}" ] || { log_error "receipt directory symlink is unsafe"; exit 2; }
|
|
mkdir -m 700 -p "${RECEIPT_DIR}"
|
|
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
|
IDENTITY_FILE="${RECEIPT_DIR}/identity.json"
|
|
INVOCATION_ID="${MODE}-$(date -u '+%Y%m%dT%H%M%SZ')-$$"
|
|
|
|
identity_content="{\"trace_id\":\"${TRACE_ID}\",\"run_id\":\"${RUN_ID}\",\"work_item_id\":\"${WORK_ITEM_ID}\",\"identity_hash\":\"${IDENTITY_HASH}\",\"operation_id\":\"${OPERATION_ID}\",\"artifact_name\":\"${ARTIFACT_NAME}\"}"
|
|
if [ -e "${IDENTITY_FILE}" ]; then
|
|
[ -f "${IDENTITY_FILE}" ] && [ ! -L "${IDENTITY_FILE}" ] \
|
|
|| { log_error "existing identity receipt is unsafe"; exit 2; }
|
|
[ "$(cat "${IDENTITY_FILE}")" = "${identity_content}" ] \
|
|
|| { log_error "run identity conflicts with durable receipt"; exit 2; }
|
|
else
|
|
identity_tmp="$(mktemp "${RECEIPT_DIR}/.identity.XXXXXX")"
|
|
printf '%s\n' "${identity_content}" > "${identity_tmp}"
|
|
chmod 600 "${identity_tmp}"
|
|
mv "${identity_tmp}" "${IDENTITY_FILE}"
|
|
fi
|
|
touch "${RECEIPT_LOG}"
|
|
RECEIPTS_READY=1
|
|
|
|
[ ! -L "${RECEIPT_DIR}/run.lock" ] || fail_closed "run_lock_symlink_unsafe"
|
|
exec 9>>"${RECEIPT_DIR}/run.lock"
|
|
flock -n 9 || fail_closed "same_run_native_backup_is_active"
|
|
|
|
trap cleanup EXIT
|
|
trap 'on_signal 1' HUP
|
|
trap 'on_signal 2' INT
|
|
trap 'on_signal 15' TERM
|
|
}
|
|
|
|
run_captured() {
|
|
local step="$1"
|
|
local stdout_hash stderr_hash terminal
|
|
shift
|
|
|
|
valid_name "${step}" || fail_closed "invalid_command_step"
|
|
COMMAND_SEQUENCE=$((COMMAND_SEQUENCE + 1))
|
|
LAST_STDOUT="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stdout"
|
|
LAST_STDERR="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stderr"
|
|
LAST_STATUS_FILE="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.status"
|
|
|
|
set +e
|
|
"$@" >"${LAST_STDOUT}" 2>"${LAST_STDERR}"
|
|
LAST_EXIT_CODE=$?
|
|
set -e
|
|
printf '%s\n' "${LAST_EXIT_CODE}" > "${LAST_STATUS_FILE}"
|
|
stdout_hash="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')"
|
|
stderr_hash="$(sha256sum "${LAST_STDERR}" | awk '{print $1}')"
|
|
terminal="pass"
|
|
[ "${LAST_EXIT_CODE}" -eq 0 ] || terminal="failed"
|
|
emit_receipt "command" "${terminal}" \
|
|
"step_${step}_exit_${LAST_EXIT_CODE}_stdout_${stdout_hash}_stderr_${stderr_hash}"
|
|
return "${LAST_EXIT_CODE}"
|
|
}
|
|
|
|
run_sql() {
|
|
local step="$1"
|
|
local query="$2"
|
|
shift 2
|
|
run_captured "${step}" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
|
clickhouse-client --format TSVRaw "$@" --query "${query}"
|
|
}
|
|
|
|
perform_check() {
|
|
local inspect_line running disk_path system_backups_count database_count backup_grant
|
|
local expected_databases_file
|
|
|
|
run_captured "container" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" docker inspect \
|
|
--format $'{{.Id}}\t{{.State.Running}}' "${CONTAINER_NAME}" \
|
|
|| fail_closed "clickhouse_container_inspect_failed"
|
|
inspect_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
|
IFS=$'\t' read -r CONTAINER_ID running <<< "${inspect_line}"
|
|
[ -n "${CONTAINER_ID}" ] && [ "${running}" = "true" ] \
|
|
|| fail_closed "clickhouse_container_not_running_or_identity_ambiguous"
|
|
|
|
run_sql "version" 'SELECT version()' || fail_closed "clickhouse_version_query_failed"
|
|
CLICKHOUSE_VERSION="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
|
[ -n "${CLICKHOUSE_VERSION}" ] || fail_closed "clickhouse_version_empty"
|
|
|
|
run_sql "system_backups" \
|
|
"SELECT count() FROM system.tables WHERE database='system' AND name='backups'" \
|
|
|| fail_closed "system_backups_capability_query_failed"
|
|
system_backups_count="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
|
[ "${system_backups_count}" = "1" ] || fail_closed "native_backup_capability_absent"
|
|
|
|
run_sql "backup_grant" 'CHECK GRANT BACKUP ON *.*' \
|
|
|| fail_closed "native_backup_grant_check_failed"
|
|
backup_grant="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
|
[ "${backup_grant}" = "1" ] || fail_closed "native_backup_grant_absent"
|
|
|
|
run_sql "backup_disk" \
|
|
'SELECT path FROM system.disks WHERE name={disk:String}' \
|
|
--param_disk "${BACKUP_DISK}" || fail_closed "backup_disk_query_failed"
|
|
disk_path="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
|
[ "${disk_path}" = "${EXPECTED_DISK_PATH}" ] \
|
|
|| fail_closed "backup_disk_missing_or_path_mismatch"
|
|
|
|
run_captured "disk_readable" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
|
test -d "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_absent"
|
|
run_captured "disk_writable" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
|
test -w "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_unwritable"
|
|
|
|
run_sql "databases" \
|
|
"SELECT name FROM system.databases WHERE name NOT IN ('system','information_schema','INFORMATION_SCHEMA','default') ORDER BY name" \
|
|
|| fail_closed "user_database_inventory_query_failed"
|
|
database_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
|
|
expected_databases_file="${RECEIPT_DIR}/${INVOCATION_ID}-expected-databases.tsv"
|
|
printf '%s\n' signoz_analytics signoz_logs signoz_metadata signoz_meter signoz_metrics signoz_traces \
|
|
> "${expected_databases_file}"
|
|
cmp -s "${expected_databases_file}" "${LAST_STDOUT}" \
|
|
|| fail_closed "signoz_database_allowlist_drift"
|
|
[ "${database_count}" -eq 6 ] || fail_closed "signoz_database_count_drift"
|
|
DATABASES_HASH="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')"
|
|
|
|
run_sql "table_inventory" \
|
|
"SELECT database,name,engine,toString(ifNull(total_rows,0)),toString(ifNull(total_bytes,0)),hex(SHA256(create_table_query)) FROM system.tables WHERE database IN ('signoz_analytics','signoz_logs','signoz_metadata','signoz_meter','signoz_metrics','signoz_traces') ORDER BY database,name" \
|
|
|| fail_closed "source_table_inventory_query_failed"
|
|
[ "$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")" -gt 0 ] \
|
|
|| fail_closed "source_table_inventory_empty"
|
|
SOURCE_INVENTORY_RECEIPT="${LAST_STDOUT}"
|
|
SOURCE_INVENTORY_HASH="$(sha256sum "${SOURCE_INVENTORY_RECEIPT}" | awk '{print $1}')"
|
|
|
|
emit_receipt "sensor_source" "pass" \
|
|
"container_${CONTAINER_ID}_version_${CLICKHOUSE_VERSION}_disk_${BACKUP_DISK}"
|
|
emit_receipt "normalized_asset_identity" "pass" \
|
|
"identity_${IDENTITY_HASH}_operation_${OPERATION_ID}_artifact_${ARTIFACT_NAME}"
|
|
emit_receipt "source_of_truth_diff" "pass" \
|
|
"configured_disk_path_exact_six_databases_hash_${DATABASES_HASH}_table_inventory_${SOURCE_INVENTORY_HASH}"
|
|
emit_receipt "risk_policy" "pass" \
|
|
"risk_medium_native_backup_async_bounded_poll_exact_allowlist_no_volume_tar_fallback"
|
|
}
|
|
|
|
query_backup_status() {
|
|
local line_count status_line
|
|
|
|
run_sql "backup_status" \
|
|
"SELECT name,toString(status),toString(num_files),toString(total_size),toString(uncompressed_size),toString(compressed_size),if(empty(error),'none',hex(substring(error,1,2048))),hex(SHA256(error)) FROM system.backups WHERE id={operation_id:String}" \
|
|
--param_operation_id "${OPERATION_ID}" || fail_closed "system_backups_status_query_failed"
|
|
line_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
|
|
if [ "${line_count}" -eq 0 ]; then
|
|
STATUS_PRESENT=0
|
|
return 0
|
|
fi
|
|
[ "${line_count}" -eq 1 ] || fail_closed "system_backups_operation_id_not_unique"
|
|
STATUS_PRESENT=1
|
|
status_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
|
IFS=$'\t' read -r BACKUP_NAME BACKUP_STATUS BACKUP_NUM_FILES BACKUP_TOTAL_SIZE \
|
|
BACKUP_UNCOMPRESSED_SIZE BACKUP_COMPRESSED_SIZE BACKUP_ERROR_HEX \
|
|
BACKUP_ERROR_HASH <<< "${status_line}"
|
|
[[ "${BACKUP_NUM_FILES}" =~ ^[0-9]+$ ]] \
|
|
&& [[ "${BACKUP_TOTAL_SIZE}" =~ ^[0-9]+$ ]] \
|
|
&& [[ "${BACKUP_UNCOMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \
|
|
&& [[ "${BACKUP_COMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \
|
|
|| fail_closed "system_backups_metrics_ambiguous"
|
|
}
|
|
|
|
server_artifact_absent() {
|
|
run_captured "artifact_absent" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
|
test ! -e "${SERVER_ARTIFACT_PATH}"
|
|
}
|
|
|
|
cleanup_server_artifact() {
|
|
run_captured "artifact_cleanup" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
|
rm -f -- "${SERVER_ARTIFACT_PATH}" || return 1
|
|
server_artifact_absent || return 1
|
|
SERVER_ARTIFACT_ACTIVE=0
|
|
emit_receipt "cleanup" "pass" "exact_server_artifact_removed_and_absent"
|
|
}
|
|
|
|
poll_backup_terminal() {
|
|
local deadline now expected_name
|
|
expected_name="Disk('${BACKUP_DISK}', '${ARTIFACT_NAME}')"
|
|
deadline=$(( $(date +%s) + BACKUP_TIMEOUT_SECONDS ))
|
|
|
|
while true; do
|
|
query_backup_status
|
|
[ "${STATUS_PRESENT}" -eq 1 ] || fail_closed "async_backup_status_disappeared"
|
|
[ "${BACKUP_NAME}" = "${expected_name}" ] || fail_closed "async_backup_name_mismatch"
|
|
emit_receipt "backup_status" "observed" \
|
|
"status_${BACKUP_STATUS}_files_${BACKUP_NUM_FILES}_total_${BACKUP_TOTAL_SIZE}_error_hash_${BACKUP_ERROR_HASH}"
|
|
|
|
case "${BACKUP_STATUS}" in
|
|
BACKUP_CREATED)
|
|
[ "${BACKUP_NUM_FILES}" -gt 0 ] \
|
|
&& [ "${BACKUP_TOTAL_SIZE}" -gt 0 ] \
|
|
&& [ "${BACKUP_UNCOMPRESSED_SIZE}" -gt 0 ] \
|
|
&& [ "${BACKUP_COMPRESSED_SIZE}" -gt 0 ] \
|
|
|| fail_closed "native_backup_terminal_metrics_empty"
|
|
SERVER_ARTIFACT_ACTIVE=1
|
|
return 0
|
|
;;
|
|
CREATING_BACKUP)
|
|
SERVER_ARTIFACT_ACTIVE=1
|
|
;;
|
|
BACKUP_FAILED|BACKUP_CANCELLED)
|
|
printf '%s\n' "${BACKUP_ERROR_HEX}" \
|
|
> "${RECEIPT_DIR}/${INVOCATION_ID}-backup-error-bounded.hex"
|
|
emit_receipt "backup_error" "retained" \
|
|
"bounded_hex_2048_bytes_hash_${BACKUP_ERROR_HASH}"
|
|
SERVER_ARTIFACT_ACTIVE=1
|
|
if ! cleanup_server_artifact; then
|
|
emit_receipt "cleanup" "failed" "failed_terminal_exact_artifact_cleanup_failed"
|
|
fi
|
|
fail_closed "native_backup_terminal_${BACKUP_STATUS}"
|
|
;;
|
|
*) fail_closed "unexpected_native_backup_status_${BACKUP_STATUS}" ;;
|
|
esac
|
|
|
|
now="$(date +%s)"
|
|
if [ "${now}" -ge "${deadline}" ]; then
|
|
fail_closed "native_backup_poll_timeout_artifact_preserved"
|
|
fi
|
|
sleep "${POLL_INTERVAL_SECONDS}"
|
|
done
|
|
}
|
|
|
|
verify_existing_local_artifact() {
|
|
local recorded_hash actual_hash inventory_hash artifact_size
|
|
local manifest_hash hook_manifest_hash hook_inventory_hash
|
|
|
|
if [ ! -e "${OUTPUT_FILE}" ] && [ ! -e "${MANIFEST_FILE}" ] \
|
|
&& [ ! -e "${HASH_FILE}" ] && [ ! -e "${INVENTORY_FILE}" ]; then
|
|
return 1
|
|
fi
|
|
[ -f "${OUTPUT_FILE}" ] && [ ! -L "${OUTPUT_FILE}" ] \
|
|
&& [ -f "${MANIFEST_FILE}" ] && [ ! -L "${MANIFEST_FILE}" ] \
|
|
&& [ -f "${HASH_FILE}" ] && [ ! -L "${HASH_FILE}" ] \
|
|
&& [ -f "${INVENTORY_FILE}" ] && [ ! -L "${INVENTORY_FILE}" ] \
|
|
|| fail_closed "local_artifact_identity_incomplete_or_unsafe"
|
|
grep -F -q "\"identity_hash\":\"${IDENTITY_HASH}\"" "${MANIFEST_FILE}" \
|
|
|| fail_closed "local_artifact_manifest_identity_mismatch"
|
|
recorded_hash="$(awk 'NR == 1 { print $1 }' "${HASH_FILE}")"
|
|
[[ "${recorded_hash}" =~ ^[0-9a-f]{64}$ ]] || fail_closed "local_artifact_hash_receipt_invalid"
|
|
actual_hash="$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')"
|
|
[ "${actual_hash}" = "${recorded_hash}" ] || fail_closed "local_artifact_hash_mismatch"
|
|
artifact_size="$(wc -c < "${OUTPUT_FILE}" | tr -d ' ')"
|
|
inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')"
|
|
manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')"
|
|
grep -F -q "\"operation_id\":\"${OPERATION_ID}\"" "${MANIFEST_FILE}" \
|
|
|| fail_closed "local_artifact_manifest_operation_mismatch"
|
|
grep -F -q "\"artifact_name\":\"${ARTIFACT_NAME}\"" "${MANIFEST_FILE}" \
|
|
|| fail_closed "local_artifact_manifest_name_mismatch"
|
|
grep -F -q "\"databases\":${EXPECTED_DATABASES_JSON}" "${MANIFEST_FILE}" \
|
|
|| fail_closed "local_artifact_manifest_database_allowlist_mismatch"
|
|
grep -F -q "\"source_inventory_sha256\":\"${inventory_hash}\"" "${MANIFEST_FILE}" \
|
|
|| fail_closed "local_source_inventory_hash_mismatch"
|
|
grep -F -q "\"sha256\":\"${actual_hash}\"" "${MANIFEST_FILE}" \
|
|
|| fail_closed "local_artifact_manifest_hash_mismatch"
|
|
grep -F -q "\"size_bytes\":${artifact_size}" "${MANIFEST_FILE}" \
|
|
|| fail_closed "local_artifact_manifest_size_mismatch"
|
|
grep -F -q '"status":"BACKUP_CREATED"' "${MANIFEST_FILE}" \
|
|
|| fail_closed "local_artifact_manifest_terminal_mismatch"
|
|
run_captured "local_zip_reuse" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_FILE}" \
|
|
|| fail_closed "local_native_archive_reuse_verifier_failed"
|
|
if [ -n "${POST_VERIFY_HOOK}" ]; then
|
|
run_captured "reuse_verify_hook_check" env \
|
|
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
|
CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \
|
|
CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \
|
|
CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \
|
|
CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
|
|
CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
|
|
CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \
|
|
"${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_reuse_check_failed"
|
|
run_captured "reuse_verify_hook" env \
|
|
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
|
CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \
|
|
CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \
|
|
CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \
|
|
CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
|
|
CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
|
|
CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \
|
|
"${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_reuse_failed"
|
|
[ "$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')" = "${actual_hash}" ] \
|
|
|| fail_closed "post_verify_hook_modified_reused_artifact"
|
|
hook_inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')"
|
|
[ "${hook_inventory_hash}" = "${inventory_hash}" ] \
|
|
|| fail_closed "post_verify_hook_modified_reused_source_inventory"
|
|
hook_manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')"
|
|
[ "${hook_manifest_hash}" = "${manifest_hash}" ] \
|
|
|| fail_closed "post_verify_hook_modified_reused_manifest"
|
|
fi
|
|
emit_receipt "idempotency" "reused" "verified_local_artifact_hash_${actual_hash}"
|
|
APPLY_PASS=1
|
|
return 0
|
|
}
|
|
|
|
copy_and_verify_artifact() {
|
|
local artifact_hash artifact_size hook_hash inventory_hash hook_inventory_hash
|
|
local manifest_hash hook_manifest_hash
|
|
|
|
rm -f -- "${OUTPUT_PARTIAL}" "${MANIFEST_PARTIAL}" "${INVENTORY_PARTIAL}"
|
|
run_captured "artifact_copy" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${BACKUP_TIMEOUT_SECONDS}" docker cp \
|
|
"${CONTAINER_NAME}:${SERVER_ARTIFACT_PATH}" "${OUTPUT_PARTIAL}" \
|
|
|| fail_closed "native_backup_artifact_copy_failed"
|
|
[ -s "${OUTPUT_PARTIAL}" ] || fail_closed "native_backup_artifact_copy_empty"
|
|
|
|
run_captured "local_zip_verify" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_PARTIAL}" \
|
|
|| fail_closed "native_backup_archive_integrity_failed"
|
|
artifact_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')"
|
|
artifact_size="$(wc -c < "${OUTPUT_PARTIAL}" | tr -d ' ')"
|
|
[[ "${artifact_size}" =~ ^[0-9]+$ ]] && [ "${artifact_size}" -gt 0 ] \
|
|
|| fail_closed "native_backup_local_size_invalid"
|
|
|
|
cp "${SOURCE_INVENTORY_RECEIPT}" "${INVENTORY_PARTIAL}"
|
|
inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')"
|
|
[ "${inventory_hash}" = "${SOURCE_INVENTORY_HASH}" ] \
|
|
|| fail_closed "copied_source_inventory_hash_mismatch"
|
|
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","identity_hash":"%s","operation_id":"%s","artifact_name":"%s","databases":%s,"source_inventory_sha256":"%s","sha256":"%s","size_bytes":%s,"status":"BACKUP_CREATED","num_files":%s,"total_size":%s,"uncompressed_size":%s,"compressed_size":%s}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${IDENTITY_HASH}" \
|
|
"${OPERATION_ID}" "${ARTIFACT_NAME}" "${EXPECTED_DATABASES_JSON}" \
|
|
"${SOURCE_INVENTORY_HASH}" "${artifact_hash}" "${artifact_size}" \
|
|
"${BACKUP_NUM_FILES}" "${BACKUP_TOTAL_SIZE}" "${BACKUP_UNCOMPRESSED_SIZE}" \
|
|
"${BACKUP_COMPRESSED_SIZE}" > "${MANIFEST_PARTIAL}"
|
|
|
|
if [ -n "${POST_VERIFY_HOOK}" ]; then
|
|
manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')"
|
|
run_captured "post_verify_hook_check" env \
|
|
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
|
CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \
|
|
CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \
|
|
CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \
|
|
CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
|
|
CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
|
|
CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \
|
|
"${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_check_failed"
|
|
run_captured "post_verify_hook" env \
|
|
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
|
CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \
|
|
CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \
|
|
CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \
|
|
CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
|
|
CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
|
|
CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \
|
|
"${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_failed"
|
|
hook_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')"
|
|
[ "${hook_hash}" = "${artifact_hash}" ] || fail_closed "post_verify_hook_modified_artifact"
|
|
hook_inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')"
|
|
[ "${hook_inventory_hash}" = "${inventory_hash}" ] \
|
|
|| fail_closed "post_verify_hook_modified_source_inventory"
|
|
hook_manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')"
|
|
[ "${hook_manifest_hash}" = "${manifest_hash}" ] \
|
|
|| fail_closed "post_verify_hook_modified_manifest"
|
|
fi
|
|
|
|
mv "${OUTPUT_PARTIAL}" "${OUTPUT_FILE}"
|
|
OUTPUT_CREATED=1
|
|
mv "${INVENTORY_PARTIAL}" "${INVENTORY_FILE}"
|
|
printf '%s %s\n' "${artifact_hash}" "$(basename "${OUTPUT_FILE}")" > "${HASH_FILE}"
|
|
mv "${MANIFEST_PARTIAL}" "${MANIFEST_FILE}"
|
|
|
|
cleanup_server_artifact || fail_closed "completed_server_artifact_cleanup_failed"
|
|
emit_receipt "independent_verifier" "pass" \
|
|
"system_status_archive_integrity_hash_${artifact_hash}_size_${artifact_size}"
|
|
APPLY_PASS=1
|
|
}
|
|
|
|
apply_backup() {
|
|
local submission_line submitted_id submitted_status
|
|
|
|
grep -F -q '"stage":"check","terminal":"check_pass_no_write"' "${RECEIPT_LOG}" \
|
|
|| fail_closed "same_run_check_pass_receipt_required_before_apply"
|
|
perform_check
|
|
emit_receipt "check" "pass" "apply_revalidation_pass"
|
|
|
|
if verify_existing_local_artifact; then
|
|
return 0
|
|
fi
|
|
|
|
query_backup_status
|
|
if [ "${STATUS_PRESENT}" -eq 0 ]; then
|
|
server_artifact_absent || fail_closed "orphan_server_artifact_without_system_status"
|
|
emit_receipt "ai_decision" "candidate" \
|
|
"submit_native_async_backup_exact_six_database_allowlist_to_exact_artifact"
|
|
SERVER_ARTIFACT_ACTIVE=1
|
|
run_sql "backup_submit" \
|
|
"BACKUP ${BACKUP_SCOPE_SQL} TO Disk({disk:String},{artifact:String}) SETTINGS id='${OPERATION_ID}' ASYNC" \
|
|
--param_disk "${BACKUP_DISK}" --param_artifact "${ARTIFACT_NAME}" \
|
|
|| fail_closed "native_backup_submit_failed"
|
|
submission_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
|
IFS=$'\t' read -r submitted_id submitted_status <<< "${submission_line}"
|
|
[ "${submitted_id}" = "${OPERATION_ID}" ] \
|
|
|| fail_closed "native_backup_submission_id_mismatch"
|
|
case "${submitted_status}" in
|
|
CREATING_BACKUP|BACKUP_CREATED) ;;
|
|
*) fail_closed "native_backup_submission_status_${submitted_status:-empty}" ;;
|
|
esac
|
|
SERVER_ARTIFACT_ACTIVE=1
|
|
emit_receipt "execution" "submitted" "operation_${OPERATION_ID}_status_${submitted_status}"
|
|
else
|
|
emit_receipt "idempotency" "resumed" "existing_operation_status_${BACKUP_STATUS}"
|
|
fi
|
|
|
|
poll_backup_terminal
|
|
copy_and_verify_artifact
|
|
}
|
|
|
|
main() {
|
|
setup "$@"
|
|
|
|
if [ "${MODE}" = "check" ]; then
|
|
perform_check
|
|
emit_receipt "ai_decision" "candidate" \
|
|
"native_backup_apply_requires_same_run_check_receipt"
|
|
emit_receipt "check" "check_pass_no_write" \
|
|
"runtime_contract_valid_no_backup_or_artifact_write"
|
|
CHECK_PASS=1
|
|
log_info "ClickHouse native backup check passed without runtime write"
|
|
return 0
|
|
fi
|
|
|
|
apply_backup
|
|
log_info "ClickHouse native backup verified: ${OUTPUT_FILE}"
|
|
}
|
|
|
|
main "$@"
|