#!/usr/bin/env bash # ============================================================================= # P0-OBS-002 - SigNoz online backup canary and legacy maintenance lease verifier # # The default native-backup path never stops the collector and therefore does # not depend on, read, or mutate the legacy monitor/cooldown contract. The # bounded lease path remains available only when an explicit legacy stop-mode # canary is requested. Neither mode reads or sources a secret/.env file. # ============================================================================= set -euo pipefail COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}" BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}" CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-/backup/scripts/clickhouse-native-restore-drill.sh}" CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-/backup/scripts/clickhouse-restore-inventory.py}" CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-/backup/config/signoz/clickhouse/config.d/backup_disk.xml}" CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml}" CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}" CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-/backup/logs/clickhouse-native-restore-drill}" MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}" MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}" MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}" COOLDOWN_FILE="${MONITOR_COOLDOWN_DIR}/${COLLECTOR_NAME}.cooldown" RECEIPT_ROOT="${SIGNOZ_CANARY_RECEIPT_ROOT:-/backup/logs/signoz-canary}" LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}" CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}" TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}" DOCKER_TIMEOUT_SECONDS="${SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS:-15}" LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}" MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}" STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}" EXPECT_COLLECTOR_STOP="${SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP:-0}" TRACE_ID="${TRACE_ID:-}" RUN_ID="${RUN_ID:-}" WORK_ITEM_ID="${WORK_ITEM_ID:-}" RECEIPT_DIR="" RECEIPT_LOG="" BACKUP_OUTPUT="" MONITOR_WINDOW="" STATE_OBSERVATIONS="" ROLLBACK_FILE="" ROLLBACK_METADATA="" RECEIPTS_READY=0 LEASE_ROLLBACK_READY=0 LEASE_MUTATION_STARTED=0 LEASE_ARMED=0 PRIOR_LEASE_PRESENT=0 PRIOR_LEASE_MODE="" PRIOR_LEASE_UID="" PRIOR_LEASE_GID="" PRIOR_LEASE_HASH="" PRIOR_LEASE_VALUE="" LEASE_TMP="" BACKUP_PID="" OBSERVER_PID="" CANARY_VERIFIED=0 CHECK_VERIFIED=0 MODE="" log_info() { printf '[INFO] %s\n' "$*"; } log_error() { printf '[ERROR] %s\n' "$*" >&2; } valid_identity() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] } valid_positive_integer() { [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ] } valid_poll_interval() { [[ "$1" =~ ^[0-9]+([.][0-9]+)?$ ]] && [[ "$1" != "0" ]] && [[ "$1" != "0.0" ]] } emit_receipt() { local stage="$1" local terminal="$2" local detail="$3" local observed_at [ "${RECEIPTS_READY}" -eq 1 ] || return 0 observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${observed_at}" \ "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}" } fail_closed() { local detail="$1" log_error "${detail}" emit_receipt "failure" "failed" "${detail}" exit 1 } require_command() { command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}" } run_collector_docker() { timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ "${DOCKER_TIMEOUT_SECONDS}" docker "$@" } require_native_restore_contract() { local label path for label in \ CLICKHOUSE_NATIVE_POST_VERIFY_HOOK \ CLICKHOUSE_RESTORE_INVENTORY_HELPER \ CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG \ CLICKHOUSE_RESTORE_CLUSTER_CONFIG; do path="${!label}" case "${path}" in /*) ;; *) fail_closed "native_restore_contract_${label}_path_not_absolute" ;; esac [ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \ || fail_closed "native_restore_contract_${label}_missing_unreadable_or_symlink" done [ -x "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" ] \ || fail_closed "native_restore_contract_post_verify_hook_not_executable" [ -x "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" ] \ || fail_closed "native_restore_contract_inventory_helper_not_executable" } acquire_canary_lock() { [ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe" if [ -e "${LOCK_FILE}" ]; then [ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file" fi exec 9>>"${LOCK_FILE}" flock -n 9 || fail_closed "another_signoz_canary_holds_lock" } require_native_receipt_contract() { local label path run_dir for label in CLICKHOUSE_NATIVE_RECEIPT_ROOT CLICKHOUSE_RESTORE_RECEIPT_ROOT; do path="${!label}" case "${path}" in /*) ;; *) fail_closed "native_receipt_contract_${label}_path_not_absolute" ;; esac [[ "${path}" != *$'\n'* ]] \ || fail_closed "native_receipt_contract_${label}_path_contains_newline" [ ! -L "${path}" ] \ || fail_closed "native_receipt_contract_${label}_root_symlink_unsafe" if [ -e "${path}" ]; then [ -d "${path}" ] && [ -r "${path}" ] && [ -w "${path}" ] \ || fail_closed "native_receipt_contract_${label}_root_not_readable_writable_directory" fi run_dir="${path}/${RUN_ID}" [ ! -e "${run_dir}" ] && [ ! -L "${run_dir}" ] \ || fail_closed "native_receipt_contract_${label}_stale_same_run_path_present" done [ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" != "${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" ] \ || fail_closed "native_and_restore_receipt_roots_must_be_distinct" } verify_same_run_native_restore_receipts() { local native_receipts restore_receipts restore_status native_receipts="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl" restore_receipts="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl" restore_status="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/status.json" for path in "${native_receipts}" "${restore_receipts}" "${restore_status}"; do [ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \ || fail_closed "same_run_native_restore_receipt_missing_unreadable_or_symlink" done python3 - "${native_receipts}" "${restore_receipts}" "${restore_status}" \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" <<'PY' \ || fail_closed "same_run_native_restore_receipt_validation_failed" import json import os import sys def reject(message: str) -> None: print(f"receipt_verifier:{message}", file=sys.stderr) raise SystemExit(1) native_path, restore_path, status_path, trace_id, run_id, work_item_id = sys.argv[1:] identity = { "trace_id": trace_id, "run_id": run_id, "work_item_id": work_item_id, } def read_jsonl(path: str, label: str) -> list[dict[str, object]]: if os.path.islink(path) or not os.path.isfile(path): reject(f"{label}_unsafe") records: list[dict[str, object]] = [] try: with open(path, encoding="utf-8") as source: for line_number, raw_line in enumerate(source, start=1): if not raw_line.strip(): continue value = json.loads(raw_line) if not isinstance(value, dict): reject(f"{label}_line_{line_number}_not_object") for key, expected in identity.items(): if value.get(key) != expected: reject(f"{label}_line_{line_number}_{key}_mismatch") invocation_id = value.get("invocation_id") if not isinstance(invocation_id, str) or not invocation_id: reject(f"{label}_line_{line_number}_invocation_id_invalid") records.append(value) except (OSError, UnicodeError, json.JSONDecodeError) as exc: reject(f"{label}_malformed_{type(exc).__name__}") if not records: reject(f"{label}_empty") return records def mode_records( records: list[dict[str, object]], mode: str, label: str ) -> list[dict[str, object]]: selected = [ record for record in records if str(record.get("invocation_id", "")).startswith(f"{mode}-") ] if not selected: reject(f"{label}_{mode}_invocation_missing") return selected def require_exact( records: list[dict[str, object]], *, label: str, stage: str, terminal: str, detail_prefix: str | None = None, ) -> None: matches = [ record for record in records if record.get("stage") == stage and record.get("terminal") == terminal and ( detail_prefix is None or str(record.get("detail", "")).startswith(detail_prefix) ) ] if len(matches) != 1: reject(f"{label}_{stage}_{terminal}_count_{len(matches)}") native_records = read_jsonl(native_path, "native_receipts") native_check = mode_records(native_records, "check", "native") native_apply = mode_records(native_records, "apply", "native") require_exact( native_check, label="native_check", stage="terminal", terminal="check_pass_no_write", ) require_exact( native_apply, label="native_apply", stage="terminal", terminal="pass", ) require_exact( native_apply, label="native_apply", stage="independent_verifier", terminal="pass", ) require_exact( native_apply, label="native_apply_hook_check", stage="command", terminal="pass", detail_prefix="step_post_verify_hook_check_exit_0_", ) require_exact( native_apply, label="native_apply_hook", stage="command", terminal="pass", detail_prefix="step_post_verify_hook_exit_0_", ) restore_records = read_jsonl(restore_path, "restore_receipts") restore_check = mode_records(restore_records, "check", "restore") restore_apply = mode_records(restore_records, "apply", "restore") require_exact( restore_check, label="restore_check", stage="terminal", terminal="check_pass_no_runtime_write", ) require_exact( restore_apply, label="restore_apply", stage="terminal", terminal="verified", ) require_exact( restore_apply, label="restore_apply", stage="independent_post_verifier", terminal="pass", ) require_exact( restore_apply, label="restore_apply", stage="cleanup", terminal="pass", ) if os.path.islink(status_path) or not os.path.isfile(status_path): reject("restore_status_unsafe") try: with open(status_path, encoding="utf-8") as source: status = json.load(source) except (OSError, UnicodeError, json.JSONDecodeError) as exc: reject(f"restore_status_malformed_{type(exc).__name__}") if not isinstance(status, dict): reject("restore_status_not_object") for key, expected in identity.items(): if status.get(key) != expected: reject(f"restore_status_{key}_mismatch") expected_strings = { "schema_version": "clickhouse_native_restore_drill_status_v1", "mode": "apply", "terminal": "verified", "restore_terminal": "RESTORED", "network_mode": "docker_internal_only", "staging_network_mode": "none", } for key, expected in expected_strings.items(): if not isinstance(status.get(key), str) or status.get(key) != expected: reject(f"restore_status_{key}_invalid") for key, expected in (("exit_code", 0), ("manifest_parity", 1)): if type(status.get(key)) is not int or status.get(key) != expected: reject(f"restore_status_{key}_invalid") for key in ( "check_pass", "apply_pass", "cleanup_verified", "artifact_staging_verified", "clickhouse_artifact_readback_verified", "ephemeral_runtime_created", "isolated_keeper", ): if status.get(key) is not True: reject(f"restore_status_{key}_not_true") for key in ( "production_network_attached", "production_restore_performed", "allow_non_empty_tables", ): if status.get(key) is not False: reject(f"restore_status_{key}_not_false") def positive_integer(key: str) -> int: value = status.get(key) if isinstance(value, bool) or not isinstance(value, int) or value <= 0: reject(f"restore_status_{key}_not_positive_integer") return value source_database_count = positive_integer("source_database_count") restored_database_count = positive_integer("restored_database_count") if source_database_count != restored_database_count: reject("restore_status_database_count_mismatch") source_table_count = positive_integer("source_table_count") restored_table_count = positive_integer("restored_table_count") if source_table_count != restored_table_count: reject("restore_status_table_count_mismatch") positive_integer("critical_nonzero_count") check_count = status.get("check_table_count") check_pass_count = status.get("check_table_pass_count") if ( isinstance(check_count, bool) or not isinstance(check_count, int) or isinstance(check_pass_count, bool) or not isinstance(check_pass_count, int) or check_count <= 0 or check_count != check_pass_count ): reject("restore_status_check_table_counts_invalid") PY emit_receipt "native_restore_receipt_verifier" "pass" \ "same_run_machine_readable_native_check_apply_hook_and_isolated_restore_status_verified" } require_monitor_fragment() { local fragment="$1" local count count="$(grep -F -c -- "${fragment}" "${MONITOR_SCRIPT}" || true)" [ "${count}" -eq 1 ] || fail_closed "monitor_contract_ambiguous_fragment_count_${count}" } require_monitor_function_fragment() { local function_name="$1" local fragment="$2" local count count="$(awk -v header="${function_name}() {" -v needle="${fragment}" ' $0 == header { inside = 1 depth = 0 } inside { original = $0 if (index(original, needle)) { count++ } open_line = original close_line = original depth += gsub(/\{/, "", open_line) depth -= gsub(/\}/, "", close_line) if (depth == 0) { print count + 0 printed = 1 exit } } END { if (!printed) { print count + 0 } } ' "${MONITOR_SCRIPT}")" [ "${count}" -eq 1 ] \ || fail_closed "monitor_${function_name}_contract_ambiguous_fragment_count_${count}" } collector_running_state() { local running if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then return 1 fi case "${running}" in true) printf '%s\n' "running" ;; false) printf '%s\n' "stopped" ;; *) return 1 ;; esac } collector_runtime_metadata() { run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \ "${COLLECTOR_NAME}" 2>/dev/null } listener_is_up() { local port="$1" ss -H -lnt | awk -v port="${port}" ' $4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) } ' } verify_collector_runtime() { local expected_id="$1" local expected_started_at="$2" local expected_restart_count="$3" local expected_health="$4" local metadata actual_id actual_running actual_started_at actual_restart_count actual_health metadata="$(collector_runtime_metadata)" || return 1 IFS=$'\t' read -r actual_id actual_running actual_started_at actual_restart_count actual_health <<< "${metadata}" [ "${actual_id}" = "${expected_id}" ] || return 1 [ "${actual_running}" = "true" ] || return 1 [ "${actual_started_at}" = "${expected_started_at}" ] || return 1 [ "${actual_restart_count}" = "${expected_restart_count}" ] || return 1 [ "${actual_health}" = "${expected_health}" ] || return 1 listener_is_up 4317 || return 1 listener_is_up 4318 || return 1 } restore_lease() { local restore_tmp restore_hash current_value [ "${LEASE_MUTATION_STARTED}" -eq 1 ] || return 0 [ "${LEASE_ROLLBACK_READY}" -eq 1 ] || return 1 if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then [ -f "${ROLLBACK_FILE}" ] || return 1 restore_tmp="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.restore.${RUN_ID}.XXXXXX")" || return 1 cp -p "${ROLLBACK_FILE}" "${restore_tmp}" || return 1 chmod "${PRIOR_LEASE_MODE}" "${restore_tmp}" || return 1 mv -f "${restore_tmp}" "${COOLDOWN_FILE}" || return 1 restore_tmp="" restore_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" || return 1 [ "${restore_hash}" = "${PRIOR_LEASE_HASH}" ] || return 1 [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] || return 1 [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] || return 1 [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] || return 1 current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")" [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] || return 1 else rm -f "${COOLDOWN_FILE}" || return 1 [ ! -e "${COOLDOWN_FILE}" ] || return 1 fi LEASE_ARMED=0 LEASE_MUTATION_STARTED=0 return 0 } terminate_children() { if [ -n "${BACKUP_PID}" ] && kill -0 "${BACKUP_PID}" 2>/dev/null; then kill -TERM "${BACKUP_PID}" 2>/dev/null || true wait "${BACKUP_PID}" 2>/dev/null || true fi if [ -n "${OBSERVER_PID}" ] && kill -0 "${OBSERVER_PID}" 2>/dev/null; then kill -TERM "${OBSERVER_PID}" 2>/dev/null || true wait "${OBSERVER_PID}" 2>/dev/null || true fi } cleanup() { local exit_code=$? local lease_terminal="not_armed" local overall_terminal="failed" trap - EXIT HUP INT TERM set +e terminate_children rm -f "${LEASE_TMP}" 2>/dev/null || true if [ "${LEASE_MUTATION_STARTED}" -eq 1 ]; then if restore_lease; then lease_terminal="restored" else lease_terminal="restore_failed" exit_code=91 fi elif [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then lease_terminal="no_write" elif [ "${MODE}" = "apply" ] && [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then lease_terminal="no_write" fi if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then overall_terminal="check_pass_no_write" elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then overall_terminal="partial_degraded" fi emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}" if [ "${overall_terminal}" = "partial_degraded" ]; then emit_receipt "runtime_operation" "pass" \ "clickhouse_native_backup_and_isolated_restore_runtime_verifier_pass" emit_receipt "closure_writeback" "partial_degraded" \ "local_durable_receipt_ack_incident_not_applicable_telegram_not_sent_km_rag_writeback_pending_offsite_false_sqlite_application_consistent_export_pending" elif [ "${overall_terminal}" = "check_pass_no_write" ]; then emit_receipt "closure_writeback" "not_applicable" \ "check_mode_no_runtime_change_local_durable_receipt_ack" else emit_receipt "closure_writeback" "deferred" \ "failed_terminal_requires_followup_no_completion_claim" fi emit_receipt "terminal" "${overall_terminal}" \ "backup_canary_${overall_terminal}_clickhouse_native_scope_only_offsite_false_sqlite_pending" printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \ "${lease_terminal}" "${exit_code}" exit "${exit_code}" } on_signal() { local signal_number="$1" emit_receipt "signal" "failed" "signal_${signal_number}" exit "$((128 + signal_number))" } setup_receipts() { valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and must be path-safe"; exit 2; } valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and must be path-safe"; exit 2; } valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and must be path-safe"; exit 2; } [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root must not be a symlink"; exit 2; } mkdir -p "${RECEIPT_ROOT}" RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" [ ! -e "${RECEIPT_DIR}" ] || { log_error "receipt directory already exists: ${RECEIPT_DIR}"; exit 2; } mkdir -m 700 "${RECEIPT_DIR}" RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl" BACKUP_OUTPUT="${RECEIPT_DIR}/backup-output.log" MONITOR_WINDOW="${RECEIPT_DIR}/monitor-window.log" STATE_OBSERVATIONS="${RECEIPT_DIR}/collector-state.tsv" ROLLBACK_FILE="${RECEIPT_DIR}/cooldown.rollback" ROLLBACK_METADATA="${RECEIPT_DIR}/cooldown.rollback.json" : > "${RECEIPT_LOG}" RECEIPTS_READY=1 trap cleanup EXIT trap 'on_signal 1' HUP trap 'on_signal 2' INT trap 'on_signal 15' TERM } verify_monitor_contract() { local cron_output cron_count cron_line exclude_count exclude_line [ -f "${MONITOR_SCRIPT}" ] && [ -r "${MONITOR_SCRIPT}" ] && [ ! -L "${MONITOR_SCRIPT}" ] \ || fail_closed "monitor_script_missing_unreadable_or_symlink" [ -f "${MONITOR_LOG}" ] && [ -r "${MONITOR_LOG}" ] && [ ! -L "${MONITOR_LOG}" ] \ || fail_closed "monitor_log_missing_unreadable_or_symlink" [ -d "${MONITOR_COOLDOWN_DIR}" ] && [ -w "${MONITOR_COOLDOWN_DIR}" ] && [ ! -L "${MONITOR_COOLDOWN_DIR}" ] \ || fail_closed "monitor_cooldown_directory_missing_unwritable_or_symlink" require_monitor_fragment "COOLDOWN_DIR:=${MONITOR_COOLDOWN_DIR}" require_monitor_fragment 'local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown"' require_monitor_fragment 'last_sent=$(cat "$cooldown_file")' # The live monitor has another cooldown path with the same elapsed # calculation. Bind this assertion to is_in_cooldown instead of requiring # a globally unique string. require_monitor_function_fragment is_in_cooldown 'elapsed=$(( now - last_sent ))' require_monitor_fragment 'if (( elapsed < ACTION_COOLDOWN_SECONDS )); then' require_monitor_fragment 'COOLDOWN: ${container}' require_monitor_fragment 'is_in_cooldown "$container_name" && continue' require_monitor_fragment 'set_cooldown "$container_name"' require_monitor_fragment 'log "AUTO_REPAIR: docker restart ${container}"' require_monitor_fragment 'if docker restart "$container"' exclude_count="$(grep -F -c 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}" || true)" [ "${exclude_count}" -eq 1 ] || fail_closed "monitor_exclude_contract_ambiguous" exclude_line="$(grep -F 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}")" [[ "${exclude_line}" != *"${COLLECTOR_NAME}"* ]] || fail_closed "collector_already_excluded_monitor_contract_changed" [ "$(grep -F -c "${COLLECTOR_NAME}" "${MONITOR_SCRIPT}" || true)" -eq 0 ] \ || fail_closed "collector_literal_present_monitor_contract_changed" cron_output="$(crontab -l 2>/dev/null)" || fail_closed "monitor_crontab_unreadable" cron_count="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" ' $0 !~ /^[[:space:]]*#/ && index($0, script) { count++ } END { print count + 0 } ')" [ "${cron_count}" -eq 1 ] || fail_closed "monitor_cron_contract_ambiguous_count_${cron_count}" cron_line="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" ' $0 !~ /^[[:space:]]*#/ && index($0, script) { print } ')" [[ "${cron_line}" == "*/5 * * * * "* ]] || fail_closed "monitor_cron_schedule_not_every_five_minutes" [[ "${cron_line}" == *">> ${MONITOR_LOG}"* ]] || fail_closed "monitor_cron_log_route_mismatch" [[ "${cron_line}" != *"COOLDOWN_DIR="* ]] || fail_closed "monitor_cron_cooldown_override_ambiguous" } inspect_prior_lease() { local current_uid current_gid current_uid="$(id -u)" current_gid="$(id -g)" if [ -e "${COOLDOWN_FILE}" ]; then [ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \ || fail_closed "existing_cooldown_receipt_unsafe" PRIOR_LEASE_PRESENT=1 PRIOR_LEASE_MODE="$(stat -c '%a' "${COOLDOWN_FILE}")" PRIOR_LEASE_UID="$(stat -c '%u' "${COOLDOWN_FILE}")" PRIOR_LEASE_GID="$(stat -c '%g' "${COOLDOWN_FILE}")" [ "${PRIOR_LEASE_UID}" = "${current_uid}" ] && [ "${PRIOR_LEASE_GID}" = "${current_gid}" ] \ || fail_closed "existing_cooldown_receipt_owner_mismatch" PRIOR_LEASE_VALUE="$(tr -d '\r\n' < "${COOLDOWN_FILE}")" [[ "${PRIOR_LEASE_VALUE}" =~ ^[0-9]+$ ]] || fail_closed "existing_cooldown_receipt_not_epoch" PRIOR_LEASE_HASH="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" else PRIOR_LEASE_PRESENT=0 PRIOR_LEASE_MODE="664" PRIOR_LEASE_UID="${current_uid}" PRIOR_LEASE_GID="${current_gid}" PRIOR_LEASE_HASH="absent" PRIOR_LEASE_VALUE="absent" fi } capture_prior_lease() { local current_hash current_value if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then [ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \ || fail_closed "cooldown_changed_between_check_and_rollback_capture" current_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")" [ "${current_hash}" = "${PRIOR_LEASE_HASH}" ] \ && [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] \ && [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] \ && [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] \ && [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] \ || fail_closed "cooldown_changed_between_check_and_rollback_capture" cp -p "${COOLDOWN_FILE}" "${ROLLBACK_FILE}" [ "$(sha256sum "${ROLLBACK_FILE}" | awk '{print $1}')" = "${PRIOR_LEASE_HASH}" ] \ || fail_closed "cooldown_changed_between_check_and_rollback_capture" else [ ! -e "${COOLDOWN_FILE}" ] \ || fail_closed "cooldown_appeared_between_check_and_rollback_capture" fi printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","prior_present":%s,"prior_mode":"%s","prior_uid":"%s","prior_gid":"%s","prior_hash":"%s","prior_value":"%s"}\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${PRIOR_LEASE_PRESENT}" \ "${PRIOR_LEASE_MODE}" "${PRIOR_LEASE_UID}" "${PRIOR_LEASE_GID}" \ "${PRIOR_LEASE_HASH}" "${PRIOR_LEASE_VALUE}" > "${ROLLBACK_METADATA}" LEASE_ROLLBACK_READY=1 emit_receipt "rollback_capture" "pass" "prior_cooldown_receipt_captured" } arm_lease() { local now lease_epoch lease_hash now="$(date +%s)" lease_epoch=$((now + CANARY_TIMEOUT_SECONDS + LEASE_MARGIN_SECONDS)) LEASE_TMP="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.lease.${RUN_ID}.XXXXXX")" printf '%s\n' "${lease_epoch}" > "${LEASE_TMP}" chmod "${PRIOR_LEASE_MODE}" "${LEASE_TMP}" LEASE_MUTATION_STARTED=1 mv -f "${LEASE_TMP}" "${COOLDOWN_FILE}" LEASE_TMP="" LEASE_ARMED=1 [ "$(tr -d '\r\n' < "${COOLDOWN_FILE}")" = "${lease_epoch}" ] \ || fail_closed "cooldown_lease_epoch_readback_mismatch" [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "$(id -u)" ] \ || fail_closed "cooldown_lease_owner_readback_mismatch" [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "$(id -g)" ] \ || fail_closed "cooldown_lease_group_readback_mismatch" lease_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" emit_receipt "lease" "armed" "future_epoch_${lease_epoch}_hash_${lease_hash}" } observe_collector() { local epoch state while kill -0 "${BACKUP_PID}" 2>/dev/null; do epoch="$(date +%s)" if state="$(collector_running_state)"; then printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}" else printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}" fi sleep "${STATE_POLL_SECONDS}" done epoch="$(date +%s)" if state="$(collector_running_state)"; then printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}" else printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}" fi } verify_monitor_window() { local log_identity_before="$1" local log_size_before="$2" local log_identity_after log_size_after start_byte local auto_repair_count detected_count cooldown_count local stop_epoch restore_epoch next_cron_epoch cron_crossed=0 log_identity_after="$(stat -c '%d:%i' "${MONITOR_LOG}")" log_size_after="$(stat -c '%s' "${MONITOR_LOG}")" [ "${log_identity_after}" = "${log_identity_before}" ] \ || fail_closed "monitor_log_rotated_during_canary" [ "${log_size_after}" -ge "${log_size_before}" ] \ || fail_closed "monitor_log_shrank_during_canary" start_byte=$((log_size_before + 1)) tail -c "+${start_byte}" "${MONITOR_LOG}" > "${MONITOR_WINDOW}" auto_repair_count="$(grep -a -F -c "AUTO_REPAIR: docker restart ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)" detected_count="$(grep -a -F -c "DETECTED: ${COLLECTOR_NAME} state=exited" "${MONITOR_WINDOW}" || true)" cooldown_count="$(grep -a -F -c "COOLDOWN: ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)" [ "${auto_repair_count}" -eq 0 ] || fail_closed "monitor_auto_repair_contaminated_canary" [ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \ || fail_closed "collector_state_observer_ambiguous" if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then [ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \ || fail_closed "online_native_backup_collector_continuity_failed" [ "${detected_count}" -eq 0 ] \ || fail_closed "online_native_backup_monitor_detected_collector" emit_receipt "monitor_verifier" "pass" \ "online_native_collector_continuous_auto_repair_0_detected_0" return 0 fi stop_epoch="$(awk '$2 == "stopped" { print $1; exit }' "${STATE_OBSERVATIONS}")" [ -n "${stop_epoch}" ] || fail_closed "collector_stop_transition_not_observed" restore_epoch="$(awk ' $2 == "stopped" { stopped_seen = 1; next } stopped_seen && $2 == "running" { print $1; exit } ' "${STATE_OBSERVATIONS}")" [ -n "${restore_epoch}" ] || fail_closed "collector_restore_transition_not_observed" next_cron_epoch=$(( ((stop_epoch / MONITOR_CRON_INTERVAL_SECONDS) + 1) * MONITOR_CRON_INTERVAL_SECONDS )) if [ "${next_cron_epoch}" -le "${restore_epoch}" ]; then cron_crossed=1 fi if [ "${detected_count}" -gt 0 ]; then [ "${cooldown_count}" -ge "${detected_count}" ] \ || fail_closed "monitor_detected_without_matching_cooldown" fi if [ "${cron_crossed}" -eq 1 ]; then [ "${detected_count}" -gt 0 ] && [ "${cooldown_count}" -gt 0 ] \ || fail_closed "cron_crossed_without_collector_cooldown_receipt" fi emit_receipt "monitor_verifier" "pass" \ "auto_repair_${auto_repair_count}_detected_${detected_count}_cooldown_${cooldown_count}_cron_crossed_${cron_crossed}" } verify_online_collector_window() { [ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \ || fail_closed "collector_state_observer_ambiguous" [ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \ || fail_closed "online_native_backup_collector_continuity_failed" emit_receipt "monitor_verifier" "pass" \ "online_native_collector_continuous_monitor_contract_not_applicable" } main() { local container_id_before container_started_at_before container_restart_count_before local container_health_before collector_metadata_before local monitor_log_identity="" monitor_log_size="" local source_diff_detail ai_decision_detail local backup_status native_restore_contract_detail="not_applicable" setup_receipts [ "$#" -eq 1 ] || fail_closed "exactly_one_mode_required_use_check_or_apply" case "$1" in --check) MODE="check" ;; --apply) MODE="apply" ;; *) fail_closed "unsupported_mode_use_check_or_apply" ;; esac # Hold the canary lock before inspecting or hashing any mutable backup or # restore toolchain path. This closes the check/apply substitution window. require_command flock acquire_canary_lock for command_name in awk date docker env grep kill pgrep python3 sha256sum sleep ss timeout tr; do require_command "${command_name}" done valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout" valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after" valid_positive_integer "${DOCKER_TIMEOUT_SECONDS}" || fail_closed "invalid_docker_timeout" valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval" case "${EXPECT_COLLECTOR_STOP}" in 0|1) ;; *) fail_closed "invalid_expect_collector_stop_mode" ;; esac [ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \ || fail_closed "backup_script_missing_unexecutable_or_symlink" if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then require_native_restore_contract require_native_receipt_contract native_restore_contract_detail="hook_$(sha256sum "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" | awk '{print $1}')_inventory_$(sha256sum "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" | awk '{print $1}')_disk_config_$(sha256sum "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" | awk '{print $1}')_cluster_config_$(sha256sum "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" | awk '{print $1}')" else for command_name in cp crontab id mktemp mv rm stat tail; do require_command "${command_name}" done valid_positive_integer "${LEASE_MARGIN_SECONDS}" || fail_closed "invalid_lease_margin" valid_positive_integer "${MONITOR_CRON_INTERVAL_SECONDS}" || fail_closed "invalid_monitor_interval" [ "${MONITOR_CRON_INTERVAL_SECONDS}" -le 300 ] || fail_closed "monitor_interval_exceeds_cron_contract" verify_monitor_contract fi if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then fail_closed "another_signoz_backup_is_running" fi collector_metadata_before="$(collector_runtime_metadata)" \ || fail_closed "collector_runtime_metadata_unreadable" IFS=$'\t' read -r container_id_before _container_running_before \ container_started_at_before container_restart_count_before container_health_before \ <<< "${collector_metadata_before}" [ -n "${container_id_before}" ] \ && [ -n "${container_started_at_before}" ] \ && [[ "${container_restart_count_before}" =~ ^[0-9]+$ ]] \ && [[ "${container_health_before}" =~ ^(healthy|none)$ ]] \ || fail_closed "collector_runtime_metadata_invalid" verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \ "${container_restart_count_before}" "${container_health_before}" \ || fail_closed "collector_preflight_runtime_failed" if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then monitor_log_identity="$(stat -c '%d:%i' "${MONITOR_LOG}")" monitor_log_size="$(stat -c '%s' "${MONITOR_LOG}")" inspect_prior_lease emit_receipt "sensor_source" "pass" \ "monitor_hash_$(sha256sum "${MONITOR_SCRIPT}" | awk '{print $1}')_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}" source_diff_detail="legacy_monitor_and_cron_contract_exact_cooldown_prior_present_${PRIOR_LEASE_PRESENT}_hash_${PRIOR_LEASE_HASH}" ai_decision_detail="arm_exact_container_future_epoch_run_legacy_stop_backup_then_restore" else PRIOR_LEASE_PRESENT=0 PRIOR_LEASE_HASH="not_applicable" emit_receipt "sensor_source" "pass" \ "online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_${native_restore_contract_detail}_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}" source_diff_detail="online_native_backup_isolated_restore_contract_bound_monitor_and_cooldown_contract_not_applicable" ai_decision_detail="run_online_native_backup_with_mandatory_isolated_restore_verifier_without_collector_or_monitor_mutation" fi emit_receipt "normalized_asset_identity" "pass" \ "asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker" emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}" emit_receipt "ai_decision" "candidate" "${ai_decision_detail}" emit_receipt "risk_policy" "pass" \ "risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_mandatory_isolated_restore_independent_verifier" emit_receipt "check" "pass" \ "mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_${native_restore_contract_detail}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent" if [ "${MODE}" = "check" ]; then CHECK_VERIFIED=1 log_info "SigNoz backup canary check passed without arming lease or launching backup" return 0 fi if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then capture_prior_lease arm_lease else emit_receipt "lease" "not_required" \ "online_native_backup_has_no_collector_stop_or_cooldown_write" fi : > "${STATE_OBSERVATIONS}" set +e timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \ env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" \ CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" \ CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" \ CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" \ CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" \ CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" \ BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \ > "${BACKUP_OUTPUT}" 2>&1 & BACKUP_PID=$! observe_collector & OBSERVER_PID=$! wait "${BACKUP_PID}" backup_status=$? BACKUP_PID="" wait "${OBSERVER_PID}" OBSERVER_PID="" set -e emit_receipt "execution" "$([ "${backup_status}" -eq 0 ] && printf pass || printf failed)" \ "backup_exit_${backup_status}" [ "${backup_status}" -eq 0 ] || fail_closed "backup_terminal_nonzero_${backup_status}" grep -a -F -q '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)' "${BACKUP_OUTPUT}" \ || fail_closed "retention_skip_receipt_missing" if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then grep -a -F -q '========== SignOz 備份完成 (' "${BACKUP_OUTPUT}" \ || fail_closed "backup_success_terminal_missing" else grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \ || fail_closed "native_backup_success_terminal_missing" verify_same_run_native_restore_receipts fi if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then verify_monitor_window "${monitor_log_identity}" "${monitor_log_size}" else verify_online_collector_window fi verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \ "${container_restart_count_before}" "${container_health_before}" \ || fail_closed "collector_post_backup_runtime_failed" emit_receipt "post_verifier" "pass" \ "collector_running_same_identity_started_at_restart_count_health_listeners_4317_4318" CANARY_VERIFIED=1 log_info "SigNoz backup canary verified; cooldown lease will now roll back" } main "$@"