feat(observability): close native SigNoz backup restore canary

scripts/backup/backup-signoz.sh

@@ -1,12 +1,12 @@
 #!/bin/bash
 # =============================================================================
-# WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
-# 版本: 1.3.0
-# 建立日期: 2026-04-05
-# 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
-# 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
-# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
-# 2026-07-15 Codex: v1.3 加入初始狀態、全 run 有界還原與 success 前獨立 verifier
+# WOOO AIOps - SigNoz ClickHouse native backup orchestrator
+# Version: 2.0.0
+#
+# ClickHouse is backed up online through its native BACKUP engine.  This script
+# never reads or archives the live ClickHouse/SQLite Docker volumes.  SigNoz
+# metadata/SQLite remains an explicit application-consistent export gap and is
+# recorded in the Restic payload without being misreported as completed.
 # =============================================================================
 
 set -euo pipefail
@@ -16,17 +16,31 @@ source "$(dirname "$0")/common.sh"
 SERVICE="signoz"
 LOCAL_REPO="${BACKUP_BASE}/signoz"
 DUMP_DIR="/tmp/signoz-backup-$$"
-COLLECTOR_NAME="signoz-otel-collector"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}"
+COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}"
 COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
-COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"
-COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}"
-ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"
-ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"
+COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}"
 TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
 BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"
-COLLECTOR_WAS_RUNNING=0
-COLLECTOR_RESTORE_PENDING=0
-COLLECTOR_RESTORE_ATTEMPTS_USED=0
+RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}"
+RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}"
+RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}"
+RESTIC_READBACK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_READBACK_TIMEOUT_SECONDS:-120}"
+RESTIC_CHECK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_CHECK_TIMEOUT_SECONDS:-1800}"
+
+BACKUP_ID_SEED="$(date -u '+%Y%m%dT%H%M%SZ')-$$"
+TRACE_ID="${TRACE_ID:-signoz-backup-${BACKUP_ID_SEED}}"
+RUN_ID="${RUN_ID:-signoz-${BACKUP_ID_SEED}}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-SIGNOZ-BACKUP}"
+export TRACE_ID RUN_ID WORK_ITEM_ID
+
+COLLECTOR_ID_BEFORE=""
+COLLECTOR_STARTED_AT_BEFORE=""
+COLLECTOR_RESTARTS_BEFORE=""
+COLLECTOR_HEALTH_BEFORE=""
+COLLECTOR_OBSERVER_PID=""
+COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
 CLEANUP_COMPLETE=0
 FAILURE_NOTIFIED=0
 
@@ -35,27 +49,9 @@ run_collector_docker() {
         "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
 }
 
-collector_running_state() {
-    local running
-    if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
-        return 1
-    fi
-
-    case "${running}" in
-        true)
-            printf '%s\n' "running"
-            ;;
-        false)
-            printf '%s\n' "stopped"
-            ;;
-        *)
-            return 1
-            ;;
-    esac
-}
-
-collector_is_running() {
-    [ "$(collector_running_state)" = "running" ]
+collector_identity_state() {
+    run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
+        "${COLLECTOR_NAME}" 2>/dev/null
 }
 
 notify_failure_once() {
@@ -67,50 +63,63 @@ notify_failure_once() {
     notify_clawbot "failed" "${SERVICE}" "${detail}" || true
 }
 
-restore_collector() {
-    if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
-        return 0
-    fi
+observe_collector_continuity() {
+    local epoch observation
 
-    while [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; do
-        COLLECTOR_RESTORE_ATTEMPTS_USED=$((COLLECTOR_RESTORE_ATTEMPTS_USED + 1))
-        log_info "重啟 ${COLLECTOR_NAME} (${COLLECTOR_RESTORE_ATTEMPTS_USED}/${COLLECTOR_RESTORE_MAX_ATTEMPTS})..."
-
-        if run_collector_docker start "${COLLECTOR_NAME}" >/dev/null 2>&1 \
-            && collector_is_running; then
-            COLLECTOR_RESTORE_PENDING=0
-            return 0
-        fi
-
-        log_warn "${COLLECTOR_NAME} 重啟或 running readback 失敗"
-        if [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; then
-            sleep "${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS}"
+    while true; do
+        epoch="$(date +%s)"
+        if observation="$(collector_identity_state)"; then
+            printf '%s\t%s\n' "${epoch}" "${observation}" >> "${COLLECTOR_OBSERVATIONS}"
+        else
+            printf '%s\tunknown\tunknown\tunknown\tunknown\tunknown\n' "${epoch}" >> "${COLLECTOR_OBSERVATIONS}"
         fi
+        sleep "${COLLECTOR_CONTINUITY_POLL_SECONDS}"
     done
-
-    log_error "${COLLECTOR_NAME} 在有界重試後仍無法恢復"
-    return 1
 }
 
-verify_collector_final_state() {
-    if [ "${COLLECTOR_WAS_RUNNING}" -ne 1 ]; then
-        log_info "${COLLECTOR_NAME} 原本即為 stopped，不執行自動啟動"
-        return 0
-    fi
+start_collector_observer() {
+    : > "${COLLECTOR_OBSERVATIONS}"
+    observe_collector_continuity &
+    COLLECTOR_OBSERVER_PID=$!
+}
 
-    if collector_is_running; then
-        log_success "${COLLECTOR_NAME} final running verifier 通過"
-        return 0
+stop_collector_observer() {
+    if [ -n "${COLLECTOR_OBSERVER_PID}" ] \
+        && kill -0 "${COLLECTOR_OBSERVER_PID}" 2>/dev/null; then
+        kill -TERM "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
+        wait "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
     fi
+    COLLECTOR_OBSERVER_PID=""
+}
 
-    log_warn "${COLLECTOR_NAME} final running verifier 首次失敗，使用剩餘有界還原額度"
-    COLLECTOR_RESTORE_PENDING=1
-    if ! restore_collector; then
+verify_collector_continuity() {
+    local observation final_id final_running final_started_at final_restarts final_health
+
+    stop_collector_observer
+    [ -s "${COLLECTOR_OBSERVATIONS}" ] || {
+        log_error "${COLLECTOR_NAME} continuity receipt 為空"
+        return 1
+    }
+    if awk -F '\t' -v expected_id="${COLLECTOR_ID_BEFORE}" \
+        -v expected_started="${COLLECTOR_STARTED_AT_BEFORE}" \
+        -v expected_restarts="${COLLECTOR_RESTARTS_BEFORE}" \
+        -v expected_health="${COLLECTOR_HEALTH_BEFORE}" \
+        '$2 == "unknown" || $3 != "true" || $2 != expected_id || \
+         $4 != expected_started || $5 != expected_restarts || $6 != expected_health { exit 1 }' \
+        "${COLLECTOR_OBSERVATIONS}"; then
+        :
+    else
+        log_error "${COLLECTOR_NAME} 在 native backup window 發生停止或 identity drift"
         return 1
     fi
 
-    # 與 docker start 的回傳值與 restore 內部 readback 分離，作為 success 前獨立 verifier。
-    collector_is_running
+    observation="$(collector_identity_state)" || return 1
+    IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${observation}"
+    [ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
+        && [ "${final_running}" = "true" ] \
+        && [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
+        && [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
+        && [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ]
 }
 
 cleanup() {
@@ -120,13 +129,9 @@ cleanup() {
         return "${exit_code}"
     fi
     CLEANUP_COMPLETE=1
-
-    # cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
     trap '' HUP INT TERM
-    if ! restore_collector; then
-        notify_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}"
-    fi
-    rm -rf "${DUMP_DIR}"
+    stop_collector_observer
+    rm -rf -- "${DUMP_DIR}"
     return "${exit_code}"
 }
 
@@ -135,166 +140,198 @@ on_signal() {
     exit "$((128 + signal_number))"
 }
 
-create_volume_archive() {
-    local volume_name="$1"
-    local output_file="$2"
-    local extra_exclude="${3:-}"
-    local partial_file="${output_file}.partial"
+run_native_clickhouse_backup() {
+    local mode="$1"
 
-    log_info "備份 volume: ${volume_name}"
-    rm -f "${partial_file}" "${output_file}"
-
-    if [ -n "${extra_exclude}" ]; then
-        if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
-            "${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
-            docker run --rm -v "${volume_name}:/data" alpine \
-            tar czf - "${extra_exclude}" /data \
-            > "${partial_file}" 2>/dev/null; then
-            log_error "  Volume ${volume_name} archive 建立失敗"
-            rm -f "${partial_file}" "${output_file}"
+    [ -f "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
+        && [ -x "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
+        && [ ! -L "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
+        || {
+            log_error "ClickHouse native backup adapter 缺失、不可執行或為 symlink"
             return 1
-        fi
-    else
-        if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
-            "${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
-            docker run --rm -v "${volume_name}:/data" alpine \
-            tar czf - /data \
-            > "${partial_file}" 2>/dev/null; then
-            log_error "  Volume ${volume_name} archive 建立失敗"
-            rm -f "${partial_file}" "${output_file}"
-            return 1
-        fi
-    fi
+        }
 
-    if [ ! -s "${partial_file}" ]; then
-        log_error "  Volume ${volume_name} 備份失敗 (空檔案)"
-        rm -f "${partial_file}" "${output_file}"
-        return 1
-    fi
-
-    mv "${partial_file}" "${output_file}"
+    CLICKHOUSE_NATIVE_OUTPUT_DIR="${DUMP_DIR}" \
+        TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+        "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" "${mode}"
 }
 
-verify_volume_archive() {
-    local volume_name="$1"
-    local output_file="$2"
+write_metadata_gap_receipt() {
+    cat > "${DUMP_DIR}/signoz-metadata-gap.json" <<EOF
+{"trace_id":"${TRACE_ID}","run_id":"${RUN_ID}","work_item_id":"${WORK_ITEM_ID}","asset":"signoz-metadata-sqlite","terminal":"pending_application_consistent_export","raw_volume_read":false,"raw_volume_archive":false,"completion_claim":false}
+EOF
+    log_warn "SigNoz metadata/SQLite 尚缺 application-consistent export；本 run 未讀取 live volume"
+}
 
-    if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
-        "${ARCHIVE_VERIFY_TIMEOUT_SECONDS}" \
-        tar -tzf "${output_file}" >/dev/null 2>&1; then
-        log_error "  Volume ${volume_name} archive integrity verifier 失敗"
-        rm -f "${output_file}" "${output_file}.partial"
-        return 1
-    fi
+write_restic_receipt() {
+    local snapshot_id="$1"
+    local artifact_hash="$2"
+    local manifest_hash="$3"
+    local inventory_hash="$4"
+    local receipt_dir receipt_tmp receipt_file
 
-    local size
-    size="$(du -h "${output_file}" | cut -f1)"
-    log_success "  Volume ${volume_name} 備份與 integrity verifier 完成 (${size})"
+    [[ "${RESTIC_RECEIPT_ROOT}" == /* ]] && [ ! -L "${RESTIC_RECEIPT_ROOT}" ] || return 1
+    mkdir -p "${RESTIC_RECEIPT_ROOT}"
+    receipt_dir="${RESTIC_RECEIPT_ROOT}/${RUN_ID}"
+    [ ! -L "${receipt_dir}" ] || return 1
+    mkdir -m 700 -p "${receipt_dir}"
+    receipt_file="${receipt_dir}/restic-snapshot.json"
+    receipt_tmp="${receipt_file}.partial.$$"
+    printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","service":"signoz","snapshot_id":"%s","artifact_sha256":"%s","manifest_sha256":"%s","source_inventory_sha256":"%s","repository_scope":"local_same_failure_domain","restic_check_read_data_subset":"1%%","offsite_verified":false,"metadata_sqlite_terminal":"pending_application_consistent_export","completion_claim":"clickhouse_native_only"}\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${snapshot_id}" \
+        "${artifact_hash}" "${manifest_hash}" "${inventory_hash}" > "${receipt_tmp}"
+    chmod 600 "${receipt_tmp}"
+    mv "${receipt_tmp}" "${receipt_file}"
 }
 
 main() {
-    local start_time=$(date +%s)
+    local start_time end_time duration snapshot_id snapshot_json tags
+    local initial_observation initial_running final_id final_running
+    local final_started_at final_restarts final_health
+    local artifact_file manifest_file inventory_file artifact_hash manifest_hash inventory_hash
 
+    start_time="$(date +%s)"
     trap cleanup EXIT
     trap 'on_signal 1' HUP
     trap 'on_signal 2' INT
     trap 'on_signal 15' TERM
 
-    log_info "========== 開始 SignOz 備份 =========="
+    log_info "========== 開始 SigNoz ClickHouse native 備份 =========="
+    [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]] \
+        && [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0" ]] \
+        && [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0.0" ]] || {
+        notify_failure_once "COLLECTOR_CONTINUITY_POLL_SECONDS 無效"
+        exit 1
+    }
+    for timeout_value in "${RESTIC_INIT_TIMEOUT_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
+        "${RESTIC_READBACK_TIMEOUT_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}"; do
+        [[ "${timeout_value}" =~ ^[0-9]+$ ]] && [ "${timeout_value}" -gt 0 ] || {
+            notify_failure_once "Restic timeout contract 無效"
+            exit 1
+        }
+    done
     mkdir -p "${DUMP_DIR}"
 
-    local timestamp=$(date "+%Y%m%d_%H%M%S")
-
-    # Step 1: 只在 Collector 原本 running 時暫停，並在 stop 前 arm 還原。
-    local collector_initial_state
-    if ! collector_initial_state="$(collector_running_state)"; then
-        log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態"
-        notify_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態"
+    # Check-mode must pass before the backup window. It writes only durable
+    # receipts and does not submit BACKUP or create a server artifact.
+    if ! run_native_clickhouse_backup --check; then
+        log_error "ClickHouse native BACKUP check-mode 失敗；沒有 volume tar fallback"
+        notify_failure_once "SigNoz ClickHouse native BACKUP runtime contract 未就緒"
         exit 1
     fi
 
-    if [ "${collector_initial_state}" = "running" ]; then
-        COLLECTOR_WAS_RUNNING=1
-        COLLECTOR_RESTORE_PENDING=1
-        log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..."
-        if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then
-            log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止"
-            notify_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}"
-            exit 1
-        fi
-    else
-        log_error "${COLLECTOR_NAME} 初始狀態為 stopped，停止備份且不自動啟動"
-        notify_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止"
-        exit 1
-    fi
-    docker stop signoz-telemetrystore-migrator 2>/dev/null || true
-
-    # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
-    local clickhouse_archive="${DUMP_DIR}/clickhouse_${timestamp}.tar.gz"
-    local sqlite_archive="${DUMP_DIR}/sqlite_${timestamp}.tar.gz"
-    create_volume_archive "signoz-clickhouse" "${clickhouse_archive}" "--exclude=/data/tmp" || {
-        log_error "ClickHouse volume 備份失敗"
-        notify_failure_once "SignOz ClickHouse 備份失敗"
+    initial_observation="$(collector_identity_state)" || {
+        notify_failure_once "無法讀取 ${COLLECTOR_NAME} 初始 identity/state"
         exit 1
     }
-
-    # Step 3: 備份 SQLite volume (SignOz metadata)
-    create_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
-        log_error "SQLite volume 備份失敗"
-        notify_failure_once "SignOz SQLite 備份失敗"
+    IFS=$'\t' read -r COLLECTOR_ID_BEFORE initial_running COLLECTOR_STARTED_AT_BEFORE \
+        COLLECTOR_RESTARTS_BEFORE COLLECTOR_HEALTH_BEFORE <<< "${initial_observation}"
+    [ -n "${COLLECTOR_ID_BEFORE}" ] && [ "${initial_running}" = "true" ] || {
+        notify_failure_once "${COLLECTOR_NAME} 初始狀態不是 running"
         exit 1
     }
+    [ -n "${COLLECTOR_STARTED_AT_BEFORE}" ] \
+        && [[ "${COLLECTOR_RESTARTS_BEFORE}" =~ ^[0-9]+$ ]] \
+        && [[ "${COLLECTOR_HEALTH_BEFORE}" =~ ^(healthy|none)$ ]] || {
+        notify_failure_once "${COLLECTOR_NAME} 初始 runtime metadata 不完整"
+        exit 1
+    }
+    start_collector_observer
 
-    # Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。
-    if ! restore_collector; then
-        notify_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}"
+    # Online native backup: no collector stop and no live-volume read.
+    if ! run_native_clickhouse_backup --apply; then
+        notify_failure_once "SigNoz ClickHouse native BACKUP 失敗"
         exit 1
     fi
 
-    # Step 5: Collector 恢復後才在 host 做有界 archive integrity verifier。
-    verify_volume_archive "signoz-clickhouse" "${clickhouse_archive}" || {
-        notify_failure_once "SignOz ClickHouse archive integrity verifier 失敗"
+    if ! verify_collector_continuity; then
+        notify_failure_once "${COLLECTOR_NAME} no-downtime continuity verifier 失敗"
         exit 1
-    }
-    verify_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
-        notify_failure_once "SignOz SQLite archive integrity verifier 失敗"
-        exit 1
-    }
+    fi
+    log_success "${COLLECTOR_NAME} no-downtime continuity verifier 通過"
+
+    write_metadata_gap_receipt
+
+    artifact_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip' -print)"
+    manifest_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip.manifest.json' -print)"
+    inventory_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.source-inventory.tsv' -print)"
+    [ "$(printf '%s\n' "${artifact_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
+        && [ "$(printf '%s\n' "${manifest_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
+        && [ "$(printf '%s\n' "${inventory_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] || {
+        log_error "ClickHouse native artifact set 不唯一或不完整"
+        exit 1
+    }
+    artifact_hash="$(sha256sum "${artifact_file}" | awk '{print $1}')"
+    manifest_hash="$(sha256sum "${manifest_file}" | awk '{print $1}')"
+    inventory_hash="$(sha256sum "${inventory_file}" | awk '{print $1}')"
 
-    # Step 6: 初始化 Restic 倉庫
     if [ ! -d "${LOCAL_REPO}/data" ]; then
         log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
-        restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
+        timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_INIT_TIMEOUT_SECONDS}" \
+            restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
             log_error "Restic 倉庫初始化失敗"
             exit 1
         }
     fi
 
-    # Step 7: Restic 備份
-    log_info "建立 Restic 備份..."
-    local tags=$(build_tags "${SERVICE}")
-    restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}"         --password-file "${RESTIC_PASSWORD_FILE}"         ${tags} 2>&1
+    log_info "建立 ClickHouse native artifact 的 Restic 備份..."
+    tags="$(build_tags "${SERVICE}")"
+    timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
+        restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
+            --password-file "${RESTIC_PASSWORD_FILE}" ${tags} \
+            --tag "trace:${TRACE_ID}" --tag "run:${RUN_ID}" \
+            --tag "work-item:${WORK_ITEM_ID}" 2>&1
 
-    local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1)
+    snapshot_json="$(timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
+        "${RESTIC_READBACK_TIMEOUT_SECONDS}" restic -r "${LOCAL_REPO}" snapshots \
+        --tag "run:${RUN_ID}" --latest 1 --json \
+        --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)"
+    snapshot_id="$(printf '%s\n' "${snapshot_json}" \
+        | sed -n 's/.*"short_id":"\([^"]*\)".*/\1/p' | head -1)"
+    [[ "${snapshot_id}" =~ ^[A-Za-z0-9]+$ ]] || {
+        log_error "Restic same-run snapshot durable readback 缺失"
+        exit 1
+    }
+    timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}" \
+        restic -r "${LOCAL_REPO}" check --read-data-subset=1% \
+            --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
+        log_error "Restic repository/data-subset verifier 失敗"
+        exit 1
+    }
+    write_restic_receipt "${snapshot_id}" "${artifact_hash}" \
+        "${manifest_hash}" "${inventory_hash}" || {
+        log_error "Restic durable receipt writeback 失敗"
+        exit 1
+    }
     log_success "Restic 備份完成: ${snapshot_id}"
 
-    # Step 8: GFS 清理；real canary 可明確跳過 destructive retention。
     if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then
         log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
     else
         cleanup_old_backups "${LOCAL_REPO}"
     fi
 
-    # Step 9: success 前獨立檢查 runtime state；失敗時不得發送 success。
-    if ! verify_collector_final_state; then
-        notify_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗"
+    # Final readback is separate from the continuity observer receipt.
+    initial_observation="$(collector_identity_state)" || {
+        notify_failure_once "${COLLECTOR_NAME} final readback 失敗"
         exit 1
-    fi
+    }
+    IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${initial_observation}"
+    [ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
+        && [ "${final_running}" = "true" ] \
+        && [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
+        && [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
+        && [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ] || {
+        notify_failure_once "${COLLECTOR_NAME} final identity/start/restart/health verifier 失敗"
+        exit 1
+    }
 
-    local end_time=$(date +%s)
-    local duration=$((end_time - start_time))
-    log_success "========== SignOz 備份完成 (${duration}s) =========="
-    notify_clawbot "success" "${SERVICE}" "SignOz 備份完成 (ClickHouse+SQLite)" "${duration}"
+    end_time="$(date +%s)"
+    duration=$((end_time - start_time))
+    log_success "========== SigNoz ClickHouse native 備份完成 (${duration}s) =========="
+    notify_clawbot "warning" "${SERVICE}" \
+        "ClickHouse native backup 完成；metadata/SQLite application-consistent export 仍 pending" \
+        "${duration}"
 }
 
 main "$@"

scripts/backup/clickhouse-native-backup.sh

@@ -0,0 +1,628 @@
+#!/usr/bin/env bash
+# =============================================================================
+# P0-OBS-002 - ClickHouse native BACKUP adapter for SigNoz
+#
+# This adapter deliberately has no live-volume tar fallback.  It requires a
+# ClickHouse Disk configured and allowed for BACKUP, records every bounded
+# command's stdout/stderr/exit status, and copies only the completed native
+# backup artifact into the caller's staging directory.
+# =============================================================================
+
+set -euo pipefail
+
+CONTAINER_NAME="${CLICKHOUSE_NATIVE_CONTAINER_NAME:-signoz-clickhouse}"
+BACKUP_DISK="${CLICKHOUSE_NATIVE_BACKUP_DISK:-backups}"
+EXPECTED_DISK_PATH="${CLICKHOUSE_NATIVE_EXPECTED_DISK_PATH:-/backups/}"
+OUTPUT_DIR="${CLICKHOUSE_NATIVE_OUTPUT_DIR:-}"
+RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}"
+COMMAND_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS:-30}"
+BACKUP_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS:-1800}"
+POLL_INTERVAL_SECONDS="${CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS:-5}"
+KILL_AFTER_SECONDS="${CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS:-30}"
+POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}"
+EXPECTED_DATABASES_CSV="signoz_analytics,signoz_logs,signoz_metadata,signoz_meter,signoz_metrics,signoz_traces"
+EXPECTED_DATABASES_JSON='["signoz_analytics","signoz_logs","signoz_metadata","signoz_meter","signoz_metrics","signoz_traces"]'
+BACKUP_SCOPE_SQL="DATABASE signoz_analytics,DATABASE signoz_logs,DATABASE signoz_metadata,DATABASE signoz_meter,DATABASE signoz_metrics,DATABASE signoz_traces"
+
+TRACE_ID="${TRACE_ID:-}"
+RUN_ID="${RUN_ID:-}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-}"
+
+MODE=""
+IDENTITY_HASH=""
+OPERATION_ID=""
+ARTIFACT_NAME=""
+SERVER_ARTIFACT_PATH=""
+OUTPUT_FILE=""
+MANIFEST_FILE=""
+HASH_FILE=""
+INVENTORY_FILE=""
+OUTPUT_PARTIAL=""
+MANIFEST_PARTIAL=""
+INVENTORY_PARTIAL=""
+RECEIPT_DIR=""
+RECEIPT_LOG=""
+IDENTITY_FILE=""
+INVOCATION_ID=""
+
+LAST_STDOUT=""
+LAST_STDERR=""
+LAST_STATUS_FILE=""
+LAST_EXIT_CODE=0
+COMMAND_SEQUENCE=0
+
+CONTAINER_ID=""
+CLICKHOUSE_VERSION=""
+DATABASES_HASH=""
+SOURCE_INVENTORY_RECEIPT=""
+SOURCE_INVENTORY_HASH=""
+STATUS_PRESENT=0
+BACKUP_NAME=""
+BACKUP_STATUS=""
+BACKUP_NUM_FILES="0"
+BACKUP_TOTAL_SIZE="0"
+BACKUP_UNCOMPRESSED_SIZE="0"
+BACKUP_COMPRESSED_SIZE="0"
+BACKUP_ERROR_HEX="none"
+BACKUP_ERROR_HASH=""
+
+RECEIPTS_READY=0
+CHECK_PASS=0
+APPLY_PASS=0
+OUTPUT_CREATED=0
+SERVER_ARTIFACT_ACTIVE=0
+
+log_info() { printf '[INFO] %s\n' "$*"; }
+log_error() { printf '[ERROR] %s\n' "$*" >&2; }
+
+valid_identity() {
+    [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
+}
+
+valid_name() {
+    [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]
+}
+
+valid_positive_integer() {
+    [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
+}
+
+emit_receipt() {
+    local stage="$1"
+    local terminal="$2"
+    local detail="$3"
+    local observed_at
+
+    [ "${RECEIPTS_READY}" -eq 1 ] || return 0
+    observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+    printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${INVOCATION_ID}" \
+        "${observed_at}" "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}"
+}
+
+fail_closed() {
+    local detail="$1"
+    log_error "${detail}"
+    emit_receipt "failure" "failed" "${detail}"
+    exit 1
+}
+
+require_command() {
+    command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
+}
+
+cleanup() {
+    local exit_code=$?
+    local terminal="failed"
+
+    trap - EXIT HUP INT TERM
+    set +e
+    [ -z "${OUTPUT_PARTIAL}" ] || rm -f -- "${OUTPUT_PARTIAL}"
+    [ -z "${MANIFEST_PARTIAL}" ] || rm -f -- "${MANIFEST_PARTIAL}"
+    [ -z "${INVENTORY_PARTIAL}" ] || rm -f -- "${INVENTORY_PARTIAL}"
+
+    if [ "${OUTPUT_CREATED}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then
+        rm -f -- "${OUTPUT_FILE}" "${MANIFEST_FILE}" "${HASH_FILE}" "${INVENTORY_FILE}"
+        emit_receipt "cleanup" "removed" "unverified_local_artifact_removed"
+    fi
+
+    if [ "${SERVER_ARTIFACT_ACTIVE}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then
+        emit_receipt "cleanup" "preserved" \
+            "active_or_unverified_server_artifact_preserved_for_safe_followup"
+    fi
+
+    if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_PASS}" -eq 1 ]; then
+        terminal="check_pass_no_write"
+    elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "apply" ] && [ "${APPLY_PASS}" -eq 1 ]; then
+        terminal="pass"
+    fi
+
+    emit_receipt "terminal" "${terminal}" "clickhouse_native_backup_${terminal}"
+    printf 'CLICKHOUSE_NATIVE_BACKUP_TERMINAL trace_id=%s run_id=%s work_item_id=%s mode=%s terminal=%s exit_code=%s\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" "${terminal}" "${exit_code}"
+    exit "${exit_code}"
+}
+
+on_signal() {
+    local signal_number="$1"
+    emit_receipt "signal" "failed" "signal_${signal_number}"
+    exit "$((128 + signal_number))"
+}
+
+setup() {
+    local identity_content identity_tmp
+
+    [ "$#" -eq 1 ] || { log_error "exactly one mode is required: --check or --apply"; exit 2; }
+    case "$1" in
+        --check) MODE="check" ;;
+        --apply) MODE="apply" ;;
+        *) log_error "unsupported mode: use --check or --apply"; exit 2 ;;
+    esac
+
+    valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and path-safe"; exit 2; }
+    valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and path-safe"; exit 2; }
+    valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and path-safe"; exit 2; }
+    valid_name "${CONTAINER_NAME}" || { log_error "invalid container name"; exit 2; }
+    valid_name "${BACKUP_DISK}" || { log_error "invalid backup disk name"; exit 2; }
+    valid_positive_integer "${COMMAND_TIMEOUT_SECONDS}" || { log_error "invalid command timeout"; exit 2; }
+    valid_positive_integer "${BACKUP_TIMEOUT_SECONDS}" || { log_error "invalid backup timeout"; exit 2; }
+    valid_positive_integer "${POLL_INTERVAL_SECONDS}" || { log_error "invalid poll interval"; exit 2; }
+    valid_positive_integer "${KILL_AFTER_SECONDS}" || { log_error "invalid kill-after timeout"; exit 2; }
+    [[ "${EXPECTED_DISK_PATH}" == /*/ ]] && [[ "${EXPECTED_DISK_PATH}" != *".."* ]] \
+        || { log_error "expected disk path must be an absolute trailing-slash path without dot-dot"; exit 2; }
+    [ -n "${OUTPUT_DIR}" ] && [[ "${OUTPUT_DIR}" == /* ]] \
+        || { log_error "CLICKHOUSE_NATIVE_OUTPUT_DIR must be absolute"; exit 2; }
+    [ -d "${OUTPUT_DIR}" ] && [ -w "${OUTPUT_DIR}" ] && [ ! -L "${OUTPUT_DIR}" ] \
+        || { log_error "output directory missing, unwritable, or symlink"; exit 2; }
+    if [ -n "${POST_VERIFY_HOOK}" ]; then
+        [[ "${POST_VERIFY_HOOK}" == /* ]] && [ -f "${POST_VERIFY_HOOK}" ] \
+            && [ -x "${POST_VERIFY_HOOK}" ] && [ ! -L "${POST_VERIFY_HOOK}" ] \
+            || { log_error "post verifier hook must be an absolute executable regular file"; exit 2; }
+    fi
+
+    for command_name in awk basename cat chmod cmp cp date docker env flock grep mkdir mktemp mv rm sha256sum sleep timeout touch tr unzip wc; do
+        command -v "${command_name}" >/dev/null 2>&1 \
+            || { log_error "required command missing: ${command_name}"; exit 2; }
+    done
+
+    IDENTITY_HASH="$(printf '%s\000%s\000%s' "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" | sha256sum | awk '{print $1}')"
+    OPERATION_ID="awoooi-${IDENTITY_HASH}"
+    ARTIFACT_NAME="signoz-${IDENTITY_HASH}.zip"
+    SERVER_ARTIFACT_PATH="${EXPECTED_DISK_PATH%/}/${ARTIFACT_NAME}"
+    OUTPUT_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.zip"
+    MANIFEST_FILE="${OUTPUT_FILE}.manifest.json"
+    HASH_FILE="${OUTPUT_FILE}.sha256"
+    INVENTORY_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.source-inventory.tsv"
+    OUTPUT_PARTIAL="${OUTPUT_FILE}.partial"
+    MANIFEST_PARTIAL="${MANIFEST_FILE}.partial"
+    INVENTORY_PARTIAL="${INVENTORY_FILE}.partial"
+
+    [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root symlink is unsafe"; exit 2; }
+    mkdir -p "${RECEIPT_ROOT}"
+    RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
+    [ ! -L "${RECEIPT_DIR}" ] || { log_error "receipt directory symlink is unsafe"; exit 2; }
+    mkdir -m 700 -p "${RECEIPT_DIR}"
+    RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
+    IDENTITY_FILE="${RECEIPT_DIR}/identity.json"
+    INVOCATION_ID="${MODE}-$(date -u '+%Y%m%dT%H%M%SZ')-$$"
+
+    identity_content="{\"trace_id\":\"${TRACE_ID}\",\"run_id\":\"${RUN_ID}\",\"work_item_id\":\"${WORK_ITEM_ID}\",\"identity_hash\":\"${IDENTITY_HASH}\",\"operation_id\":\"${OPERATION_ID}\",\"artifact_name\":\"${ARTIFACT_NAME}\"}"
+    if [ -e "${IDENTITY_FILE}" ]; then
+        [ -f "${IDENTITY_FILE}" ] && [ ! -L "${IDENTITY_FILE}" ] \
+            || { log_error "existing identity receipt is unsafe"; exit 2; }
+        [ "$(cat "${IDENTITY_FILE}")" = "${identity_content}" ] \
+            || { log_error "run identity conflicts with durable receipt"; exit 2; }
+    else
+        identity_tmp="$(mktemp "${RECEIPT_DIR}/.identity.XXXXXX")"
+        printf '%s\n' "${identity_content}" > "${identity_tmp}"
+        chmod 600 "${identity_tmp}"
+        mv "${identity_tmp}" "${IDENTITY_FILE}"
+    fi
+    touch "${RECEIPT_LOG}"
+    RECEIPTS_READY=1
+
+    [ ! -L "${RECEIPT_DIR}/run.lock" ] || fail_closed "run_lock_symlink_unsafe"
+    exec 9>>"${RECEIPT_DIR}/run.lock"
+    flock -n 9 || fail_closed "same_run_native_backup_is_active"
+
+    trap cleanup EXIT
+    trap 'on_signal 1' HUP
+    trap 'on_signal 2' INT
+    trap 'on_signal 15' TERM
+}
+
+run_captured() {
+    local step="$1"
+    local stdout_hash stderr_hash terminal
+    shift
+
+    valid_name "${step}" || fail_closed "invalid_command_step"
+    COMMAND_SEQUENCE=$((COMMAND_SEQUENCE + 1))
+    LAST_STDOUT="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stdout"
+    LAST_STDERR="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stderr"
+    LAST_STATUS_FILE="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.status"
+
+    set +e
+    "$@" >"${LAST_STDOUT}" 2>"${LAST_STDERR}"
+    LAST_EXIT_CODE=$?
+    set -e
+    printf '%s\n' "${LAST_EXIT_CODE}" > "${LAST_STATUS_FILE}"
+    stdout_hash="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')"
+    stderr_hash="$(sha256sum "${LAST_STDERR}" | awk '{print $1}')"
+    terminal="pass"
+    [ "${LAST_EXIT_CODE}" -eq 0 ] || terminal="failed"
+    emit_receipt "command" "${terminal}" \
+        "step_${step}_exit_${LAST_EXIT_CODE}_stdout_${stdout_hash}_stderr_${stderr_hash}"
+    return "${LAST_EXIT_CODE}"
+}
+
+run_sql() {
+    local step="$1"
+    local query="$2"
+    shift 2
+    run_captured "${step}" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+        clickhouse-client --format TSVRaw "$@" --query "${query}"
+}
+
+perform_check() {
+    local inspect_line running disk_path system_backups_count database_count backup_grant
+    local expected_databases_file
+
+    run_captured "container" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" docker inspect \
+        --format $'{{.Id}}\t{{.State.Running}}' "${CONTAINER_NAME}" \
+        || fail_closed "clickhouse_container_inspect_failed"
+    inspect_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+    IFS=$'\t' read -r CONTAINER_ID running <<< "${inspect_line}"
+    [ -n "${CONTAINER_ID}" ] && [ "${running}" = "true" ] \
+        || fail_closed "clickhouse_container_not_running_or_identity_ambiguous"
+
+    run_sql "version" 'SELECT version()' || fail_closed "clickhouse_version_query_failed"
+    CLICKHOUSE_VERSION="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+    [ -n "${CLICKHOUSE_VERSION}" ] || fail_closed "clickhouse_version_empty"
+
+    run_sql "system_backups" \
+        "SELECT count() FROM system.tables WHERE database='system' AND name='backups'" \
+        || fail_closed "system_backups_capability_query_failed"
+    system_backups_count="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+    [ "${system_backups_count}" = "1" ] || fail_closed "native_backup_capability_absent"
+
+    run_sql "backup_grant" 'CHECK GRANT BACKUP ON *.*' \
+        || fail_closed "native_backup_grant_check_failed"
+    backup_grant="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+    [ "${backup_grant}" = "1" ] || fail_closed "native_backup_grant_absent"
+
+    run_sql "backup_disk" \
+        'SELECT path FROM system.disks WHERE name={disk:String}' \
+        --param_disk "${BACKUP_DISK}" || fail_closed "backup_disk_query_failed"
+    disk_path="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+    [ "${disk_path}" = "${EXPECTED_DISK_PATH}" ] \
+        || fail_closed "backup_disk_missing_or_path_mismatch"
+
+    run_captured "disk_readable" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+        test -d "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_absent"
+    run_captured "disk_writable" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+        test -w "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_unwritable"
+
+    run_sql "databases" \
+        "SELECT name FROM system.databases WHERE name NOT IN ('system','information_schema','INFORMATION_SCHEMA','default') ORDER BY name" \
+        || fail_closed "user_database_inventory_query_failed"
+    database_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
+    expected_databases_file="${RECEIPT_DIR}/${INVOCATION_ID}-expected-databases.tsv"
+    printf '%s\n' signoz_analytics signoz_logs signoz_metadata signoz_meter signoz_metrics signoz_traces \
+        > "${expected_databases_file}"
+    cmp -s "${expected_databases_file}" "${LAST_STDOUT}" \
+        || fail_closed "signoz_database_allowlist_drift"
+    [ "${database_count}" -eq 6 ] || fail_closed "signoz_database_count_drift"
+    DATABASES_HASH="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')"
+
+    run_sql "table_inventory" \
+        "SELECT database,name,engine,toString(ifNull(total_rows,0)),toString(ifNull(total_bytes,0)),hex(SHA256(create_table_query)) FROM system.tables WHERE database IN ('signoz_analytics','signoz_logs','signoz_metadata','signoz_meter','signoz_metrics','signoz_traces') ORDER BY database,name" \
+        || fail_closed "source_table_inventory_query_failed"
+    [ "$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")" -gt 0 ] \
+        || fail_closed "source_table_inventory_empty"
+    SOURCE_INVENTORY_RECEIPT="${LAST_STDOUT}"
+    SOURCE_INVENTORY_HASH="$(sha256sum "${SOURCE_INVENTORY_RECEIPT}" | awk '{print $1}')"
+
+    emit_receipt "sensor_source" "pass" \
+        "container_${CONTAINER_ID}_version_${CLICKHOUSE_VERSION}_disk_${BACKUP_DISK}"
+    emit_receipt "normalized_asset_identity" "pass" \
+        "identity_${IDENTITY_HASH}_operation_${OPERATION_ID}_artifact_${ARTIFACT_NAME}"
+    emit_receipt "source_of_truth_diff" "pass" \
+        "configured_disk_path_exact_six_databases_hash_${DATABASES_HASH}_table_inventory_${SOURCE_INVENTORY_HASH}"
+    emit_receipt "risk_policy" "pass" \
+        "risk_medium_native_backup_async_bounded_poll_exact_allowlist_no_volume_tar_fallback"
+}
+
+query_backup_status() {
+    local line_count status_line
+
+    run_sql "backup_status" \
+        "SELECT name,toString(status),toString(num_files),toString(total_size),toString(uncompressed_size),toString(compressed_size),if(empty(error),'none',hex(substring(error,1,2048))),hex(SHA256(error)) FROM system.backups WHERE id={operation_id:String}" \
+        --param_operation_id "${OPERATION_ID}" || fail_closed "system_backups_status_query_failed"
+    line_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
+    if [ "${line_count}" -eq 0 ]; then
+        STATUS_PRESENT=0
+        return 0
+    fi
+    [ "${line_count}" -eq 1 ] || fail_closed "system_backups_operation_id_not_unique"
+    STATUS_PRESENT=1
+    status_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+    IFS=$'\t' read -r BACKUP_NAME BACKUP_STATUS BACKUP_NUM_FILES BACKUP_TOTAL_SIZE \
+        BACKUP_UNCOMPRESSED_SIZE BACKUP_COMPRESSED_SIZE BACKUP_ERROR_HEX \
+        BACKUP_ERROR_HASH <<< "${status_line}"
+    [[ "${BACKUP_NUM_FILES}" =~ ^[0-9]+$ ]] \
+        && [[ "${BACKUP_TOTAL_SIZE}" =~ ^[0-9]+$ ]] \
+        && [[ "${BACKUP_UNCOMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \
+        && [[ "${BACKUP_COMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \
+        || fail_closed "system_backups_metrics_ambiguous"
+}
+
+server_artifact_absent() {
+    run_captured "artifact_absent" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+        test ! -e "${SERVER_ARTIFACT_PATH}"
+}
+
+cleanup_server_artifact() {
+    run_captured "artifact_cleanup" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+        rm -f -- "${SERVER_ARTIFACT_PATH}" || return 1
+    server_artifact_absent || return 1
+    SERVER_ARTIFACT_ACTIVE=0
+    emit_receipt "cleanup" "pass" "exact_server_artifact_removed_and_absent"
+}
+
+poll_backup_terminal() {
+    local deadline now expected_name
+    expected_name="Disk('${BACKUP_DISK}', '${ARTIFACT_NAME}')"
+    deadline=$(( $(date +%s) + BACKUP_TIMEOUT_SECONDS ))
+
+    while true; do
+        query_backup_status
+        [ "${STATUS_PRESENT}" -eq 1 ] || fail_closed "async_backup_status_disappeared"
+        [ "${BACKUP_NAME}" = "${expected_name}" ] || fail_closed "async_backup_name_mismatch"
+        emit_receipt "backup_status" "observed" \
+            "status_${BACKUP_STATUS}_files_${BACKUP_NUM_FILES}_total_${BACKUP_TOTAL_SIZE}_error_hash_${BACKUP_ERROR_HASH}"
+
+        case "${BACKUP_STATUS}" in
+            BACKUP_CREATED)
+                [ "${BACKUP_NUM_FILES}" -gt 0 ] \
+                    && [ "${BACKUP_TOTAL_SIZE}" -gt 0 ] \
+                    && [ "${BACKUP_UNCOMPRESSED_SIZE}" -gt 0 ] \
+                    && [ "${BACKUP_COMPRESSED_SIZE}" -gt 0 ] \
+                    || fail_closed "native_backup_terminal_metrics_empty"
+                SERVER_ARTIFACT_ACTIVE=1
+                return 0
+                ;;
+            CREATING_BACKUP)
+                SERVER_ARTIFACT_ACTIVE=1
+                ;;
+            BACKUP_FAILED|BACKUP_CANCELLED)
+                printf '%s\n' "${BACKUP_ERROR_HEX}" \
+                    > "${RECEIPT_DIR}/${INVOCATION_ID}-backup-error-bounded.hex"
+                emit_receipt "backup_error" "retained" \
+                    "bounded_hex_2048_bytes_hash_${BACKUP_ERROR_HASH}"
+                SERVER_ARTIFACT_ACTIVE=1
+                if ! cleanup_server_artifact; then
+                    emit_receipt "cleanup" "failed" "failed_terminal_exact_artifact_cleanup_failed"
+                fi
+                fail_closed "native_backup_terminal_${BACKUP_STATUS}"
+                ;;
+            *) fail_closed "unexpected_native_backup_status_${BACKUP_STATUS}" ;;
+        esac
+
+        now="$(date +%s)"
+        if [ "${now}" -ge "${deadline}" ]; then
+            fail_closed "native_backup_poll_timeout_artifact_preserved"
+        fi
+        sleep "${POLL_INTERVAL_SECONDS}"
+    done
+}
+
+verify_existing_local_artifact() {
+    local recorded_hash actual_hash inventory_hash artifact_size
+    local manifest_hash hook_manifest_hash hook_inventory_hash
+
+    if [ ! -e "${OUTPUT_FILE}" ] && [ ! -e "${MANIFEST_FILE}" ] \
+        && [ ! -e "${HASH_FILE}" ] && [ ! -e "${INVENTORY_FILE}" ]; then
+        return 1
+    fi
+    [ -f "${OUTPUT_FILE}" ] && [ ! -L "${OUTPUT_FILE}" ] \
+        && [ -f "${MANIFEST_FILE}" ] && [ ! -L "${MANIFEST_FILE}" ] \
+        && [ -f "${HASH_FILE}" ] && [ ! -L "${HASH_FILE}" ] \
+        && [ -f "${INVENTORY_FILE}" ] && [ ! -L "${INVENTORY_FILE}" ] \
+        || fail_closed "local_artifact_identity_incomplete_or_unsafe"
+    grep -F -q "\"identity_hash\":\"${IDENTITY_HASH}\"" "${MANIFEST_FILE}" \
+        || fail_closed "local_artifact_manifest_identity_mismatch"
+    recorded_hash="$(awk 'NR == 1 { print $1 }' "${HASH_FILE}")"
+    [[ "${recorded_hash}" =~ ^[0-9a-f]{64}$ ]] || fail_closed "local_artifact_hash_receipt_invalid"
+    actual_hash="$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')"
+    [ "${actual_hash}" = "${recorded_hash}" ] || fail_closed "local_artifact_hash_mismatch"
+    artifact_size="$(wc -c < "${OUTPUT_FILE}" | tr -d ' ')"
+    inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')"
+    manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')"
+    grep -F -q "\"operation_id\":\"${OPERATION_ID}\"" "${MANIFEST_FILE}" \
+        || fail_closed "local_artifact_manifest_operation_mismatch"
+    grep -F -q "\"artifact_name\":\"${ARTIFACT_NAME}\"" "${MANIFEST_FILE}" \
+        || fail_closed "local_artifact_manifest_name_mismatch"
+    grep -F -q "\"databases\":${EXPECTED_DATABASES_JSON}" "${MANIFEST_FILE}" \
+        || fail_closed "local_artifact_manifest_database_allowlist_mismatch"
+    grep -F -q "\"source_inventory_sha256\":\"${inventory_hash}\"" "${MANIFEST_FILE}" \
+        || fail_closed "local_source_inventory_hash_mismatch"
+    grep -F -q "\"sha256\":\"${actual_hash}\"" "${MANIFEST_FILE}" \
+        || fail_closed "local_artifact_manifest_hash_mismatch"
+    grep -F -q "\"size_bytes\":${artifact_size}" "${MANIFEST_FILE}" \
+        || fail_closed "local_artifact_manifest_size_mismatch"
+    grep -F -q '"status":"BACKUP_CREATED"' "${MANIFEST_FILE}" \
+        || fail_closed "local_artifact_manifest_terminal_mismatch"
+    run_captured "local_zip_reuse" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_FILE}" \
+        || fail_closed "local_native_archive_reuse_verifier_failed"
+    if [ -n "${POST_VERIFY_HOOK}" ]; then
+        run_captured "reuse_verify_hook_check" env \
+            TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+            CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \
+            CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \
+            CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \
+            CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
+            CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
+            CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \
+            "${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_reuse_check_failed"
+        run_captured "reuse_verify_hook" env \
+            TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+            CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \
+            CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \
+            CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \
+            CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
+            CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
+            CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \
+            "${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_reuse_failed"
+        [ "$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')" = "${actual_hash}" ] \
+            || fail_closed "post_verify_hook_modified_reused_artifact"
+        hook_inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')"
+        [ "${hook_inventory_hash}" = "${inventory_hash}" ] \
+            || fail_closed "post_verify_hook_modified_reused_source_inventory"
+        hook_manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')"
+        [ "${hook_manifest_hash}" = "${manifest_hash}" ] \
+            || fail_closed "post_verify_hook_modified_reused_manifest"
+    fi
+    emit_receipt "idempotency" "reused" "verified_local_artifact_hash_${actual_hash}"
+    APPLY_PASS=1
+    return 0
+}
+
+copy_and_verify_artifact() {
+    local artifact_hash artifact_size hook_hash inventory_hash hook_inventory_hash
+    local manifest_hash hook_manifest_hash
+
+    rm -f -- "${OUTPUT_PARTIAL}" "${MANIFEST_PARTIAL}" "${INVENTORY_PARTIAL}"
+    run_captured "artifact_copy" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${BACKUP_TIMEOUT_SECONDS}" docker cp \
+        "${CONTAINER_NAME}:${SERVER_ARTIFACT_PATH}" "${OUTPUT_PARTIAL}" \
+        || fail_closed "native_backup_artifact_copy_failed"
+    [ -s "${OUTPUT_PARTIAL}" ] || fail_closed "native_backup_artifact_copy_empty"
+
+    run_captured "local_zip_verify" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+        "${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_PARTIAL}" \
+        || fail_closed "native_backup_archive_integrity_failed"
+    artifact_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')"
+    artifact_size="$(wc -c < "${OUTPUT_PARTIAL}" | tr -d ' ')"
+    [[ "${artifact_size}" =~ ^[0-9]+$ ]] && [ "${artifact_size}" -gt 0 ] \
+        || fail_closed "native_backup_local_size_invalid"
+
+    cp "${SOURCE_INVENTORY_RECEIPT}" "${INVENTORY_PARTIAL}"
+    inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')"
+    [ "${inventory_hash}" = "${SOURCE_INVENTORY_HASH}" ] \
+        || fail_closed "copied_source_inventory_hash_mismatch"
+    printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","identity_hash":"%s","operation_id":"%s","artifact_name":"%s","databases":%s,"source_inventory_sha256":"%s","sha256":"%s","size_bytes":%s,"status":"BACKUP_CREATED","num_files":%s,"total_size":%s,"uncompressed_size":%s,"compressed_size":%s}\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${IDENTITY_HASH}" \
+        "${OPERATION_ID}" "${ARTIFACT_NAME}" "${EXPECTED_DATABASES_JSON}" \
+        "${SOURCE_INVENTORY_HASH}" "${artifact_hash}" "${artifact_size}" \
+        "${BACKUP_NUM_FILES}" "${BACKUP_TOTAL_SIZE}" "${BACKUP_UNCOMPRESSED_SIZE}" \
+        "${BACKUP_COMPRESSED_SIZE}" > "${MANIFEST_PARTIAL}"
+
+    if [ -n "${POST_VERIFY_HOOK}" ]; then
+        manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')"
+        run_captured "post_verify_hook_check" env \
+            TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+            CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \
+            CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \
+            CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \
+            CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
+            CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
+            CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \
+            "${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_check_failed"
+        run_captured "post_verify_hook" env \
+            TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+            CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \
+            CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \
+            CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \
+            CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
+            CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
+            CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \
+            "${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_failed"
+        hook_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')"
+        [ "${hook_hash}" = "${artifact_hash}" ] || fail_closed "post_verify_hook_modified_artifact"
+        hook_inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')"
+        [ "${hook_inventory_hash}" = "${inventory_hash}" ] \
+            || fail_closed "post_verify_hook_modified_source_inventory"
+        hook_manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')"
+        [ "${hook_manifest_hash}" = "${manifest_hash}" ] \
+            || fail_closed "post_verify_hook_modified_manifest"
+    fi
+
+    mv "${OUTPUT_PARTIAL}" "${OUTPUT_FILE}"
+    OUTPUT_CREATED=1
+    mv "${INVENTORY_PARTIAL}" "${INVENTORY_FILE}"
+    printf '%s  %s\n' "${artifact_hash}" "$(basename "${OUTPUT_FILE}")" > "${HASH_FILE}"
+    mv "${MANIFEST_PARTIAL}" "${MANIFEST_FILE}"
+
+    cleanup_server_artifact || fail_closed "completed_server_artifact_cleanup_failed"
+    emit_receipt "independent_verifier" "pass" \
+        "system_status_archive_integrity_hash_${artifact_hash}_size_${artifact_size}"
+    APPLY_PASS=1
+}
+
+apply_backup() {
+    local submission_line submitted_id submitted_status
+
+    grep -F -q '"stage":"check","terminal":"check_pass_no_write"' "${RECEIPT_LOG}" \
+        || fail_closed "same_run_check_pass_receipt_required_before_apply"
+    perform_check
+    emit_receipt "check" "pass" "apply_revalidation_pass"
+
+    if verify_existing_local_artifact; then
+        return 0
+    fi
+
+    query_backup_status
+    if [ "${STATUS_PRESENT}" -eq 0 ]; then
+        server_artifact_absent || fail_closed "orphan_server_artifact_without_system_status"
+        emit_receipt "ai_decision" "candidate" \
+            "submit_native_async_backup_exact_six_database_allowlist_to_exact_artifact"
+        SERVER_ARTIFACT_ACTIVE=1
+        run_sql "backup_submit" \
+            "BACKUP ${BACKUP_SCOPE_SQL} TO Disk({disk:String},{artifact:String}) SETTINGS id='${OPERATION_ID}' ASYNC" \
+            --param_disk "${BACKUP_DISK}" --param_artifact "${ARTIFACT_NAME}" \
+            || fail_closed "native_backup_submit_failed"
+        submission_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+        IFS=$'\t' read -r submitted_id submitted_status <<< "${submission_line}"
+        [ "${submitted_id}" = "${OPERATION_ID}" ] \
+            || fail_closed "native_backup_submission_id_mismatch"
+        case "${submitted_status}" in
+            CREATING_BACKUP|BACKUP_CREATED) ;;
+            *) fail_closed "native_backup_submission_status_${submitted_status:-empty}" ;;
+        esac
+        SERVER_ARTIFACT_ACTIVE=1
+        emit_receipt "execution" "submitted" "operation_${OPERATION_ID}_status_${submitted_status}"
+    else
+        emit_receipt "idempotency" "resumed" "existing_operation_status_${BACKUP_STATUS}"
+    fi
+
+    poll_backup_terminal
+    copy_and_verify_artifact
+}
+
+main() {
+    setup "$@"
+
+    if [ "${MODE}" = "check" ]; then
+        perform_check
+        emit_receipt "ai_decision" "candidate" \
+            "native_backup_apply_requires_same_run_check_receipt"
+        emit_receipt "check" "check_pass_no_write" \
+            "runtime_contract_valid_no_backup_or_artifact_write"
+        CHECK_PASS=1
+        log_info "ClickHouse native backup check passed without runtime write"
+        return 0
+    fi
+
+    apply_backup
+    log_info "ClickHouse native backup verified: ${OUTPUT_FILE}"
+}
+
+main "$@"

scripts/backup/clickhouse-restore-inventory.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""Validate SigNoz ClickHouse native-backup inventories without data access.
+
+The shell restore-drill driver uses this helper for two narrow jobs:
+
+* prove that a local artifact is a completed, hashed native BACKUP with the
+  exact six-database source inventory expected by P0-OBS-002; and
+* compare the restored database/table/engine set with that source inventory.
+
+Row and byte values are recorded as bounded aggregate evidence, not exact
+parity gates, because the source inventory is captured while online ingestion
+can still advance before the native BACKUP snapshot is committed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+EXPECTED_DATABASES = (
+    "signoz_analytics",
+    "signoz_logs",
+    "signoz_metadata",
+    "signoz_meter",
+    "signoz_metrics",
+    "signoz_traces",
+)
+CRITICAL_TABLES = (
+    ("signoz_logs", "logs_v2"),
+    ("signoz_metrics", "time_series_v4"),
+    ("signoz_metrics", "samples_v4"),
+    ("signoz_traces", "signoz_index_v3"),
+)
+SAFE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+SAFE_OPERATION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
+SHA256 = re.compile(r"^[0-9a-f]{64}$")
+SCHEMA_SHA256 = re.compile(r"^[0-9A-F]{64}$")
+
+
+class ContractError(ValueError):
+    """A bounded, non-secret backup or inventory contract failure."""
+
+
+@dataclass(frozen=True)
+class TableReceipt:
+    database: str
+    table: str
+    engine: str
+    rows: int
+    bytes: int
+    schema_hash: str
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def parse_inventory(path: Path) -> dict[tuple[str, str], TableReceipt]:
+    receipts: dict[tuple[str, str], TableReceipt] = {}
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except (OSError, UnicodeError) as exc:
+        raise ContractError("inventory_read_failed") from exc
+
+    if not lines:
+        raise ContractError("inventory_empty")
+
+    for line_number, line in enumerate(lines, start=1):
+        fields = line.split("\t")
+        if len(fields) != 6:
+            raise ContractError(f"inventory_field_count_invalid_line_{line_number}")
+        database, table, engine, rows_text, bytes_text, schema_hash = fields
+        if not SAFE_IDENTIFIER.fullmatch(database):
+            raise ContractError(
+                f"inventory_database_identifier_unsafe_line_{line_number}"
+            )
+        if not SAFE_IDENTIFIER.fullmatch(table):
+            raise ContractError(f"inventory_table_identifier_unsafe_line_{line_number}")
+        if not SAFE_IDENTIFIER.fullmatch(engine):
+            raise ContractError(
+                f"inventory_engine_identifier_unsafe_line_{line_number}"
+            )
+        if not rows_text.isdecimal() or not bytes_text.isdecimal():
+            raise ContractError(f"inventory_counter_invalid_line_{line_number}")
+        if not SCHEMA_SHA256.fullmatch(schema_hash):
+            raise ContractError(f"inventory_schema_hash_invalid_line_{line_number}")
+        key = (database, table)
+        if key in receipts:
+            raise ContractError(f"inventory_duplicate_table_line_{line_number}")
+        receipts[key] = TableReceipt(
+            database=database,
+            table=table,
+            engine=engine,
+            rows=int(rows_text),
+            bytes=int(bytes_text),
+            schema_hash=schema_hash,
+        )
+    return receipts
+
+
+def require_database_set(receipts: dict[tuple[str, str], TableReceipt]) -> None:
+    observed = tuple(sorted({receipt.database for receipt in receipts.values()}))
+    if observed != EXPECTED_DATABASES:
+        raise ContractError("signoz_database_set_mismatch")
+
+
+def require_critical_tables(
+    receipts: dict[tuple[str, str], TableReceipt], *, require_nonzero: bool
+) -> None:
+    for key in CRITICAL_TABLES:
+        receipt = receipts.get(key)
+        if receipt is None:
+            raise ContractError(f"critical_table_missing_{key[0]}_{key[1]}")
+        if require_nonzero and receipt.rows <= 0:
+            raise ContractError(f"critical_table_empty_{key[0]}_{key[1]}")
+
+
+def emit(values: list[tuple[str, object]]) -> None:
+    for key, value in values:
+        print(f"{key}={value}")
+
+
+def preflight(args: argparse.Namespace) -> None:
+    manifest_path = Path(args.manifest)
+    artifact_path = Path(args.artifact)
+    inventory_path = Path(args.source_inventory)
+    try:
+        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+        raise ContractError("backup_manifest_read_failed") from exc
+    if not isinstance(manifest, dict):
+        raise ContractError("backup_manifest_not_object")
+    if manifest.get("status") != "BACKUP_CREATED":
+        raise ContractError("backup_terminal_not_BACKUP_CREATED")
+
+    artifact_sha256 = sha256_file(artifact_path)
+    inventory_sha256 = sha256_file(inventory_path)
+    artifact_size = artifact_path.stat().st_size
+    manifest_sha256 = manifest.get("sha256")
+    manifest_inventory_sha256 = manifest.get("source_inventory_sha256")
+    manifest_size = manifest.get("size_bytes")
+    if not isinstance(manifest_sha256, str) or not SHA256.fullmatch(manifest_sha256):
+        raise ContractError("backup_manifest_sha256_invalid")
+    if manifest_sha256 != artifact_sha256:
+        raise ContractError("backup_artifact_sha256_mismatch")
+    if manifest_inventory_sha256 != inventory_sha256:
+        raise ContractError("source_inventory_sha256_mismatch")
+    if not isinstance(manifest_size, int) or manifest_size <= 0:
+        raise ContractError("backup_manifest_size_invalid")
+    if manifest_size != artifact_size:
+        raise ContractError("backup_artifact_size_mismatch")
+    if tuple(manifest.get("databases", ())) != EXPECTED_DATABASES:
+        raise ContractError("backup_manifest_database_allowlist_mismatch")
+
+    operation_id = manifest.get("operation_id")
+    if not isinstance(operation_id, str) or not SAFE_OPERATION_ID.fullmatch(
+        operation_id
+    ):
+        raise ContractError("backup_operation_id_invalid")
+
+    receipts = parse_inventory(inventory_path)
+    require_database_set(receipts)
+    require_critical_tables(receipts, require_nonzero=True)
+    replicated_count = sum(
+        receipt.engine.startswith("Replicated") for receipt in receipts.values()
+    )
+    distributed_count = sum(
+        receipt.engine == "Distributed" for receipt in receipts.values()
+    )
+    physical_count = sum("MergeTree" in receipt.engine for receipt in receipts.values())
+    if replicated_count <= 0:
+        raise ContractError("replicated_table_precondition_missing")
+    if distributed_count <= 0:
+        raise ContractError("distributed_table_precondition_missing")
+    if physical_count <= 0:
+        raise ContractError("physical_table_precondition_missing")
+
+    emit(
+        [
+            ("artifact_sha256", artifact_sha256),
+            ("artifact_size_bytes", artifact_size),
+            ("source_inventory_sha256", inventory_sha256),
+            ("source_database_count", len(EXPECTED_DATABASES)),
+            ("source_table_count", len(receipts)),
+            ("replicated_table_count", replicated_count),
+            ("distributed_table_count", distributed_count),
+            ("physical_table_count", physical_count),
+            ("backup_operation_id", operation_id),
+        ]
+    )
+
+
+def compare(args: argparse.Namespace) -> None:
+    source = parse_inventory(Path(args.source_inventory))
+    restored = parse_inventory(Path(args.restored_inventory))
+    require_database_set(source)
+    require_database_set(restored)
+    require_critical_tables(source, require_nonzero=True)
+    require_critical_tables(restored, require_nonzero=True)
+
+    source_keys = set(source)
+    restored_keys = set(restored)
+    if source_keys != restored_keys:
+        raise ContractError("database_table_manifest_set_mismatch")
+    for key in sorted(source_keys):
+        if source[key].engine != restored[key].engine:
+            raise ContractError(f"table_engine_mismatch_{key[0]}_{key[1]}")
+        if source[key].schema_hash != restored[key].schema_hash:
+            raise ContractError(f"table_schema_mismatch_{key[0]}_{key[1]}")
+
+    source_rows = sum(receipt.rows for receipt in source.values())
+    restored_rows = sum(receipt.rows for receipt in restored.values())
+    source_bytes = sum(receipt.bytes for receipt in source.values())
+    restored_bytes = sum(receipt.bytes for receipt in restored.values())
+    physical_count = sum("MergeTree" in receipt.engine for receipt in restored.values())
+    emit(
+        [
+            ("manifest_parity", 1),
+            ("restored_database_count", len(EXPECTED_DATABASES)),
+            ("restored_table_count", len(restored)),
+            ("physical_table_count", physical_count),
+            ("critical_nonzero_count", len(CRITICAL_TABLES)),
+            ("source_total_rows", source_rows),
+            ("restored_total_rows", restored_rows),
+            ("row_delta", restored_rows - source_rows),
+            ("source_total_bytes", source_bytes),
+            ("restored_total_bytes", restored_bytes),
+            ("byte_delta", restored_bytes - source_bytes),
+        ]
+    )
+
+
+def check_list(args: argparse.Namespace) -> None:
+    receipts = parse_inventory(Path(args.inventory))
+    require_database_set(receipts)
+    for key in sorted(receipts):
+        receipt = receipts[key]
+        if "MergeTree" in receipt.engine:
+            print(f"{receipt.database}\t{receipt.table}")
+
+
+def parser() -> argparse.ArgumentParser:
+    root = argparse.ArgumentParser()
+    commands = root.add_subparsers(dest="command", required=True)
+
+    preflight_parser = commands.add_parser("preflight")
+    preflight_parser.add_argument("--manifest", required=True)
+    preflight_parser.add_argument("--artifact", required=True)
+    preflight_parser.add_argument("--source-inventory", required=True)
+    preflight_parser.set_defaults(handler=preflight)
+
+    compare_parser = commands.add_parser("compare")
+    compare_parser.add_argument("--source-inventory", required=True)
+    compare_parser.add_argument("--restored-inventory", required=True)
+    compare_parser.set_defaults(handler=compare)
+
+    check_parser = commands.add_parser("check-list")
+    check_parser.add_argument("--inventory", required=True)
+    check_parser.set_defaults(handler=check_list)
+    return root
+
+
+def main() -> int:
+    args = parser().parse_args()
+    try:
+        args.handler(args)
+    except (ContractError, OSError) as exc:
+        print(f"ERROR={exc}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/backup/run-signoz-backup-canary.sh

@@ -0,0 +1,665 @@
+#!/usr/bin/env bash
+# =============================================================================
+# P0-OBS-002 - SigNoz online backup canary and legacy maintenance lease verifier
+#
+# The default native-backup path never stops the collector and therefore does
+# not depend on, read, or mutate the legacy monitor/cooldown contract.  The
+# bounded lease path remains available only when an explicit legacy stop-mode
+# canary is requested.  Neither mode reads or sources a secret/.env file.
+# =============================================================================
+
+set -euo pipefail
+
+COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}"
+BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}"
+MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}"
+MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}"
+MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}"
+COOLDOWN_FILE="${MONITOR_COOLDOWN_DIR}/${COLLECTOR_NAME}.cooldown"
+RECEIPT_ROOT="${SIGNOZ_CANARY_RECEIPT_ROOT:-/backup/logs/signoz-canary}"
+LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}"
+
+CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}"
+TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}"
+LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}"
+MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}"
+STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}"
+EXPECT_COLLECTOR_STOP="${SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP:-0}"
+
+TRACE_ID="${TRACE_ID:-}"
+RUN_ID="${RUN_ID:-}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-}"
+
+RECEIPT_DIR=""
+RECEIPT_LOG=""
+BACKUP_OUTPUT=""
+MONITOR_WINDOW=""
+STATE_OBSERVATIONS=""
+ROLLBACK_FILE=""
+ROLLBACK_METADATA=""
+
+RECEIPTS_READY=0
+LEASE_ROLLBACK_READY=0
+LEASE_MUTATION_STARTED=0
+LEASE_ARMED=0
+PRIOR_LEASE_PRESENT=0
+PRIOR_LEASE_MODE=""
+PRIOR_LEASE_UID=""
+PRIOR_LEASE_GID=""
+PRIOR_LEASE_HASH=""
+PRIOR_LEASE_VALUE=""
+LEASE_TMP=""
+BACKUP_PID=""
+OBSERVER_PID=""
+CANARY_VERIFIED=0
+CHECK_VERIFIED=0
+MODE=""
+
+log_info() { printf '[INFO] %s\n' "$*"; }
+log_error() { printf '[ERROR] %s\n' "$*" >&2; }
+
+valid_identity() {
+    [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
+}
+
+valid_positive_integer() {
+    [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
+}
+
+valid_poll_interval() {
+    [[ "$1" =~ ^[0-9]+([.][0-9]+)?$ ]] && [[ "$1" != "0" ]] && [[ "$1" != "0.0" ]]
+}
+
+emit_receipt() {
+    local stage="$1"
+    local terminal="$2"
+    local detail="$3"
+    local observed_at
+
+    [ "${RECEIPTS_READY}" -eq 1 ] || return 0
+    observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+    printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${observed_at}" \
+        "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}"
+}
+
+fail_closed() {
+    local detail="$1"
+    log_error "${detail}"
+    emit_receipt "failure" "failed" "${detail}"
+    exit 1
+}
+
+require_command() {
+    command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
+}
+
+require_monitor_fragment() {
+    local fragment="$1"
+    local count
+    count="$(grep -F -c -- "${fragment}" "${MONITOR_SCRIPT}" || true)"
+    [ "${count}" -eq 1 ] || fail_closed "monitor_contract_ambiguous_fragment_count_${count}"
+}
+
+require_monitor_function_fragment() {
+    local function_name="$1"
+    local fragment="$2"
+    local count
+
+    count="$(awk -v header="${function_name}() {" -v needle="${fragment}" '
+        $0 == header {
+            inside = 1
+            depth = 0
+        }
+        inside {
+            original = $0
+            if (index(original, needle)) {
+                count++
+            }
+            open_line = original
+            close_line = original
+            depth += gsub(/\{/, "", open_line)
+            depth -= gsub(/\}/, "", close_line)
+            if (depth == 0) {
+                print count + 0
+                printed = 1
+                exit
+            }
+        }
+        END {
+            if (!printed) {
+                print count + 0
+            }
+        }
+    ' "${MONITOR_SCRIPT}")"
+    [ "${count}" -eq 1 ] \
+        || fail_closed "monitor_${function_name}_contract_ambiguous_fragment_count_${count}"
+}
+
+collector_running_state() {
+    local running
+    if ! running="$(docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
+        return 1
+    fi
+    case "${running}" in
+        true) printf '%s\n' "running" ;;
+        false) printf '%s\n' "stopped" ;;
+        *) return 1 ;;
+    esac
+}
+
+collector_runtime_metadata() {
+    docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
+        "${COLLECTOR_NAME}" 2>/dev/null
+}
+
+listener_is_up() {
+    local port="$1"
+    ss -H -lnt | awk -v port="${port}" '
+        $4 ~ (":" port "$") { found = 1 }
+        END { exit(found ? 0 : 1) }
+    '
+}
+
+verify_collector_runtime() {
+    local expected_id="$1"
+    local expected_started_at="$2"
+    local expected_restart_count="$3"
+    local expected_health="$4"
+    local metadata actual_id actual_running actual_started_at actual_restart_count actual_health
+
+    metadata="$(collector_runtime_metadata)" || return 1
+    IFS=$'\t' read -r actual_id actual_running actual_started_at actual_restart_count actual_health <<< "${metadata}"
+    [ "${actual_id}" = "${expected_id}" ] || return 1
+    [ "${actual_running}" = "true" ] || return 1
+    [ "${actual_started_at}" = "${expected_started_at}" ] || return 1
+    [ "${actual_restart_count}" = "${expected_restart_count}" ] || return 1
+    [ "${actual_health}" = "${expected_health}" ] || return 1
+    listener_is_up 4317 || return 1
+    listener_is_up 4318 || return 1
+}
+
+restore_lease() {
+    local restore_tmp restore_hash current_value
+
+    [ "${LEASE_MUTATION_STARTED}" -eq 1 ] || return 0
+    [ "${LEASE_ROLLBACK_READY}" -eq 1 ] || return 1
+
+    if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then
+        [ -f "${ROLLBACK_FILE}" ] || return 1
+        restore_tmp="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.restore.${RUN_ID}.XXXXXX")" || return 1
+        cp -p "${ROLLBACK_FILE}" "${restore_tmp}" || return 1
+        chmod "${PRIOR_LEASE_MODE}" "${restore_tmp}" || return 1
+        mv -f "${restore_tmp}" "${COOLDOWN_FILE}" || return 1
+        restore_tmp=""
+
+        restore_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" || return 1
+        [ "${restore_hash}" = "${PRIOR_LEASE_HASH}" ] || return 1
+        [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] || return 1
+        [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] || return 1
+        [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] || return 1
+        current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
+        [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] || return 1
+    else
+        rm -f "${COOLDOWN_FILE}" || return 1
+        [ ! -e "${COOLDOWN_FILE}" ] || return 1
+    fi
+
+    LEASE_ARMED=0
+    LEASE_MUTATION_STARTED=0
+    return 0
+}
+
+terminate_children() {
+    if [ -n "${BACKUP_PID}" ] && kill -0 "${BACKUP_PID}" 2>/dev/null; then
+        kill -TERM "${BACKUP_PID}" 2>/dev/null || true
+        wait "${BACKUP_PID}" 2>/dev/null || true
+    fi
+    if [ -n "${OBSERVER_PID}" ] && kill -0 "${OBSERVER_PID}" 2>/dev/null; then
+        kill -TERM "${OBSERVER_PID}" 2>/dev/null || true
+        wait "${OBSERVER_PID}" 2>/dev/null || true
+    fi
+}
+
+cleanup() {
+    local exit_code=$?
+    local lease_terminal="not_armed"
+    local overall_terminal="failed"
+
+    trap - EXIT HUP INT TERM
+    set +e
+    terminate_children
+    rm -f "${LEASE_TMP}" 2>/dev/null || true
+
+    if [ "${LEASE_MUTATION_STARTED}" -eq 1 ]; then
+        if restore_lease; then
+            lease_terminal="restored"
+        else
+            lease_terminal="restore_failed"
+            exit_code=91
+        fi
+    elif [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then
+        lease_terminal="no_write"
+    elif [ "${MODE}" = "apply" ] && [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
+        lease_terminal="no_write"
+    fi
+
+    if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then
+        overall_terminal="check_pass_no_write"
+    elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then
+        overall_terminal="pass"
+    fi
+
+    emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}"
+    if [ "${overall_terminal}" = "pass" ]; then
+        emit_receipt "closure_writeback" "pass" \
+            "no_incident_notification_or_learning_delta_local_durable_receipt_ack"
+    elif [ "${overall_terminal}" = "check_pass_no_write" ]; then
+        emit_receipt "closure_writeback" "not_applicable" \
+            "check_mode_no_runtime_change_local_durable_receipt_ack"
+    else
+        emit_receipt "closure_writeback" "deferred" \
+            "failed_terminal_requires_followup_no_completion_claim"
+    fi
+    emit_receipt "terminal" "${overall_terminal}" "backup_canary_${overall_terminal}"
+    printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \
+        "${lease_terminal}" "${exit_code}"
+    exit "${exit_code}"
+}
+
+on_signal() {
+    local signal_number="$1"
+    emit_receipt "signal" "failed" "signal_${signal_number}"
+    exit "$((128 + signal_number))"
+}
+
+setup_receipts() {
+    valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and must be path-safe"; exit 2; }
+    valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and must be path-safe"; exit 2; }
+    valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and must be path-safe"; exit 2; }
+
+    [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root must not be a symlink"; exit 2; }
+    mkdir -p "${RECEIPT_ROOT}"
+    RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
+    [ ! -e "${RECEIPT_DIR}" ] || { log_error "receipt directory already exists: ${RECEIPT_DIR}"; exit 2; }
+    mkdir -m 700 "${RECEIPT_DIR}"
+
+    RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
+    BACKUP_OUTPUT="${RECEIPT_DIR}/backup-output.log"
+    MONITOR_WINDOW="${RECEIPT_DIR}/monitor-window.log"
+    STATE_OBSERVATIONS="${RECEIPT_DIR}/collector-state.tsv"
+    ROLLBACK_FILE="${RECEIPT_DIR}/cooldown.rollback"
+    ROLLBACK_METADATA="${RECEIPT_DIR}/cooldown.rollback.json"
+    : > "${RECEIPT_LOG}"
+    RECEIPTS_READY=1
+
+    trap cleanup EXIT
+    trap 'on_signal 1' HUP
+    trap 'on_signal 2' INT
+    trap 'on_signal 15' TERM
+}
+
+verify_monitor_contract() {
+    local cron_output cron_count cron_line exclude_count exclude_line
+
+    [ -f "${MONITOR_SCRIPT}" ] && [ -r "${MONITOR_SCRIPT}" ] && [ ! -L "${MONITOR_SCRIPT}" ] \
+        || fail_closed "monitor_script_missing_unreadable_or_symlink"
+    [ -f "${MONITOR_LOG}" ] && [ -r "${MONITOR_LOG}" ] && [ ! -L "${MONITOR_LOG}" ] \
+        || fail_closed "monitor_log_missing_unreadable_or_symlink"
+    [ -d "${MONITOR_COOLDOWN_DIR}" ] && [ -w "${MONITOR_COOLDOWN_DIR}" ] && [ ! -L "${MONITOR_COOLDOWN_DIR}" ] \
+        || fail_closed "monitor_cooldown_directory_missing_unwritable_or_symlink"
+
+    require_monitor_fragment "COOLDOWN_DIR:=${MONITOR_COOLDOWN_DIR}"
+    require_monitor_fragment 'local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown"'
+    require_monitor_fragment 'last_sent=$(cat "$cooldown_file")'
+    # The live monitor has another cooldown path with the same elapsed
+    # calculation. Bind this assertion to is_in_cooldown instead of requiring
+    # a globally unique string.
+    require_monitor_function_fragment is_in_cooldown 'elapsed=$(( now - last_sent ))'
+    require_monitor_fragment 'if (( elapsed < ACTION_COOLDOWN_SECONDS )); then'
+    require_monitor_fragment 'COOLDOWN: ${container}'
+    require_monitor_fragment 'is_in_cooldown "$container_name" && continue'
+    require_monitor_fragment 'set_cooldown "$container_name"'
+    require_monitor_fragment 'log "AUTO_REPAIR: docker restart ${container}"'
+    require_monitor_fragment 'if docker restart "$container"'
+
+    exclude_count="$(grep -F -c 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}" || true)"
+    [ "${exclude_count}" -eq 1 ] || fail_closed "monitor_exclude_contract_ambiguous"
+    exclude_line="$(grep -F 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}")"
+    [[ "${exclude_line}" != *"${COLLECTOR_NAME}"* ]] || fail_closed "collector_already_excluded_monitor_contract_changed"
+    [ "$(grep -F -c "${COLLECTOR_NAME}" "${MONITOR_SCRIPT}" || true)" -eq 0 ] \
+        || fail_closed "collector_literal_present_monitor_contract_changed"
+
+    cron_output="$(crontab -l 2>/dev/null)" || fail_closed "monitor_crontab_unreadable"
+    cron_count="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" '
+        $0 !~ /^[[:space:]]*#/ && index($0, script) { count++ }
+        END { print count + 0 }
+    ')"
+    [ "${cron_count}" -eq 1 ] || fail_closed "monitor_cron_contract_ambiguous_count_${cron_count}"
+    cron_line="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" '
+        $0 !~ /^[[:space:]]*#/ && index($0, script) { print }
+    ')"
+    [[ "${cron_line}" == "*/5 * * * * "* ]] || fail_closed "monitor_cron_schedule_not_every_five_minutes"
+    [[ "${cron_line}" == *">> ${MONITOR_LOG}"* ]] || fail_closed "monitor_cron_log_route_mismatch"
+    [[ "${cron_line}" != *"COOLDOWN_DIR="* ]] || fail_closed "monitor_cron_cooldown_override_ambiguous"
+
+}
+
+inspect_prior_lease() {
+    local current_uid current_gid
+    current_uid="$(id -u)"
+    current_gid="$(id -g)"
+
+    if [ -e "${COOLDOWN_FILE}" ]; then
+        [ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \
+            || fail_closed "existing_cooldown_receipt_unsafe"
+        PRIOR_LEASE_PRESENT=1
+        PRIOR_LEASE_MODE="$(stat -c '%a' "${COOLDOWN_FILE}")"
+        PRIOR_LEASE_UID="$(stat -c '%u' "${COOLDOWN_FILE}")"
+        PRIOR_LEASE_GID="$(stat -c '%g' "${COOLDOWN_FILE}")"
+        [ "${PRIOR_LEASE_UID}" = "${current_uid}" ] && [ "${PRIOR_LEASE_GID}" = "${current_gid}" ] \
+            || fail_closed "existing_cooldown_receipt_owner_mismatch"
+        PRIOR_LEASE_VALUE="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
+        [[ "${PRIOR_LEASE_VALUE}" =~ ^[0-9]+$ ]] || fail_closed "existing_cooldown_receipt_not_epoch"
+        PRIOR_LEASE_HASH="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
+    else
+        PRIOR_LEASE_PRESENT=0
+        PRIOR_LEASE_MODE="664"
+        PRIOR_LEASE_UID="${current_uid}"
+        PRIOR_LEASE_GID="${current_gid}"
+        PRIOR_LEASE_HASH="absent"
+        PRIOR_LEASE_VALUE="absent"
+    fi
+
+}
+
+capture_prior_lease() {
+    local current_hash current_value
+
+    if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then
+        [ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \
+            || fail_closed "cooldown_changed_between_check_and_rollback_capture"
+        current_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
+        current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
+        [ "${current_hash}" = "${PRIOR_LEASE_HASH}" ] \
+            && [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] \
+            && [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] \
+            && [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] \
+            && [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] \
+            || fail_closed "cooldown_changed_between_check_and_rollback_capture"
+        cp -p "${COOLDOWN_FILE}" "${ROLLBACK_FILE}"
+        [ "$(sha256sum "${ROLLBACK_FILE}" | awk '{print $1}')" = "${PRIOR_LEASE_HASH}" ] \
+            || fail_closed "cooldown_changed_between_check_and_rollback_capture"
+    else
+        [ ! -e "${COOLDOWN_FILE}" ] \
+            || fail_closed "cooldown_appeared_between_check_and_rollback_capture"
+    fi
+
+    printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","prior_present":%s,"prior_mode":"%s","prior_uid":"%s","prior_gid":"%s","prior_hash":"%s","prior_value":"%s"}\n' \
+        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${PRIOR_LEASE_PRESENT}" \
+        "${PRIOR_LEASE_MODE}" "${PRIOR_LEASE_UID}" "${PRIOR_LEASE_GID}" \
+        "${PRIOR_LEASE_HASH}" "${PRIOR_LEASE_VALUE}" > "${ROLLBACK_METADATA}"
+    LEASE_ROLLBACK_READY=1
+    emit_receipt "rollback_capture" "pass" "prior_cooldown_receipt_captured"
+}
+
+arm_lease() {
+    local now lease_epoch lease_hash
+    now="$(date +%s)"
+    lease_epoch=$((now + CANARY_TIMEOUT_SECONDS + LEASE_MARGIN_SECONDS))
+    LEASE_TMP="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.lease.${RUN_ID}.XXXXXX")"
+    printf '%s\n' "${lease_epoch}" > "${LEASE_TMP}"
+    chmod "${PRIOR_LEASE_MODE}" "${LEASE_TMP}"
+
+    LEASE_MUTATION_STARTED=1
+    mv -f "${LEASE_TMP}" "${COOLDOWN_FILE}"
+    LEASE_TMP=""
+    LEASE_ARMED=1
+
+    [ "$(tr -d '\r\n' < "${COOLDOWN_FILE}")" = "${lease_epoch}" ] \
+        || fail_closed "cooldown_lease_epoch_readback_mismatch"
+    [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "$(id -u)" ] \
+        || fail_closed "cooldown_lease_owner_readback_mismatch"
+    [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "$(id -g)" ] \
+        || fail_closed "cooldown_lease_group_readback_mismatch"
+    lease_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
+    emit_receipt "lease" "armed" "future_epoch_${lease_epoch}_hash_${lease_hash}"
+}
+
+observe_collector() {
+    local epoch state
+    while kill -0 "${BACKUP_PID}" 2>/dev/null; do
+        epoch="$(date +%s)"
+        if state="$(collector_running_state)"; then
+            printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}"
+        else
+            printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}"
+        fi
+        sleep "${STATE_POLL_SECONDS}"
+    done
+
+    epoch="$(date +%s)"
+    if state="$(collector_running_state)"; then
+        printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}"
+    else
+        printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}"
+    fi
+}
+
+verify_monitor_window() {
+    local log_identity_before="$1"
+    local log_size_before="$2"
+    local log_identity_after log_size_after start_byte
+    local auto_repair_count detected_count cooldown_count
+    local stop_epoch restore_epoch next_cron_epoch cron_crossed=0
+
+    log_identity_after="$(stat -c '%d:%i' "${MONITOR_LOG}")"
+    log_size_after="$(stat -c '%s' "${MONITOR_LOG}")"
+    [ "${log_identity_after}" = "${log_identity_before}" ] \
+        || fail_closed "monitor_log_rotated_during_canary"
+    [ "${log_size_after}" -ge "${log_size_before}" ] \
+        || fail_closed "monitor_log_shrank_during_canary"
+
+    start_byte=$((log_size_before + 1))
+    tail -c "+${start_byte}" "${MONITOR_LOG}" > "${MONITOR_WINDOW}"
+    auto_repair_count="$(grep -a -F -c "AUTO_REPAIR: docker restart ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)"
+    detected_count="$(grep -a -F -c "DETECTED: ${COLLECTOR_NAME} state=exited" "${MONITOR_WINDOW}" || true)"
+    cooldown_count="$(grep -a -F -c "COOLDOWN: ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)"
+
+    [ "${auto_repair_count}" -eq 0 ] || fail_closed "monitor_auto_repair_contaminated_canary"
+    [ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
+        || fail_closed "collector_state_observer_ambiguous"
+
+    if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
+        [ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
+            || fail_closed "online_native_backup_collector_continuity_failed"
+        [ "${detected_count}" -eq 0 ] \
+            || fail_closed "online_native_backup_monitor_detected_collector"
+        emit_receipt "monitor_verifier" "pass" \
+            "online_native_collector_continuous_auto_repair_0_detected_0"
+        return 0
+    fi
+
+    stop_epoch="$(awk '$2 == "stopped" { print $1; exit }' "${STATE_OBSERVATIONS}")"
+    [ -n "${stop_epoch}" ] || fail_closed "collector_stop_transition_not_observed"
+    restore_epoch="$(awk '
+        $2 == "stopped" { stopped_seen = 1; next }
+        stopped_seen && $2 == "running" { print $1; exit }
+    ' "${STATE_OBSERVATIONS}")"
+    [ -n "${restore_epoch}" ] || fail_closed "collector_restore_transition_not_observed"
+
+    next_cron_epoch=$(( ((stop_epoch / MONITOR_CRON_INTERVAL_SECONDS) + 1) * MONITOR_CRON_INTERVAL_SECONDS ))
+    if [ "${next_cron_epoch}" -le "${restore_epoch}" ]; then
+        cron_crossed=1
+    fi
+
+    if [ "${detected_count}" -gt 0 ]; then
+        [ "${cooldown_count}" -ge "${detected_count}" ] \
+            || fail_closed "monitor_detected_without_matching_cooldown"
+    fi
+    if [ "${cron_crossed}" -eq 1 ]; then
+        [ "${detected_count}" -gt 0 ] && [ "${cooldown_count}" -gt 0 ] \
+            || fail_closed "cron_crossed_without_collector_cooldown_receipt"
+    fi
+
+    emit_receipt "monitor_verifier" "pass" \
+        "auto_repair_${auto_repair_count}_detected_${detected_count}_cooldown_${cooldown_count}_cron_crossed_${cron_crossed}"
+}
+
+verify_online_collector_window() {
+    [ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
+        || fail_closed "collector_state_observer_ambiguous"
+    [ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
+        || fail_closed "online_native_backup_collector_continuity_failed"
+    emit_receipt "monitor_verifier" "pass" \
+        "online_native_collector_continuous_monitor_contract_not_applicable"
+}
+
+main() {
+    local container_id_before container_started_at_before container_restart_count_before
+    local container_health_before collector_metadata_before
+    local monitor_log_identity="" monitor_log_size=""
+    local source_diff_detail ai_decision_detail
+    local backup_status
+
+    setup_receipts
+
+    [ "$#" -eq 1 ] || fail_closed "exactly_one_mode_required_use_check_or_apply"
+    case "$1" in
+        --check) MODE="check" ;;
+        --apply) MODE="apply" ;;
+        *) fail_closed "unsupported_mode_use_check_or_apply" ;;
+    esac
+
+    for command_name in awk date docker env flock grep kill pgrep sha256sum sleep ss timeout tr; do
+        require_command "${command_name}"
+    done
+    valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout"
+    valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after"
+    valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval"
+    case "${EXPECT_COLLECTOR_STOP}" in
+        0|1) ;;
+        *) fail_closed "invalid_expect_collector_stop_mode" ;;
+    esac
+
+    [ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \
+        || fail_closed "backup_script_missing_unexecutable_or_symlink"
+    if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+        for command_name in cp crontab id mktemp mv rm stat tail; do
+            require_command "${command_name}"
+        done
+        valid_positive_integer "${LEASE_MARGIN_SECONDS}" || fail_closed "invalid_lease_margin"
+        valid_positive_integer "${MONITOR_CRON_INTERVAL_SECONDS}" || fail_closed "invalid_monitor_interval"
+        [ "${MONITOR_CRON_INTERVAL_SECONDS}" -le 300 ] || fail_closed "monitor_interval_exceeds_cron_contract"
+        verify_monitor_contract
+    fi
+
+    [ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe"
+    if [ -e "${LOCK_FILE}" ]; then
+        [ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file"
+    fi
+    exec 9>>"${LOCK_FILE}"
+    flock -n 9 || fail_closed "another_signoz_canary_holds_lock"
+    if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
+        fail_closed "another_signoz_backup_is_running"
+    fi
+
+    collector_metadata_before="$(collector_runtime_metadata)" \
+        || fail_closed "collector_runtime_metadata_unreadable"
+    IFS=$'\t' read -r container_id_before _container_running_before \
+        container_started_at_before container_restart_count_before container_health_before \
+        <<< "${collector_metadata_before}"
+    [ -n "${container_id_before}" ] \
+        && [ -n "${container_started_at_before}" ] \
+        && [[ "${container_restart_count_before}" =~ ^[0-9]+$ ]] \
+        && [[ "${container_health_before}" =~ ^(healthy|none)$ ]] \
+        || fail_closed "collector_runtime_metadata_invalid"
+    verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \
+        "${container_restart_count_before}" "${container_health_before}" \
+        || fail_closed "collector_preflight_runtime_failed"
+
+    if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+        monitor_log_identity="$(stat -c '%d:%i' "${MONITOR_LOG}")"
+        monitor_log_size="$(stat -c '%s' "${MONITOR_LOG}")"
+        inspect_prior_lease
+        emit_receipt "sensor_source" "pass" \
+            "monitor_hash_$(sha256sum "${MONITOR_SCRIPT}" | awk '{print $1}')_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
+        source_diff_detail="legacy_monitor_and_cron_contract_exact_cooldown_prior_present_${PRIOR_LEASE_PRESENT}_hash_${PRIOR_LEASE_HASH}"
+        ai_decision_detail="arm_exact_container_future_epoch_run_legacy_stop_backup_then_restore"
+    else
+        PRIOR_LEASE_PRESENT=0
+        PRIOR_LEASE_HASH="not_applicable"
+        emit_receipt "sensor_source" "pass" \
+            "online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
+        source_diff_detail="online_native_backup_monitor_and_cooldown_contract_not_applicable"
+        ai_decision_detail="run_online_native_backup_without_collector_or_monitor_mutation"
+    fi
+    emit_receipt "normalized_asset_identity" "pass" \
+        "asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker"
+    emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}"
+    emit_receipt "ai_decision" "candidate" "${ai_decision_detail}"
+    emit_receipt "risk_policy" "pass" \
+        "risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_independent_verifier"
+    emit_receipt "check" "pass" \
+        "mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent"
+
+    if [ "${MODE}" = "check" ]; then
+        CHECK_VERIFIED=1
+        log_info "SigNoz backup canary check passed without arming lease or launching backup"
+        return 0
+    fi
+
+    if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+        capture_prior_lease
+        arm_lease
+    else
+        emit_receipt "lease" "not_required" \
+            "online_native_backup_has_no_collector_stop_or_cooldown_write"
+    fi
+
+    : > "${STATE_OBSERVATIONS}"
+    set +e
+    timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \
+        env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+        BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \
+        > "${BACKUP_OUTPUT}" 2>&1 &
+    BACKUP_PID=$!
+    observe_collector &
+    OBSERVER_PID=$!
+    wait "${BACKUP_PID}"
+    backup_status=$?
+    BACKUP_PID=""
+    wait "${OBSERVER_PID}"
+    OBSERVER_PID=""
+    set -e
+
+    emit_receipt "execution" "$([ "${backup_status}" -eq 0 ] && printf pass || printf failed)" \
+        "backup_exit_${backup_status}"
+    [ "${backup_status}" -eq 0 ] || fail_closed "backup_terminal_nonzero_${backup_status}"
+    grep -a -F -q '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)' "${BACKUP_OUTPUT}" \
+        || fail_closed "retention_skip_receipt_missing"
+    if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+        grep -a -F -q '========== SignOz 備份完成 (' "${BACKUP_OUTPUT}" \
+            || fail_closed "backup_success_terminal_missing"
+    else
+        grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \
+            || fail_closed "native_backup_success_terminal_missing"
+    fi
+
+    if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+        verify_monitor_window "${monitor_log_identity}" "${monitor_log_size}"
+    else
+        verify_online_collector_window
+    fi
+    verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \
+        "${container_restart_count_before}" "${container_health_before}" \
+        || fail_closed "collector_post_backup_runtime_failed"
+    emit_receipt "post_verifier" "pass" \
+        "collector_running_same_identity_started_at_restart_count_health_listeners_4317_4318"
+
+    CANARY_VERIFIED=1
+    log_info "SigNoz backup canary verified; cooldown lease will now roll back"
+}
+
+main "$@"

scripts/backup/tests/test_backup_signoz_collector_restore.py

@@ -5,6 +5,7 @@ import shutil
 import subprocess
 from pathlib import Path
 
+
 ROOT = Path(__file__).resolve().parents[3]
 SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
 BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh"
@@ -20,27 +21,24 @@ def _run_backup(
     tmp_path: Path,
     *,
     initial_running: bool = True,
-    docker_run_mode: str = "success",
-    docker_run_target: str = "signoz-clickhouse",
-    archive_timeout_target: str = "",
-    tar_verify_mode: str = "success",
-    tar_verify_target: str = "clickhouse_",
-    tar_timeout_target: str = "",
-    docker_start_failures: int = 0,
+    native_check_status: int = 0,
+    native_apply_status: int = 0,
+    continuity_mode: str = "stable",
     restic_backup_status: int = 0,
-    restore_max_attempts: int = 3,
+    skip_retention: bool = False,
 ) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
     harness = tmp_path / "backup"
     fake_bin = tmp_path / "bin"
     backup_base = tmp_path / "repository"
     event_log = tmp_path / "events.log"
+    collector_id = tmp_path / "collector.id"
     collector_state = tmp_path / "collector.state"
-    start_count = tmp_path / "start.count"
     harness.mkdir()
     fake_bin.mkdir()
     (backup_base / "signoz" / "data").mkdir(parents=True)
     shutil.copy2(SCRIPT, harness / SCRIPT.name)
     event_log.touch()
+    collector_id.write_text("sha256:collector-stable\n", encoding="utf-8")
     collector_state.write_text(
         "true\n" if initial_running else "false\n",
         encoding="utf-8",
@@ -50,37 +48,62 @@ def _run_backup(
         """\
 BACKUP_BASE="${TEST_BACKUP_BASE:?}"
 RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password"
-log_info() { :; }
-log_warn() { :; }
-log_error() { :; }
-log_success() { :; }
+log_info() { printf 'info %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
+log_warn() { printf 'warn %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
+log_error() { printf 'error %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
+log_success() { printf 'success %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
 notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
-build_tags() { :; }
-cleanup_old_backups() { :; }
+build_tags() { printf '%s' '--tag signoz'; }
+cleanup_old_backups() { printf 'retention %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
 """,
         encoding="utf-8",
     )
-
+    _write_executable(
+        harness / "clickhouse-native-backup.sh",
+        """\
+#!/bin/bash
+set -eu
+printf 'native %s trace=%s run=%s work=%s\\n' \
+    "${1:-}" "${TRACE_ID:-}" "${RUN_ID:-}" "${WORK_ITEM_ID:-}" \
+    >> "${TEST_EVENT_LOG:?}"
+case "${1:-}" in
+    --check)
+        exit "${FAKE_NATIVE_CHECK_STATUS:-0}"
+        ;;
+    --apply)
+        status="${FAKE_NATIVE_APPLY_STATUS:-0}"
+        [ "$status" -eq 0 ] || exit "$status"
+        case "${FAKE_CONTINUITY_MODE:-stable}" in
+            stopped)
+                printf 'false\\n' > "${TEST_COLLECTOR_STATE:?}"
+                sleep 0.08
+                printf 'true\\n' > "${TEST_COLLECTOR_STATE:?}"
+                ;;
+            identity)
+                printf 'sha256:collector-replaced\\n' > "${TEST_COLLECTOR_ID:?}"
+                sleep 0.08
+                ;;
+            *) sleep 0.08 ;;
+        esac
+        printf 'native-zip\\n' \
+            > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip"
+        printf '{}\\n' \
+            > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip.manifest.json"
+        printf 'db\\ttable\\tengine\\trows\\tbytes\\t%s\\n' \
+            "$(printf 'A%.0s' {1..64})" \
+            > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.source-inventory.tsv"
+        ;;
+    *) exit 64 ;;
+esac
+""",
+    )
     _write_executable(
         fake_bin / "timeout",
         """\
 #!/bin/bash
 set -eu
-printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
-while [[ "${1:-}" == --kill-after=* ]]; do
-    shift
-done
+while [[ "${1:-}" == --kill-after=* ]]; do shift; done
 shift
-if [ -n "${FAKE_ARCHIVE_TIMEOUT_TARGET:-}" ] \
-   && [ "${1:-}" = "docker" ] \
-   && [[ " $* " == *"${FAKE_ARCHIVE_TIMEOUT_TARGET}"* ]]; then
-    exit 124
-fi
-if [ -n "${FAKE_TAR_TIMEOUT_TARGET:-}" ] \
-   && [ "${1:-}" = "tar" ] \
-   && [[ " $* " == *"${FAKE_TAR_TIMEOUT_TARGET}"* ]]; then
-    exit 124
-fi
 exec "$@"
 """,
     )
@@ -89,61 +112,12 @@ exec "$@"
         """\
 #!/bin/bash
 set -eu
-printf 'docker %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
-case "${1:-}" in
-    inspect)
-        cat "${TEST_COLLECTOR_STATE:?}"
-        ;;
-    stop)
-        if [ "${2:-}" = "signoz-otel-collector" ]; then
-            printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
-        fi
-        ;;
-    start)
-        count=0
-        if [ -e "${TEST_START_COUNT:?}" ]; then
-            count=$(cat "${TEST_START_COUNT}")
-        fi
-        count=$((count + 1))
-        printf '%s\n' "$count" > "${TEST_START_COUNT}"
-        if [ "$count" -le "${FAKE_DOCKER_START_FAILURES:-0}" ]; then
-            exit 1
-        fi
-        printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
-        ;;
-    run)
-        mode=success
-        if [[ " $* " == *"${FAKE_DOCKER_RUN_TARGET:-signoz-clickhouse}"* ]]; then
-            mode="${FAKE_DOCKER_RUN_MODE:-success}"
-        fi
-        if [ "$mode" = "signal" ]; then
-            kill -TERM "${PPID}"
-            sleep 0.1
-            exit 0
-        fi
-        if [ "$mode" = "empty" ]; then
-            exit 0
-        fi
-        if [ "$mode" = "partial_nonzero" ]; then
-            printf 'partial-archive-data\n'
-            exit 23
-        fi
-        printf 'archive-data\n'
-        ;;
-esac
-""",
-    )
-    _write_executable(
-        fake_bin / "tar",
-        """\
-#!/bin/bash
-set -eu
-printf 'tar %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
-if [[ " $* " == *"${FAKE_TAR_VERIFY_TARGET:-signoz-clickhouse}"* ]] \
-   && [ "${FAKE_TAR_VERIFY_MODE:-success}" = "corrupt" ]; then
-    exit 2
-fi
-exit 0
+printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
+[ "${1:-}" = "inspect" ] || exit 70
+printf '%s\\t%s\\t%s\\t%s\\t%s\\n' \
+    "$(tr -d '\\r\\n' < "${TEST_COLLECTOR_ID:?}")" \
+    "$(tr -d '\\r\\n' < "${TEST_COLLECTOR_STATE:?}")" \
+    '2026-07-15T00:00:00Z' '0' 'healthy'
 """,
     )
     _write_executable(
@@ -151,18 +125,20 @@ exit 0
         """\
 #!/bin/bash
 set -eu
-printf 'restic %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
+printf 'restic %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
 case " $* " in
-    *" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
-    *" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
+    *" backup "*)
+        dump_dir="${4:?}"
+        if [ -f "$dump_dir/signoz-metadata-gap.json" ]; then
+            printf 'gap %s\\n' "$(cat "$dump_dir/signoz-metadata-gap.json")" \
+                >> "${TEST_EVENT_LOG:?}"
+        fi
+        exit "${FAKE_RESTIC_BACKUP_STATUS:-0}"
+        ;;
+    *" snapshots "*)
+        printf '[{"short_id":"native123"}]\\n'
+        ;;
 esac
-""",
-    )
-    _write_executable(
-        fake_bin / "du",
-        """\
-#!/bin/bash
-printf '4K\t%s\n' "${2:-unknown}"
 """,
     )
 
@@ -171,22 +147,20 @@ printf '4K\t%s\n' "${2:-unknown}"
         "PATH": f"{fake_bin}:{os.environ['PATH']}",
         "TEST_BACKUP_BASE": str(backup_base),
         "TEST_EVENT_LOG": str(event_log),
+        "TEST_COLLECTOR_ID": str(collector_id),
         "TEST_COLLECTOR_STATE": str(collector_state),
-        "TEST_START_COUNT": str(start_count),
-        "FAKE_DOCKER_RUN_MODE": docker_run_mode,
-        "FAKE_DOCKER_RUN_TARGET": docker_run_target,
-        "FAKE_ARCHIVE_TIMEOUT_TARGET": archive_timeout_target,
-        "FAKE_TAR_VERIFY_MODE": tar_verify_mode,
-        "FAKE_TAR_VERIFY_TARGET": tar_verify_target,
-        "FAKE_TAR_TIMEOUT_TARGET": tar_timeout_target,
-        "FAKE_DOCKER_START_FAILURES": str(docker_start_failures),
+        "FAKE_NATIVE_CHECK_STATUS": str(native_check_status),
+        "FAKE_NATIVE_APPLY_STATUS": str(native_apply_status),
+        "FAKE_CONTINUITY_MODE": continuity_mode,
         "FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
-        "COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts),
-        "COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0",
+        "TRACE_ID": "trace-P0-OBS-002",
+        "RUN_ID": f"run-{tmp_path.name}",
+        "WORK_ITEM_ID": "P0-OBS-002",
+        "SIGNOZ_BACKUP_RECEIPT_ROOT": str(tmp_path / "receipts"),
+        "COLLECTOR_CONTINUITY_POLL_SECONDS": "0.01",
         "COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
-        "ARCHIVE_CREATE_TIMEOUT_SECONDS": "1",
-        "ARCHIVE_VERIFY_TIMEOUT_SECONDS": "1",
         "TIMEOUT_KILL_AFTER_SECONDS": "1",
+        "BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0",
     }
     process = subprocess.Popen(
         ["bash", str(harness / SCRIPT.name)],
@@ -210,274 +184,155 @@ def _events(events: list[str], prefix: str) -> list[str]:
     return [event for event in events if event.startswith(prefix)]
 
 
-def _run_backup_all_with_signoz_failure(
+def test_source_sunsets_raw_volumes_and_collector_mutation() -> None:
+    source = SCRIPT.read_text(encoding="utf-8")
+
+    assert "signoz-clickhouse" not in source
+    assert "signoz-sqlite" not in source
+    assert "docker run" not in source
+    assert "docker stop" not in source
+    assert "docker start" not in source
+    assert "create_volume_archive" not in source
+    assert "pending_application_consistent_export" in source
+    assert '"raw_volume_read":false' in source
+    assert "{{.State.StartedAt}}" in source
+    assert "{{.RestartCount}}" in source
+    assert '{{if (index .State "Health")}}' in source
+    assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
+    assert 'notify_clawbot "success"' not in source
+    assert "no-downtime continuity verifier" in source
+
+
+def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
     tmp_path: Path,
-) -> tuple[subprocess.CompletedProcess[str], list[str]]:
-    harness = tmp_path / "backup-all"
-    fake_scripts = tmp_path / "deployed-scripts"
-    event_log = tmp_path / "backup-all-events.log"
-    harness.mkdir()
-    fake_scripts.mkdir()
-    event_log.touch()
+) -> None:
+    result, events, dump_dir = _run_backup(tmp_path)
 
-    source = BACKUP_ALL.read_text(encoding="utf-8").replace(
-        "/backup/scripts",
-        str(fake_scripts),
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert len(_events(events, "native --check")) == 1
+    assert len(_events(events, "native --apply")) == 1
+    check_identity = _events(events, "native --check")[0].split(" trace=", 1)[1]
+    apply_identity = _events(events, "native --apply")[0].split(" trace=", 1)[1]
+    assert check_identity == apply_identity
+    assert not _events(events, "docker stop")
+    assert not _events(events, "docker start")
+    assert not _events(events, "docker run")
+    assert any("pending_application_consistent_export" in event for event in events)
+    assert len(_events(events, "notify warning signoz ")) == 1
+    assert not _events(events, "notify success ")
+    assert any("SigNoz ClickHouse native 備份完成" in event for event in events)
+    assert any(
+        "--tag run:run-" in event for event in events if event.startswith("restic ")
     )
-    backup_all = harness / "backup-all.sh"
-    _write_executable(backup_all, source)
-    (harness / "common.sh").write_text(
-        """\
-log_info() { :; }
-log_warn() { :; }
-log_error() { :; }
-log_success() { :; }
-notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
-""",
-        encoding="utf-8",
-    )
-
-    for name in (
-        "backup-gitea.sh",
-        "backup-momo.sh",
-        "backup-harbor.sh",
-        "backup-awoooi.sh",
-        "backup-langfuse.sh",
-        "backup-monitoring.sh",
-        "backup-signoz.sh",
-        "backup-open-webui.sh",
-        "backup-clawbot.sh",
-    ):
-        status = 17 if name == "backup-signoz.sh" else 0
-        _write_executable(
-            fake_scripts / name,
-            f"#!/bin/bash\nprintf 'run {name}\\n' >> \"${{TEST_EVENT_LOG:?}}\"\nexit {status}\n",
-        )
-
-    result = subprocess.run(
-        ["bash", str(backup_all)],
-        env={**os.environ, "TEST_EVENT_LOG": str(event_log)},
-        text=True,
-        capture_output=True,
-        timeout=10,
-    )
-    return result, event_log.read_text(encoding="utf-8").splitlines()
+    assert any("restic -r" in event and " check " in event for event in events)
+    receipts = list((tmp_path / "receipts").glob("*/restic-snapshot.json"))
+    assert len(receipts) == 1
+    receipt = receipts[0].read_text(encoding="utf-8")
+    assert '"snapshot_id":"native123"' in receipt
+    assert '"offsite_verified":false' in receipt
+    assert not dump_dir.exists()
 
 
-def test_backup_jobs_deploys_the_guarded_signoz_script() -> None:
-    playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
-
-    assert 'dest: "/backup/scripts/{{ item }}"' in playbook
-    assert "- backup-signoz.sh" in playbook
-    assert "tags: backup_jobs" in playbook
-
-
-def test_restore_policy_is_bounded_timed_and_independently_verified() -> None:
-    source = SCRIPT.read_text(encoding="utf-8")
-
-    assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source
-    assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source
-    assert 'ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"' in source
-    assert 'ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"' in source
-    assert 'TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"' in source
-    assert 'timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}"' in source
-    assert 'tar -tzf "${output_file}"' in source
-    assert "docker run" in source
-    assert "|| true" not in source.split("create_volume_archive()", 1)[1].split(
-        "verify_volume_archive()",
-        1,
-    )[0]
-    assert "verify_collector_final_state" in source
-    assert source.index("verify_collector_final_state") < source.rindex(
-        'notify_clawbot "success"'
-    )
-
-
-def test_canary_retention_skip_is_explicit_and_defaults_off() -> None:
-    source = SCRIPT.read_text(encoding="utf-8")
-
-    assert 'BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"' in source
-    assert 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' in source
-    assert source.index('restic -r "${LOCAL_REPO}" backup') < source.index(
-        'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]'
-    )
-    assert source.index('if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]') < source.index(
-        "if ! verify_collector_final_state"
-    )
-
-
-def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None:
-    result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal")
+def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None:
+    result, events, dump_dir = _run_backup(tmp_path, native_check_status=19)
 
     assert result.returncode != 0
-    assert _events(events, "docker stop signoz-otel-collector")
-    assert len(_events(events, "docker start signoz-otel-collector")) == 1
-    assert not _events(events, "notify success ")
+    assert _events(events, "native --check")
+    assert not _events(events, "native --apply")
+    assert not _events(events, "restic ")
+    assert len(_events(events, "notify failed signoz ")) == 1
     assert not dump_dir.exists()
 
 
-def test_collector_is_restored_when_clickhouse_backup_fails(tmp_path: Path) -> None:
-    result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="empty")
+def test_native_apply_failure_never_reports_scoped_success(tmp_path: Path) -> None:
+    result, events, dump_dir = _run_backup(tmp_path, native_apply_status=23)
 
-    assert result.returncode == 1
-    assert len(_events(events, "docker start signoz-otel-collector")) == 1
+    assert result.returncode != 0
+    assert _events(events, "native --apply")
+    assert not _events(events, "restic ")
+    assert not _events(events, "notify warning ")
     assert not _events(events, "notify success ")
+    assert len(_events(events, "notify failed signoz ")) == 1
     assert not dump_dir.exists()
 
 
-def test_exit_cleanup_does_not_restart_an_already_restored_collector(
+def test_initially_stopped_collector_fails_without_apply(tmp_path: Path) -> None:
+    result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
+
+    assert result.returncode != 0
+    assert _events(events, "native --check")
+    assert not _events(events, "native --apply")
+    assert not _events(events, "restic ")
+    assert not dump_dir.exists()
+
+
+def test_continuity_observer_rejects_stopped_sample(tmp_path: Path) -> None:
+    result, events, dump_dir = _run_backup(tmp_path, continuity_mode="stopped")
+
+    assert result.returncode != 0
+    assert any("continuity verifier 失敗" in event for event in events)
+    assert not _events(events, "restic ")
+    assert not _events(events, "notify warning ")
+    assert not dump_dir.exists()
+
+
+def test_continuity_observer_rejects_container_identity_drift(
+    tmp_path: Path,
+) -> None:
+    result, events, dump_dir = _run_backup(tmp_path, continuity_mode="identity")
+
+    assert result.returncode != 0
+    assert any("continuity verifier 失敗" in event for event in events)
+    assert not _events(events, "restic ")
+    assert not dump_dir.exists()
+
+
+def test_restic_failure_is_nonzero_and_no_completion_notification(
     tmp_path: Path,
 ) -> None:
     result, events, dump_dir = _run_backup(tmp_path, restic_backup_status=17)
 
     assert result.returncode == 17
-    assert len(_events(events, "docker start signoz-otel-collector")) == 1
+    assert _events(events, "restic ")
+    assert not _events(events, "notify warning ")
     assert not _events(events, "notify success ")
     assert not dump_dir.exists()
 
 
-def test_fail_once_restores_within_budget_and_succeeds(tmp_path: Path) -> None:
-    result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=1)
+def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None:
+    result, events, _dump_dir = _run_backup(tmp_path, skip_retention=True)
 
     assert result.returncode == 0
-    assert len(_events(events, "docker start signoz-otel-collector")) == 2
-    assert len(_events(events, "notify success ")) == 1
-    assert not _events(events, "notify failed ")
-    assert not dump_dir.exists()
+    assert any("BACKUP_SKIP_RETENTION_CLEANUP=1" in event for event in events)
+    assert not _events(events, "retention ")
 
 
-def test_permanent_restore_failure_is_nonzero_and_never_notifies_success(
-    tmp_path: Path,
-) -> None:
-    result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=99)
+def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None:
+    playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
 
-    assert result.returncode != 0
-    assert len(_events(events, "docker start signoz-otel-collector")) == 3
-    assert len(_events(events, "notify failed ")) == 1
-    assert not _events(events, "notify success ")
-    assert not dump_dir.exists()
-
-
-def test_initially_stopped_fails_closed_without_stop_start_or_success(
-    tmp_path: Path,
-) -> None:
-    result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
-
-    assert result.returncode != 0
-    assert not _events(events, "docker stop signoz-otel-collector")
-    assert not _events(events, "docker start signoz-otel-collector")
-    assert len(_events(events, "notify failed ")) == 1
-    assert not _events(events, "notify success ")
-    assert not dump_dir.exists()
-
-
-def test_success_notification_follows_independent_final_running_readback(
-    tmp_path: Path,
-) -> None:
-    result, events, _dump_dir = _run_backup(tmp_path)
-
-    assert result.returncode == 0
-    inspect_positions = [
-        index
-        for index, event in enumerate(events)
-        if event.startswith("docker inspect --format")
-    ]
-    success_position = next(
-        index
-        for index, event in enumerate(events)
-        if event.startswith("notify success ")
+    assert 'dest: "/backup/scripts/{{ item }}"' in playbook
+    assert "- backup-signoz.sh" in playbook
+    assert "- clickhouse-native-backup.sh" in playbook
+    assert "- clickhouse-native-restore-drill.sh" in playbook
+    assert "- clickhouse-restore-inventory.py" in playbook
+    assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in playbook
+    assert (
+        "/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
+        in playbook
     )
-    assert len(inspect_positions) >= 3
-    assert inspect_positions[-1] < success_position
+    assert "tags: backup_jobs" in playbook
 
-    start_position = next(
-        index
-        for index, event in enumerate(events)
-        if event.startswith("docker start signoz-otel-collector")
+
+def test_scripts_are_valid_bash() -> None:
+    result = subprocess.run(
+        [
+            "bash",
+            "-n",
+            str(SCRIPT),
+            str(ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"),
+        ],
+        text=True,
+        capture_output=True,
+        check=False,
     )
-    tar_positions = [
-        index for index, event in enumerate(events) if event.startswith("tar -tzf")
-    ]
-    assert len(tar_positions) == 2
-    assert start_position < tar_positions[0]
-
-
-def test_archive_create_timeout_is_bounded_restores_and_notifies_once(
-    tmp_path: Path,
-) -> None:
-    result, events, dump_dir = _run_backup(
-        tmp_path,
-        archive_timeout_target="signoz-clickhouse",
-    )
-
-    assert result.returncode != 0
-    assert len(_events(events, "docker start signoz-otel-collector")) == 1
-    assert len(_events(events, "notify failed ")) == 1
-    assert not _events(events, "notify success ")
-    assert not dump_dir.exists()
-
-
-def test_partial_sqlite_archive_nonzero_fails_overall_and_removes_output(
-    tmp_path: Path,
-) -> None:
-    result, events, dump_dir = _run_backup(
-        tmp_path,
-        docker_run_mode="partial_nonzero",
-        docker_run_target="signoz-sqlite",
-    )
-
-    assert result.returncode != 0
-    assert len(_events(events, "docker start signoz-otel-collector")) == 1
-    assert len(_events(events, "notify failed ")) == 1
-    assert not _events(events, "notify success ")
-    assert not dump_dir.exists()
-
-
-def test_corrupt_archive_fails_after_collector_restore_and_notifies_once(
-    tmp_path: Path,
-) -> None:
-    result, events, dump_dir = _run_backup(
-        tmp_path,
-        tar_verify_mode="corrupt",
-        tar_verify_target="clickhouse_",
-    )
-
-    assert result.returncode != 0
-    start_position = next(
-        index
-        for index, event in enumerate(events)
-        if event.startswith("docker start signoz-otel-collector")
-    )
-    tar_position = next(
-        index for index, event in enumerate(events) if event.startswith("tar -tzf")
-    )
-    assert start_position < tar_position
-    assert len(_events(events, "notify failed ")) == 1
-    assert not _events(events, "notify success ")
-    assert not dump_dir.exists()
-
-
-def test_integrity_verifier_timeout_is_bounded_and_never_reports_success(
-    tmp_path: Path,
-) -> None:
-    result, events, dump_dir = _run_backup(
-        tmp_path,
-        tar_timeout_target="clickhouse_",
-    )
-
-    assert result.returncode != 0
-    assert len(_events(events, "docker start signoz-otel-collector")) == 1
-    assert len(_events(events, "notify failed ")) == 1
-    assert not _events(events, "notify success ")
-    assert not dump_dir.exists()
-
-
-def test_backup_all_propagates_signoz_failure_to_nonzero_terminal(
-    tmp_path: Path,
-) -> None:
-    result, events = _run_backup_all_with_signoz_failure(tmp_path)
-
-    assert result.returncode == 1
-    assert "run backup-signoz.sh" in events
-    assert any(event.startswith("notify warning all ") for event in events)
-    assert not any(event.startswith("notify success all ") for event in events)
+    assert result.returncode == 0, result.stderr

scripts/backup/tests/test_clickhouse_native_backup.py

@@ -0,0 +1,478 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"
+PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
+
+
+def _write_executable(path: Path, text: str) -> None:
+    path.write_text(text, encoding="utf-8")
+    path.chmod(0o755)
+
+
+def _install_fakes(fake_bin: Path) -> None:
+    _write_executable(
+        fake_bin / "timeout",
+        """\
+#!/bin/bash
+set -eu
+while [[ "${1:-}" == --kill-after=* ]]; do shift; done
+shift
+exec "$@"
+""",
+    )
+    _write_executable(fake_bin / "flock", "#!/bin/bash\nexit 0\n")
+    _write_executable(
+        fake_bin / "sha256sum",
+        """\
+#!/usr/bin/env python3
+import hashlib
+import sys
+
+if len(sys.argv) == 1:
+    payload = sys.stdin.buffer.read()
+    print(f"{hashlib.sha256(payload).hexdigest()}  -")
+else:
+    with open(sys.argv[1], "rb") as source:
+        payload = source.read()
+    print(f"{hashlib.sha256(payload).hexdigest()}  {sys.argv[1]}")
+""",
+    )
+    _write_executable(
+        fake_bin / "unzip",
+        """\
+#!/bin/bash
+set -eu
+printf 'unzip %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
+exit "${FAKE_UNZIP_STATUS:-0}"
+""",
+    )
+    _write_executable(
+        fake_bin / "docker",
+        """\
+#!/bin/bash
+set -eu
+printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
+
+find_arg() {
+    local wanted="$1"
+    shift
+    while [ "$#" -gt 0 ]; do
+        if [ "$1" = "$wanted" ]; then
+            printf '%s\\n' "${2:-}"
+            return 0
+        fi
+        shift
+    done
+    return 1
+}
+
+case "${1:-}" in
+    inspect)
+        printf 'sha256:clickhouse-test-id\\ttrue\\n'
+        ;;
+    cp)
+        [ -e "${TEST_SERVER_ARTIFACT:?}" ] || exit 44
+        printf 'clickhouse-native-zip-bytes\\n' > "${3:?}"
+        ;;
+    exec)
+        shift
+        container="${1:?}"
+        shift
+        case "${1:-}" in
+            test)
+                shift
+                if [ "${1:-}" = "!" ]; then
+                    [ ! -e "${TEST_SERVER_ARTIFACT:?}" ]
+                elif [ "${1:-}" = "-d" ] || [ "${1:-}" = "-w" ]; then
+                    [ "${FAKE_DISK_READY:-1}" = "1" ]
+                    [ "${2:-}" = "${FAKE_DISK_PATH:-/backups/}" ]
+                else
+                    exit 45
+                fi
+                ;;
+            rm)
+                rm -f "${TEST_SERVER_ARTIFACT:?}"
+                ;;
+            clickhouse-client)
+                query="$(find_arg --query "$@")"
+                operation_id="$(find_arg --param_operation_id "$@" || true)"
+                artifact="$(find_arg --param_artifact "$@" || true)"
+                disk="$(find_arg --param_disk "$@" || true)"
+                case "$query" in
+                    'SELECT version()')
+                        printf '25.5.1.1\\n'
+                        ;;
+                    *"database='system' AND name='backups'"*)
+                        printf '%s\\n' "${FAKE_SYSTEM_BACKUPS_COUNT:-1}"
+                        ;;
+                    'CHECK GRANT BACKUP ON *.*')
+                        printf '%s\\n' "${FAKE_BACKUP_GRANT:-1}"
+                        ;;
+                    *"FROM system.disks"*)
+                        printf '%s\\n' "${FAKE_DISK_PATH:-/backups/}"
+                        ;;
+                    *"FROM system.databases"*)
+                        printf '%s\\n' signoz_analytics signoz_logs signoz_metadata \
+                            signoz_meter signoz_metrics signoz_traces
+                        if [ "${FAKE_EXTRA_DATABASE:-0}" = "1" ]; then
+                            printf 'unexpected_db\\n'
+                        fi
+                        ;;
+                    *"FROM system.tables WHERE database IN"*)
+                        schema_hash="$(printf 'A%.0s' {1..64})"
+                        printf 'signoz_analytics\\tanalytics\\tReplicatedMergeTree\\t10\\t1000\\t%s\\n' "$schema_hash"
+                        printf 'signoz_logs\\tlogs_v2\\tReplicatedMergeTree\\t20\\t2000\\t%s\\n' "$schema_hash"
+                        printf 'signoz_metadata\\tmetadata\\tMergeTree\\t3\\t300\\t%s\\n' "$schema_hash"
+                        printf 'signoz_meter\\tmeter\\tDistributed\\t0\\t0\\t%s\\n' "$schema_hash"
+                        printf 'signoz_metrics\\tsamples_v4\\tReplicatedMergeTree\\t40\\t4000\\t%s\\n' "$schema_hash"
+                        printf 'signoz_traces\\tsignoz_index_v3\\tReplicatedMergeTree\\t50\\t5000\\t%s\\n' "$schema_hash"
+                        ;;
+                    BACKUP*)
+                        case "$query" in
+                            *" SETTINGS id='${EXPECTED_FAKE_OPERATION_ID:?}' ASYNC")
+                                operation_id="${EXPECTED_FAKE_OPERATION_ID}"
+                                ;;
+                            *) exit 49 ;;
+                        esac
+                        printf '%s\\n' "$operation_id" > "${TEST_OPERATION_ID:?}"
+                        printf '%s\\n' "$artifact" > "${TEST_ARTIFACT_NAME:?}"
+                        printf '%s\\n' "$disk" > "${TEST_DISK_NAME:?}"
+                        : > "${TEST_SUBMITTED:?}"
+                        : > "${TEST_SERVER_ARTIFACT:?}"
+                        printf '%s\\tCREATING_BACKUP\\n' "$operation_id"
+                        ;;
+                    *"FROM system.backups WHERE id="*)
+                        mode="${FAKE_STATUS_MODE:-success}"
+                        if [ "$mode" = "existing_success" ] || [ -e "${TEST_SUBMITTED:?}" ]; then
+                            artifact_name="$(cat "${TEST_ARTIFACT_NAME:?}" 2>/dev/null || true)"
+                            [ -n "$artifact_name" ] || artifact_name="${EXPECTED_FAKE_ARTIFACT:?}"
+                            backup_name="Disk('backups', '$artifact_name')"
+                            case "$mode" in
+                                failed)
+                                    printf '%s\\tBACKUP_FAILED\\t0\\t0\\t0\\t0\\t434F44453A20353938\\t%s\\n' \
+                                        "$backup_name" "$(printf '1%.0s' {1..64})"
+                                    ;;
+                                *)
+                                    : > "${TEST_SERVER_ARTIFACT:?}"
+                                    printf '%s\\tBACKUP_CREATED\\t12\\t1200\\t2400\\t1100\\tnone\\t%s\\n' \
+                                        "$backup_name" "$(printf '0%.0s' {1..64})"
+                                    ;;
+                            esac
+                        fi
+                        ;;
+                    *) exit 46 ;;
+                esac
+                ;;
+            *) exit 47 ;;
+        esac
+        ;;
+    *) exit 48 ;;
+esac
+""",
+    )
+
+
+def _harness(
+    tmp_path: Path,
+    *,
+    status_mode: str = "success",
+    extra_database: bool = False,
+    disk_ready: bool = True,
+    unzip_status: int = 0,
+    with_hook: bool = False,
+) -> tuple[dict[str, str], dict[str, Path]]:
+    fake_bin = tmp_path / "bin"
+    output = tmp_path / "output"
+    receipts = tmp_path / "receipts"
+    fake_bin.mkdir()
+    output.mkdir()
+    receipts.mkdir()
+    _install_fakes(fake_bin)
+
+    event_log = tmp_path / "events.log"
+    event_log.touch()
+    paths = {
+        "output": output,
+        "receipts": receipts,
+        "events": event_log,
+        "submitted": tmp_path / "submitted",
+        "server_artifact": tmp_path / "server-artifact",
+        "operation_id": tmp_path / "operation-id",
+        "artifact_name": tmp_path / "artifact-name",
+        "disk_name": tmp_path / "disk-name",
+        "hook_events": tmp_path / "hook-events.log",
+    }
+    identity_hash = (
+        subprocess.check_output(
+            [str(fake_bin / "sha256sum")],
+            input=b"trace-native\0run-native\0P0-OBS-002",
+        )
+        .decode()
+        .split()[0]
+    )
+    expected_artifact = f"signoz-{identity_hash}.zip"
+
+    env = {
+        **os.environ,
+        "PATH": f"{fake_bin}:{os.environ['PATH']}",
+        "TRACE_ID": "trace-native",
+        "RUN_ID": "run-native",
+        "WORK_ITEM_ID": "P0-OBS-002",
+        "CLICKHOUSE_NATIVE_OUTPUT_DIR": str(output),
+        "CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(receipts),
+        "CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS": "2",
+        "CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS": "2",
+        "CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS": "1",
+        "CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS": "1",
+        "TEST_EVENT_LOG": str(event_log),
+        "TEST_SUBMITTED": str(paths["submitted"]),
+        "TEST_SERVER_ARTIFACT": str(paths["server_artifact"]),
+        "TEST_OPERATION_ID": str(paths["operation_id"]),
+        "TEST_ARTIFACT_NAME": str(paths["artifact_name"]),
+        "TEST_DISK_NAME": str(paths["disk_name"]),
+        "EXPECTED_FAKE_ARTIFACT": expected_artifact,
+        "EXPECTED_FAKE_OPERATION_ID": f"awoooi-{identity_hash}",
+        "FAKE_STATUS_MODE": status_mode,
+        "FAKE_EXTRA_DATABASE": "1" if extra_database else "0",
+        "FAKE_DISK_READY": "1" if disk_ready else "0",
+        "FAKE_UNZIP_STATUS": str(unzip_status),
+    }
+    if with_hook:
+        hook = tmp_path / "restore-verifier-hook.sh"
+        _write_executable(
+            hook,
+            """\
+#!/bin/bash
+set -eu
+if [ "$1" = "--apply" ]; then
+    test -f "$CLICKHOUSE_NATIVE_ARTIFACT_PATH"
+    test -f "$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH"
+    test -f "$CLICKHOUSE_NATIVE_MANIFEST_PATH"
+    grep -q '"status":"BACKUP_CREATED"' "$CLICKHOUSE_NATIVE_MANIFEST_PATH"
+fi
+printf '%s trace=%s run=%s work=%s dbs=%s inventory=%s manifest=%s\\n' \
+    "$1" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
+    "$CLICKHOUSE_NATIVE_EXPECTED_DATABASES" \
+    "$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH" \
+    "${CLICKHOUSE_NATIVE_MANIFEST_PATH:-none}" \
+    >> "${TEST_HOOK_EVENTS:?}"
+""",
+        )
+        env["CLICKHOUSE_NATIVE_POST_VERIFY_HOOK"] = str(hook)
+        env["TEST_HOOK_EVENTS"] = str(paths["hook_events"])
+    return env, paths
+
+
+def _run(env: dict[str, str], mode: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["bash", str(SCRIPT), mode],
+        env=env,
+        text=True,
+        capture_output=True,
+        timeout=15,
+        check=False,
+    )
+
+
+def _receipts(path: Path) -> list[dict[str, object]]:
+    receipt_file = path / "run-native" / "receipts.jsonl"
+    return [
+        json.loads(line)
+        for line in receipt_file.read_text(encoding="utf-8").splitlines()
+    ]
+
+
+def test_check_validates_exact_scope_without_backup_or_artifact_write(
+    tmp_path: Path,
+) -> None:
+    env, paths = _harness(tmp_path)
+    result = _run(env, "--check")
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    events = paths["events"].read_text(encoding="utf-8")
+    assert "BACKUP DATABASE" not in events
+    assert "docker cp" not in events
+    assert "docker exec signoz-clickhouse rm" not in events
+    assert not list(paths["output"].iterdir())
+    receipts = _receipts(paths["receipts"])
+    assert receipts[-1]["terminal"] == "check_pass_no_write"
+
+
+def test_apply_requires_same_run_check_receipt(tmp_path: Path) -> None:
+    env, paths = _harness(tmp_path)
+    result = _run(env, "--apply")
+
+    assert result.returncode != 0
+    assert "same_run_check_pass_receipt_required_before_apply" in result.stderr
+    assert paths["events"].read_text(encoding="utf-8") == ""
+
+
+def test_apply_submits_exact_six_database_backup_and_verifies_zip(
+    tmp_path: Path,
+) -> None:
+    env, paths = _harness(tmp_path)
+    assert _run(env, "--check").returncode == 0
+    result = _run(env, "--apply")
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    events = paths["events"].read_text(encoding="utf-8")
+    assert "BACKUP DATABASE signoz_analytics,DATABASE signoz_logs" in events
+    assert "DATABASE signoz_metadata,DATABASE signoz_meter" in events
+    assert "DATABASE signoz_metrics,DATABASE signoz_traces" in events
+    assert "BACKUP ALL" not in events
+    backup_line = next(
+        line for line in events.splitlines() if "--query BACKUP " in line
+    )
+    assert " SETTINGS id='awoooi-" in backup_line
+    assert "id={operation_id:String}" not in backup_line
+    assert "--param_operation_id" not in backup_line
+    assert "docker cp signoz-clickhouse:/backups/signoz-" in events
+    assert "unzip -tq" in events
+    assert "docker exec signoz-clickhouse rm -f -- /backups/signoz-" in events
+    assert not paths["server_artifact"].exists()
+    outputs = list(paths["output"].glob("clickhouse-native-*.zip"))
+    assert len(outputs) == 1
+    manifest = json.loads(Path(str(outputs[0]) + ".manifest.json").read_text())
+    assert manifest["databases"] == [
+        "signoz_analytics",
+        "signoz_logs",
+        "signoz_metadata",
+        "signoz_meter",
+        "signoz_metrics",
+        "signoz_traces",
+    ]
+    inventory = list(paths["output"].glob("*.source-inventory.tsv"))
+    assert len(inventory) == 1
+    assert "ReplicatedMergeTree" in inventory[0].read_text(encoding="utf-8")
+    assert "Distributed" in inventory[0].read_text(encoding="utf-8")
+
+
+def test_database_allowlist_drift_fails_check(tmp_path: Path) -> None:
+    env, _paths = _harness(tmp_path, extra_database=True)
+    result = _run(env, "--check")
+    assert result.returncode != 0
+    assert "signoz_database_allowlist_drift" in result.stderr
+
+
+def test_missing_backup_grant_fails_check_before_submission(tmp_path: Path) -> None:
+    env, paths = _harness(tmp_path)
+    env["FAKE_BACKUP_GRANT"] = "0"
+
+    result = _run(env, "--check")
+
+    assert result.returncode != 0
+    assert "native_backup_grant_absent" in result.stderr
+    assert not paths["submitted"].exists()
+
+
+def test_missing_runtime_backup_disk_fails_check(tmp_path: Path) -> None:
+    env, _paths = _harness(tmp_path, disk_ready=False)
+    result = _run(env, "--check")
+    assert result.returncode != 0
+    assert "backup_disk_directory_absent" in result.stderr
+
+
+def test_failed_async_terminal_has_exact_cleanup_and_no_local_artifact(
+    tmp_path: Path,
+) -> None:
+    env, paths = _harness(tmp_path, status_mode="failed")
+    assert _run(env, "--check").returncode == 0
+    result = _run(env, "--apply")
+
+    assert result.returncode != 0
+    assert "native_backup_terminal_BACKUP_FAILED" in result.stderr
+    assert not paths["server_artifact"].exists()
+    assert not list(paths["output"].iterdir())
+    receipt_dir = paths["receipts"] / "run-native"
+    assert list(receipt_dir.glob("*.stderr"))
+    assert list(receipt_dir.glob("*.status"))
+    bounded_errors = list(receipt_dir.glob("*-backup-error-bounded.hex"))
+    assert len(bounded_errors) == 1
+    assert bounded_errors[0].read_text(encoding="utf-8") == ("434F44453A20353938\n")
+
+
+def test_zip_failure_removes_local_partial_and_preserves_server_evidence(
+    tmp_path: Path,
+) -> None:
+    env, paths = _harness(tmp_path, unzip_status=7)
+    assert _run(env, "--check").returncode == 0
+    result = _run(env, "--apply")
+
+    assert result.returncode != 0
+    assert "native_backup_archive_integrity_failed" in result.stderr
+    assert paths["server_artifact"].exists()
+    assert not list(paths["output"].iterdir())
+    receipts = _receipts(paths["receipts"])
+    assert any(
+        item["stage"] == "cleanup" and item["terminal"] == "preserved"
+        for item in receipts
+    )
+
+
+def test_restore_verifier_hook_uses_check_and_apply_contract(tmp_path: Path) -> None:
+    env, paths = _harness(tmp_path, with_hook=True)
+    assert _run(env, "--check").returncode == 0
+    result = _run(env, "--apply")
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    hook_events = paths["hook_events"].read_text(encoding="utf-8").splitlines()
+    assert sum(event.startswith("--check ") for event in hook_events) == 1
+    assert sum(event.startswith("--apply ") for event in hook_events) == 1
+    assert all("signoz_analytics,signoz_logs" in event for event in hook_events)
+    assert any(
+        "manifest=" in event and "manifest=none" not in event
+        for event in hook_events
+        if event.startswith("--apply ")
+    )
+
+
+def test_verified_local_artifact_is_idempotently_reused(tmp_path: Path) -> None:
+    env, paths = _harness(tmp_path)
+    assert _run(env, "--check").returncode == 0
+    assert _run(env, "--apply").returncode == 0
+    submissions_before = (
+        paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE")
+    )
+
+    result = _run(env, "--apply")
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    submissions_after = (
+        paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE")
+    )
+    assert submissions_after == submissions_before
+    receipts = _receipts(paths["receipts"])
+    assert any(
+        item["stage"] == "idempotency" and item["terminal"] == "reused"
+        for item in receipts
+    )
+
+
+def test_source_contract_has_no_raw_volume_fallback_and_is_deployed() -> None:
+    source = SCRIPT.read_text(encoding="utf-8")
+    playbook = PLAYBOOK.read_text(encoding="utf-8")
+
+    assert "BACKUP ${BACKUP_SCOPE_SQL}" in source
+    assert "SETTINGS id='${OPERATION_ID}' ASYNC" in source
+    assert "SETTINGS id={operation_id:String}" not in source
+    assert "BACKUP ALL" not in source
+    assert ".zip" in source
+    assert "unzip -tq" in source
+    assert "--format $'{{.Id}}\\t{{.State.Running}}'" in source
+    assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
+    assert "docker run" not in source
+    assert "clickhouse-native-backup.sh" in playbook
+    result = subprocess.run(
+        ["bash", "-n", str(SCRIPT)],
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr

scripts/backup/tests/test_clickhouse_native_restore_drill.py

@@ -0,0 +1,848 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import zipfile
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "clickhouse-native-restore-drill.sh"
+INVENTORY_HELPER = ROOT / "clickhouse-restore-inventory.py"
+REPO_ROOT = ROOT.parents[1]
+CLICKHOUSE_IMAGE_ID = (
+    "sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6"
+)
+ZOOKEEPER_IMAGE_ID = (
+    "sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0"
+)
+EMPTY_SHA256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"
+EXPECTED_DATABASES = [
+    "signoz_analytics",
+    "signoz_logs",
+    "signoz_metadata",
+    "signoz_meter",
+    "signoz_metrics",
+    "signoz_traces",
+]
+
+
+def _inventory(
+    *,
+    restored_delta: int = 0,
+    engine_mismatch: bool = False,
+    schema_mismatch: bool = False,
+) -> str:
+    rows = [
+        ("signoz_analytics", "events", "ReplicatedMergeTree", 10, 100),
+        ("signoz_analytics", "events_all", "Distributed", 10, 100),
+        ("signoz_logs", "logs_v2", "ReplicatedMergeTree", 20, 200),
+        ("signoz_metadata", "resource_attrs", "ReplicatedMergeTree", 2, 20),
+        ("signoz_meter", "meter", "ReplicatedMergeTree", 3, 30),
+        ("signoz_metrics", "samples_v4", "ReplicatedMergeTree", 30, 300),
+        ("signoz_metrics", "time_series_v4", "ReplicatedMergeTree", 12, 120),
+        ("signoz_traces", "signoz_index_v3", "ReplicatedMergeTree", 40, 400),
+    ]
+    rendered: list[str] = []
+    for index, (database, table, engine, row_count, byte_count) in enumerate(rows):
+        if engine_mismatch and table == "meter":
+            engine = "MergeTree"
+        schema_hash = (
+            hashlib.sha256(f"{database}.{table}:{engine}".encode("utf-8"))
+            .hexdigest()
+            .upper()
+        )
+        if schema_mismatch and table == "meter":
+            schema_hash = "F" * 64
+        rendered.append(
+            "\t".join(
+                (
+                    database,
+                    table,
+                    engine,
+                    str(row_count + (restored_delta if index == 0 else 0)),
+                    str(byte_count + (restored_delta if index == 0 else 0)),
+                    schema_hash,
+                )
+            )
+        )
+    return "\n".join(rendered) + "\n"
+
+
+def _write_backup_fixture(tmp_path: Path) -> tuple[Path, Path, Path]:
+    artifact = tmp_path / "clickhouse-native.zip.partial"
+    with zipfile.ZipFile(artifact, "w") as archive:
+        archive.writestr(".backup", "bounded native backup fixture\n")
+    inventory = tmp_path / "source-inventory.tsv.partial"
+    inventory.write_text(_inventory(), encoding="utf-8")
+    artifact_sha = hashlib.sha256(artifact.read_bytes()).hexdigest()
+    inventory_sha = hashlib.sha256(inventory.read_bytes()).hexdigest()
+    manifest = tmp_path / "manifest.json.partial"
+    manifest.write_text(
+        json.dumps(
+            {
+                "trace_id": "backup-trace",
+                "run_id": "backup-run",
+                "work_item_id": "P0-OBS-002",
+                "operation_id": "awoooi-backup-operation",
+                "databases": EXPECTED_DATABASES,
+                "source_inventory_sha256": inventory_sha,
+                "sha256": artifact_sha,
+                "size_bytes": artifact.stat().st_size,
+                "status": "BACKUP_CREATED",
+            },
+            separators=(",", ":"),
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    return artifact, manifest, inventory
+
+
+def _write_fake_docker(tmp_path: Path) -> tuple[Path, Path, Path]:
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    state_dir = tmp_path / "docker-state"
+    state_dir.mkdir()
+    log_path = tmp_path / "docker.log"
+    docker = bin_dir / "docker"
+    docker.write_text(
+        r"""#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\n' "$*" >> "${FAKE_DOCKER_LOG:?}"
+state="${FAKE_DOCKER_STATE:?}"
+cmd="${1:?}"
+shift
+last_arg() { eval "printf '%s' \"\${$#}\""; }
+
+case "$cmd" in
+  info)
+    [ "${FAKE_DOCKER_INFO_FAIL:-0}" = 0 ] || exit 78
+    printf '25.0.0\\n'
+    ;;
+  image)
+    sub="${1:?}"
+    shift
+    [ "$sub" = inspect ] || exit 90
+    image="$(last_arg "$@")"
+    case "$image" in
+      clickhouse/clickhouse-server:25.5.6) printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" ;;
+      signoz/zookeeper:3.7.1) printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" ;;
+      *) exit 1 ;;
+    esac
+    ;;
+  container)
+    sub="${1:?}"
+    shift
+    case "$sub" in
+      inspect)
+        format=""
+        if [ "${1:-}" = --format ]; then
+          format="$2"
+          shift 2
+        fi
+        name="${1:?}"
+        [ -f "$state/container-$name" ] || exit 1
+        if [[ "$format" == *Labels* ]]; then
+          if [ "${FAKE_LABEL_INSPECT_HANG:-0}" = 1 ] && [[ "$name" == *ch-restore* ]]; then
+            sleep 10
+          fi
+          cat "$state/container-label-$name"
+        fi
+        ;;
+      ls)
+        for item in "$state"/container-*; do
+          [ -e "$item" ] || continue
+          case "$item" in
+            *-label-*|*-network-*|*-image-*) continue ;;
+          esac
+          basename "$item" | sed 's/^container-//'
+        done
+        ;;
+      *) exit 90 ;;
+    esac
+    ;;
+  inspect)
+    format=""
+    if [ "${1:-}" = --format ]; then
+      format="$2"
+      shift 2
+    fi
+    name="${1:?}"
+    [ -f "$state/container-$name" ] || exit 1
+    case "$format" in
+      *State.Running*) printf 'true\n' ;;
+      *HostConfig.NetworkMode*) cat "$state/container-network-$name" ;;
+      *.Image*) cat "$state/container-image-$name" ;;
+      *) printf '%s\n' "fake-$name" ;;
+    esac
+    ;;
+  network)
+    sub="${1:?}"
+    shift
+    case "$sub" in
+      create)
+        name="$(last_arg "$@")"
+        label=""
+        while [ "$#" -gt 0 ]; do
+          if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi
+        done
+        touch "$state/network-$name"
+        printf '%s\n' "$label" > "$state/network-label-$name"
+        printf '%s\n' "$name" > "$state/network-name"
+        printf '%s\n' "$name"
+        ;;
+      inspect)
+        format=""
+        if [ "${1:-}" = --format ]; then
+          format="$2"
+          shift 2
+        fi
+        name="${1:?}"
+        [ -f "$state/network-$name" ] || exit 1
+        if [[ "$format" == *Internal* ]]; then printf 'true\n'; fi
+        if [[ "$format" == *Labels* ]]; then cat "$state/network-label-$name"; fi
+        ;;
+      ls)
+        for item in "$state"/network-*; do
+          [ -e "$item" ] || continue
+          case "$item" in *-label-*|*/network-name) continue ;; esac
+          basename "$item" | sed 's/^network-//'
+        done
+        ;;
+      rm)
+        name="${1:?}"
+        rm -f "$state/network-$name" "$state/network-label-$name" "$state/network-name"
+        printf '%s\n' "$name"
+        ;;
+      *) exit 90 ;;
+    esac
+    ;;
+  volume)
+    sub="${1:?}"
+    shift
+    case "$sub" in
+      create)
+        name="$(last_arg "$@")"
+        label=""
+        while [ "$#" -gt 0 ]; do
+          if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi
+        done
+        touch "$state/volume-$name"
+        printf '%s\n' "$label" > "$state/volume-label-$name"
+        printf '%s\n' "$name"
+        ;;
+      inspect)
+        format=""
+        if [ "${1:-}" = --format ]; then format="$2"; shift 2; fi
+        name="${1:?}"
+        [ -f "$state/volume-$name" ] || exit 1
+        if [[ "$format" == *Labels* ]]; then cat "$state/volume-label-$name"; fi
+        ;;
+      ls)
+        for item in "$state"/volume-*; do
+          [ -e "$item" ] || continue
+          case "$item" in *-label-*) continue ;; esac
+          basename "$item" | sed 's/^volume-//'
+        done
+        ;;
+      rm)
+        name="${1:?}"
+        rm -f "$state/volume-$name" "$state/volume-label-$name"
+        if [[ "$name" == *-backup ]]; then
+          rm -f "$state/staged-artifact-sha256"
+        fi
+        printf '%s\n' "$name"
+        ;;
+      *) exit 90 ;;
+    esac
+    ;;
+  run)
+    name=""
+    replica=""
+    label=""
+    network=""
+    image=""
+    expected_artifact_sha256=""
+    while [ "$#" -gt 0 ]; do
+      case "$1" in
+        --name)
+          name="$2"
+          shift 2
+          ;;
+        --env)
+          if [[ "$2" == CLICKHOUSE_RESTORE_REPLICA=* ]]; then
+            replica="${2#*=}"
+          fi
+          if [[ "$2" == EXPECTED_ARTIFACT_SHA256=* ]]; then
+            expected_artifact_sha256="${2#*=}"
+          fi
+          shift 2
+          ;;
+        --label)
+          label="${2#*=}"
+          shift 2
+          ;;
+        --network)
+          network="$2"
+          shift 2
+          ;;
+        --network-alias|--restart|--mount|--user|--entrypoint)
+          shift 2
+          ;;
+        --detach|--pull=never)
+          shift
+          ;;
+        clickhouse/clickhouse-server:25.5.6|signoz/zookeeper:3.7.1)
+          image="$1"
+          shift
+          ;;
+        *)
+          shift
+          ;;
+      esac
+    done
+    [ -n "$name" ] || exit 91
+    touch "$state/container-$name"
+    printf '%s\n' "$label" > "$state/container-label-$name"
+    printf '%s\n' "$network" > "$state/container-network-$name"
+    case "$image" in
+      clickhouse/clickhouse-server:25.5.6)
+        printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" > "$state/container-image-$name"
+        ;;
+      signoz/zookeeper:3.7.1)
+        printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" > "$state/container-image-$name"
+        ;;
+      *) exit 96 ;;
+    esac
+    if [ -n "$replica" ]; then printf '%s\n' "$replica" > "$state/replica"; fi
+    if [ "$network" = none ]; then
+      if [ "${FAKE_STAGING_HANG:-0}" = 1 ]; then sleep 10; fi
+      [ "${FAKE_STAGING_FAIL:-0}" = 0 ] || exit 94
+      staged_hash="$expected_artifact_sha256"
+      if [ "${FAKE_STAGING_HASH_MISMATCH:-0}" = 1 ]; then
+        staged_hash="$(printf '0%.0s' {1..64})"
+      fi
+      printf '%s\n' "$staged_hash" > "$state/staged-artifact-sha256"
+      printf '%s\n' "$staged_hash"
+    else
+      printf 'fake-container-%s\n' "$name"
+    fi
+    ;;
+  exec)
+    container="${1:?}"
+    shift
+    [ -f "$state/container-$container" ] || exit 1
+    if [ "${1:-}" = sha256sum ]; then
+      [ -f "$state/staged-artifact-sha256" ] || exit 97
+      artifact_hash="$(cat "$state/staged-artifact-sha256")"
+      if [ "${FAKE_CLICKHOUSE_HASH_MISMATCH:-0}" = 1 ]; then
+        artifact_hash="$(printf 'f%.0s' {1..64})"
+      fi
+      printf '%s  /backups/native-backup.zip\n' "$artifact_hash"
+      exit 0
+    fi
+    query=""
+    operation=""
+    while [ "$#" -gt 0 ]; do
+      case "$1" in
+        --query)
+          query="$2"
+          shift 2
+          ;;
+        --param_operation_id)
+          operation="$2"
+          shift 2
+          ;;
+        *) shift ;;
+      esac
+    done
+    case "$query" in
+      CHECK\ TABLE*)
+        if [ "${FAKE_CHECK_TABLE_FAIL:-0}" = 1 ]; then printf '0\n'; else printf '1\n'; fi
+        ;;
+      RESTORE\ *)
+        case "$query" in
+          *" SETTINGS id='${EXPECTED_FAKE_RESTORE_OPERATION_ID:?}' ASYNC")
+            operation="${EXPECTED_FAKE_RESTORE_OPERATION_ID}"
+            ;;
+          *) exit 93 ;;
+        esac
+        touch "$state/restored"
+        printf '%s\tRESTORING\n' "$operation"
+        ;;
+      *FROM\ system.backups*)
+        printf "Disk('backups', 'native-backup.zip')\tRESTORED\t100\t1000\t%s\n" "${EMPTY_SHA256:?}"
+        ;;
+      *FROM\ system.disks*) printf '/backups/\n' ;;
+      *FROM\ system.macros*)
+        printf 'replica\t%s\nshard\t01\n' "$(cat "$state/replica")"
+        ;;
+      *FROM\ system.clusters*) printf 'cluster\trestore-clickhouse\t9000\n' ;;
+      *FROM\ system.zookeeper*) printf '1\n' ;;
+      *"SELECT count() FROM system.databases"*) printf '0\n' ;;
+      *"SELECT count() FROM system.tables"*) printf '0\n' ;;
+      *"SELECT database,name,engine"*) cat "${FAKE_RESTORED_INVENTORY:?}" ;;
+      "SELECT 1") printf '1\n' ;;
+      *) printf 'UNHANDLED QUERY: %s\n' "$query" >&2; exit 92 ;;
+    esac
+    ;;
+  logs)
+    printf 'bounded fake container log\n'
+    ;;
+  rm)
+    if [ "${1:-}" = -f ]; then shift; fi
+    name="${1:?}"
+    rm -f "$state/container-$name" "$state/container-label-$name" \
+      "$state/container-network-$name" "$state/container-image-$name"
+    printf '%s\n' "$name"
+    ;;
+  *) exit 99 ;;
+esac
+""",
+        encoding="utf-8",
+    )
+    docker.chmod(0o755)
+    return bin_dir, state_dir, log_path
+
+
+def _environment(
+    tmp_path: Path,
+    artifact: Path,
+    manifest: Path,
+    inventory: Path,
+    restored_inventory: Path,
+) -> dict[str, str]:
+    bin_dir, state_dir, log_path = _write_fake_docker(tmp_path)
+    return os.environ | {
+        "PATH": f"{bin_dir}:{os.environ['PATH']}",
+        "TRACE_ID": "trace-restore-test",
+        "RUN_ID": "run-restore-test",
+        "WORK_ITEM_ID": "P0-OBS-002",
+        "CLICKHOUSE_NATIVE_ARTIFACT_PATH": str(artifact),
+        "CLICKHOUSE_NATIVE_MANIFEST_PATH": str(manifest),
+        "CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH": str(inventory),
+        "CLICKHOUSE_NATIVE_ARTIFACT_SHA256": hashlib.sha256(
+            artifact.read_bytes()
+        ).hexdigest(),
+        "CLICKHOUSE_NATIVE_OPERATION_ID": "awoooi-backup-operation",
+        "CLICKHOUSE_NATIVE_EXPECTED_DATABASES": ",".join(EXPECTED_DATABASES),
+        "CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(tmp_path / "receipts"),
+        "CLICKHOUSE_RESTORE_POLL_INTERVAL_SECONDS": "1",
+        "CLICKHOUSE_RESTORE_STARTUP_TIMEOUT_SECONDS": "5",
+        "CLICKHOUSE_RESTORE_TIMEOUT_SECONDS": "5",
+        "CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS": "5",
+        "FAKE_DOCKER_LOG": str(log_path),
+        "FAKE_DOCKER_STATE": str(state_dir),
+        "FAKE_CLICKHOUSE_IMAGE_ID": CLICKHOUSE_IMAGE_ID,
+        "FAKE_ZOOKEEPER_IMAGE_ID": ZOOKEEPER_IMAGE_ID,
+        "FAKE_RESTORED_INVENTORY": str(restored_inventory),
+        "EXPECTED_FAKE_RESTORE_OPERATION_ID": "restore-"
+        + hashlib.sha256(
+            b"trace-restore-test\0run-restore-test\0P0-OBS-002"
+        ).hexdigest(),
+        "EMPTY_SHA256": EMPTY_SHA256,
+    }
+
+
+def _run(mode: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["bash", str(SCRIPT), mode],
+        text=True,
+        capture_output=True,
+        env=env,
+    )
+
+
+def _status(tmp_path: Path) -> dict[str, object]:
+    return json.loads(
+        (tmp_path / "receipts" / "run-restore-test" / "status.json").read_text(
+            encoding="utf-8"
+        )
+    )
+
+
+def _assert_ephemeral_resources_absent(state_dir: Path) -> None:
+    assert not list(state_dir.glob("container-*"))
+    assert not list(state_dir.glob("volume-*"))
+    assert not list(state_dir.glob("network-*"))
+    assert not (state_dir / "staged-artifact-sha256").exists()
+
+
+def test_hook_check_uses_env_contract_without_runtime_creation(tmp_path: Path) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(restored_delta=3), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+
+    result = _run("--check", env)
+
+    assert result.returncode == 0, result.stderr
+    assert "terminal=check_pass_no_runtime_write" in result.stdout
+    docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8")
+    assert "image inspect" in docker_log
+    assert "network create" not in docker_log
+    assert "volume create" not in docker_log
+    assert not any(line.startswith("run ") for line in docker_log.splitlines())
+    status = _status(tmp_path)
+    assert status["check_pass"] is True
+    assert status["ephemeral_runtime_created"] is False
+    assert status["production_network_attached"] is False
+    assert status["production_restore_performed"] is False
+
+
+def test_apply_restores_on_internal_pair_and_cleans_every_resource(
+    tmp_path: Path,
+) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(restored_delta=3), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    check = _run("--check", env)
+    assert check.returncode == 0, check.stderr
+
+    result = _run("--apply", env)
+
+    assert result.returncode == 0, result.stderr
+    assert "terminal=verified" in result.stdout
+    docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines()
+    network_create = next(
+        line for line in docker_lines if line.startswith("network create")
+    )
+    assert "--internal" in network_create
+    run_lines = [line for line in docker_lines if line.startswith("run ")]
+    assert len(run_lines) == 3
+    assert all("--pull=never" in line for line in run_lines)
+    assert all("--publish" not in line and " -p " not in line for line in run_lines)
+
+    staging_run = next(line for line in run_lines if "--network none" in line)
+    zookeeper_run = next(line for line in run_lines if "signoz/zookeeper:3.7.1" in line)
+    clickhouse_run = next(
+        line for line in run_lines if "--network-alias restore-clickhouse" in line
+    )
+    assert "--user 0:0" in staging_run
+    assert "--entrypoint /bin/sh" in staging_run
+    assert (
+        f"type=bind,src={artifact},dst=/tmp/awoooi-source-native-backup.zip,readonly"
+        in staging_run
+    )
+    assert sum(f"src={artifact}" in line for line in run_lines) == 1
+    assert "type=volume,src=awoooi-ch-restore-" in staging_run
+    assert "-backup,dst=/backups" in staging_run
+    assert "ALLOW_ANONYMOUS_LOGIN=yes" in zookeeper_run
+    assert "--network awoooi-ch-restore-" in zookeeper_run
+    assert "--network awoooi-ch-restore-" in clickhouse_run
+    assert str(artifact) not in clickhouse_run
+    assert "type=volume,src=awoooi-ch-restore-" in clickhouse_run
+    assert "-backup,dst=/backups" in clickhouse_run
+    assert "dst=/backups,readonly" not in clickhouse_run
+    assert (
+        "dst=/etc/clickhouse-server/config.d/awoooi-backup-disk.xml,readonly"
+        in clickhouse_run
+    )
+    assert (
+        "dst=/etc/clickhouse-server/config.d/awoooi-restore-isolation.xml,readonly"
+        in clickhouse_run
+    )
+    hash_readback = next(
+        line
+        for line in docker_lines
+        if line.startswith("exec ") and "sha256sum /backups/native-backup.zip" in line
+    )
+    assert "exec awoooi-ch-restore-" in hash_readback
+    restore_line = next(line for line in docker_lines if "RESTORE DATABASE" in line)
+    assert "allow_non_empty_tables" not in restore_line
+    assert " SETTINGS id='restore-" in restore_line
+    assert "id={operation_id:String}" not in restore_line
+    macro_line = next(line for line in docker_lines if "FROM system.macros" in line)
+    assert "SELECT macro,substitution" in macro_line
+    assert "SELECT name,value" not in macro_line
+    state_dir = Path(env["FAKE_DOCKER_STATE"])
+    _assert_ephemeral_resources_absent(state_dir)
+    status = _status(tmp_path)
+    assert status["terminal"] == "verified"
+    assert status["check_pass"] is True
+    assert status["restore_terminal"] == "RESTORED"
+    assert status["manifest_parity"] == 1
+    assert status["check_table_count"] == status["check_table_pass_count"] == 7
+    assert status["critical_nonzero_count"] == 4
+    assert status["row_delta"] == 3
+    expected_hash = hashlib.sha256(artifact.read_bytes()).hexdigest()
+    assert status["artifact_sha256"] == expected_hash
+    assert status["staged_artifact_sha256"] == expected_hash
+    assert status["clickhouse_artifact_sha256"] == expected_hash
+    assert status["artifact_staging_verified"] is True
+    assert status["clickhouse_artifact_readback_verified"] is True
+    assert status["staging_network_mode"] == "none"
+    assert status["artifact_source_mount"] == "read_only_staging_only"
+    assert status["backup_volume_mount"] == "writable_ephemeral"
+    assert status["cleanup_verified"] is True
+
+
+def test_apply_failure_still_cleans_pair_network_and_volumes(tmp_path: Path) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    assert _run("--check", env).returncode == 0
+    env["FAKE_CHECK_TABLE_FAIL"] = "1"
+
+    result = _run("--apply", env)
+
+    assert result.returncode == 1
+    assert "terminal=failed" in result.stdout
+    state_dir = Path(env["FAKE_DOCKER_STATE"])
+    _assert_ephemeral_resources_absent(state_dir)
+    status = _status(tmp_path)
+    assert status["terminal"] == "failed"
+    assert status["cleanup_verified"] is True
+    assert status["apply_pass"] is False
+
+
+def test_artifact_staging_failure_cleans_exact_resources(tmp_path: Path) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    assert _run("--check", env).returncode == 0
+    env["FAKE_STAGING_FAIL"] = "1"
+
+    result = _run("--apply", env)
+
+    assert result.returncode == 1
+    _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
+    status = _status(tmp_path)
+    assert status["terminal"] == "failed"
+    assert status["reason"] == "isolated_artifact_staging_failed"
+    assert status["artifact_staging_verified"] is False
+    assert status["cleanup_verified"] is True
+
+
+def test_artifact_staging_hang_is_bounded_and_cleans_exact_resources(
+    tmp_path: Path,
+) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1"
+    assert _run("--check", env).returncode == 0
+    env["FAKE_STAGING_HANG"] = "1"
+
+    result = subprocess.run(
+        ["bash", str(SCRIPT), "--apply"],
+        text=True,
+        capture_output=True,
+        env=env,
+        timeout=12,
+    )
+
+    assert result.returncode == 1
+    _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
+    status = _status(tmp_path)
+    assert status["terminal"] == "failed"
+    assert status["reason"] == "isolated_artifact_staging_failed"
+    assert status["artifact_staging_verified"] is False
+    assert status["cleanup_verified"] is True
+
+
+def test_staged_artifact_hash_mismatch_fails_before_isolated_services(
+    tmp_path: Path,
+) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    assert _run("--check", env).returncode == 0
+    env["FAKE_STAGING_HASH_MISMATCH"] = "1"
+
+    result = _run("--apply", env)
+
+    assert result.returncode == 1
+    docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines()
+    assert not any(
+        "signoz/zookeeper:3.7.1" in line
+        for line in docker_lines
+        if line.startswith("run ")
+    )
+    _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
+    status = _status(tmp_path)
+    assert status["reason"] == "staged_artifact_hash_mismatch"
+    assert status["cleanup_verified"] is True
+
+
+def test_clickhouse_artifact_hash_readback_mismatch_fails_and_cleans(
+    tmp_path: Path,
+) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    assert _run("--check", env).returncode == 0
+    env["FAKE_CLICKHOUSE_HASH_MISMATCH"] = "1"
+
+    result = _run("--apply", env)
+
+    assert result.returncode == 1
+    _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
+    status = _status(tmp_path)
+    assert status["reason"] == "isolated_clickhouse_artifact_hash_mismatch"
+    assert status["artifact_staging_verified"] is True
+    assert status["clickhouse_artifact_readback_verified"] is False
+    assert status["clickhouse_artifact_sha256"] == "f" * 64
+    assert status["cleanup_verified"] is True
+
+
+def test_docker_daemon_failure_cannot_claim_cleanup_verified(tmp_path: Path) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    assert _run("--check", env).returncode == 0
+    env["FAKE_DOCKER_INFO_FAIL"] = "1"
+
+    result = _run("--apply", env)
+
+    assert result.returncode == 1
+    status = _status(tmp_path)
+    assert status["terminal"] == "failed"
+    assert status["cleanup_verified"] is False
+    assert status["reason"] == "ephemeral_resource_cleanup_failed"
+
+
+def test_cleanup_label_inspect_hang_is_bounded_and_fails_closed(tmp_path: Path) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1"
+    assert _run("--check", env).returncode == 0
+    env["FAKE_CHECK_TABLE_FAIL"] = "1"
+    env["FAKE_LABEL_INSPECT_HANG"] = "1"
+
+    result = subprocess.run(
+        ["bash", str(SCRIPT), "--apply"],
+        text=True,
+        capture_output=True,
+        env=env,
+        timeout=12,
+    )
+
+    assert result.returncode == 1
+    status = _status(tmp_path)
+    assert status["terminal"] == "failed"
+    assert status["cleanup_verified"] is False
+    assert status["reason"] == "ephemeral_resource_cleanup_failed"
+
+
+def test_check_fails_closed_on_local_image_id_drift(tmp_path: Path) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    env["FAKE_CLICKHOUSE_IMAGE_ID"] = "sha256:" + "0" * 64
+
+    result = _run("--check", env)
+
+    assert result.returncode == 1
+    assert "clickhouse_local_image_id_mismatch" in result.stderr
+    docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8")
+    assert "network create" not in docker_log
+    assert "volume create" not in docker_log
+
+
+def test_flat_host_deploy_accepts_reviewed_absolute_helper_and_config_paths(
+    tmp_path: Path,
+) -> None:
+    artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+    restored = tmp_path / "restored.tsv"
+    restored.write_text(_inventory(), encoding="utf-8")
+    env = _environment(tmp_path, artifact, manifest, inventory, restored)
+    host_root = tmp_path / "host-backup"
+    host_root.mkdir()
+    helper = host_root / "clickhouse-restore-inventory.py"
+    backup_config = host_root / "backup_disk.xml"
+    cluster_config = host_root / "isolated-cluster.xml"
+    shutil.copyfile(INVENTORY_HELPER, helper)
+    shutil.copyfile(
+        REPO_ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml", backup_config
+    )
+    shutil.copyfile(
+        REPO_ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
+        cluster_config,
+    )
+    env.update(
+        {
+            "CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(helper),
+            "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_config),
+            "CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config),
+        }
+    )
+
+    result = _run("--check", env)
+
+    assert result.returncode == 0, result.stderr
+    assert "terminal=check_pass_no_runtime_write" in result.stdout
+
+
+def test_inventory_compare_allows_online_row_delta_but_not_engine_drift(
+    tmp_path: Path,
+) -> None:
+    source = tmp_path / "source.tsv"
+    restored = tmp_path / "restored.tsv"
+    source.write_text(_inventory(), encoding="utf-8")
+    restored.write_text(_inventory(restored_delta=9), encoding="utf-8")
+
+    accepted = subprocess.run(
+        [
+            "python3",
+            str(INVENTORY_HELPER),
+            "compare",
+            "--source-inventory",
+            str(source),
+            "--restored-inventory",
+            str(restored),
+        ],
+        text=True,
+        capture_output=True,
+    )
+    assert accepted.returncode == 0, accepted.stderr
+    assert "manifest_parity=1" in accepted.stdout
+    assert "row_delta=9" in accepted.stdout
+
+    restored.write_text(_inventory(engine_mismatch=True), encoding="utf-8")
+    rejected = subprocess.run(
+        [
+            "python3",
+            str(INVENTORY_HELPER),
+            "compare",
+            "--source-inventory",
+            str(source),
+            "--restored-inventory",
+            str(restored),
+        ],
+        text=True,
+        capture_output=True,
+    )
+    assert rejected.returncode == 1
+    assert "table_engine_mismatch_signoz_meter_meter" in rejected.stderr
+
+    restored.write_text(_inventory(schema_mismatch=True), encoding="utf-8")
+    schema_rejected = subprocess.run(
+        [
+            "python3",
+            str(INVENTORY_HELPER),
+            "compare",
+            "--source-inventory",
+            str(source),
+            "--restored-inventory",
+            str(restored),
+        ],
+        text=True,
+        capture_output=True,
+    )
+    assert schema_rejected.returncode == 1
+    assert "table_schema_mismatch_signoz_meter_meter" in schema_rejected.stderr

scripts/backup/tests/test_signoz_backup_canary_wrapper.py

@@ -0,0 +1,528 @@
+from __future__ import annotations
+
+import json
+import os
+import stat
+import subprocess
+import time
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh"
+PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
+ANSIBLE_VALIDATE = ROOT / "scripts" / "ops" / "ansible-validate.sh"
+READINESS_AUDIT = (
+    ROOT / "scripts" / "reboot-recovery" / "reboot-recovery-readiness-audit.sh"
+)
+
+
+def _write_executable(path: Path, text: str) -> None:
+    path.write_text(text, encoding="utf-8")
+    path.chmod(0o755)
+
+
+def _monitor_source(cooldown_dir: Path, *, omit_contract: bool = False) -> str:
+    cooldown_log = (
+        'log "COOLDOWN: ${container} skipped"'
+        if not omit_contract
+        else 'log "maintenance skip for ${container}"'
+    )
+    return f"""\
+#!/bin/bash
+: "${{ACTION_COOLDOWN_SECONDS:=${{SEND_COOLDOWN_SECONDS}}}}"
+: "${{COOLDOWN_DIR:={cooldown_dir}}}"
+: "${{EXCLUDE_CONTAINERS:=signoz-clickhouse}}"
+is_in_cooldown() {{
+    local container="$1"
+    local cooldown_file="${{COOLDOWN_DIR}}/${{container}}.cooldown"
+    local last_sent now elapsed
+    last_sent=$(cat "$cooldown_file")
+    now=$(date +%s)
+    elapsed=$(( now - last_sent ))
+    if (( elapsed < ACTION_COOLDOWN_SECONDS )); then
+        {cooldown_log}
+        return 0
+    fi
+    return 1
+}}
+set_cooldown() {{ :; }}
+# A second monitor path may use the same elapsed expression. The wrapper must
+# validate the expression inside is_in_cooldown, not require global uniqueness.
+# elapsed=$(( now - last_sent ))
+while read -r container_name; do
+    is_in_cooldown "$container_name" && continue
+    set_cooldown "$container_name"
+    container="$container_name"
+    log "AUTO_REPAIR: docker restart ${{container}}"
+    if docker restart "$container"; then
+        :
+    fi
+done
+"""
+
+
+def _install_fake_commands(fake_bin: Path) -> None:
+    _write_executable(
+        fake_bin / "crontab",
+        """\
+#!/bin/bash
+set -eu
+[ "${1:-}" = "-l" ]
+printf '*/5 * * * * %s >> %s 2>&1\n' \
+    "${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
+if [ "${FAKE_DUPLICATE_CRON:-0}" = "1" ]; then
+    printf '*/5 * * * * %s >> %s 2>&1\n' \
+        "${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
+fi
+""",
+    )
+    _write_executable(
+        fake_bin / "docker",
+        """\
+#!/bin/bash
+set -eu
+[ "${1:-}" = "inspect" ] || exit 70
+case " $* " in
+    *"{{.State.StartedAt}}"*)
+        printf 'sha256:collector-test-id\\t%s\\t2026-07-15T00:00:00Z\\t0\\thealthy\\n' \
+            "$(cat "${TEST_COLLECTOR_STATE:?}")"
+        ;;
+    *"{{.Id}}"*) printf 'sha256:collector-test-id\n' ;;
+    *"{{.State.Running}}"*) cat "${TEST_COLLECTOR_STATE:?}" ;;
+    *) exit 71 ;;
+esac
+""",
+    )
+    _write_executable(
+        fake_bin / "ss",
+        """\
+#!/bin/bash
+set -eu
+while IFS= read -r port; do
+    [ -n "$port" ] || continue
+    printf 'LISTEN 0 4096 *:%s *:*\n' "$port"
+done < "${TEST_LISTENERS:?}"
+""",
+    )
+    _write_executable(
+        fake_bin / "timeout",
+        """\
+#!/bin/bash
+set -eu
+while [[ "${1:-}" == --kill-after=* ]]; do
+    shift
+done
+[ "$#" -gt 1 ]
+shift
+exec "$@"
+""",
+    )
+    _write_executable(
+        fake_bin / "flock",
+        """\
+#!/bin/bash
+exit 0
+""",
+    )
+    _write_executable(
+        fake_bin / "pgrep",
+        """\
+#!/bin/bash
+exit 1
+""",
+    )
+    _write_executable(
+        fake_bin / "stat",
+        """\
+#!/usr/bin/env python3
+import os
+import stat
+import sys
+
+if len(sys.argv) != 4 or sys.argv[1] != "-c":
+    raise SystemExit(64)
+value = os.stat(sys.argv[3])
+formats = {
+    "%a": format(stat.S_IMODE(value.st_mode), "o"),
+    "%u": str(value.st_uid),
+    "%g": str(value.st_gid),
+    "%s": str(value.st_size),
+    "%d:%i": f"{value.st_dev}:{value.st_ino}",
+}
+if sys.argv[2] not in formats:
+    raise SystemExit(65)
+print(formats[sys.argv[2]])
+""",
+    )
+    _write_executable(
+        fake_bin / "sha256sum",
+        """\
+#!/usr/bin/env python3
+import hashlib
+import sys
+
+if len(sys.argv) != 2:
+    raise SystemExit(64)
+with open(sys.argv[1], "rb") as source:
+    digest = hashlib.sha256(source.read()).hexdigest()
+print(f"{digest}  {sys.argv[1]}")
+""",
+    )
+
+
+def _backup_source() -> str:
+    return """\
+#!/bin/bash
+set -eu
+[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80
+[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ]
+printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
+    > "${TEST_BACKUP_IDENTITIES:?}"
+if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
+    cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}"
+    printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
+    case "${FAKE_MONITOR_MODE:-cooldown}" in
+        cooldown)
+            printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
+                >> "${TEST_MONITOR_LOG:?}"
+            printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
+                >> "${TEST_MONITOR_LOG:?}"
+            ;;
+        auto_repair)
+            printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
+                >> "${TEST_MONITOR_LOG:?}"
+            printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
+                >> "${TEST_MONITOR_LOG:?}"
+            printf '[test] AUTO_REPAIR: docker restart signoz-otel-collector\n' \
+                >> "${TEST_MONITOR_LOG:?}"
+            ;;
+        none) : ;;
+        *) exit 81 ;;
+    esac
+    sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
+    printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
+else
+    sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
+fi
+if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then
+    printf '4317\n' > "${TEST_LISTENERS:?}"
+fi
+printf '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)\n'
+if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
+    printf '[SUCCESS] ========== SignOz 備份完成 (1s) ==========\n'
+else
+    printf '[SUCCESS] ========== SigNoz ClickHouse native 備份完成 (1s) ==========\n'
+fi
+"""
+
+
+def _run_canary(
+    tmp_path: Path,
+    *,
+    mode: str = "apply",
+    prior_lease: bool = True,
+    monitor_mode: str = "cooldown",
+    backup_sleep_seconds: float = 0.2,
+    monitor_interval_seconds: int = 300,
+    omit_monitor_contract: bool = False,
+    duplicate_cron: bool = False,
+    drop_listener: bool = False,
+    expect_collector_stop: bool = True,
+) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]:
+    fake_bin = tmp_path / "bin"
+    cooldown_dir = tmp_path / "cooldown"
+    receipt_root = tmp_path / "receipts"
+    fake_bin.mkdir()
+    cooldown_dir.mkdir()
+    receipt_root.mkdir()
+
+    monitor_script = tmp_path / "docker-health-monitor.sh"
+    monitor_log = tmp_path / "monitor.log"
+    backup_script = tmp_path / "backup-signoz.sh"
+    collector_state = tmp_path / "collector.state"
+    listeners = tmp_path / "listeners"
+    identities = tmp_path / "backup-identities.tsv"
+    lease_observed = tmp_path / "lease-observed.txt"
+    cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown"
+    run_id = f"canary-{tmp_path.name}"
+
+    _write_executable(
+        monitor_script,
+        _monitor_source(cooldown_dir, omit_contract=omit_monitor_contract),
+    )
+    _write_executable(backup_script, _backup_source())
+    _install_fake_commands(fake_bin)
+    monitor_log.write_text("[test] monitor seed\n", encoding="utf-8")
+    collector_state.write_text("true\n", encoding="utf-8")
+    listeners.write_text("4317\n4318\n", encoding="utf-8")
+    if prior_lease:
+        cooldown_file.write_text("123\n", encoding="utf-8")
+        cooldown_file.chmod(0o640)
+
+    env = {
+        **os.environ,
+        "PATH": f"{fake_bin}:{os.environ['PATH']}",
+        "TRACE_ID": "trace-P0-OBS-002",
+        "RUN_ID": run_id,
+        "WORK_ITEM_ID": "P0-OBS-002",
+        "SIGNOZ_CANARY_BACKUP_SCRIPT": str(backup_script),
+        "SIGNOZ_CANARY_MONITOR_SCRIPT": str(monitor_script),
+        "SIGNOZ_CANARY_MONITOR_LOG": str(monitor_log),
+        "SIGNOZ_CANARY_COOLDOWN_DIR": str(cooldown_dir),
+        "SIGNOZ_CANARY_RECEIPT_ROOT": str(receipt_root),
+        "SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"),
+        "SIGNOZ_CANARY_TIMEOUT_SECONDS": "5",
+        "SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2",
+        "SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2",
+        "SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds),
+        "SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05",
+        "SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"),
+        "TEST_MONITOR_SCRIPT": str(monitor_script),
+        "TEST_MONITOR_LOG": str(monitor_log),
+        "TEST_COLLECTOR_STATE": str(collector_state),
+        "TEST_LISTENERS": str(listeners),
+        "TEST_BACKUP_IDENTITIES": str(identities),
+        "TEST_COOLDOWN_FILE": str(cooldown_file),
+        "TEST_LEASE_OBSERVED": str(lease_observed),
+        "FAKE_MONITOR_MODE": monitor_mode,
+        "FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds),
+        "FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0",
+        "FAKE_DROP_LISTENER": "1" if drop_listener else "0",
+        "FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0",
+    }
+    started_at = int(time.time())
+    result = subprocess.run(
+        ["bash", str(WRAPPER), f"--{mode}"],
+        env=env,
+        text=True,
+        capture_output=True,
+        timeout=15,
+        check=False,
+    )
+    paths = {
+        "cooldown": cooldown_file,
+        "receipts": receipt_root / run_id / "receipts.jsonl",
+        "rollback_metadata": receipt_root / run_id / "cooldown.rollback.json",
+        "identities": identities,
+        "lease_observed": lease_observed,
+        "collector_state": collector_state,
+        "listeners": listeners,
+    }
+    return result, paths, str(started_at)
+
+
+def _receipts(path: Path) -> list[dict[str, object]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
+
+
+def _terminal(receipts: list[dict[str, object]], stage: str) -> str:
+    return str([item for item in receipts if item["stage"] == stage][-1]["terminal"])
+
+
+def test_existing_lease_is_atomically_replaced_and_exactly_restored(
+    tmp_path: Path,
+) -> None:
+    result, paths, started_at = _run_canary(tmp_path)
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+    assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
+    assert int(paths["lease_observed"].read_text(encoding="utf-8")) >= (
+        int(started_at) + 7
+    )
+    identities = paths["identities"].read_text(encoding="utf-8").split("\t")
+    assert identities[0] == "trace-P0-OBS-002"
+    assert identities[1].startswith("canary-")
+    assert identities[2] == "P0-OBS-002\n"
+    receipts = _receipts(paths["receipts"])
+    assert _terminal(receipts, "rollback") == "restored"
+    assert _terminal(receipts, "terminal") == "pass"
+    metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8"))
+    assert metadata["trace_id"] == "trace-P0-OBS-002"
+    assert metadata["prior_present"] == 1
+
+    stages = [str(item["stage"]) for item in receipts]
+    ordered = [
+        "sensor_source",
+        "normalized_asset_identity",
+        "source_of_truth_diff",
+        "ai_decision",
+        "risk_policy",
+        "check",
+        "execution",
+        "post_verifier",
+        "rollback",
+        "closure_writeback",
+        "terminal",
+    ]
+    assert [stages.index(stage) for stage in ordered] == sorted(
+        stages.index(stage) for stage in ordered
+    )
+
+
+def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None:
+    result, paths, _ = _run_canary(tmp_path, prior_lease=False)
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert not paths["cooldown"].exists()
+    receipts = _receipts(paths["receipts"])
+    assert _terminal(receipts, "rollback") == "restored"
+    assert _terminal(receipts, "terminal") == "pass"
+
+
+def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
+    tmp_path: Path,
+) -> None:
+    result, paths, _ = _run_canary(tmp_path, expect_collector_stop=False)
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+    assert not paths["lease_observed"].exists()
+    assert paths["collector_state"].read_text(encoding="utf-8") == "true\n"
+    receipts = _receipts(paths["receipts"])
+    assert any(
+        item["stage"] == "lease" and item["terminal"] == "not_required"
+        for item in receipts
+    )
+    assert _terminal(receipts, "rollback") == "no_write"
+    assert _terminal(receipts, "terminal") == "pass"
+
+
+def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
+    tmp_path: Path,
+) -> None:
+    result, paths, _ = _run_canary(
+        tmp_path,
+        expect_collector_stop=False,
+        omit_monitor_contract=True,
+        duplicate_cron=True,
+    )
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert not paths["lease_observed"].exists()
+    receipts = _receipts(paths["receipts"])
+    source_diff = [
+        item for item in receipts if item["stage"] == "source_of_truth_diff"
+    ][-1]
+    assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"]
+    assert _terminal(receipts, "terminal") == "pass"
+
+
+def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None:
+    result, paths, _ = _run_canary(tmp_path, monitor_mode="auto_repair")
+
+    assert result.returncode != 0
+    assert "monitor_auto_repair_contaminated_canary" in result.stderr
+    assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+    receipts = _receipts(paths["receipts"])
+    assert _terminal(receipts, "rollback") == "restored"
+    assert _terminal(receipts, "terminal") == "failed"
+
+
+def test_cron_crossing_requires_monitor_cooldown_receipt(tmp_path: Path) -> None:
+    result, paths, _ = _run_canary(
+        tmp_path,
+        monitor_mode="none",
+        backup_sleep_seconds=1.2,
+        monitor_interval_seconds=1,
+    )
+
+    assert result.returncode != 0
+    assert "cron_crossed_without_collector_cooldown_receipt" in result.stderr
+    assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+
+
+def test_cron_crossing_with_detected_and_cooldown_receipts_passes(
+    tmp_path: Path,
+) -> None:
+    result, paths, _ = _run_canary(
+        tmp_path,
+        monitor_mode="cooldown",
+        backup_sleep_seconds=1.2,
+        monitor_interval_seconds=1,
+    )
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    receipts = _receipts(paths["receipts"])
+    monitor_receipt = [
+        item for item in receipts if item["stage"] == "monitor_verifier"
+    ][-1]
+    assert monitor_receipt["terminal"] == "pass"
+    assert "cron_crossed_1" in str(monitor_receipt["detail"])
+
+
+def test_missing_monitor_contract_fails_before_lease_or_backup(tmp_path: Path) -> None:
+    result, paths, _ = _run_canary(tmp_path, omit_monitor_contract=True)
+
+    assert result.returncode != 0
+    assert "monitor_contract_ambiguous_fragment_count_0" in result.stderr
+    assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+    assert not paths["lease_observed"].exists()
+    receipts = _receipts(paths["receipts"])
+    assert _terminal(receipts, "rollback") == "not_armed"
+    assert _terminal(receipts, "terminal") == "failed"
+
+
+def test_ambiguous_cron_contract_fails_before_lease_or_backup(
+    tmp_path: Path,
+) -> None:
+    result, paths, _ = _run_canary(tmp_path, duplicate_cron=True)
+
+    assert result.returncode != 0
+    assert "monitor_cron_contract_ambiguous_count_2" in result.stderr
+    assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+    assert not paths["lease_observed"].exists()
+
+
+def test_post_backup_listener_failure_fails_and_restores_lease(
+    tmp_path: Path,
+) -> None:
+    result, paths, _ = _run_canary(tmp_path, drop_listener=True)
+
+    assert result.returncode != 0
+    assert "collector_post_backup_runtime_failed" in result.stderr
+    assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+    receipts = _receipts(paths["receipts"])
+    assert _terminal(receipts, "rollback") == "restored"
+    assert _terminal(receipts, "terminal") == "failed"
+
+
+def test_wrapper_deployment_and_static_contracts_are_owned() -> None:
+    wrapper = WRAPPER.read_text(encoding="utf-8")
+    assert "BACKUP_SKIP_RETENTION_CLEANUP=1" in wrapper
+    assert 'TRACE_ID="${TRACE_ID:-}"' in wrapper
+    assert 'RUN_ID="${RUN_ID:-}"' in wrapper
+    assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper
+    assert "monitor_auto_repair_contaminated_canary" in wrapper
+    assert "collector_post_backup_runtime_failed" in wrapper
+    assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8")
+    assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text(
+        encoding="utf-8"
+    )
+    assert str(WRAPPER.relative_to(ROOT)) in READINESS_AUDIT.read_text(encoding="utf-8")
+
+
+def test_check_mode_proves_contract_without_lease_or_backup_write(
+    tmp_path: Path,
+) -> None:
+    result, paths, _ = _run_canary(tmp_path, mode="check")
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+    assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
+    assert not paths["identities"].exists()
+    assert not paths["lease_observed"].exists()
+    check_receipts = _receipts(paths["receipts"])
+    assert _terminal(check_receipts, "rollback") == "no_write"
+    assert _terminal(check_receipts, "terminal") == "check_pass_no_write"
+
+
+def test_wrapper_is_valid_bash() -> None:
+    result = subprocess.run(
+        ["bash", "-n", str(WRAPPER)],
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stderr