fix(observability): harden native backup controlled apply

2026-07-15 09:43:40 +08:00
parent 322e812d93
commit 032be5f400
12 changed files with 1849 additions and 190 deletions
--- a/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh
+++ b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh
@@ -11,7 +11,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml"
 LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"

-TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
+TARGET_HOST="wooo@192.168.0.110"
 REMOTE_DIR="/home/wooo/signoz/deploy/docker"
 REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
 REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
@@ -31,6 +31,12 @@ RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
 API_URL="http://127.0.0.1:8080/api/v1/health"
 SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 SAFE_HOME="/home/wooo"
+LOCAL_KILL_AFTER_SECONDS=15
+REMOTE_CHECK_TIMEOUT_SECONDS=180
+REMOTE_STAGE_TIMEOUT_SECONDS=60
+REMOTE_TRANSFER_TIMEOUT_SECONDS=60
+REMOTE_APPLY_TIMEOUT_SECONDS=3600
+REMOTE_CLEANUP_TIMEOUT_SECONDS=60

 SSH_OPTS=(
  -o BatchMode=yes
@@ -79,7 +85,7 @@ for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
  }
 done

-for command_name in python3 scp ssh; do
+for command_name in python3 scp ssh timeout; do
  command -v "${command_name}" >/dev/null 2>&1 || {
    printf 'required local command missing: %s\n' "${command_name}" >&2
    exit 69
@@ -94,6 +100,18 @@ hash_file() {
  fi
 }

+bounded_ssh() {
+  local timeout_seconds="$1"
+  shift
+  timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
+    "${timeout_seconds}" ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" "$@"
+}
+
+bounded_scp() {
+  timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
+    "${REMOTE_TRANSFER_TIMEOUT_SECONDS}" scp -q "${SSH_OPTS[@]}" "$@"
+}
+
 validate_local_sources() {
  [ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ]
  [ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ]
@@ -136,13 +154,13 @@ local_receipt() {
 }

 remote_candidate_compose_check() {
-  ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" \
-    "env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
+  bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" \
+    "timeout --signal=TERM --kill-after=15 120 env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
    < "${LOCAL_OVERRIDE}"
 }

 remote_read_only_check() {
-  ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
+  bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" bash -s -- \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
    "${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK'
 set -euo pipefail
@@ -159,6 +177,24 @@ REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
 HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
 SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 SAFE_HOME="/home/wooo"
+COMMAND_TIMEOUT_SECONDS=30
+COMPOSE_TIMEOUT_SECONDS=120
+KILL_AFTER_SECONDS=15
+
+for command_name in docker timeout; do
+    command -v "${command_name}" >/dev/null 2>&1
+done
+
+docker_cmd() {
+    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" docker "$@"
+}
+
+compose_cmd() {
+    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMPOSE_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+        docker compose --env-file /dev/null -p signoz "$@"
+}

 file_hash_or_absent() {
    if [ -f "$1" ] && [ ! -L "$1" ]; then
@@ -172,14 +208,10 @@ file_hash_or_absent() {

 [ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
 for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
-    docker inspect "${container}" >/dev/null
+    docker_cmd inspect "${container}" >/dev/null
 done
-env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
-    docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
-    config -q >/dev/null
-env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
-    docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
-    config --services | grep -qx clickhouse
+compose_cmd -f "${REMOTE_BASE}" config -q >/dev/null
+compose_cmd -f "${REMOTE_BASE}" config --services | grep -qx clickhouse

 override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")"
 config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")"
@@ -195,13 +227,13 @@ elif [ -e "${HOST_BACKUP_DIR}" ]; then
    host_dir=unsafe
 fi

-disk_count="$(docker exec signoz-clickhouse clickhouse-client --query \
+disk_count="$(docker_cmd exec signoz-clickhouse clickhouse-client --query \
    "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
-mount_count="$(docker inspect --format \
+mount_count="$(docker_cmd inspect --format \
    '{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \
    signoz-clickhouse | tr -cd '1' | wc -c | xargs)"
-collector_running="$(docker inspect --format '{{.State.Running}}' signoz-otel-collector)"
-collector_restarts="$(docker inspect --format '{{.RestartCount}}' signoz-otel-collector)"
+collector_running="$(docker_cmd inspect --format '{{.State.Running}}' signoz-otel-collector)"
+collector_restarts="$(docker_cmd inspect --format '{{.RestartCount}}' signoz-otel-collector)"

 printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \
@@ -223,20 +255,58 @@ fi

 STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}"
 STAGE_CREATED=0
+LOCAL_APPLY_VERIFIED=0

 cleanup_stage() {
-  if [ "${STAGE_CREATED}" -eq 1 ]; then
-    ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
-set -u
+  local cleanup_rc
+  [ "${STAGE_CREATED}" -eq 1 ] || return 0
+  bounded_ssh "${REMOTE_CLEANUP_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1
+set -euo pipefail
 stage_dir="$1"
+if [ ! -e "${stage_dir}" ]; then
+  exit 0
+fi
+[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
 rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml"
-rmdir "${stage_dir}" 2>/dev/null || true
+rmdir "${stage_dir}"
+[ ! -e "${stage_dir}" ]
 REMOTE_CLEANUP
-  fi
+  cleanup_rc=$?
+  [ "${cleanup_rc}" -eq 0 ] || return "${cleanup_rc}"
+  STAGE_CREATED=0
 }
-trap cleanup_stage EXIT

-ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
+on_local_exit() {
+  local rc=$?
+  local cleanup_rc=0 cleanup_terminal=not_created terminal=failed
+  trap - EXIT HUP INT TERM
+  set +e
+  if [ "${STAGE_CREATED}" -eq 1 ]; then
+    cleanup_stage
+    cleanup_rc=$?
+    if [ "${cleanup_rc}" -eq 0 ]; then
+      cleanup_terminal=pass_absence_verified
+    else
+      cleanup_terminal=failed_or_residue_present
+      rc=92
+    fi
+  fi
+  local_receipt cleanup "$([ "${cleanup_rc}" -eq 0 ] && printf pass || printf failed)" \
+    "remote_stage_${cleanup_terminal}"
+  if [ "${rc}" -eq 0 ] && [ "${LOCAL_APPLY_VERIFIED}" -eq 1 ]; then
+    terminal=pass
+  fi
+  local_receipt terminal "${terminal}" \
+    "remote_controlled_apply_${terminal}_stage_${cleanup_terminal}_exit_${rc}"
+  exit "${rc}"
+}
+
+trap on_local_exit EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+bounded_ssh "${REMOTE_STAGE_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
 set -euo pipefail
 stage_dir="$1"
 [ ! -e "${stage_dir}" ]
@@ -245,12 +315,12 @@ mkdir -m 0700 "${stage_dir}"
 REMOTE_STAGE
 STAGE_CREATED=1

-scp -q "${SSH_OPTS[@]}" "${LOCAL_OVERRIDE}" \
+bounded_scp "${LOCAL_OVERRIDE}" \
  "${TARGET_HOST}:${STAGE_DIR}/override.yaml"
-scp -q "${SSH_OPTS[@]}" "${LOCAL_CONFIG}" \
+bounded_scp "${LOCAL_CONFIG}" \
  "${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml"

-ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
+bounded_ssh "${REMOTE_APPLY_TIMEOUT_SECONDS}" bash -s -- \
  "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
  "${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY'
 set -euo pipefail
@@ -282,12 +352,26 @@ API_URL="http://127.0.0.1:8080/api/v1/health"
 SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 SAFE_HOME="/home/wooo"
 LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock"
+BACKUP_OPERATION_LOCK_FILE="/tmp/awoooi-signoz-backup-operation.lock"
+CANARY_LOCK_FILE="/tmp/awoooi-signoz-backup-canary.lock"
+COMMAND_TIMEOUT_SECONDS=30
+COMPOSE_CONFIG_TIMEOUT_SECONDS=120
+COMPOSE_APPLY_TIMEOUT_SECONDS=300
+KILL_AFTER_SECONDS=15

 APPLY_STARTED=0
 DEPLOY_VERIFIED=0
 HOST_DIR_CREATED=0
 PRIOR_OVERRIDE_PRESENT=0
 PRIOR_CONFIG_PRESENT=0
+PRIOR_OVERRIDE_HASH="absent"
+PRIOR_OVERRIDE_MODE="absent"
+PRIOR_OVERRIDE_UID="absent"
+PRIOR_OVERRIDE_GID="absent"
+PRIOR_CONFIG_HASH="absent"
+PRIOR_CONFIG_MODE="absent"
+PRIOR_CONFIG_UID="absent"
+PRIOR_CONFIG_GID="absent"
 PRIOR_DISK_COUNT=0
 BEFORE_IMAGE_ID=""
 BEFORE_DATA_VOLUME=""
@@ -307,38 +391,60 @@ receipt() {
    printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
 }

+docker_cmd() {
+    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" docker "$@"
+}
+
+bounded_cmd() {
+    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" "$@"
+}
+
+compose_config_cmd() {
+    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMPOSE_CONFIG_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
+}
+
+compose_apply_cmd() {
+    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMPOSE_APPLY_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
+}
+
+pgrep_cmd() {
+    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" pgrep "$@"
+}
+
+ss_cmd() {
+    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
+        "${COMMAND_TIMEOUT_SECONDS}" ss "$@"
+}
+
 compose_base_config() {
-    env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
-        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-        -f "${REMOTE_BASE}" config -q >/dev/null 2>&1
+    compose_config_cmd -f "${REMOTE_BASE}" config -q >/dev/null 2>&1
 }

 compose_managed_config() {
-    env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
-        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-        -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" config -q >/dev/null 2>&1
+    compose_config_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
+        config -q >/dev/null 2>&1
 }

 compose_stage_config() {
-    env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
-        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-        -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config -q >/dev/null 2>&1
+    compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" \
+        config -q >/dev/null 2>&1
 }

 recreate_managed() {
-    timeout --signal=TERM --kill-after=30 300 \
-        env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
-        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-        -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
+    compose_apply_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
        up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
        >/dev/null 2>&1
 }

 recreate_base() {
-    timeout --signal=TERM --kill-after=30 300 \
-        env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
-        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-        -f "${REMOTE_BASE}" \
+    compose_apply_cmd -f "${REMOTE_BASE}" \
        up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
        >/dev/null 2>&1
 }
@@ -346,11 +452,11 @@ recreate_base() {
 wait_clickhouse() {
    local attempt running health
    for attempt in $(seq 1 36); do
-        running="$(docker inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
-        health="$(docker inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
+        running="$(docker_cmd inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
+        health="$(docker_cmd inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
            "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
        if [ "${running}" = true ] && [ "${health}" = healthy ] \
-            && timeout --kill-after=5 10 docker exec "${CLICKHOUSE_CONTAINER}" \
+            && docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
                clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then
            return 0
        fi
@@ -361,7 +467,7 @@ wait_clickhouse() {

 listener_up() {
    local port="$1"
-    ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
+    ss_cmd -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
 }

 wait_api_and_listeners() {
@@ -376,25 +482,72 @@ wait_api_and_listeners() {
    return 1
 }

+file_matches_metadata() {
+    local path="$1" expected_hash="$2" expected_mode="$3" expected_uid="$4" expected_gid="$5"
+    [ -f "${path}" ] && [ ! -L "${path}" ] \
+        && [ "$(sha256sum "${path}" | awk '{print $1}')" = "${expected_hash}" ] \
+        && [ "$(stat -c '%a' "${path}")" = "${expected_mode}" ] \
+        && [ "$(stat -c '%u' "${path}")" = "${expected_uid}" ] \
+        && [ "$(stat -c '%g' "${path}")" = "${expected_gid}" ]
+}
+
+restore_exact_file() {
+    local source="$1" target="$2" expected_hash="$3" expected_mode="$4"
+    local expected_uid="$5" expected_gid="$6" tmp="$7"
+    local current_uid current_gid
+
+    bounded_cmd install -m "${expected_mode}" "${source}" "${tmp}" || return 1
+    current_uid="$(stat -c '%u' "${tmp}")" || return 1
+    current_gid="$(stat -c '%g' "${tmp}")" || return 1
+    if [ "${current_uid}" != "${expected_uid}" ] || [ "${current_gid}" != "${expected_gid}" ]; then
+        bounded_cmd sudo -n chown "${expected_uid}:${expected_gid}" "${tmp}" || return 1
+    fi
+    bounded_cmd mv -f "${tmp}" "${target}" || return 1
+    file_matches_metadata "${target}" "${expected_hash}" "${expected_mode}" \
+        "${expected_uid}" "${expected_gid}"
+}
+
 restore_managed_files() {
    local tmp
    if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
        tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}"
-        install -m 0644 "${RECEIPT_DIR}/prior-override.yaml" "${tmp}"
-        mv -f "${tmp}" "${REMOTE_OVERRIDE}"
+        restore_exact_file "${RECEIPT_DIR}/prior-override.yaml" "${REMOTE_OVERRIDE}" \
+            "${PRIOR_OVERRIDE_HASH}" "${PRIOR_OVERRIDE_MODE}" \
+            "${PRIOR_OVERRIDE_UID}" "${PRIOR_OVERRIDE_GID}" "${tmp}" || return 1
    else
-        rm -f "${REMOTE_OVERRIDE}"
+        bounded_cmd rm -f "${REMOTE_OVERRIDE}" || return 1
+        [ ! -e "${REMOTE_OVERRIDE}" ] || return 1
    fi

    if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then
        tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}"
-        install -m 0644 "${RECEIPT_DIR}/prior-backup-disk.xml" "${tmp}"
-        mv -f "${tmp}" "${REMOTE_CONFIG}"
+        restore_exact_file "${RECEIPT_DIR}/prior-backup-disk.xml" "${REMOTE_CONFIG}" \
+            "${PRIOR_CONFIG_HASH}" "${PRIOR_CONFIG_MODE}" \
+            "${PRIOR_CONFIG_UID}" "${PRIOR_CONFIG_GID}" "${tmp}" || return 1
    else
-        rm -f "${REMOTE_CONFIG}"
+        bounded_cmd rm -f "${REMOTE_CONFIG}" || return 1
+        [ ! -e "${REMOTE_CONFIG}" ] || return 1
    fi
 }

+cleanup_owned_residue() {
+    local path
+    for path in \
+        "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" \
+        "${REMOTE_CONFIG}.candidate.${RUN_ID}" \
+        "${REMOTE_OVERRIDE}.rollback.${RUN_ID}" \
+        "${REMOTE_CONFIG}.rollback.${RUN_ID}"; do
+        bounded_cmd rm -f "${path}" || return 1
+        [ ! -e "${path}" ] || return 1
+    done
+    if [ -e "${STAGE_DIR}" ]; then
+        [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
+        bounded_cmd rm -f "${STAGE_OVERRIDE}" "${STAGE_CONFIG}" || return 1
+        bounded_cmd rmdir "${STAGE_DIR}" || return 1
+    fi
+    [ ! -e "${STAGE_DIR}" ] || return 1
+}
+
 rollback_deploy() {
    local disk_count rollback_dir_terminal=retained
    restore_managed_files || return 1
@@ -409,26 +562,27 @@ rollback_deploy() {
    wait_clickhouse || return 1
    wait_api_and_listeners || return 1

-    disk_count="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+    disk_count="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
        "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1
    [ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1
-    [ "$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
-    [ "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
-    [ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
-    [ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1
+    [ "$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
+    [ "$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
+    [ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
+    [ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1

    if [ "${HOST_DIR_CREATED}" -eq 1 ]; then
-        if sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
+        if bounded_cmd sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
            rollback_dir_terminal=removed_empty
        else
            rollback_dir_terminal=retained_nonempty_no_delete
        fi
    fi
-    receipt rollback pass "prior_compose_state_restored_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
+    receipt rollback pass \
+        "prior_compose_state_restored_exact_hash_mode_uid_gid_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
 }

 on_exit() {
-    local rc=$? rollback_rc=0 terminal=failed
+    local rc=$? rollback_rc=0 cleanup_rc=0 terminal=failed
    trap - EXIT HUP INT TERM
    set +e
    if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
@@ -439,7 +593,15 @@ on_exit() {
            rc=91
        fi
    fi
-    if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
+    cleanup_owned_residue
+    cleanup_rc=$?
+    if [ "${cleanup_rc}" -ne 0 ]; then
+        receipt cleanup failed "exact_stage_candidate_or_rollback_residue_cleanup_failed"
+        rc=92
+    else
+        receipt cleanup pass "exact_stage_candidate_and_rollback_residue_absence_verified"
+    fi
+    if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ] && [ "${cleanup_rc}" -eq 0 ]; then
        terminal=pass
    fi
    receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}"
@@ -455,10 +617,22 @@ trap 'exit 129' HUP
 trap 'exit 130' INT
 trap 'exit 143' TERM

+for lock_path in "${LOCK_FILE}" "${BACKUP_OPERATION_LOCK_FILE}" "${CANARY_LOCK_FILE}"; do
+    [ ! -L "${lock_path}" ] \
+        || { receipt check failed operation_lock_symlink_unsafe; exit 75; }
+    if [ -e "${lock_path}" ]; then
+        [ -f "${lock_path}" ] \
+            || { receipt check failed operation_lock_not_regular; exit 75; }
+    fi
+done
 exec 9>>"${LOCK_FILE}"
 flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; }
+exec 7>>"${CANARY_LOCK_FILE}"
+flock -n 7 || { receipt check failed backup_canary_lock_held; exit 75; }
+exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"
+flock -n 8 || { receipt check failed backup_operation_lock_held; exit 75; }

-for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss sudo timeout; do
+for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss stat sudo timeout; do
    command -v "${command_name}" >/dev/null 2>&1 \
        || { receipt check failed "required_command_missing_${command_name}"; exit 69; }
 done
@@ -474,21 +648,39 @@ done

 compose_base_config
 compose_stage_config
-env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
-    docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-    -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
+compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
    | grep -qx "${REMOTE_SERVICE}"

-if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
-    receipt check failed active_signoz_backup_detected
-    exit 75
-fi
-active_native_backups="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
-    "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
-[ "${active_native_backups}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
+require_backup_idle() {
+    local phase="$1" active_native_backups process_rc
+    set +e
+    pgrep_cmd -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' \
+        >/dev/null 2>&1
+    process_rc=$?
+    set -e
+    case "${process_rc}" in
+        0)
+            receipt check failed "active_signoz_backup_or_restore_process_detected_${phase}"
+            return 75
+            ;;
+        1) ;;
+        *)
+            receipt check failed "signoz_backup_process_probe_failed_${phase}_exit_${process_rc}"
+            return 69
+            ;;
+    esac
+    active_native_backups="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+        "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" \
+        || return 1
+    [ "${active_native_backups}" = 0 ] \
+        || { receipt check failed "active_native_backup_or_restore_${phase}"; return 75; }
+    receipt check pass "backup_idle_${phase}_shared_operation_and_canary_locks_held"
+}
+
+require_backup_idle preflight

 for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do
-    [ "$(docker inspect --format '{{.State.Running}}' "${container}")" = true ]
+    [ "$(docker_cmd inspect --format '{{.State.Running}}' "${container}")" = true ]
 done
 wait_clickhouse
 wait_api_and_listeners
@@ -505,42 +697,56 @@ fi
    || { receipt check failed partial_managed_file_drift; exit 78; }
 if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
    compose_managed_config
-    cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
-    cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
+    PRIOR_OVERRIDE_HASH="$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')"
+    PRIOR_OVERRIDE_MODE="$(stat -c '%a' "${REMOTE_OVERRIDE}")"
+    PRIOR_OVERRIDE_UID="$(stat -c '%u' "${REMOTE_OVERRIDE}")"
+    PRIOR_OVERRIDE_GID="$(stat -c '%g' "${REMOTE_OVERRIDE}")"
+    PRIOR_CONFIG_HASH="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
+    PRIOR_CONFIG_MODE="$(stat -c '%a' "${REMOTE_CONFIG}")"
+    PRIOR_CONFIG_UID="$(stat -c '%u' "${REMOTE_CONFIG}")"
+    PRIOR_CONFIG_GID="$(stat -c '%g' "${REMOTE_CONFIG}")"
+    bounded_cmd cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
+    bounded_cmd cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
+    [ "$(sha256sum "${RECEIPT_DIR}/prior-override.yaml" | awk '{print $1}')" = "${PRIOR_OVERRIDE_HASH}" ]
+    [ "$(sha256sum "${RECEIPT_DIR}/prior-backup-disk.xml" | awk '{print $1}')" = "${PRIOR_CONFIG_HASH}" ]
 fi

-PRIOR_DISK_COUNT="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+PRIOR_DISK_COUNT="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
    "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
-BEFORE_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
-BEFORE_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
+BEFORE_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
+BEFORE_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
 [ -n "${BEFORE_DATA_VOLUME}" ]
-BEFORE_COLLECTOR_ID="$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
-BEFORE_COLLECTOR_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
-BEFORE_SIGNOZ_ID="$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
-BEFORE_SIGNOZ_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
-BEFORE_ZOOKEEPER_ID="$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
-BEFORE_ZOOKEEPER_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"
+BEFORE_COLLECTOR_ID="$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
+BEFORE_COLLECTOR_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
+BEFORE_SIGNOZ_ID="$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
+BEFORE_SIGNOZ_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
+BEFORE_ZOOKEEPER_ID="$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
+BEFORE_ZOOKEEPER_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"

 receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid"
 receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}"
-receipt source_of_truth_diff pass "prior_override_${PRIOR_OVERRIDE_PRESENT}_prior_config_${PRIOR_CONFIG_PRESENT}_prior_disk_count_${PRIOR_DISK_COUNT}"
+receipt source_of_truth_diff pass \
+    "prior_override_${PRIOR_OVERRIDE_PRESENT}_hash_${PRIOR_OVERRIDE_HASH}_mode_${PRIOR_OVERRIDE_MODE}_uid_${PRIOR_OVERRIDE_UID}_gid_${PRIOR_OVERRIDE_GID}_prior_config_${PRIOR_CONFIG_PRESENT}_hash_${PRIOR_CONFIG_HASH}_mode_${PRIOR_CONFIG_MODE}_uid_${PRIOR_CONFIG_UID}_gid_${PRIOR_CONFIG_GID}_prior_disk_count_${PRIOR_DISK_COUNT}"
 receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only"
 receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback"
-receipt check pass "compose_config_q_stage_hashes_runtime_preflight_backup_idle"
+receipt check pass \
+    "compose_config_q_stage_hashes_runtime_preflight_dual_locks_backup_idle"
+
+require_backup_idle pre_mutation

 APPLY_STARTED=1
 if [ -e "${HOST_BACKUP_DIR}" ]; then
    [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]
    [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
 else
-    sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
+    bounded_cmd sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
    HOST_DIR_CREATED=1
 fi

-install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
-install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
-mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
-mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
+bounded_cmd install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
+bounded_cmd install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
+bounded_cmd mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
+bounded_cmd mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
 [ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
 [ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
 compose_managed_config
@@ -548,39 +754,39 @@ compose_managed_config
 recreate_managed
 wait_clickhouse

-AFTER_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
-AFTER_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
+AFTER_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
+AFTER_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
 [ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ]
 [ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ]

-backup_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
-config_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
+backup_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
+config_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
 [ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ]
 [ "${config_mount}" = false ]
 [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
-server_uid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
+server_uid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
    awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)"
-server_gid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
+server_gid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
    awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)"
 [ "${server_uid}" = 101 ]
 [ "${server_gid}" = 101 ]
-docker exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
+docker_cmd exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
    'test -d /backups && test -w /backups'

-disk_state="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+disk_state="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
    "SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
 [ "${disk_state}" = "Local|0|0|0|1" ]
-[ "$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+[ "$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
    "EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ]
-docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
+docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
    | grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)'

-[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
-[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
-[ "$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
-[ "$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
-[ "$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
-[ "$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
+[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
+[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
+[ "$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
+[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
+[ "$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
+[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
 wait_api_and_listeners

 receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged"
@@ -589,6 +795,4 @@ receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_execute
 DEPLOY_VERIFIED=1
 REMOTE_APPLY

-trap - EXIT
-cleanup_stage
-local_receipt terminal pass "remote_controlled_apply_and_postverify_complete"
+LOCAL_APPLY_VERIFIED=1
--- a/scripts/ops/deploy-signoz-native-backup-toolchain.sh
+++ b/scripts/ops/deploy-signoz-native-backup-toolchain.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# P0-OBS-002 - atomically deploy the host110 SigNoz native backup toolchain.
+# P0-OBS-002 - bounded controlled deploy of the host110 SigNoz native backup toolchain.
 #
 # This deployer owns seven repo-sourced files only. It does not run a backup or
 # restore, restart a container, pull an image, or inspect a secret-bearing
@@ -12,6 +12,13 @@ TARGET_HOST="wooo@192.168.0.110"
 SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 SAFE_HOME="/home/wooo"
 API_URL="http://127.0.0.1:8080/api/v1/health"
+CANARY_LOCK="/tmp/awoooi-signoz-backup-canary.lock"
+OPERATION_LOCK="/tmp/awoooi-signoz-backup-operation.lock"
+
+SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"
+SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"
+DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"
+TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"

 REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"
 REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}"
@@ -61,10 +68,18 @@ Optional safe absolute path environment:
  SIGNOZ_TOOLCHAIN_RECEIPT_ROOT
  SIGNOZ_TOOLCHAIN_STAGE_ROOT

+Optional bounded-command timeout environment (positive integer seconds):
+  SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS
+  SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS
+  SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS
+  SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS
+
 --check validates local sources and performs a remote read-only target diff and
 runtime readiness check. --apply stages all seven sources, validates hashes and
-syntax, atomically replaces exact targets, and rolls every target back to its
-prior hash/mode/owner if any bounded verifier fails.
+syntax, uses a same-filesystem atomic replacement for each exact target, and
+rolls every target back to its prior hash/mode/owner if any bounded verifier
+fails. SSH, SCP, and Docker calls are bounded, and apply holds both the SigNoz
+canary lock and backup operation lock through post-verification or rollback.

 Flat restore-driver invocation must set:
  CLICKHOUSE_RESTORE_INVENTORY_HELPER=<script-dir>/clickhouse-restore-inventory.py
@@ -89,6 +104,10 @@ valid_identity() {
  [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
 }

+valid_positive_integer() {
+  [[ "$1" =~ ^[1-9][0-9]*$ ]]
+}
+
 safe_absolute_path() {
  local path="$1"
  [[ "${path}" == /* ]] || return 1
@@ -111,6 +130,24 @@ for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
  }
 done

+for value in \
+  "${SSH_COMMAND_TIMEOUT_SECONDS}" \
+  "${SCP_COMMAND_TIMEOUT_SECONDS}" \
+  "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \
+  "${TIMEOUT_KILL_AFTER_SECONDS}"; do
+  valid_positive_integer "${value}" || {
+    printf 'toolchain command timeouts must be positive integer seconds\n' >&2
+    exit 64
+  }
+done
+[ "${SSH_COMMAND_TIMEOUT_SECONDS}" -le 1200 ] \
+  && [ "${SCP_COMMAND_TIMEOUT_SECONDS}" -le 600 ] \
+  && [ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" -le 300 ] \
+  && [ "${TIMEOUT_KILL_AFTER_SECONDS}" -le 60 ] || {
+  printf 'toolchain command timeout exceeds bounded maximum\n' >&2
+  exit 64
+}
+
 for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
  safe_absolute_path "${path}" || {
    printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2
@@ -133,13 +170,25 @@ if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \
  exit 64
 fi

-for command_name in bash python3 scp ssh; do
+for command_name in bash python3 scp ssh timeout; do
  command -v "${command_name}" >/dev/null 2>&1 || {
    printf 'required local command missing: %s\n' "${command_name}" >&2
    exit 69
  }
 done

+
+bounded_ssh() {
+  timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
+    "${SSH_COMMAND_TIMEOUT_SECONDS}" ssh "$@"
+}
+
+
+bounded_scp() {
+  timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
+    "${SCP_COMMAND_TIMEOUT_SECONDS}" scp "$@"
+}
+
 hash_file() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum "$1" | awk '{print $1}'
@@ -241,9 +290,10 @@ local_receipt() {
 }

 remote_read_only_check() {
-  ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
+  bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
    "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \
+    "${DOCKER_COMMAND_TIMEOUT_SECONDS}" "${TIMEOUT_KILL_AFTER_SECONDS}" \
    "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
 set -euo pipefail

@@ -254,7 +304,9 @@ REMOTE_SCRIPT_DIR="$4"
 REMOTE_CONFIG_ROOT="$5"
 RECEIPT_ROOT="$6"
 STAGE_ROOT="$7"
-shift 7
+DOCKER_COMMAND_TIMEOUT_SECONDS="$8"
+TIMEOUT_KILL_AFTER_SECONDS="$9"
+shift 9
 EXPECTED_HASHES=("$@")
 LABELS=(
  backup-signoz.sh
@@ -279,10 +331,18 @@ API_URL="http://127.0.0.1:8080/api/v1/health"

 [ "${#EXPECTED_HASHES[@]}" -eq 7 ]
 [ -d /backup ] && [ ! -L /backup ]
-for command_name in awk curl docker pgrep sha256sum ss stat; do
+for command_name in awk curl docker pgrep sha256sum ss stat timeout; do
  command -v "${command_name}" >/dev/null 2>&1
 done

+[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]
+[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]]
+
+bounded_docker() {
+  timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
+    "${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@"
+}
+
 listener_up() {
  local port="$1"
  ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
@@ -305,7 +365,7 @@ validate_existing_ancestors() {

 runtime_line() {
  local container="$1" value running health
-  value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
+  value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
  IFS='|' read -r _ _ _ running health <<<"${value}"
  [ "${running}" = true ]
  case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
@@ -322,7 +382,7 @@ validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
 validate_existing_ancestors "${RECEIPT_ROOT}"
 validate_existing_ancestors "${STAGE_ROOT}"
 [ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ]
-active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
+active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \
  "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
 [ "${active_operations}" = 0 ]

@@ -373,7 +433,7 @@ fi
 STAGE_CREATED=0
 cleanup_stage() {
  if [ "${STAGE_CREATED}" -eq 1 ]; then
-    ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
+    bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
 set -u
 stage_dir="$1"
 if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
@@ -393,7 +453,7 @@ REMOTE_CLEANUP
 }
 trap cleanup_stage EXIT

-ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
+bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
 set -euo pipefail
 stage_dir="$1"
 [ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
@@ -405,13 +465,15 @@ REMOTE_STAGE
 STAGE_CREATED=1

 for index in "${!LOCAL_SOURCES[@]}"; do
-  scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
+  bounded_scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
    "${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"
 done

-ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
+bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
  "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
  "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \
+  "${CANARY_LOCK}" "${OPERATION_LOCK}" "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \
+  "${TIMEOUT_KILL_AFTER_SECONDS}" \
  "${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
 set -euo pipefail

@@ -422,7 +484,11 @@ REMOTE_SCRIPT_DIR="$4"
 REMOTE_CONFIG_ROOT="$5"
 RECEIPT_ROOT="$6"
 STAGE_DIR="$7"
-shift 7
+CANARY_LOCK="$8"
+OPERATION_LOCK="$9"
+DOCKER_COMMAND_TIMEOUT_SECONDS="${10}"
+TIMEOUT_KILL_AFTER_SECONDS="${11}"
+shift 11
 EXPECTED_HASHES=("$@")
 LABELS=(
  backup-signoz.sh
@@ -472,6 +538,8 @@ APPLY_STARTED=0
 DEPLOY_VERIFIED=0
 RUNTIME_BASELINE_CAPTURED=0
 LOCK_HELD=0
+CANARY_LOCK_HELD=0
+OPERATION_LOCK_HELD=0
 REMOTE_UID="$(id -u)"
 REMOTE_GID="$(id -g)"
 CREATED_DIRS=()
@@ -481,7 +549,7 @@ PRIOR_UIDS=()
 PRIOR_GIDS=()
 PRIOR_HASHES=()

-for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat; do
+for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat timeout; do
  command -v "${command_name}" >/dev/null 2>&1 || {
    printf 'required remote command missing: %s\n' "${command_name}" >&2
    exit 69
@@ -493,6 +561,10 @@ if [ "${REMOTE_UID}" -ne 0 ]; then
 fi
 [ "${#EXPECTED_HASHES[@]}" -eq 7 ]
 [ -d /backup ] && [ ! -L /backup ]
+[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]
+[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]]
+[ "${CANARY_LOCK}" = "/tmp/awoooi-signoz-backup-canary.lock" ]
+[ "${OPERATION_LOCK}" = "/tmp/awoooi-signoz-backup-operation.lock" ]

 run_root() {
  if [ "${REMOTE_UID}" -eq 0 ]; then
@@ -502,6 +574,11 @@ run_root() {
  fi
 }

+bounded_docker() {
+  timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
+    "${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@"
+}
+
 root_hash() {
  if [ "${REMOTE_UID}" -eq 0 ]; then
    sha256sum "$1" | awk '{print $1}'
@@ -550,7 +627,7 @@ listener_up() {
 runtime_snapshot() {
  local container value running health port api_code
  for container in "${CONTAINERS[@]}"; do
-    value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
+    value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
    IFS='|' read -r _ _ _ running health <<<"${value}"
    [ "${running}" = true ]
    case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
@@ -673,6 +750,14 @@ on_exit() {
  if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then
    rc=92
  fi
+  if [ "${OPERATION_LOCK_HELD}" -eq 1 ]; then
+    flock -u 7 >/dev/null 2>&1
+    exec 7>&-
+  fi
+  if [ "${CANARY_LOCK_HELD}" -eq 1 ]; then
+    flock -u 8 >/dev/null 2>&1
+    exec 8>&-
+  fi
  if [ "${LOCK_HELD}" -eq 1 ]; then
    flock -u 9 >/dev/null 2>&1
    exec 9>&-
@@ -694,11 +779,33 @@ exec 9>>"${LOCK_FILE}"
 flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; }
 LOCK_HELD=1

+[ ! -L "${CANARY_LOCK}" ] \
+  || { receipt check failed canary_lock_symlink_unsafe; exit 75; }
+if [ -e "${CANARY_LOCK}" ]; then
+  [ -f "${CANARY_LOCK}" ] \
+    || { receipt check failed canary_lock_not_regular_file; exit 75; }
+fi
+exec 8>>"${CANARY_LOCK}"
+flock -n 8 \
+  || { receipt check failed another_signoz_canary_holds_canary_lock; exit 75; }
+CANARY_LOCK_HELD=1
+
+[ ! -L "${OPERATION_LOCK}" ] \
+  || { receipt check failed backup_operation_lock_symlink_unsafe; exit 75; }
+if [ -e "${OPERATION_LOCK}" ]; then
+  [ -f "${OPERATION_LOCK}" ] \
+    || { receipt check failed backup_operation_lock_not_regular_file; exit 75; }
+fi
+exec 7>>"${OPERATION_LOCK}"
+flock -n 7 \
+  || { receipt check failed another_signoz_backup_holds_operation_lock; exit 75; }
+OPERATION_LOCK_HELD=1
+
 if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
  receipt check failed active_signoz_backup_toolchain_process
  exit 75
 fi
-active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
+active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \
  "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
 [ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }

@@ -744,9 +851,9 @@ done
 receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources"
 receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
 receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}"
-receipt ai_decision candidate "atomic_install_native_backup_toolchain_only"
-receipt risk_policy pass "risk_medium_bounded_files_only_no_runtime_execution_no_container_change_full_rollback"
-receipt check pass "runtime_baseline_active_operations_0_hash_bash_xml_py_compile"
+receipt ai_decision candidate "same_filesystem_candidate_replace_native_backup_toolchain_only"
+receipt risk_policy pass "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_backup_operation_locks_no_runtime_execution_no_container_change_full_rollback"
+receipt check pass "dual_signoz_canary_and_backup_operation_locks_held_runtime_baseline_active_operations_0_hash_bash_xml_py_compile"

 APPLY_STARTED=1
 for directory in "${TARGET_DIRS[@]}"; do
@@ -790,7 +897,7 @@ done

 runtime_snapshot > "${RUNTIME_AFTER}"
 cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"
-receipt execution pass "seven_candidates_atomically_replaced_expected_hashes_and_modes"
+receipt execution pass "seven_same_filesystem_candidates_replaced_with_full_rollback_expected_hashes_and_modes"
 receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged"
 receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull"
 DEPLOY_VERIFIED=1
--- a/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
+++ b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
@@ -2,6 +2,7 @@ from __future__ import annotations

 import os
 from pathlib import Path
+import re
 import subprocess
 import xml.etree.ElementTree as ET

@@ -10,6 +11,12 @@ ROOT = Path(__file__).resolve().parents[3]
 CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
 OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
 DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
+BACKUP_SCRIPT = ROOT / "scripts/backup/backup-signoz.sh"
+
+
+def write_executable(path: Path, text: str) -> None:
+    path.write_text(text, encoding="utf-8")
+    path.chmod(0o755)


 def executable_text() -> str:
@@ -105,9 +112,9 @@ def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> No
    text = DEPLOYER.read_text(encoding="utf-8")
    check_terminal = text.index('if [ "${MODE}" = "--check" ]')
    stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
-    first_scp = text.index("scp -q")
+    first_transfer = text.index('bounded_scp "${LOCAL_OVERRIDE}"')

-    assert check_terminal < stage_assignment < first_scp
+    assert check_terminal < stage_assignment < first_transfer
    assert "-f '${REMOTE_BASE}' -f - config -q" in text
    assert '< "${LOCAL_OVERRIDE}"' in text
    assert "check_pass_no_write" in text
@@ -120,7 +127,12 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
    assert 'REMOTE_SERVICE="clickhouse"' in text
    assert "--env-file /dev/null" in text
    assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
-    assert "timeout --signal=TERM --kill-after=30 300" in text
+    assert "bounded_ssh" in text
+    assert "bounded_scp" in text
+    assert "docker_cmd" in text
+    assert "compose_config_cmd" in text
+    assert "compose_apply_cmd" in text
+    assert "REMOTE_APPLY_TIMEOUT_SECONDS=3600" in text
    assert "up -d --no-deps --pull never --force-recreate" in text
    assert '"${REMOTE_SERVICE}"' in text
    assert "docker compose down" not in executable
@@ -133,15 +145,135 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:

 def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
    text = DEPLOYER.read_text(encoding="utf-8")
-    assert "active_signoz_backup_detected" in text
+    assert "active_signoz_backup_or_restore_process_detected" in text
    assert "active_native_backup_or_restore" in text
+    assert "signoz_backup_process_probe_failed" in text
    assert "partial_managed_file_drift" in text
    assert "another_deploy_holds_lock" in text
-    assert "backup-signoz[.]sh" in text
+    for process in (
+        "backup-signoz",
+        "clickhouse-native-backup",
+        "clickhouse-native-restore-drill",
+        "run-signoz-backup-canary",
+    ):
+        assert process in text
+    probe_block = text[
+        text.index("require_backup_idle()") : text.index(
+            "require_backup_idle preflight"
+        )
+    ]
+    assert "process_rc=$?" in probe_block
+    assert 'case "${process_rc}" in' in probe_block
+    assert "1) ;;" in probe_block
+    assert "return 69" in probe_block
    assert "CREATING_BACKUP" in text
    assert "RESTORING" in text


+def test_target_host_is_fixed_and_ignores_ambient_override(tmp_path: Path) -> None:
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    ssh_log = tmp_path / "ssh.log"
+    write_executable(
+        fake_bin / "ssh",
+        """#!/bin/bash
+printf '%s\\n' "$*" >> "${FAKE_SSH_LOG:?}"
+cat >/dev/null
+""",
+    )
+    write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
+    env = os.environ.copy()
+    env.update(
+        {
+            "PATH": f"{fake_bin}:{env['PATH']}",
+            "TRACE_ID": "P0-OBS-002-test",
+            "RUN_ID": "fixed-host-test",
+            "WORK_ITEM_ID": "P0-OBS-002",
+            "TARGET_HOST": "attacker@example.invalid",
+            "FAKE_SSH_LOG": str(ssh_log),
+        }
+    )
+
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+
+    assert result.returncode == 0, result.stderr
+    invocations = ssh_log.read_text(encoding="utf-8").splitlines()
+    assert len(invocations) == 2
+    assert all("wooo@192.168.0.110" in line for line in invocations)
+    assert all("attacker@example.invalid" not in line for line in invocations)
+
+
+def test_all_transport_and_remote_docker_operations_are_bounded() -> None:
+    text = DEPLOYER.read_text(encoding="utf-8")
+    executable = executable_text()
+
+    assert executable.count(' ssh "${SSH_OPTS[@]}"') == 1
+    assert executable.count(' scp -q "${SSH_OPTS[@]}"') == 1
+    assert not re.search(r"\bdocker (inspect|exec)\b", executable)
+    assert 'timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}"' in text
+    assert 'docker_cmd exec "${CLICKHOUSE_CONTAINER}"' in text
+    assert "ss_cmd -H -lnt" in text
+    for mutation in (
+        "bounded_cmd cp -p",
+        "bounded_cmd install -m",
+        "bounded_cmd mv -f",
+        "bounded_cmd sudo -n install",
+        "bounded_cmd rm -f",
+    ):
+        assert mutation in text
+
+
+def test_stage_cleanup_failure_changes_apply_terminal_to_failed(tmp_path: Path) -> None:
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    ssh_count = tmp_path / "ssh-count"
+    write_executable(
+        fake_bin / "ssh",
+        """#!/bin/bash
+count=0
+if [ -f "${FAKE_SSH_COUNT:?}" ]; then
+  count="$(cat "${FAKE_SSH_COUNT}")"
+fi
+count=$((count + 1))
+printf '%s\\n' "${count}" > "${FAKE_SSH_COUNT}"
+cat >/dev/null
+if [ "${count}" -eq 5 ]; then
+  exit 1
+fi
+""",
+    )
+    write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
+    env = os.environ.copy()
+    env.update(
+        {
+            "PATH": f"{fake_bin}:{env['PATH']}",
+            "TRACE_ID": "P0-OBS-002-test",
+            "RUN_ID": "cleanup-failure-test",
+            "WORK_ITEM_ID": "P0-OBS-002",
+            "FAKE_SSH_COUNT": str(ssh_count),
+        }
+    )
+
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--apply"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+    )
+
+    assert result.returncode == 92
+    assert '"phase":"cleanup","result":"failed"' in result.stdout
+    assert '"phase":"terminal","result":"failed"' in result.stdout
+    assert "failed_or_residue_present" in result.stdout
+
+
 def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
    text = DEPLOYER.read_text(encoding="utf-8")
    assert "BEFORE_IMAGE_ID" in text
@@ -166,13 +298,13 @@ def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
    assert "101:101:750" in text
    assert "server_uid" in text
    assert "server_gid" in text
-    assert "docker exec -u 101:101" in text
+    assert "docker_cmd exec -u 101:101" in text
    assert "test -w /backups" in text
    assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
    assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
    identity_start = text.index('server_uid="')
    identity_block = text[
-        identity_start : text.index("docker exec -u 101:101", identity_start)
+        identity_start : text.index("docker_cmd exec -u 101:101", identity_start)
    ]
    assert "sh -c" not in identity_block
    assert "concurrency_disabled" in text
@@ -194,6 +326,48 @@ def test_failed_apply_restores_prior_managed_compose_state() -> None:
    assert "retained_nonempty_no_delete" in text
    assert "rc=91" in text
    assert "trap on_exit EXIT" in text
+    for token in (
+        "PRIOR_OVERRIDE_HASH",
+        "PRIOR_OVERRIDE_MODE",
+        "PRIOR_OVERRIDE_UID",
+        "PRIOR_OVERRIDE_GID",
+        "PRIOR_CONFIG_HASH",
+        "PRIOR_CONFIG_MODE",
+        "PRIOR_CONFIG_UID",
+        "PRIOR_CONFIG_GID",
+        "file_matches_metadata",
+        "restore_exact_file",
+        "prior_compose_state_restored_exact_hash_mode_uid_gid",
+    ):
+        assert token in text
+    assert "exact_stage_candidate_and_rollback_residue_absence_verified" in text
+
+
+def test_deployer_and_backup_execution_share_operation_lock_contract() -> None:
+    deployer = DEPLOYER.read_text(encoding="utf-8")
+    backup = BACKUP_SCRIPT.read_text(encoding="utf-8")
+    operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
+    canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
+
+    assert operation_lock in deployer
+    assert operation_lock in backup
+    assert canary_lock in deployer
+    assert 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' in deployer
+    assert 'exec 7>>"${CANARY_LOCK_FILE}"' in deployer
+    assert "flock -n 8 || { receipt check failed backup_operation_lock_held" in deployer
+    assert "flock -n 7 || { receipt check failed backup_canary_lock_held" in deployer
+    assert 'exec 8>>"${OPERATION_LOCK_FILE}"' in backup
+    assert "require_backup_idle pre_mutation" in deployer
+    assert deployer.index('exec 7>>"${CANARY_LOCK_FILE}"') < deployer.index(
+        'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"'
+    )
+    assert deployer.index('exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"') < deployer.index(
+        "require_backup_idle pre_mutation"
+    )
+    remote_cleanup_call = deployer.rindex("    cleanup_owned_residue")
+    assert remote_cleanup_call < deployer.index(
+        'receipt terminal "${terminal}"', remote_cleanup_call
+    )


 def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:
--- a/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
+++ b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
@@ -3,10 +3,13 @@ from __future__ import annotations
 import os
 from pathlib import Path
 import subprocess
+import time


 ROOT = Path(__file__).resolve().parents[3]
 DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
+BACKUP = ROOT / "scripts/backup/backup-signoz.sh"
+CANARY = ROOT / "scripts/backup/run-signoz-backup-canary.sh"


 def deployer_text() -> str:
@@ -119,6 +122,33 @@ def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> Non
    assert "must not overlap" in result.stderr


+def test_timeout_overrides_fail_closed_before_network_access() -> None:
+    for key, value, expected in (
+        (
+            "SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS",
+            "0",
+            "positive integer seconds",
+        ),
+        (
+            "SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS",
+            "301",
+            "exceeds bounded maximum",
+        ),
+    ):
+        env = base_env()
+        env[key] = value
+        result = subprocess.run(
+            ["bash", str(DEPLOYER), "--check"],
+            check=False,
+            capture_output=True,
+            text=True,
+            env=env,
+        )
+        assert result.returncode == 64
+        assert expected in result.stderr
+        assert "ssh:" not in result.stderr
+
+
 def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
    text = deployer_text()
    assert 'TARGET_HOST="wooo@192.168.0.110"' in text
@@ -229,6 +259,70 @@ def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
    assert "check_pass_no_write" in result.stdout


+def test_hung_ssh_is_bounded_and_cannot_claim_check_pass(tmp_path: Path) -> None:
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    ssh = fake_bin / "ssh"
+    ssh.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8")
+    scp = fake_bin / "scp"
+    scp.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8")
+    ssh.chmod(0o755)
+    scp.chmod(0o755)
+
+    env = base_env()
+    env["PATH"] = f"{fake_bin}:{env['PATH']}"
+    env["SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS"] = "1"
+    env["SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS"] = "1"
+    started = time.monotonic()
+    result = subprocess.run(
+        ["bash", str(DEPLOYER), "--check"],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=env,
+        timeout=5,
+    )
+    elapsed = time.monotonic() - started
+
+    assert result.returncode == 124
+    assert elapsed < 4
+    assert "check_pass_no_write" not in result.stdout
+
+
+def test_all_ssh_scp_and_docker_calls_use_bounded_wrappers() -> None:
+    text = deployer_text()
+    executable = executable_text()
+
+    assert (
+        'SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"'
+        in text
+    )
+    assert (
+        'SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"'
+        in text
+    )
+    assert (
+        'DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"'
+        in text
+    )
+    assert (
+        'TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"'
+        in text
+    )
+    assert "bounded_ssh()" in text
+    assert "bounded_scp()" in text
+    assert text.count('ssh "$@"') == 1
+    assert text.count('scp "$@"') == 1
+    assert executable.count("bounded_docker()") == 2
+    assert "$(docker inspect" not in executable
+    assert "$(docker exec" not in executable
+    assert "\n  docker inspect" not in executable
+    assert "\n  docker exec" not in executable
+    assert executable.count("bounded_docker inspect") == 2
+    assert executable.count("bounded_docker exec") == 2
+    assert "timeout --signal=TERM" in executable
+
+
 def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
    text = deployer_text()
    for token in (
@@ -250,18 +344,60 @@ def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
    assert "bounded_execution_not_started_targets_unchanged" in text


-def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
+def test_apply_uses_same_filesystem_candidates_and_accurate_replace_receipt() -> None:
    text = deployer_text()
    assert '"${target}.candidate.${RUN_ID}"' in text
    assert (
        'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
    ) in text
    assert "deployed-manifest.tsv" in text
-    assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
+    assert (
+        "seven_same_filesystem_candidates_replaced_with_full_rollback_"
+        "expected_hashes_and_modes"
+    ) in text
+    assert "seven_candidates_atomically_replaced" not in text
+    assert "atomically replaces exact targets" not in text
    assert "stage_candidate_residue_0" in text
    assert "verify_residue_zero" in text


+def test_apply_holds_dual_backup_locks_through_postverify_or_rollback() -> None:
+    text = deployer_text()
+    backup_text = BACKUP.read_text(encoding="utf-8")
+    canary_text = CANARY.read_text(encoding="utf-8")
+    canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
+    operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
+
+    assert f'CANARY_LOCK="{canary_lock}"' in text
+    assert f'OPERATION_LOCK="{operation_lock}"' in text
+    assert canary_lock in canary_text
+    assert operation_lock in backup_text
+    canary_acquire = text.index('exec 8>>"${CANARY_LOCK}"')
+    operation_acquire = text.index('exec 7>>"${OPERATION_LOCK}"')
+    active_check = text.index("if pgrep -af", operation_acquire)
+    apply_started = text.index("APPLY_STARTED=1", operation_acquire)
+    target_replace = text.index(
+        'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}"', apply_started
+    )
+    assert canary_acquire < operation_acquire < active_check < apply_started
+    assert apply_started < target_replace
+
+    on_exit = text[text.index("on_exit() {") : text.index("trap on_exit EXIT")]
+    rollback = on_exit.index("rollback_targets")
+    release_operation = on_exit.index("flock -u 7")
+    release_canary = on_exit.index("flock -u 8")
+    assert rollback < release_operation < release_canary
+    assert 'rm -f "${OPERATION_LOCK}"' not in text
+    assert 'rm -f "${CANARY_LOCK}"' not in text
+    assert "another_signoz_backup_holds_operation_lock" in text
+    assert "another_signoz_canary_holds_canary_lock" in text
+    assert (
+        "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_"
+        "backup_operation_locks_"
+        "no_runtime_execution_no_container_change_full_rollback"
+    ) in text
+
+
 def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
    text = deployer_text()
    for container in (