#!/usr/bin/env bash
# P0-OBS-002 - controlled deployment of the host110 ClickHouse native BACKUP disk.
#
# This script owns only an additive Compose override and one ClickHouse config
# fragment. It never runs BACKUP/RESTORE, reads a dotenv/secret source, pulls an
# image, or recreates another service.

set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml"
LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"

TARGET_HOST="wooo@192.168.0.110"
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
REMOTE_PROJECT="signoz"
REMOTE_SERVICE="clickhouse"
CLICKHOUSE_CONTAINER="signoz-clickhouse"
COLLECTOR_CONTAINER="signoz-otel-collector"
SIGNOZ_CONTAINER="signoz"
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
CONTAINER_BACKUP_DIR="/backups"
CLICKHOUSE_DISK="backups"
CLICKHOUSE_UID="101"
CLICKHOUSE_GID="101"
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
API_URL="http://127.0.0.1:8080/api/v1/health"
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SAFE_HOME="/home/wooo"
LOCAL_KILL_AFTER_SECONDS=15
REMOTE_CHECK_TIMEOUT_SECONDS=180
REMOTE_STAGE_TIMEOUT_SECONDS=60
REMOTE_TRANSFER_TIMEOUT_SECONDS=60
REMOTE_APPLY_TIMEOUT_SECONDS=3600
REMOTE_CLEANUP_TIMEOUT_SECONDS=60

SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout=8
  -o ConnectionAttempts=1
  -o ServerAliveInterval=5
  -o ServerAliveCountMax=2
)

usage() {
  cat <<'USAGE'
Usage: deploy-signoz-clickhouse-backup-disk.sh [--check|--apply]

Required environment:
  TRACE_ID       Path-safe controlled-apply trace identifier
  RUN_ID         Unique path-safe execution identifier
  WORK_ITEM_ID   Path-safe work-item identifier

--check performs local source validation and remote read-only Compose/runtime
checks. --apply atomically installs the two repo-owned artifacts, recreates only
the ClickHouse service with --pull never, independently verifies runtime state,
and restores the exact prior managed Compose state on failure.
USAGE
}

MODE="${1:---check}"
case "${MODE}" in
  --check|--apply) ;;
  -h|--help) usage; exit 0 ;;
  *) usage >&2; exit 64 ;;
esac
[ "$#" -le 1 ] || { usage >&2; exit 64; }

TRACE_ID="${TRACE_ID:-}"
RUN_ID="${RUN_ID:-}"
WORK_ITEM_ID="${WORK_ITEM_ID:-}"

valid_identity() {
  [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
}

for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
  valid_identity "${value}" || {
    printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
    exit 64
  }
done

for command_name in python3 scp ssh timeout; do
  command -v "${command_name}" >/dev/null 2>&1 || {
    printf 'required local command missing: %s\n' "${command_name}" >&2
    exit 69
  }
done

hash_file() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum "$1" | awk '{print $1}'
  else
    shasum -a 256 "$1" | awk '{print $1}'
  fi
}

bounded_ssh() {
  local timeout_seconds="$1"
  shift
  timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
    "${timeout_seconds}" ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" "$@"
}

bounded_scp() {
  timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
    "${REMOTE_TRANSFER_TIMEOUT_SECONDS}" scp -q "${SSH_OPTS[@]}" "$@"
}

validate_local_sources() {
  [ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ]
  [ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ]

  python3 - "${LOCAL_CONFIG}" <<'PY'
from pathlib import Path
import sys
import xml.etree.ElementTree as ET

path = Path(sys.argv[1])
root = ET.parse(path).getroot()
assert root.tag == "clickhouse"
disk = root.find("./storage_configuration/disks/backups")
assert disk is not None
assert disk.findtext("type") == "local"
assert disk.findtext("path") == "/backups/"
backups = root.find("./backups")
assert backups is not None
assert backups.findtext("allowed_disk") == "backups"
assert backups.findtext("allowed_path") == "/backups/"
assert backups.findtext("allow_concurrent_backups") == "false"
assert backups.findtext("allow_concurrent_restores") == "false"
PY

  [ "$(grep -F -c '  clickhouse:' "${LOCAL_OVERRIDE}")" -eq 1 ]
  [ "$(grep -F -c 'source: /backup/staging/signoz-clickhouse' "${LOCAL_OVERRIDE}")" -eq 1 ]
  [ "$(grep -F -c 'target: /backups' "${LOCAL_OVERRIDE}")" -eq 1 ]
  [ "$(grep -F -c 'target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml' "${LOCAL_OVERRIDE}")" -eq 1 ]
  ! grep -Eq '^[[:space:]]*(image|build|command|environment):' "${LOCAL_OVERRIDE}"
}

validate_local_sources
OVERRIDE_SHA="$(hash_file "${LOCAL_OVERRIDE}")"
CONFIG_SHA="$(hash_file "${LOCAL_CONFIG}")"

local_receipt() {
  local phase="$1" result="$2" detail="$3"
  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}\n' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
}

remote_candidate_compose_check() {
  bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" \
    "timeout --signal=TERM --kill-after=15 120 env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
    < "${LOCAL_OVERRIDE}"
}

remote_read_only_check() {
  bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" bash -s -- \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
    "${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK'
set -euo pipefail

TRACE_ID="$1"
RUN_ID="$2"
WORK_ITEM_ID="$3"
EXPECTED_OVERRIDE_SHA="$4"
EXPECTED_CONFIG_SHA="$5"
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SAFE_HOME="/home/wooo"
COMMAND_TIMEOUT_SECONDS=30
COMPOSE_TIMEOUT_SECONDS=120
KILL_AFTER_SECONDS=15

for command_name in docker timeout; do
    command -v "${command_name}" >/dev/null 2>&1
done

docker_cmd() {
    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
        "${COMMAND_TIMEOUT_SECONDS}" docker "$@"
}

compose_cmd() {
    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
        "${COMPOSE_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
        docker compose --env-file /dev/null -p signoz "$@"
}

file_hash_or_absent() {
    if [ -f "$1" ] && [ ! -L "$1" ]; then
        sha256sum "$1" | awk '{print $1}'
    elif [ ! -e "$1" ]; then
        printf 'absent\n'
    else
        printf 'unsafe\n'
    fi
}

[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
    docker_cmd inspect "${container}" >/dev/null
done
compose_cmd -f "${REMOTE_BASE}" config -q >/dev/null
compose_cmd -f "${REMOTE_BASE}" config --services | grep -qx clickhouse

override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")"
config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")"
override_match=false
config_match=false
[ "${override_hash}" = "${EXPECTED_OVERRIDE_SHA}" ] && override_match=true
[ "${config_hash}" = "${EXPECTED_CONFIG_SHA}" ] && config_match=true

host_dir=absent
if [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]; then
    host_dir=present
elif [ -e "${HOST_BACKUP_DIR}" ]; then
    host_dir=unsafe
fi

disk_count="$(docker_cmd exec signoz-clickhouse clickhouse-client --query \
    "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
mount_count="$(docker_cmd inspect --format \
    '{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \
    signoz-clickhouse | tr -cd '1' | wc -c | xargs)"
collector_running="$(docker_cmd inspect --format '{{.State.Running}}' signoz-otel-collector)"
collector_restarts="$(docker_cmd inspect --format '{{.RestartCount}}' signoz-otel-collector)"

printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \
    "${config_match}" "${host_dir}" "${disk_count}" "${mount_count}" \
    "${collector_running}" "${collector_restarts}"
REMOTE_CHECK
}

local_receipt sensor_source pass "override_sha_${OVERRIDE_SHA}_config_sha_${CONFIG_SHA}"
local_receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_disk_backups"
remote_candidate_compose_check
remote_read_only_check
local_receipt check pass "compose_config_q_no_output_no_write"

if [ "${MODE}" = "--check" ]; then
  local_receipt terminal check_pass_no_write "source_and_remote_preflight_valid"
  exit 0
fi

STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}"
STAGE_CREATED=0
LOCAL_APPLY_VERIFIED=0

cleanup_stage() {
  local cleanup_rc
  [ "${STAGE_CREATED}" -eq 1 ] || return 0
  bounded_ssh "${REMOTE_CLEANUP_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1
set -euo pipefail
stage_dir="$1"
if [ ! -e "${stage_dir}" ]; then
  exit 0
fi
[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml"
rmdir "${stage_dir}"
[ ! -e "${stage_dir}" ]
REMOTE_CLEANUP
  cleanup_rc=$?
  [ "${cleanup_rc}" -eq 0 ] || return "${cleanup_rc}"
  STAGE_CREATED=0
}

on_local_exit() {
  local rc=$?
  local cleanup_rc=0 cleanup_terminal=not_created terminal=failed
  trap - EXIT HUP INT TERM
  set +e
  if [ "${STAGE_CREATED}" -eq 1 ]; then
    cleanup_stage
    cleanup_rc=$?
    if [ "${cleanup_rc}" -eq 0 ]; then
      cleanup_terminal=pass_absence_verified
    else
      cleanup_terminal=failed_or_residue_present
      rc=92
    fi
  fi
  local_receipt cleanup "$([ "${cleanup_rc}" -eq 0 ] && printf pass || printf failed)" \
    "remote_stage_${cleanup_terminal}"
  if [ "${rc}" -eq 0 ] && [ "${LOCAL_APPLY_VERIFIED}" -eq 1 ]; then
    terminal=pass
  fi
  local_receipt terminal "${terminal}" \
    "remote_controlled_apply_${terminal}_stage_${cleanup_terminal}_exit_${rc}"
  exit "${rc}"
}

trap on_local_exit EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM

bounded_ssh "${REMOTE_STAGE_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
set -euo pipefail
stage_dir="$1"
[ ! -e "${stage_dir}" ]
umask 077
mkdir -m 0700 "${stage_dir}"
REMOTE_STAGE
STAGE_CREATED=1

bounded_scp "${LOCAL_OVERRIDE}" \
  "${TARGET_HOST}:${STAGE_DIR}/override.yaml"
bounded_scp "${LOCAL_CONFIG}" \
  "${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml"

bounded_ssh "${REMOTE_APPLY_TIMEOUT_SECONDS}" bash -s -- \
  "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
  "${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY'
set -euo pipefail

TRACE_ID="$1"
RUN_ID="$2"
WORK_ITEM_ID="$3"
EXPECTED_OVERRIDE_SHA="$4"
EXPECTED_CONFIG_SHA="$5"
STAGE_DIR="$6"

REMOTE_DIR="/home/wooo/signoz/deploy/docker"
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
STAGE_OVERRIDE="${STAGE_DIR}/override.yaml"
STAGE_CONFIG="${STAGE_DIR}/backup_disk.xml"
REMOTE_PROJECT="signoz"
REMOTE_SERVICE="clickhouse"
CLICKHOUSE_CONTAINER="signoz-clickhouse"
COLLECTOR_CONTAINER="signoz-otel-collector"
SIGNOZ_CONTAINER="signoz"
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
API_URL="http://127.0.0.1:8080/api/v1/health"
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SAFE_HOME="/home/wooo"
LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock"
BACKUP_OPERATION_LOCK_FILE="/tmp/awoooi-signoz-backup-operation.lock"
CANARY_LOCK_FILE="/tmp/awoooi-signoz-backup-canary.lock"
COMMAND_TIMEOUT_SECONDS=30
COMPOSE_CONFIG_TIMEOUT_SECONDS=120
COMPOSE_APPLY_TIMEOUT_SECONDS=300
KILL_AFTER_SECONDS=15

APPLY_STARTED=0
DEPLOY_VERIFIED=0
HOST_DIR_CREATED=0
PRIOR_OVERRIDE_PRESENT=0
PRIOR_CONFIG_PRESENT=0
PRIOR_OVERRIDE_HASH="absent"
PRIOR_OVERRIDE_MODE="absent"
PRIOR_OVERRIDE_UID="absent"
PRIOR_OVERRIDE_GID="absent"
PRIOR_CONFIG_HASH="absent"
PRIOR_CONFIG_MODE="absent"
PRIOR_CONFIG_UID="absent"
PRIOR_CONFIG_GID="absent"
PRIOR_DISK_COUNT=0
BEFORE_IMAGE_ID=""
BEFORE_DATA_VOLUME=""
BEFORE_COLLECTOR_ID=""
BEFORE_COLLECTOR_RESTARTS=""
BEFORE_SIGNOZ_ID=""
BEFORE_SIGNOZ_RESTARTS=""
BEFORE_ZOOKEEPER_ID=""
BEFORE_ZOOKEEPER_RESTARTS=""

receipt() {
    local phase="$1" result="$2" detail="$3"
    local line
    line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}' \
        "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
    printf '%s\n' "${line}"
    printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
}

docker_cmd() {
    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
        "${COMMAND_TIMEOUT_SECONDS}" docker "$@"
}

bounded_cmd() {
    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
        "${COMMAND_TIMEOUT_SECONDS}" "$@"
}

compose_config_cmd() {
    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
        "${COMPOSE_CONFIG_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
}

compose_apply_cmd() {
    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
        "${COMPOSE_APPLY_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
        docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
}

pgrep_cmd() {
    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
        "${COMMAND_TIMEOUT_SECONDS}" pgrep "$@"
}

ss_cmd() {
    timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
        "${COMMAND_TIMEOUT_SECONDS}" ss "$@"
}

compose_base_config() {
    compose_config_cmd -f "${REMOTE_BASE}" config -q >/dev/null 2>&1
}

compose_managed_config() {
    compose_config_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
        config -q >/dev/null 2>&1
}

compose_stage_config() {
    compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" \
        config -q >/dev/null 2>&1
}

recreate_managed() {
    compose_apply_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
        up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
        >/dev/null 2>&1
}

recreate_base() {
    compose_apply_cmd -f "${REMOTE_BASE}" \
        up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
        >/dev/null 2>&1
}

wait_clickhouse() {
    local attempt running health
    for attempt in $(seq 1 36); do
        running="$(docker_cmd inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
        health="$(docker_cmd inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
            "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
        if [ "${running}" = true ] && [ "${health}" = healthy ] \
            && docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
                clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then
            return 0
        fi
        sleep 5
    done
    return 1
}

listener_up() {
    local port="$1"
    ss_cmd -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
}

wait_api_and_listeners() {
    local attempt api_code
    for attempt in $(seq 1 30); do
        api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}" 2>/dev/null || true)"
        if [ "${api_code}" = 200 ] && listener_up 4317 && listener_up 4318 && listener_up 8080; then
            return 0
        fi
        sleep 5
    done
    return 1
}

file_matches_metadata() {
    local path="$1" expected_hash="$2" expected_mode="$3" expected_uid="$4" expected_gid="$5"
    [ -f "${path}" ] && [ ! -L "${path}" ] \
        && [ "$(sha256sum "${path}" | awk '{print $1}')" = "${expected_hash}" ] \
        && [ "$(stat -c '%a' "${path}")" = "${expected_mode}" ] \
        && [ "$(stat -c '%u' "${path}")" = "${expected_uid}" ] \
        && [ "$(stat -c '%g' "${path}")" = "${expected_gid}" ]
}

restore_exact_file() {
    local source="$1" target="$2" expected_hash="$3" expected_mode="$4"
    local expected_uid="$5" expected_gid="$6" tmp="$7"
    local current_uid current_gid

    bounded_cmd install -m "${expected_mode}" "${source}" "${tmp}" || return 1
    current_uid="$(stat -c '%u' "${tmp}")" || return 1
    current_gid="$(stat -c '%g' "${tmp}")" || return 1
    if [ "${current_uid}" != "${expected_uid}" ] || [ "${current_gid}" != "${expected_gid}" ]; then
        bounded_cmd sudo -n chown "${expected_uid}:${expected_gid}" "${tmp}" || return 1
    fi
    bounded_cmd mv -f "${tmp}" "${target}" || return 1
    file_matches_metadata "${target}" "${expected_hash}" "${expected_mode}" \
        "${expected_uid}" "${expected_gid}"
}

restore_managed_files() {
    local tmp
    if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
        tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}"
        restore_exact_file "${RECEIPT_DIR}/prior-override.yaml" "${REMOTE_OVERRIDE}" \
            "${PRIOR_OVERRIDE_HASH}" "${PRIOR_OVERRIDE_MODE}" \
            "${PRIOR_OVERRIDE_UID}" "${PRIOR_OVERRIDE_GID}" "${tmp}" || return 1
    else
        bounded_cmd rm -f "${REMOTE_OVERRIDE}" || return 1
        [ ! -e "${REMOTE_OVERRIDE}" ] || return 1
    fi

    if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then
        tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}"
        restore_exact_file "${RECEIPT_DIR}/prior-backup-disk.xml" "${REMOTE_CONFIG}" \
            "${PRIOR_CONFIG_HASH}" "${PRIOR_CONFIG_MODE}" \
            "${PRIOR_CONFIG_UID}" "${PRIOR_CONFIG_GID}" "${tmp}" || return 1
    else
        bounded_cmd rm -f "${REMOTE_CONFIG}" || return 1
        [ ! -e "${REMOTE_CONFIG}" ] || return 1
    fi
}

cleanup_owned_residue() {
    local path
    for path in \
        "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" \
        "${REMOTE_CONFIG}.candidate.${RUN_ID}" \
        "${REMOTE_OVERRIDE}.rollback.${RUN_ID}" \
        "${REMOTE_CONFIG}.rollback.${RUN_ID}"; do
        bounded_cmd rm -f "${path}" || return 1
        [ ! -e "${path}" ] || return 1
    done
    if [ -e "${STAGE_DIR}" ]; then
        [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
        bounded_cmd rm -f "${STAGE_OVERRIDE}" "${STAGE_CONFIG}" || return 1
        bounded_cmd rmdir "${STAGE_DIR}" || return 1
    fi
    [ ! -e "${STAGE_DIR}" ] || return 1
}

rollback_deploy() {
    local disk_count rollback_dir_terminal=retained
    restore_managed_files || return 1

    if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
        compose_managed_config || return 1
        recreate_managed || return 1
    else
        compose_base_config || return 1
        recreate_base || return 1
    fi
    wait_clickhouse || return 1
    wait_api_and_listeners || return 1

    disk_count="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
        "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1
    [ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1
    [ "$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
    [ "$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
    [ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
    [ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1

    if [ "${HOST_DIR_CREATED}" -eq 1 ]; then
        if bounded_cmd sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
            rollback_dir_terminal=removed_empty
        else
            rollback_dir_terminal=retained_nonempty_no_delete
        fi
    fi
    receipt rollback pass \
        "prior_compose_state_restored_exact_hash_mode_uid_gid_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
}

on_exit() {
    local rc=$? rollback_rc=0 cleanup_rc=0 terminal=failed
    trap - EXIT HUP INT TERM
    set +e
    if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
        rollback_deploy
        rollback_rc=$?
        if [ "${rollback_rc}" -ne 0 ]; then
            receipt rollback failed "prior_compose_state_restore_failed"
            rc=91
        fi
    fi
    cleanup_owned_residue
    cleanup_rc=$?
    if [ "${cleanup_rc}" -ne 0 ]; then
        receipt cleanup failed "exact_stage_candidate_or_rollback_residue_cleanup_failed"
        rc=92
    else
        receipt cleanup pass "exact_stage_candidate_and_rollback_residue_absence_verified"
    fi
    if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ] && [ "${cleanup_rc}" -eq 0 ]; then
        terminal=pass
    fi
    receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}"
    exit "${rc}"
}

mkdir -p "${RECEIPT_ROOT}"
[ ! -e "${RECEIPT_DIR}" ]
mkdir -m 0700 "${RECEIPT_DIR}"
: > "${RECEIPT_LOG}"
trap on_exit EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM

for lock_path in "${LOCK_FILE}" "${BACKUP_OPERATION_LOCK_FILE}" "${CANARY_LOCK_FILE}"; do
    [ ! -L "${lock_path}" ] \
        || { receipt check failed operation_lock_symlink_unsafe; exit 75; }
    if [ -e "${lock_path}" ]; then
        [ -f "${lock_path}" ] \
            || { receipt check failed operation_lock_not_regular; exit 75; }
    fi
done
exec 9>>"${LOCK_FILE}"
flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; }
exec 7>>"${CANARY_LOCK_FILE}"
flock -n 7 || { receipt check failed backup_canary_lock_held; exit 75; }
exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"
flock -n 8 || { receipt check failed backup_operation_lock_held; exit 75; }

for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss stat sudo timeout; do
    command -v "${command_name}" >/dev/null 2>&1 \
        || { receipt check failed "required_command_missing_${command_name}"; exit 69; }
done

[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
[ -f "${STAGE_OVERRIDE}" ] && [ ! -L "${STAGE_OVERRIDE}" ]
[ -f "${STAGE_CONFIG}" ] && [ ! -L "${STAGE_CONFIG}" ]
[ "$(sha256sum "${STAGE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
[ "$(sha256sum "${STAGE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
[ -d "${REMOTE_DIR}" ] && [ ! -L "${REMOTE_DIR}" ] && [ -w "${REMOTE_DIR}" ]
[ -d /backup ] && [ ! -L /backup ]

compose_base_config
compose_stage_config
compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
    | grep -qx "${REMOTE_SERVICE}"

require_backup_idle() {
    local phase="$1" active_native_backups process_rc
    set +e
    pgrep_cmd -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' \
        >/dev/null 2>&1
    process_rc=$?
    set -e
    case "${process_rc}" in
        0)
            receipt check failed "active_signoz_backup_or_restore_process_detected_${phase}"
            return 75
            ;;
        1) ;;
        *)
            receipt check failed "signoz_backup_process_probe_failed_${phase}_exit_${process_rc}"
            return 69
            ;;
    esac
    active_native_backups="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
        "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" \
        || return 1
    [ "${active_native_backups}" = 0 ] \
        || { receipt check failed "active_native_backup_or_restore_${phase}"; return 75; }
    receipt check pass "backup_idle_${phase}_shared_operation_and_canary_locks_held"
}

require_backup_idle preflight

for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do
    [ "$(docker_cmd inspect --format '{{.State.Running}}' "${container}")" = true ]
done
wait_clickhouse
wait_api_and_listeners

if [ -e "${REMOTE_OVERRIDE}" ]; then
    [ -f "${REMOTE_OVERRIDE}" ] && [ ! -L "${REMOTE_OVERRIDE}" ]
    PRIOR_OVERRIDE_PRESENT=1
fi
if [ -e "${REMOTE_CONFIG}" ]; then
    [ -f "${REMOTE_CONFIG}" ] && [ ! -L "${REMOTE_CONFIG}" ]
    PRIOR_CONFIG_PRESENT=1
fi
[ "${PRIOR_OVERRIDE_PRESENT}" -eq "${PRIOR_CONFIG_PRESENT}" ] \
    || { receipt check failed partial_managed_file_drift; exit 78; }
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
    compose_managed_config
    PRIOR_OVERRIDE_HASH="$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')"
    PRIOR_OVERRIDE_MODE="$(stat -c '%a' "${REMOTE_OVERRIDE}")"
    PRIOR_OVERRIDE_UID="$(stat -c '%u' "${REMOTE_OVERRIDE}")"
    PRIOR_OVERRIDE_GID="$(stat -c '%g' "${REMOTE_OVERRIDE}")"
    PRIOR_CONFIG_HASH="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
    PRIOR_CONFIG_MODE="$(stat -c '%a' "${REMOTE_CONFIG}")"
    PRIOR_CONFIG_UID="$(stat -c '%u' "${REMOTE_CONFIG}")"
    PRIOR_CONFIG_GID="$(stat -c '%g' "${REMOTE_CONFIG}")"
    bounded_cmd cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
    bounded_cmd cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
    [ "$(sha256sum "${RECEIPT_DIR}/prior-override.yaml" | awk '{print $1}')" = "${PRIOR_OVERRIDE_HASH}" ]
    [ "$(sha256sum "${RECEIPT_DIR}/prior-backup-disk.xml" | awk '{print $1}')" = "${PRIOR_CONFIG_HASH}" ]
fi

PRIOR_DISK_COUNT="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
    "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
BEFORE_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
BEFORE_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
[ -n "${BEFORE_DATA_VOLUME}" ]
BEFORE_COLLECTOR_ID="$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
BEFORE_COLLECTOR_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
BEFORE_SIGNOZ_ID="$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
BEFORE_SIGNOZ_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
BEFORE_ZOOKEEPER_ID="$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
BEFORE_ZOOKEEPER_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"

receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid"
receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}"
receipt source_of_truth_diff pass \
    "prior_override_${PRIOR_OVERRIDE_PRESENT}_hash_${PRIOR_OVERRIDE_HASH}_mode_${PRIOR_OVERRIDE_MODE}_uid_${PRIOR_OVERRIDE_UID}_gid_${PRIOR_OVERRIDE_GID}_prior_config_${PRIOR_CONFIG_PRESENT}_hash_${PRIOR_CONFIG_HASH}_mode_${PRIOR_CONFIG_MODE}_uid_${PRIOR_CONFIG_UID}_gid_${PRIOR_CONFIG_GID}_prior_disk_count_${PRIOR_DISK_COUNT}"
receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only"
receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback"
receipt check pass \
    "compose_config_q_stage_hashes_runtime_preflight_dual_locks_backup_idle"

require_backup_idle pre_mutation

APPLY_STARTED=1
if [ -e "${HOST_BACKUP_DIR}" ]; then
    [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]
    [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
else
    bounded_cmd sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
    HOST_DIR_CREATED=1
fi

bounded_cmd install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
bounded_cmd install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
bounded_cmd mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
bounded_cmd mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
[ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
[ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
compose_managed_config

recreate_managed
wait_clickhouse

AFTER_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
AFTER_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
[ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ]
[ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ]

backup_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
config_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
[ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ]
[ "${config_mount}" = false ]
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
server_uid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
    awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)"
server_gid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
    awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)"
[ "${server_uid}" = 101 ]
[ "${server_gid}" = 101 ]
docker_cmd exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
    'test -d /backups && test -w /backups'

disk_state="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
    "SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
[ "${disk_state}" = "Local|0|0|0|1" ]
[ "$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
    "EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ]
docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
    | grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)'

[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
[ "$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
[ "$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
wait_api_and_listeners

receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged"
receipt post_verifier pass "disk_backups_local_rw_healthy_server_uid101_write_access_concurrency_disabled_collector_unchanged_listeners_4317_4318_8080_api_200"
receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_executed"
DEPLOY_VERIFIED=1
REMOTE_APPLY

LOCAL_APPLY_VERIFIED=1