#!/usr/bin/env bash
# P0-OBS-002 - bounded controlled deploy of the host110 SigNoz native backup toolchain.
#
# This deployer owns seven repo-sourced files only. It does not run a backup or
# restore, restart a container, pull an image, or inspect a secret-bearing
# runtime source.

set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TARGET_HOST="wooo@192.168.0.110"
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SAFE_HOME="/home/wooo"
API_URL="http://127.0.0.1:8080/api/v1/health"
CANARY_LOCK="/tmp/awoooi-signoz-backup-canary.lock"
OPERATION_LOCK="/tmp/awoooi-signoz-backup-operation.lock"

SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"
SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"
DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"
TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"

REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"
REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}"
RECEIPT_ROOT="${SIGNOZ_TOOLCHAIN_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-native-backup-toolchain}"
STAGE_ROOT="${SIGNOZ_TOOLCHAIN_STAGE_ROOT:-/tmp}"

LABELS=(
  backup-signoz.sh
  clickhouse-native-backup.sh
  clickhouse-native-restore-drill.sh
  clickhouse-restore-inventory.py
  run-signoz-backup-canary.sh
  backup_disk.xml
  isolated-cluster.xml
)
LOCAL_SOURCES=(
  "${ROOT_DIR}/scripts/backup/backup-signoz.sh"
  "${ROOT_DIR}/scripts/backup/clickhouse-native-backup.sh"
  "${ROOT_DIR}/scripts/backup/clickhouse-native-restore-drill.sh"
  "${ROOT_DIR}/scripts/backup/clickhouse-restore-inventory.py"
  "${ROOT_DIR}/scripts/backup/run-signoz-backup-canary.sh"
  "${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
  "${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
)
MODES=(0755 0755 0755 0755 0755 0644 0644)

SSH_OPTS=(
  -o BatchMode=yes
  -o ConnectTimeout=8
  -o ConnectionAttempts=1
  -o ServerAliveInterval=5
  -o ServerAliveCountMax=2
)

usage() {
  cat <<'USAGE'
Usage: deploy-signoz-native-backup-toolchain.sh [--check|--apply]

Required environment:
  TRACE_ID       Path-safe controlled-apply trace identifier
  RUN_ID         Unique path-safe execution identifier
  WORK_ITEM_ID   Path-safe work-item identifier

Optional safe absolute path environment:
  SIGNOZ_TOOLCHAIN_SCRIPT_DIR
  SIGNOZ_TOOLCHAIN_CONFIG_ROOT
  SIGNOZ_TOOLCHAIN_RECEIPT_ROOT
  SIGNOZ_TOOLCHAIN_STAGE_ROOT

Optional bounded-command timeout environment (positive integer seconds):
  SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS
  SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS
  SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS
  SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS

--check validates local sources and performs a remote read-only target diff and
runtime readiness check. --apply stages all seven sources, validates hashes and
syntax, uses a same-filesystem atomic replacement for each exact target, and
rolls every target back to its prior hash/mode/owner if any bounded verifier
fails. SSH, SCP, and Docker calls are bounded, and apply holds both the SigNoz
canary lock and backup operation lock through post-verification or rollback.

Flat restore-driver invocation must set:
  CLICKHOUSE_RESTORE_INVENTORY_HELPER=<script-dir>/clickhouse-restore-inventory.py
  CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG=<config-root>/config.d/backup_disk.xml
  CLICKHOUSE_RESTORE_CLUSTER_CONFIG=<config-root>/restore-drill/config.d/isolated-cluster.xml
USAGE
}

MODE="${1:---check}"
case "${MODE}" in
  --check|--apply) ;;
  -h|--help) usage; exit 0 ;;
  *) usage >&2; exit 64 ;;
esac
[ "$#" -le 1 ] || { usage >&2; exit 64; }

TRACE_ID="${TRACE_ID:-}"
RUN_ID="${RUN_ID:-}"
WORK_ITEM_ID="${WORK_ITEM_ID:-}"

valid_identity() {
  [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
}

valid_positive_integer() {
  [[ "$1" =~ ^[1-9][0-9]*$ ]]
}

safe_absolute_path() {
  local path="$1"
  [[ "${path}" == /* ]] || return 1
  [[ "${path}" != / && "${path}" != */ ]] || return 1
  [[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1
  case "/${path#/}/" in
    *'//'*|*'/../'*|*'/./'*) return 1 ;;
  esac
}

paths_overlap() {
  local left="$1" right="$2"
  [[ "${left}" = "${right}" || "${left}" == "${right}"/* || "${right}" == "${left}"/* ]]
}

for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
  valid_identity "${value}" || {
    printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
    exit 64
  }
done

for value in \
  "${SSH_COMMAND_TIMEOUT_SECONDS}" \
  "${SCP_COMMAND_TIMEOUT_SECONDS}" \
  "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \
  "${TIMEOUT_KILL_AFTER_SECONDS}"; do
  valid_positive_integer "${value}" || {
    printf 'toolchain command timeouts must be positive integer seconds\n' >&2
    exit 64
  }
done
[ "${SSH_COMMAND_TIMEOUT_SECONDS}" -le 1200 ] \
  && [ "${SCP_COMMAND_TIMEOUT_SECONDS}" -le 600 ] \
  && [ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" -le 300 ] \
  && [ "${TIMEOUT_KILL_AFTER_SECONDS}" -le 60 ] || {
  printf 'toolchain command timeout exceeds bounded maximum\n' >&2
  exit 64
}

for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
  safe_absolute_path "${path}" || {
    printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2
    exit 64
  }
done
case "${REMOTE_SCRIPT_DIR}" in /backup/*) ;; *) printf 'script dir must stay under /backup\n' >&2; exit 64 ;; esac
case "${REMOTE_CONFIG_ROOT}" in /backup/*) ;; *) printf 'config root must stay under /backup\n' >&2; exit 64 ;; esac
case "${RECEIPT_ROOT}" in /backup/*) ;; *) printf 'receipt root must stay under /backup\n' >&2; exit 64 ;; esac
case "${STAGE_ROOT}" in /tmp|/tmp/*|/backup/deploy-staging|/backup/deploy-staging/*) ;;
  *) printf 'stage root must stay under /tmp or /backup/deploy-staging\n' >&2; exit 64 ;;
esac
if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \
  || paths_overlap "${REMOTE_SCRIPT_DIR}" "${RECEIPT_ROOT}" \
  || paths_overlap "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" \
  || paths_overlap "${STAGE_ROOT}" "${REMOTE_SCRIPT_DIR}" \
  || paths_overlap "${STAGE_ROOT}" "${REMOTE_CONFIG_ROOT}" \
  || paths_overlap "${STAGE_ROOT}" "${RECEIPT_ROOT}"; then
  printf 'toolchain target, receipt, and stage paths must not overlap\n' >&2
  exit 64
fi

for command_name in bash python3 scp ssh timeout; do
  command -v "${command_name}" >/dev/null 2>&1 || {
    printf 'required local command missing: %s\n' "${command_name}" >&2
    exit 69
  }
done


bounded_ssh() {
  timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
    "${SSH_COMMAND_TIMEOUT_SECONDS}" ssh "$@"
}


bounded_scp() {
  timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
    "${SCP_COMMAND_TIMEOUT_SECONDS}" scp "$@"
}

hash_file() {
  if command -v sha256sum >/dev/null 2>&1; then
    sha256sum "$1" | awk '{print $1}'
  else
    shasum -a 256 "$1" | awk '{print $1}'
  fi
}

validate_xml_pair() {
  python3 - "$1" "$2" <<'PY'
from pathlib import Path
import sys
import xml.etree.ElementTree as ET

backup_path = Path(sys.argv[1])
cluster_path = Path(sys.argv[2])
backup = ET.parse(backup_path).getroot()
cluster = ET.parse(cluster_path).getroot()
assert backup.tag == "clickhouse"
assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
assert backup.findtext("./backups/allowed_disk") == "backups"
assert backup.findtext("./backups/allowed_path") == "/backups/"
assert backup.findtext("./backups/allow_concurrent_backups") == "false"
assert backup.findtext("./backups/allow_concurrent_restores") == "false"
assert cluster.tag == "clickhouse"
assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
assert cluster.findtext("./zookeeper/node/port") == "2181"
assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
assert cluster.findtext("./remote_servers/cluster/shard/replica/port") == "9000"
assert cluster.findtext("./macros/shard") == "01"
replica = cluster.find("./macros/replica")
assert replica is not None
assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
PY
}

validate_local_sources() {
  local index source
  for index in "${!LOCAL_SOURCES[@]}"; do
    source="${LOCAL_SOURCES[index]}"
    [ -f "${source}" ] && [ ! -L "${source}" ]
    [ -s "${source}" ]
  done

  for index in 0 1 2 4; do
    bash -n "${LOCAL_SOURCES[index]}"
  done
  python3 - "${LOCAL_SOURCES[3]}" <<'PY'
from pathlib import Path
import sys

path = Path(sys.argv[1])
compile(path.read_text(encoding="utf-8"), str(path), "exec")
PY
  validate_xml_pair "${LOCAL_SOURCES[5]}" "${LOCAL_SOURCES[6]}"

  python3 - "${LOCAL_SOURCES[2]}" <<'PY'
from pathlib import Path
import sys

text = Path(sys.argv[1]).read_text(encoding="utf-8")
for token in (
    "CLICKHOUSE_RESTORE_INVENTORY_HELPER",
    "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG",
    "CLICKHOUSE_RESTORE_CLUSTER_CONFIG",
):
    assert token in text
PY
}

validate_local_sources
SOURCE_HASHES=()
SOURCE_DETAIL=""
for index in "${!LOCAL_SOURCES[@]}"; do
  source_hash="$(hash_file "${LOCAL_SOURCES[index]}")"
  SOURCE_HASHES+=("${source_hash}")
  SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_"
done
SOURCE_DETAIL="${SOURCE_DETAIL%_}"

CONFIG_DIR="${REMOTE_CONFIG_ROOT}/config.d"
RESTORE_CONFIG_DIR="${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
TARGET_PATHS=(
  "${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
  "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
  "${CONFIG_DIR}/backup_disk.xml"
  "${RESTORE_CONFIG_DIR}/isolated-cluster.xml"
)
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-native-backup-toolchain.${RUN_ID}"

local_receipt() {
  local phase="$1" result="$2" detail="$3"
  printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}\n' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
}

remote_read_only_check() {
  bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
    "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \
    "${DOCKER_COMMAND_TIMEOUT_SECONDS}" "${TIMEOUT_KILL_AFTER_SECONDS}" \
    "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
set -euo pipefail

TRACE_ID="$1"
RUN_ID="$2"
WORK_ITEM_ID="$3"
REMOTE_SCRIPT_DIR="$4"
REMOTE_CONFIG_ROOT="$5"
RECEIPT_ROOT="$6"
STAGE_ROOT="$7"
DOCKER_COMMAND_TIMEOUT_SECONDS="$8"
TIMEOUT_KILL_AFTER_SECONDS="$9"
shift 9
EXPECTED_HASHES=("$@")
LABELS=(
  backup-signoz.sh
  clickhouse-native-backup.sh
  clickhouse-native-restore-drill.sh
  clickhouse-restore-inventory.py
  run-signoz-backup-canary.sh
  backup_disk.xml
  isolated-cluster.xml
)
MODES=(0755 0755 0755 0755 0755 0644 0644)
TARGETS=(
  "${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
  "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
  "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
  "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
)
API_URL="http://127.0.0.1:8080/api/v1/health"

[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
[ -d /backup ] && [ ! -L /backup ]
for command_name in awk curl docker pgrep sha256sum ss stat timeout; do
  command -v "${command_name}" >/dev/null 2>&1
done

[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]
[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]]

bounded_docker() {
  timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
    "${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@"
}

listener_up() {
  local port="$1"
  ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
}

validate_existing_ancestors() {
  local requested="$1" current="" component
  local -a components
  IFS='/' read -r -a components <<<"${requested#/}"
  for component in "${components[@]}"; do
    [ -n "${component}" ] || continue
    current="${current}/${component}"
    if [ -e "${current}" ] || [ -L "${current}" ]; then
      [ -d "${current}" ] && [ ! -L "${current}" ]
    else
      break
    fi
  done
}

runtime_line() {
  local container="$1" value running health
  value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
  IFS='|' read -r _ _ _ running health <<<"${value}"
  [ "${running}" = true ]
  case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
  printf '%s' "${value}"
}

if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
  printf 'active SigNoz backup toolchain process detected\n' >&2
  exit 75
fi
validate_existing_ancestors "${REMOTE_SCRIPT_DIR}"
validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/config.d"
validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
validate_existing_ancestors "${RECEIPT_ROOT}"
validate_existing_ancestors "${STAGE_ROOT}"
[ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ]
active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \
  "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
[ "${active_operations}" = 0 ]

runtime_summary=""
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
  runtime_summary+="${container}_$(runtime_line "${container}")_"
done
for port in 4317 4318 8080; do listener_up "${port}"; done
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
[ "${api_code}" = 200 ]

matching=0
drift=0
absent=0
for index in "${!TARGETS[@]}"; do
  target="${TARGETS[index]}"
  if [ -e "${target}" ] || [ -L "${target}" ]; then
    [ -f "${target}" ] && [ ! -L "${target}" ]
    current_hash="$(sha256sum "${target}" | awk '{print $1}')"
    current_mode="$(stat -c '%a' "${target}")"
    if [ "${current_hash}" = "${EXPECTED_HASHES[index]}" ] && [ "0${current_mode}" = "${MODES[index]}" ]; then
      matching=$((matching + 1))
    else
      drift=$((drift + 1))
    fi
  else
    absent=$((absent + 1))
  fi
done

printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"source_of_truth_diff","result":"pass","detail":"matching_%s_drift_%s_absent_%s_active_operations_0_api_200_ports_4317_4318_8080"}\n' \
  "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${matching}" "${drift}" "${absent}"
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"runtime_readiness","result":"pass","detail":"container_identity_started_at_restart_count_health_captured_%s"}\n' \
  "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${runtime_summary%_}"
REMOTE_CHECK
}

local_receipt sensor_source pass "${SOURCE_DETAIL}"
local_receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
remote_read_only_check
local_receipt check pass "local_syntax_xml_python_compile_and_remote_read_only_preflight"

if [ "${MODE}" = --check ]; then
  local_receipt terminal check_pass_no_write "remote_target_unchanged_no_stage_no_candidate"
  exit 0
fi

STAGE_CREATED=0
cleanup_stage() {
  if [ "${STAGE_CREATED}" -eq 1 ]; then
    bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
set -u
stage_dir="$1"
if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
  rm -rf "${stage_dir}/pycache" "${stage_dir}/candidate-pycache"
  rm -f \
    "${stage_dir}/backup-signoz.sh" \
    "${stage_dir}/clickhouse-native-backup.sh" \
    "${stage_dir}/clickhouse-native-restore-drill.sh" \
    "${stage_dir}/clickhouse-restore-inventory.py" \
    "${stage_dir}/run-signoz-backup-canary.sh" \
    "${stage_dir}/backup_disk.xml" \
    "${stage_dir}/isolated-cluster.xml"
  rmdir "${stage_dir}" 2>/dev/null || true
fi
REMOTE_CLEANUP
  fi
}
trap cleanup_stage EXIT

bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
set -euo pipefail
stage_dir="$1"
[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
stage_root="${stage_dir%/*}"
[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ]
umask 077
mkdir -m 0700 "${stage_dir}"
REMOTE_STAGE
STAGE_CREATED=1

for index in "${!LOCAL_SOURCES[@]}"; do
  bounded_scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
    "${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"
done

bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
  "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
  "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \
  "${CANARY_LOCK}" "${OPERATION_LOCK}" "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \
  "${TIMEOUT_KILL_AFTER_SECONDS}" \
  "${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
set -euo pipefail

TRACE_ID="$1"
RUN_ID="$2"
WORK_ITEM_ID="$3"
REMOTE_SCRIPT_DIR="$4"
REMOTE_CONFIG_ROOT="$5"
RECEIPT_ROOT="$6"
STAGE_DIR="$7"
CANARY_LOCK="$8"
OPERATION_LOCK="$9"
DOCKER_COMMAND_TIMEOUT_SECONDS="${10}"
TIMEOUT_KILL_AFTER_SECONDS="${11}"
shift 11
EXPECTED_HASHES=("$@")
LABELS=(
  backup-signoz.sh
  clickhouse-native-backup.sh
  clickhouse-native-restore-drill.sh
  clickhouse-restore-inventory.py
  run-signoz-backup-canary.sh
  backup_disk.xml
  isolated-cluster.xml
)
MODES=(0755 0755 0755 0755 0755 0644 0644)
STAGED=(
  "${STAGE_DIR}/backup-signoz.sh"
  "${STAGE_DIR}/clickhouse-native-backup.sh"
  "${STAGE_DIR}/clickhouse-native-restore-drill.sh"
  "${STAGE_DIR}/clickhouse-restore-inventory.py"
  "${STAGE_DIR}/run-signoz-backup-canary.sh"
  "${STAGE_DIR}/backup_disk.xml"
  "${STAGE_DIR}/isolated-cluster.xml"
)
TARGETS=(
  "${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
  "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
  "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
  "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
  "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
)
TARGET_DIRS=(
  "${REMOTE_SCRIPT_DIR}"
  "${REMOTE_CONFIG_ROOT}/config.d"
  "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
)
CONTAINERS=(signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1)
API_URL="http://127.0.0.1:8080/api/v1/health"
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
PRIOR_DIR="${RECEIPT_DIR}/prior"
PRIOR_MANIFEST="${RECEIPT_DIR}/prior-manifest.tsv"
DEPLOYED_MANIFEST="${RECEIPT_DIR}/deployed-manifest.tsv"
RUNTIME_BEFORE="${RECEIPT_DIR}/runtime-before.tsv"
RUNTIME_AFTER="${RECEIPT_DIR}/runtime-after.tsv"
LOCK_FILE="/tmp/awoooi-signoz-native-backup-toolchain.lock"

APPLY_STARTED=0
DEPLOY_VERIFIED=0
RUNTIME_BASELINE_CAPTURED=0
LOCK_HELD=0
CANARY_LOCK_HELD=0
OPERATION_LOCK_HELD=0
REMOTE_UID="$(id -u)"
REMOTE_GID="$(id -g)"
CREATED_DIRS=()
PRIOR_PRESENT=()
PRIOR_MODES=()
PRIOR_UIDS=()
PRIOR_GIDS=()
PRIOR_HASHES=()

for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat timeout; do
  command -v "${command_name}" >/dev/null 2>&1 || {
    printf 'required remote command missing: %s\n' "${command_name}" >&2
    exit 69
  }
done
if [ "${REMOTE_UID}" -ne 0 ]; then
  command -v sudo >/dev/null 2>&1 || { printf 'required remote command missing: sudo\n' >&2; exit 69; }
  sudo -n true
fi
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
[ -d /backup ] && [ ! -L /backup ]
[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]
[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]]
[ "${CANARY_LOCK}" = "/tmp/awoooi-signoz-backup-canary.lock" ]
[ "${OPERATION_LOCK}" = "/tmp/awoooi-signoz-backup-operation.lock" ]

run_root() {
  if [ "${REMOTE_UID}" -eq 0 ]; then
    "$@"
  else
    sudo -n "$@"
  fi
}

bounded_docker() {
  timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
    "${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@"
}

root_hash() {
  if [ "${REMOTE_UID}" -eq 0 ]; then
    sha256sum "$1" | awk '{print $1}'
  else
    sudo -n sha256sum "$1" | awk '{print $1}'
  fi
}

ensure_dir_tree() {
  local requested="$1" record_created="$2" current="" component
  local -a components
  IFS='/' read -r -a components <<<"${requested#/}"
  for component in "${components[@]}"; do
    [ -n "${component}" ] || continue
    current="${current}/${component}"
    if [ -e "${current}" ] || [ -L "${current}" ]; then
      [ -d "${current}" ] && [ ! -L "${current}" ]
    else
      run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0755 "${current}"
      if [ "${record_created}" = true ]; then
        CREATED_DIRS+=("${current}")
      fi
    fi
  done
}

ensure_dir_tree "${RECEIPT_ROOT}" false
[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]
run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0700 "${RECEIPT_DIR}"
install -d -m 0700 "${PRIOR_DIR}"
: > "${RECEIPT_LOG}"

receipt() {
  local phase="$1" result="$2" detail="$3" line
  line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}' \
    "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
  printf '%s\n' "${line}"
  printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
}

listener_up() {
  local port="$1"
  ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
}

runtime_snapshot() {
  local container value running health port api_code
  for container in "${CONTAINERS[@]}"; do
    value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
    IFS='|' read -r _ _ _ running health <<<"${value}"
    [ "${running}" = true ]
    case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
    printf 'container\t%s\t%s\n' "${container}" "${value}"
  done
  for port in 4317 4318 8080; do
    listener_up "${port}"
    printf 'listener\t%s\tup\n' "${port}"
  done
  api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
  [ "${api_code}" = 200 ]
  printf 'api\thealth\t%s\n' "${api_code}"
}

validate_xml_pair() {
  python3 - "$1" "$2" <<'PY'
from pathlib import Path
import sys
import xml.etree.ElementTree as ET

backup_path = Path(sys.argv[1])
cluster_path = Path(sys.argv[2])
backup = ET.parse(backup_path).getroot()
cluster = ET.parse(cluster_path).getroot()
assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
assert backup.findtext("./backups/allowed_disk") == "backups"
assert backup.findtext("./backups/allowed_path") == "/backups/"
assert backup.findtext("./backups/allow_concurrent_backups") == "false"
assert backup.findtext("./backups/allow_concurrent_restores") == "false"
assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
assert cluster.findtext("./macros/shard") == "01"
replica = cluster.find("./macros/replica")
assert replica is not None
assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
PY
}

cleanup_transients() {
  local index rc=0
  for index in "${!TARGETS[@]}"; do
    run_root rm -f \
      "${TARGETS[index]}.candidate.${RUN_ID}" \
      "${TARGETS[index]}.rollback.${RUN_ID}" || rc=1
  done
  if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then
    rm -rf "${STAGE_DIR}/pycache" "${STAGE_DIR}/candidate-pycache" || rc=1
    rm -f "${STAGED[@]}" || rc=1
    rmdir "${STAGE_DIR}" 2>/dev/null || rc=1
  else
    rc=1
  fi
  return "${rc}"
}

verify_residue_zero() {
  local index
  [ ! -e "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
  for index in "${!TARGETS[@]}"; do
    [ ! -e "${TARGETS[index]}.candidate.${RUN_ID}" ] \
      && [ ! -L "${TARGETS[index]}.candidate.${RUN_ID}" ] || return 1
    [ ! -e "${TARGETS[index]}.rollback.${RUN_ID}" ] \
      && [ ! -L "${TARGETS[index]}.rollback.${RUN_ID}" ] || return 1
  done
}

rollback_targets() (
  set -euo pipefail
  local index target rollback_candidate actual_mode actual_hash
  for index in "${!TARGETS[@]}"; do
    target="${TARGETS[index]}"
    if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
      rollback_candidate="${target}.rollback.${RUN_ID}"
      run_root install \
        -o "${PRIOR_UIDS[index]}" -g "${PRIOR_GIDS[index]}" \
        -m "${PRIOR_MODES[index]}" \
        "${PRIOR_DIR}/${LABELS[index]}" "${rollback_candidate}"
      [ "$(root_hash "${rollback_candidate}")" = "${PRIOR_HASHES[index]}" ]
      run_root mv -f "${rollback_candidate}" "${target}"
      actual_mode="$(stat -c '%a' "${target}")"
      actual_hash="$(root_hash "${target}")"
      [ "${actual_mode}" = "${PRIOR_MODES[index]}" ]
      [ "${actual_hash}" = "${PRIOR_HASHES[index]}" ]
    else
      run_root rm -f "${target}"
      [ ! -e "${target}" ] && [ ! -L "${target}" ]
    fi
  done

  for ((index=${#CREATED_DIRS[@]} - 1; index >= 0; index--)); do
    run_root rmdir "${CREATED_DIRS[index]}"
  done

  if [ "${RUNTIME_BASELINE_CAPTURED}" -eq 1 ]; then
    runtime_snapshot > "${RECEIPT_DIR}/runtime-rollback.tsv"
    cmp -s "${RUNTIME_BEFORE}" "${RECEIPT_DIR}/runtime-rollback.tsv"
  fi
  receipt rollback pass "all_prior_files_modes_hashes_owners_and_runtime_restored"
)

on_exit() {
  local rc=$? rollback_rc=0 cleanup_rc=0 residue_rc=0 terminal=failed
  trap - EXIT HUP INT TERM
  set +e
  if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
    rollback_targets
    rollback_rc=$?
    if [ "${rollback_rc}" -ne 0 ]; then
      receipt rollback failed "prior_state_restore_or_runtime_verifier_failed"
      rc=91
    fi
  elif [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 0 ]; then
    receipt rollback no_write "bounded_execution_not_started_targets_unchanged"
  fi
  cleanup_transients
  cleanup_rc=$?
  verify_residue_zero
  residue_rc=$?
  if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then
    rc=92
  fi
  if [ "${OPERATION_LOCK_HELD}" -eq 1 ]; then
    flock -u 7 >/dev/null 2>&1
    exec 7>&-
  fi
  if [ "${CANARY_LOCK_HELD}" -eq 1 ]; then
    flock -u 8 >/dev/null 2>&1
    exec 8>&-
  fi
  if [ "${LOCK_HELD}" -eq 1 ]; then
    flock -u 9 >/dev/null 2>&1
    exec 9>&-
    rm -f "${LOCK_FILE}"
  fi
  if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
    terminal=pass
  fi
  receipt terminal "${terminal}" "exit_${rc}_stage_candidate_residue_0_receipt_ref_${RECEIPT_LOG}"
  exit "${rc}"
}

trap on_exit EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM

exec 9>>"${LOCK_FILE}"
flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; }
LOCK_HELD=1

[ ! -L "${CANARY_LOCK}" ] \
  || { receipt check failed canary_lock_symlink_unsafe; exit 75; }
if [ -e "${CANARY_LOCK}" ]; then
  [ -f "${CANARY_LOCK}" ] \
    || { receipt check failed canary_lock_not_regular_file; exit 75; }
fi
exec 8>>"${CANARY_LOCK}"
flock -n 8 \
  || { receipt check failed another_signoz_canary_holds_canary_lock; exit 75; }
CANARY_LOCK_HELD=1

[ ! -L "${OPERATION_LOCK}" ] \
  || { receipt check failed backup_operation_lock_symlink_unsafe; exit 75; }
if [ -e "${OPERATION_LOCK}" ]; then
  [ -f "${OPERATION_LOCK}" ] \
    || { receipt check failed backup_operation_lock_not_regular_file; exit 75; }
fi
exec 7>>"${OPERATION_LOCK}"
flock -n 7 \
  || { receipt check failed another_signoz_backup_holds_operation_lock; exit 75; }
OPERATION_LOCK_HELD=1

if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
  receipt check failed active_signoz_backup_toolchain_process
  exit 75
fi
active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \
  "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
[ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }

runtime_snapshot > "${RUNTIME_BEFORE}"
RUNTIME_BASELINE_CAPTURED=1

[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
for index in "${!STAGED[@]}"; do
  [ -f "${STAGED[index]}" ] && [ ! -L "${STAGED[index]}" ]
  [ "$(sha256sum "${STAGED[index]}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
done
for index in 0 1 2 4; do bash -n "${STAGED[index]}"; done
PYTHONPYCACHEPREFIX="${STAGE_DIR}/pycache" python3 -m py_compile "${STAGED[3]}"
validate_xml_pair "${STAGED[5]}" "${STAGED[6]}"

: > "${PRIOR_MANIFEST}"
for index in "${!TARGETS[@]}"; do
  target="${TARGETS[index]}"
  [ ! -e "${target}.candidate.${RUN_ID}" ] && [ ! -L "${target}.candidate.${RUN_ID}" ]
  [ ! -e "${target}.rollback.${RUN_ID}" ] && [ ! -L "${target}.rollback.${RUN_ID}" ]
  if [ -e "${target}" ] || [ -L "${target}" ]; then
    [ -f "${target}" ] && [ ! -L "${target}" ]
    PRIOR_PRESENT[index]=1
    PRIOR_MODES[index]="$(stat -c '%a' "${target}")"
    PRIOR_UIDS[index]="$(stat -c '%u' "${target}")"
    PRIOR_GIDS[index]="$(stat -c '%g' "${target}")"
    PRIOR_HASHES[index]="$(root_hash "${target}")"
    run_root cp -p "${target}" "${PRIOR_DIR}/${LABELS[index]}"
    [ "$(root_hash "${PRIOR_DIR}/${LABELS[index]}")" = "${PRIOR_HASHES[index]}" ]
  else
    PRIOR_PRESENT[index]=0
    PRIOR_MODES[index]=absent
    PRIOR_UIDS[index]=absent
    PRIOR_GIDS[index]=absent
    PRIOR_HASHES[index]=absent
  fi
  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
    "${LABELS[index]}" "${target}" "${PRIOR_PRESENT[index]}" \
    "${PRIOR_MODES[index]}" "${PRIOR_UIDS[index]}" "${PRIOR_GIDS[index]}" \
    "${PRIOR_HASHES[index]}" >> "${PRIOR_MANIFEST}"
done

receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources"
receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}"
receipt ai_decision candidate "same_filesystem_candidate_replace_native_backup_toolchain_only"
receipt risk_policy pass "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_backup_operation_locks_no_runtime_execution_no_container_change_full_rollback"
receipt check pass "dual_signoz_canary_and_backup_operation_locks_held_runtime_baseline_active_operations_0_hash_bash_xml_py_compile"

APPLY_STARTED=1
for directory in "${TARGET_DIRS[@]}"; do
  ensure_dir_tree "${directory}" true
done

for index in "${!TARGETS[@]}"; do
  target="${TARGETS[index]}"
  if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
    target_uid="${PRIOR_UIDS[index]}"
    target_gid="${PRIOR_GIDS[index]}"
  else
    target_uid="${REMOTE_UID}"
    target_gid="${REMOTE_GID}"
  fi
  run_root install -o "${target_uid}" -g "${target_gid}" -m "${MODES[index]}" \
    "${STAGED[index]}" "${target}.candidate.${RUN_ID}"
  [ "$(root_hash "${target}.candidate.${RUN_ID}")" = "${EXPECTED_HASHES[index]}" ]
  [ "0$(stat -c '%a' "${target}.candidate.${RUN_ID}")" = "${MODES[index]}" ]
done

for index in "${!TARGETS[@]}"; do
  run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"
done

for index in "${!TARGETS[@]}"; do
  [ -f "${TARGETS[index]}" ] && [ ! -L "${TARGETS[index]}" ]
  [ "$(root_hash "${TARGETS[index]}")" = "${EXPECTED_HASHES[index]}" ]
  [ "0$(stat -c '%a' "${TARGETS[index]}")" = "${MODES[index]}" ]
done
for index in 0 1 2 4; do bash -n "${TARGETS[index]}"; done
PYTHONPYCACHEPREFIX="${STAGE_DIR}/candidate-pycache" python3 -m py_compile "${TARGETS[3]}"
validate_xml_pair "${TARGETS[5]}" "${TARGETS[6]}"

: > "${DEPLOYED_MANIFEST}"
for index in "${!TARGETS[@]}"; do
  printf '%s\t%s\t%s\t%s\n' \
    "${LABELS[index]}" "${TARGETS[index]}" "${MODES[index]}" \
    "${EXPECTED_HASHES[index]}" >> "${DEPLOYED_MANIFEST}"
done

runtime_snapshot > "${RUNTIME_AFTER}"
cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"
receipt execution pass "seven_same_filesystem_candidates_replaced_with_full_rollback_expected_hashes_and_modes"
receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged"
receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull"
DEPLOY_VERIFIED=1
REMOTE_APPLY

trap - EXIT
cleanup_stage
local_receipt terminal pass "remote_controlled_apply_postverify_and_stage_candidate_residue_0"