595 lines
24 KiB
Bash
Executable File
595 lines
24 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# P0-OBS-002 - controlled deployment of the host110 ClickHouse native BACKUP disk.
|
|
#
|
|
# This script owns only an additive Compose override and one ClickHouse config
|
|
# fragment. It never runs BACKUP/RESTORE, reads a dotenv/secret source, pulls an
|
|
# image, or recreates another service.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml"
|
|
LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
|
|
|
|
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
|
|
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
|
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
|
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
|
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
|
REMOTE_PROJECT="signoz"
|
|
REMOTE_SERVICE="clickhouse"
|
|
CLICKHOUSE_CONTAINER="signoz-clickhouse"
|
|
COLLECTOR_CONTAINER="signoz-otel-collector"
|
|
SIGNOZ_CONTAINER="signoz"
|
|
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
|
|
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
|
CONTAINER_BACKUP_DIR="/backups"
|
|
CLICKHOUSE_DISK="backups"
|
|
CLICKHOUSE_UID="101"
|
|
CLICKHOUSE_GID="101"
|
|
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
|
|
API_URL="http://127.0.0.1:8080/api/v1/health"
|
|
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
SAFE_HOME="/home/wooo"
|
|
|
|
SSH_OPTS=(
|
|
-o BatchMode=yes
|
|
-o ConnectTimeout=8
|
|
-o ConnectionAttempts=1
|
|
-o ServerAliveInterval=5
|
|
-o ServerAliveCountMax=2
|
|
)
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: deploy-signoz-clickhouse-backup-disk.sh [--check|--apply]
|
|
|
|
Required environment:
|
|
TRACE_ID Path-safe controlled-apply trace identifier
|
|
RUN_ID Unique path-safe execution identifier
|
|
WORK_ITEM_ID Path-safe work-item identifier
|
|
|
|
--check performs local source validation and remote read-only Compose/runtime
|
|
checks. --apply atomically installs the two repo-owned artifacts, recreates only
|
|
the ClickHouse service with --pull never, independently verifies runtime state,
|
|
and restores the exact prior managed Compose state on failure.
|
|
USAGE
|
|
}
|
|
|
|
MODE="${1:---check}"
|
|
case "${MODE}" in
|
|
--check|--apply) ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) usage >&2; exit 64 ;;
|
|
esac
|
|
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
|
|
|
TRACE_ID="${TRACE_ID:-}"
|
|
RUN_ID="${RUN_ID:-}"
|
|
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
|
|
|
valid_identity() {
|
|
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
|
}
|
|
|
|
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
|
valid_identity "${value}" || {
|
|
printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
|
|
exit 64
|
|
}
|
|
done
|
|
|
|
for command_name in python3 scp ssh; do
|
|
command -v "${command_name}" >/dev/null 2>&1 || {
|
|
printf 'required local command missing: %s\n' "${command_name}" >&2
|
|
exit 69
|
|
}
|
|
done
|
|
|
|
hash_file() {
|
|
if command -v sha256sum >/dev/null 2>&1; then
|
|
sha256sum "$1" | awk '{print $1}'
|
|
else
|
|
shasum -a 256 "$1" | awk '{print $1}'
|
|
fi
|
|
}
|
|
|
|
validate_local_sources() {
|
|
[ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ]
|
|
[ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ]
|
|
|
|
python3 - "${LOCAL_CONFIG}" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
path = Path(sys.argv[1])
|
|
root = ET.parse(path).getroot()
|
|
assert root.tag == "clickhouse"
|
|
disk = root.find("./storage_configuration/disks/backups")
|
|
assert disk is not None
|
|
assert disk.findtext("type") == "local"
|
|
assert disk.findtext("path") == "/backups/"
|
|
backups = root.find("./backups")
|
|
assert backups is not None
|
|
assert backups.findtext("allowed_disk") == "backups"
|
|
assert backups.findtext("allowed_path") == "/backups/"
|
|
assert backups.findtext("allow_concurrent_backups") == "false"
|
|
assert backups.findtext("allow_concurrent_restores") == "false"
|
|
PY
|
|
|
|
[ "$(grep -F -c ' clickhouse:' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
|
[ "$(grep -F -c 'source: /backup/staging/signoz-clickhouse' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
|
[ "$(grep -F -c 'target: /backups' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
|
[ "$(grep -F -c 'target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
|
! grep -Eq '^[[:space:]]*(image|build|command|environment):' "${LOCAL_OVERRIDE}"
|
|
}
|
|
|
|
validate_local_sources
|
|
OVERRIDE_SHA="$(hash_file "${LOCAL_OVERRIDE}")"
|
|
CONFIG_SHA="$(hash_file "${LOCAL_CONFIG}")"
|
|
|
|
local_receipt() {
|
|
local phase="$1" result="$2" detail="$3"
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
|
|
}
|
|
|
|
remote_candidate_compose_check() {
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" \
|
|
"env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
|
|
< "${LOCAL_OVERRIDE}"
|
|
}
|
|
|
|
remote_read_only_check() {
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
|
"${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
EXPECTED_OVERRIDE_SHA="$4"
|
|
EXPECTED_CONFIG_SHA="$5"
|
|
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
|
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
|
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
|
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
|
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
|
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
SAFE_HOME="/home/wooo"
|
|
|
|
file_hash_or_absent() {
|
|
if [ -f "$1" ] && [ ! -L "$1" ]; then
|
|
sha256sum "$1" | awk '{print $1}'
|
|
elif [ ! -e "$1" ]; then
|
|
printf 'absent\n'
|
|
else
|
|
printf 'unsafe\n'
|
|
fi
|
|
}
|
|
|
|
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
|
|
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
|
|
docker inspect "${container}" >/dev/null
|
|
done
|
|
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
|
|
config -q >/dev/null
|
|
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
|
|
config --services | grep -qx clickhouse
|
|
|
|
override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")"
|
|
config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")"
|
|
override_match=false
|
|
config_match=false
|
|
[ "${override_hash}" = "${EXPECTED_OVERRIDE_SHA}" ] && override_match=true
|
|
[ "${config_hash}" = "${EXPECTED_CONFIG_SHA}" ] && config_match=true
|
|
|
|
host_dir=absent
|
|
if [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]; then
|
|
host_dir=present
|
|
elif [ -e "${HOST_BACKUP_DIR}" ]; then
|
|
host_dir=unsafe
|
|
fi
|
|
|
|
disk_count="$(docker exec signoz-clickhouse clickhouse-client --query \
|
|
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
|
mount_count="$(docker inspect --format \
|
|
'{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \
|
|
signoz-clickhouse | tr -cd '1' | wc -c | xargs)"
|
|
collector_running="$(docker inspect --format '{{.State.Running}}' signoz-otel-collector)"
|
|
collector_restarts="$(docker inspect --format '{{.RestartCount}}' signoz-otel-collector)"
|
|
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \
|
|
"${config_match}" "${host_dir}" "${disk_count}" "${mount_count}" \
|
|
"${collector_running}" "${collector_restarts}"
|
|
REMOTE_CHECK
|
|
}
|
|
|
|
local_receipt sensor_source pass "override_sha_${OVERRIDE_SHA}_config_sha_${CONFIG_SHA}"
|
|
local_receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_disk_backups"
|
|
remote_candidate_compose_check
|
|
remote_read_only_check
|
|
local_receipt check pass "compose_config_q_no_output_no_write"
|
|
|
|
if [ "${MODE}" = "--check" ]; then
|
|
local_receipt terminal check_pass_no_write "source_and_remote_preflight_valid"
|
|
exit 0
|
|
fi
|
|
|
|
STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}"
|
|
STAGE_CREATED=0
|
|
|
|
cleanup_stage() {
|
|
if [ "${STAGE_CREATED}" -eq 1 ]; then
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
|
|
set -u
|
|
stage_dir="$1"
|
|
rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml"
|
|
rmdir "${stage_dir}" 2>/dev/null || true
|
|
REMOTE_CLEANUP
|
|
fi
|
|
}
|
|
trap cleanup_stage EXIT
|
|
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
|
|
set -euo pipefail
|
|
stage_dir="$1"
|
|
[ ! -e "${stage_dir}" ]
|
|
umask 077
|
|
mkdir -m 0700 "${stage_dir}"
|
|
REMOTE_STAGE
|
|
STAGE_CREATED=1
|
|
|
|
scp -q "${SSH_OPTS[@]}" "${LOCAL_OVERRIDE}" \
|
|
"${TARGET_HOST}:${STAGE_DIR}/override.yaml"
|
|
scp -q "${SSH_OPTS[@]}" "${LOCAL_CONFIG}" \
|
|
"${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml"
|
|
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
|
"${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
EXPECTED_OVERRIDE_SHA="$4"
|
|
EXPECTED_CONFIG_SHA="$5"
|
|
STAGE_DIR="$6"
|
|
|
|
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
|
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
|
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
|
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
|
STAGE_OVERRIDE="${STAGE_DIR}/override.yaml"
|
|
STAGE_CONFIG="${STAGE_DIR}/backup_disk.xml"
|
|
REMOTE_PROJECT="signoz"
|
|
REMOTE_SERVICE="clickhouse"
|
|
CLICKHOUSE_CONTAINER="signoz-clickhouse"
|
|
COLLECTOR_CONTAINER="signoz-otel-collector"
|
|
SIGNOZ_CONTAINER="signoz"
|
|
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
|
|
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
|
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
|
|
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
|
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
|
API_URL="http://127.0.0.1:8080/api/v1/health"
|
|
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
SAFE_HOME="/home/wooo"
|
|
LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock"
|
|
|
|
APPLY_STARTED=0
|
|
DEPLOY_VERIFIED=0
|
|
HOST_DIR_CREATED=0
|
|
PRIOR_OVERRIDE_PRESENT=0
|
|
PRIOR_CONFIG_PRESENT=0
|
|
PRIOR_DISK_COUNT=0
|
|
BEFORE_IMAGE_ID=""
|
|
BEFORE_DATA_VOLUME=""
|
|
BEFORE_COLLECTOR_ID=""
|
|
BEFORE_COLLECTOR_RESTARTS=""
|
|
BEFORE_SIGNOZ_ID=""
|
|
BEFORE_SIGNOZ_RESTARTS=""
|
|
BEFORE_ZOOKEEPER_ID=""
|
|
BEFORE_ZOOKEEPER_RESTARTS=""
|
|
|
|
receipt() {
|
|
local phase="$1" result="$2" detail="$3"
|
|
local line
|
|
line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
|
|
printf '%s\n' "${line}"
|
|
printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
|
|
}
|
|
|
|
compose_base_config() {
|
|
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
|
-f "${REMOTE_BASE}" config -q >/dev/null 2>&1
|
|
}
|
|
|
|
compose_managed_config() {
|
|
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
|
-f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" config -q >/dev/null 2>&1
|
|
}
|
|
|
|
compose_stage_config() {
|
|
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
|
-f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config -q >/dev/null 2>&1
|
|
}
|
|
|
|
recreate_managed() {
|
|
timeout --signal=TERM --kill-after=30 300 \
|
|
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
|
-f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
|
|
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
recreate_base() {
|
|
timeout --signal=TERM --kill-after=30 300 \
|
|
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
|
-f "${REMOTE_BASE}" \
|
|
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
wait_clickhouse() {
|
|
local attempt running health
|
|
for attempt in $(seq 1 36); do
|
|
running="$(docker inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
|
|
health="$(docker inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
|
|
"${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
|
|
if [ "${running}" = true ] && [ "${health}" = healthy ] \
|
|
&& timeout --kill-after=5 10 docker exec "${CLICKHOUSE_CONTAINER}" \
|
|
clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
listener_up() {
|
|
local port="$1"
|
|
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
|
}
|
|
|
|
wait_api_and_listeners() {
|
|
local attempt api_code
|
|
for attempt in $(seq 1 30); do
|
|
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}" 2>/dev/null || true)"
|
|
if [ "${api_code}" = 200 ] && listener_up 4317 && listener_up 4318 && listener_up 8080; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
restore_managed_files() {
|
|
local tmp
|
|
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
|
tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}"
|
|
install -m 0644 "${RECEIPT_DIR}/prior-override.yaml" "${tmp}"
|
|
mv -f "${tmp}" "${REMOTE_OVERRIDE}"
|
|
else
|
|
rm -f "${REMOTE_OVERRIDE}"
|
|
fi
|
|
|
|
if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then
|
|
tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}"
|
|
install -m 0644 "${RECEIPT_DIR}/prior-backup-disk.xml" "${tmp}"
|
|
mv -f "${tmp}" "${REMOTE_CONFIG}"
|
|
else
|
|
rm -f "${REMOTE_CONFIG}"
|
|
fi
|
|
}
|
|
|
|
rollback_deploy() {
|
|
local disk_count rollback_dir_terminal=retained
|
|
restore_managed_files || return 1
|
|
|
|
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
|
compose_managed_config || return 1
|
|
recreate_managed || return 1
|
|
else
|
|
compose_base_config || return 1
|
|
recreate_base || return 1
|
|
fi
|
|
wait_clickhouse || return 1
|
|
wait_api_and_listeners || return 1
|
|
|
|
disk_count="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1
|
|
[ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1
|
|
[ "$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
|
|
[ "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
|
|
[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
|
|
[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1
|
|
|
|
if [ "${HOST_DIR_CREATED}" -eq 1 ]; then
|
|
if sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
|
|
rollback_dir_terminal=removed_empty
|
|
else
|
|
rollback_dir_terminal=retained_nonempty_no_delete
|
|
fi
|
|
fi
|
|
receipt rollback pass "prior_compose_state_restored_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
|
|
}
|
|
|
|
on_exit() {
|
|
local rc=$? rollback_rc=0 terminal=failed
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
|
rollback_deploy
|
|
rollback_rc=$?
|
|
if [ "${rollback_rc}" -ne 0 ]; then
|
|
receipt rollback failed "prior_compose_state_restore_failed"
|
|
rc=91
|
|
fi
|
|
fi
|
|
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
|
|
terminal=pass
|
|
fi
|
|
receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}"
|
|
exit "${rc}"
|
|
}
|
|
|
|
mkdir -p "${RECEIPT_ROOT}"
|
|
[ ! -e "${RECEIPT_DIR}" ]
|
|
mkdir -m 0700 "${RECEIPT_DIR}"
|
|
: > "${RECEIPT_LOG}"
|
|
trap on_exit EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
exec 9>>"${LOCK_FILE}"
|
|
flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; }
|
|
|
|
for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss sudo timeout; do
|
|
command -v "${command_name}" >/dev/null 2>&1 \
|
|
|| { receipt check failed "required_command_missing_${command_name}"; exit 69; }
|
|
done
|
|
|
|
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
|
[ -f "${STAGE_OVERRIDE}" ] && [ ! -L "${STAGE_OVERRIDE}" ]
|
|
[ -f "${STAGE_CONFIG}" ] && [ ! -L "${STAGE_CONFIG}" ]
|
|
[ "$(sha256sum "${STAGE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
|
|
[ "$(sha256sum "${STAGE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
|
|
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
|
|
[ -d "${REMOTE_DIR}" ] && [ ! -L "${REMOTE_DIR}" ] && [ -w "${REMOTE_DIR}" ]
|
|
[ -d /backup ] && [ ! -L /backup ]
|
|
|
|
compose_base_config
|
|
compose_stage_config
|
|
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
|
-f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
|
|
| grep -qx "${REMOTE_SERVICE}"
|
|
|
|
if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
|
receipt check failed active_signoz_backup_detected
|
|
exit 75
|
|
fi
|
|
active_native_backups="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
|
|
[ "${active_native_backups}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
|
|
|
|
for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do
|
|
[ "$(docker inspect --format '{{.State.Running}}' "${container}")" = true ]
|
|
done
|
|
wait_clickhouse
|
|
wait_api_and_listeners
|
|
|
|
if [ -e "${REMOTE_OVERRIDE}" ]; then
|
|
[ -f "${REMOTE_OVERRIDE}" ] && [ ! -L "${REMOTE_OVERRIDE}" ]
|
|
PRIOR_OVERRIDE_PRESENT=1
|
|
fi
|
|
if [ -e "${REMOTE_CONFIG}" ]; then
|
|
[ -f "${REMOTE_CONFIG}" ] && [ ! -L "${REMOTE_CONFIG}" ]
|
|
PRIOR_CONFIG_PRESENT=1
|
|
fi
|
|
[ "${PRIOR_OVERRIDE_PRESENT}" -eq "${PRIOR_CONFIG_PRESENT}" ] \
|
|
|| { receipt check failed partial_managed_file_drift; exit 78; }
|
|
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
|
compose_managed_config
|
|
cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
|
|
cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
|
|
fi
|
|
|
|
PRIOR_DISK_COUNT="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
|
BEFORE_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
|
|
BEFORE_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
|
[ -n "${BEFORE_DATA_VOLUME}" ]
|
|
BEFORE_COLLECTOR_ID="$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
|
|
BEFORE_COLLECTOR_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
|
|
BEFORE_SIGNOZ_ID="$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
|
|
BEFORE_SIGNOZ_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
|
|
BEFORE_ZOOKEEPER_ID="$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
|
|
BEFORE_ZOOKEEPER_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"
|
|
|
|
receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid"
|
|
receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}"
|
|
receipt source_of_truth_diff pass "prior_override_${PRIOR_OVERRIDE_PRESENT}_prior_config_${PRIOR_CONFIG_PRESENT}_prior_disk_count_${PRIOR_DISK_COUNT}"
|
|
receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only"
|
|
receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback"
|
|
receipt check pass "compose_config_q_stage_hashes_runtime_preflight_backup_idle"
|
|
|
|
APPLY_STARTED=1
|
|
if [ -e "${HOST_BACKUP_DIR}" ]; then
|
|
[ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]
|
|
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
|
|
else
|
|
sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
|
|
HOST_DIR_CREATED=1
|
|
fi
|
|
|
|
install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
|
|
install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
|
|
mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
|
|
mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
|
|
[ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
|
|
[ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
|
|
compose_managed_config
|
|
|
|
recreate_managed
|
|
wait_clickhouse
|
|
|
|
AFTER_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
|
|
AFTER_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
|
[ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ]
|
|
[ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ]
|
|
|
|
backup_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
|
config_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
|
[ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ]
|
|
[ "${config_mount}" = false ]
|
|
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
|
|
server_uid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
|
|
awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)"
|
|
server_gid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
|
|
awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)"
|
|
[ "${server_uid}" = 101 ]
|
|
[ "${server_gid}" = 101 ]
|
|
docker exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
|
|
'test -d /backups && test -w /backups'
|
|
|
|
disk_state="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
|
[ "${disk_state}" = "Local|0|0|0|1" ]
|
|
[ "$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ]
|
|
docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
|
|
| grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)'
|
|
|
|
[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
|
|
[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
|
|
[ "$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
|
|
[ "$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
|
|
[ "$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
|
|
[ "$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
|
|
wait_api_and_listeners
|
|
|
|
receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged"
|
|
receipt post_verifier pass "disk_backups_local_rw_healthy_server_uid101_write_access_concurrency_disabled_collector_unchanged_listeners_4317_4318_8080_api_200"
|
|
receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_executed"
|
|
DEPLOY_VERIFIED=1
|
|
REMOTE_APPLY
|
|
|
|
trap - EXIT
|
|
cleanup_stage
|
|
local_receipt terminal pass "remote_controlled_apply_and_postverify_complete"
|