799 lines
31 KiB
Bash
Executable File
799 lines
31 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# P0-OBS-002 - controlled deployment of the host110 ClickHouse native BACKUP disk.
|
|
#
|
|
# This script owns only an additive Compose override and one ClickHouse config
|
|
# fragment. It never runs BACKUP/RESTORE, reads a dotenv/secret source, pulls an
|
|
# image, or recreates another service.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml"
|
|
LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
|
|
|
|
TARGET_HOST="wooo@192.168.0.110"
|
|
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
|
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
|
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
|
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
|
REMOTE_PROJECT="signoz"
|
|
REMOTE_SERVICE="clickhouse"
|
|
CLICKHOUSE_CONTAINER="signoz-clickhouse"
|
|
COLLECTOR_CONTAINER="signoz-otel-collector"
|
|
SIGNOZ_CONTAINER="signoz"
|
|
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
|
|
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
|
CONTAINER_BACKUP_DIR="/backups"
|
|
CLICKHOUSE_DISK="backups"
|
|
CLICKHOUSE_UID="101"
|
|
CLICKHOUSE_GID="101"
|
|
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
|
|
API_URL="http://127.0.0.1:8080/api/v1/health"
|
|
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
SAFE_HOME="/home/wooo"
|
|
LOCAL_KILL_AFTER_SECONDS=15
|
|
REMOTE_CHECK_TIMEOUT_SECONDS=180
|
|
REMOTE_STAGE_TIMEOUT_SECONDS=60
|
|
REMOTE_TRANSFER_TIMEOUT_SECONDS=60
|
|
REMOTE_APPLY_TIMEOUT_SECONDS=3600
|
|
REMOTE_CLEANUP_TIMEOUT_SECONDS=60
|
|
|
|
SSH_OPTS=(
|
|
-o BatchMode=yes
|
|
-o ConnectTimeout=8
|
|
-o ConnectionAttempts=1
|
|
-o ServerAliveInterval=5
|
|
-o ServerAliveCountMax=2
|
|
)
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: deploy-signoz-clickhouse-backup-disk.sh [--check|--apply]
|
|
|
|
Required environment:
|
|
TRACE_ID Path-safe controlled-apply trace identifier
|
|
RUN_ID Unique path-safe execution identifier
|
|
WORK_ITEM_ID Path-safe work-item identifier
|
|
|
|
--check performs local source validation and remote read-only Compose/runtime
|
|
checks. --apply atomically installs the two repo-owned artifacts, recreates only
|
|
the ClickHouse service with --pull never, independently verifies runtime state,
|
|
and restores the exact prior managed Compose state on failure.
|
|
USAGE
|
|
}
|
|
|
|
MODE="${1:---check}"
|
|
case "${MODE}" in
|
|
--check|--apply) ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) usage >&2; exit 64 ;;
|
|
esac
|
|
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
|
|
|
TRACE_ID="${TRACE_ID:-}"
|
|
RUN_ID="${RUN_ID:-}"
|
|
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
|
|
|
valid_identity() {
|
|
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
|
}
|
|
|
|
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
|
valid_identity "${value}" || {
|
|
printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
|
|
exit 64
|
|
}
|
|
done
|
|
|
|
for command_name in python3 scp ssh timeout; do
|
|
command -v "${command_name}" >/dev/null 2>&1 || {
|
|
printf 'required local command missing: %s\n' "${command_name}" >&2
|
|
exit 69
|
|
}
|
|
done
|
|
|
|
hash_file() {
|
|
if command -v sha256sum >/dev/null 2>&1; then
|
|
sha256sum "$1" | awk '{print $1}'
|
|
else
|
|
shasum -a 256 "$1" | awk '{print $1}'
|
|
fi
|
|
}
|
|
|
|
bounded_ssh() {
|
|
local timeout_seconds="$1"
|
|
shift
|
|
timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
|
|
"${timeout_seconds}" ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" "$@"
|
|
}
|
|
|
|
bounded_scp() {
|
|
timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
|
|
"${REMOTE_TRANSFER_TIMEOUT_SECONDS}" scp -q "${SSH_OPTS[@]}" "$@"
|
|
}
|
|
|
|
validate_local_sources() {
|
|
[ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ]
|
|
[ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ]
|
|
|
|
python3 - "${LOCAL_CONFIG}" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
path = Path(sys.argv[1])
|
|
root = ET.parse(path).getroot()
|
|
assert root.tag == "clickhouse"
|
|
disk = root.find("./storage_configuration/disks/backups")
|
|
assert disk is not None
|
|
assert disk.findtext("type") == "local"
|
|
assert disk.findtext("path") == "/backups/"
|
|
backups = root.find("./backups")
|
|
assert backups is not None
|
|
assert backups.findtext("allowed_disk") == "backups"
|
|
assert backups.findtext("allowed_path") == "/backups/"
|
|
assert backups.findtext("allow_concurrent_backups") == "false"
|
|
assert backups.findtext("allow_concurrent_restores") == "false"
|
|
PY
|
|
|
|
[ "$(grep -F -c ' clickhouse:' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
|
[ "$(grep -F -c 'source: /backup/staging/signoz-clickhouse' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
|
[ "$(grep -F -c 'target: /backups' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
|
[ "$(grep -F -c 'target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
|
! grep -Eq '^[[:space:]]*(image|build|command|environment):' "${LOCAL_OVERRIDE}"
|
|
}
|
|
|
|
validate_local_sources
|
|
OVERRIDE_SHA="$(hash_file "${LOCAL_OVERRIDE}")"
|
|
CONFIG_SHA="$(hash_file "${LOCAL_CONFIG}")"
|
|
|
|
local_receipt() {
|
|
local phase="$1" result="$2" detail="$3"
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
|
|
}
|
|
|
|
remote_candidate_compose_check() {
|
|
bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" \
|
|
"timeout --signal=TERM --kill-after=15 120 env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
|
|
< "${LOCAL_OVERRIDE}"
|
|
}
|
|
|
|
remote_read_only_check() {
|
|
bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" bash -s -- \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
|
"${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
EXPECTED_OVERRIDE_SHA="$4"
|
|
EXPECTED_CONFIG_SHA="$5"
|
|
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
|
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
|
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
|
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
|
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
|
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
SAFE_HOME="/home/wooo"
|
|
COMMAND_TIMEOUT_SECONDS=30
|
|
COMPOSE_TIMEOUT_SECONDS=120
|
|
KILL_AFTER_SECONDS=15
|
|
|
|
for command_name in docker timeout; do
|
|
command -v "${command_name}" >/dev/null 2>&1
|
|
done
|
|
|
|
docker_cmd() {
|
|
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" docker "$@"
|
|
}
|
|
|
|
compose_cmd() {
|
|
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMPOSE_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p signoz "$@"
|
|
}
|
|
|
|
file_hash_or_absent() {
|
|
if [ -f "$1" ] && [ ! -L "$1" ]; then
|
|
sha256sum "$1" | awk '{print $1}'
|
|
elif [ ! -e "$1" ]; then
|
|
printf 'absent\n'
|
|
else
|
|
printf 'unsafe\n'
|
|
fi
|
|
}
|
|
|
|
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
|
|
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
|
|
docker_cmd inspect "${container}" >/dev/null
|
|
done
|
|
compose_cmd -f "${REMOTE_BASE}" config -q >/dev/null
|
|
compose_cmd -f "${REMOTE_BASE}" config --services | grep -qx clickhouse
|
|
|
|
override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")"
|
|
config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")"
|
|
override_match=false
|
|
config_match=false
|
|
[ "${override_hash}" = "${EXPECTED_OVERRIDE_SHA}" ] && override_match=true
|
|
[ "${config_hash}" = "${EXPECTED_CONFIG_SHA}" ] && config_match=true
|
|
|
|
host_dir=absent
|
|
if [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]; then
|
|
host_dir=present
|
|
elif [ -e "${HOST_BACKUP_DIR}" ]; then
|
|
host_dir=unsafe
|
|
fi
|
|
|
|
disk_count="$(docker_cmd exec signoz-clickhouse clickhouse-client --query \
|
|
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
|
mount_count="$(docker_cmd inspect --format \
|
|
'{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \
|
|
signoz-clickhouse | tr -cd '1' | wc -c | xargs)"
|
|
collector_running="$(docker_cmd inspect --format '{{.State.Running}}' signoz-otel-collector)"
|
|
collector_restarts="$(docker_cmd inspect --format '{{.RestartCount}}' signoz-otel-collector)"
|
|
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \
|
|
"${config_match}" "${host_dir}" "${disk_count}" "${mount_count}" \
|
|
"${collector_running}" "${collector_restarts}"
|
|
REMOTE_CHECK
|
|
}
|
|
|
|
local_receipt sensor_source pass "override_sha_${OVERRIDE_SHA}_config_sha_${CONFIG_SHA}"
|
|
local_receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_disk_backups"
|
|
remote_candidate_compose_check
|
|
remote_read_only_check
|
|
local_receipt check pass "compose_config_q_no_output_no_write"
|
|
|
|
if [ "${MODE}" = "--check" ]; then
|
|
local_receipt terminal check_pass_no_write "source_and_remote_preflight_valid"
|
|
exit 0
|
|
fi
|
|
|
|
STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}"
|
|
STAGE_CREATED=0
|
|
LOCAL_APPLY_VERIFIED=0
|
|
|
|
cleanup_stage() {
|
|
local cleanup_rc
|
|
[ "${STAGE_CREATED}" -eq 1 ] || return 0
|
|
bounded_ssh "${REMOTE_CLEANUP_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1
|
|
set -euo pipefail
|
|
stage_dir="$1"
|
|
if [ ! -e "${stage_dir}" ]; then
|
|
exit 0
|
|
fi
|
|
[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
|
rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml"
|
|
rmdir "${stage_dir}"
|
|
[ ! -e "${stage_dir}" ]
|
|
REMOTE_CLEANUP
|
|
cleanup_rc=$?
|
|
[ "${cleanup_rc}" -eq 0 ] || return "${cleanup_rc}"
|
|
STAGE_CREATED=0
|
|
}
|
|
|
|
on_local_exit() {
|
|
local rc=$?
|
|
local cleanup_rc=0 cleanup_terminal=not_created terminal=failed
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
if [ "${STAGE_CREATED}" -eq 1 ]; then
|
|
cleanup_stage
|
|
cleanup_rc=$?
|
|
if [ "${cleanup_rc}" -eq 0 ]; then
|
|
cleanup_terminal=pass_absence_verified
|
|
else
|
|
cleanup_terminal=failed_or_residue_present
|
|
rc=92
|
|
fi
|
|
fi
|
|
local_receipt cleanup "$([ "${cleanup_rc}" -eq 0 ] && printf pass || printf failed)" \
|
|
"remote_stage_${cleanup_terminal}"
|
|
if [ "${rc}" -eq 0 ] && [ "${LOCAL_APPLY_VERIFIED}" -eq 1 ]; then
|
|
terminal=pass
|
|
fi
|
|
local_receipt terminal "${terminal}" \
|
|
"remote_controlled_apply_${terminal}_stage_${cleanup_terminal}_exit_${rc}"
|
|
exit "${rc}"
|
|
}
|
|
|
|
trap on_local_exit EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
bounded_ssh "${REMOTE_STAGE_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
|
|
set -euo pipefail
|
|
stage_dir="$1"
|
|
[ ! -e "${stage_dir}" ]
|
|
umask 077
|
|
mkdir -m 0700 "${stage_dir}"
|
|
REMOTE_STAGE
|
|
STAGE_CREATED=1
|
|
|
|
bounded_scp "${LOCAL_OVERRIDE}" \
|
|
"${TARGET_HOST}:${STAGE_DIR}/override.yaml"
|
|
bounded_scp "${LOCAL_CONFIG}" \
|
|
"${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml"
|
|
|
|
bounded_ssh "${REMOTE_APPLY_TIMEOUT_SECONDS}" bash -s -- \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
|
"${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
EXPECTED_OVERRIDE_SHA="$4"
|
|
EXPECTED_CONFIG_SHA="$5"
|
|
STAGE_DIR="$6"
|
|
|
|
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
|
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
|
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
|
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
|
STAGE_OVERRIDE="${STAGE_DIR}/override.yaml"
|
|
STAGE_CONFIG="${STAGE_DIR}/backup_disk.xml"
|
|
REMOTE_PROJECT="signoz"
|
|
REMOTE_SERVICE="clickhouse"
|
|
CLICKHOUSE_CONTAINER="signoz-clickhouse"
|
|
COLLECTOR_CONTAINER="signoz-otel-collector"
|
|
SIGNOZ_CONTAINER="signoz"
|
|
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
|
|
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
|
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
|
|
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
|
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
|
API_URL="http://127.0.0.1:8080/api/v1/health"
|
|
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
SAFE_HOME="/home/wooo"
|
|
LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock"
|
|
BACKUP_OPERATION_LOCK_FILE="/tmp/awoooi-signoz-backup-operation.lock"
|
|
CANARY_LOCK_FILE="/tmp/awoooi-signoz-backup-canary.lock"
|
|
COMMAND_TIMEOUT_SECONDS=30
|
|
COMPOSE_CONFIG_TIMEOUT_SECONDS=120
|
|
COMPOSE_APPLY_TIMEOUT_SECONDS=300
|
|
KILL_AFTER_SECONDS=15
|
|
|
|
APPLY_STARTED=0
|
|
DEPLOY_VERIFIED=0
|
|
HOST_DIR_CREATED=0
|
|
PRIOR_OVERRIDE_PRESENT=0
|
|
PRIOR_CONFIG_PRESENT=0
|
|
PRIOR_OVERRIDE_HASH="absent"
|
|
PRIOR_OVERRIDE_MODE="absent"
|
|
PRIOR_OVERRIDE_UID="absent"
|
|
PRIOR_OVERRIDE_GID="absent"
|
|
PRIOR_CONFIG_HASH="absent"
|
|
PRIOR_CONFIG_MODE="absent"
|
|
PRIOR_CONFIG_UID="absent"
|
|
PRIOR_CONFIG_GID="absent"
|
|
PRIOR_DISK_COUNT=0
|
|
BEFORE_IMAGE_ID=""
|
|
BEFORE_DATA_VOLUME=""
|
|
BEFORE_COLLECTOR_ID=""
|
|
BEFORE_COLLECTOR_RESTARTS=""
|
|
BEFORE_SIGNOZ_ID=""
|
|
BEFORE_SIGNOZ_RESTARTS=""
|
|
BEFORE_ZOOKEEPER_ID=""
|
|
BEFORE_ZOOKEEPER_RESTARTS=""
|
|
|
|
receipt() {
|
|
local phase="$1" result="$2" detail="$3"
|
|
local line
|
|
line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
|
|
printf '%s\n' "${line}"
|
|
printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
|
|
}
|
|
|
|
docker_cmd() {
|
|
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" docker "$@"
|
|
}
|
|
|
|
bounded_cmd() {
|
|
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" "$@"
|
|
}
|
|
|
|
compose_config_cmd() {
|
|
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMPOSE_CONFIG_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
|
|
}
|
|
|
|
compose_apply_cmd() {
|
|
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMPOSE_APPLY_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
|
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
|
|
}
|
|
|
|
pgrep_cmd() {
|
|
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" pgrep "$@"
|
|
}
|
|
|
|
ss_cmd() {
|
|
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${COMMAND_TIMEOUT_SECONDS}" ss "$@"
|
|
}
|
|
|
|
compose_base_config() {
|
|
compose_config_cmd -f "${REMOTE_BASE}" config -q >/dev/null 2>&1
|
|
}
|
|
|
|
compose_managed_config() {
|
|
compose_config_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
|
|
config -q >/dev/null 2>&1
|
|
}
|
|
|
|
compose_stage_config() {
|
|
compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" \
|
|
config -q >/dev/null 2>&1
|
|
}
|
|
|
|
recreate_managed() {
|
|
compose_apply_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
|
|
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
recreate_base() {
|
|
compose_apply_cmd -f "${REMOTE_BASE}" \
|
|
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
|
|
>/dev/null 2>&1
|
|
}
|
|
|
|
wait_clickhouse() {
|
|
local attempt running health
|
|
for attempt in $(seq 1 36); do
|
|
running="$(docker_cmd inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
|
|
health="$(docker_cmd inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
|
|
"${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
|
|
if [ "${running}" = true ] && [ "${health}" = healthy ] \
|
|
&& docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
|
|
clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
listener_up() {
|
|
local port="$1"
|
|
ss_cmd -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
|
}
|
|
|
|
wait_api_and_listeners() {
|
|
local attempt api_code
|
|
for attempt in $(seq 1 30); do
|
|
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}" 2>/dev/null || true)"
|
|
if [ "${api_code}" = 200 ] && listener_up 4317 && listener_up 4318 && listener_up 8080; then
|
|
return 0
|
|
fi
|
|
sleep 5
|
|
done
|
|
return 1
|
|
}
|
|
|
|
file_matches_metadata() {
|
|
local path="$1" expected_hash="$2" expected_mode="$3" expected_uid="$4" expected_gid="$5"
|
|
[ -f "${path}" ] && [ ! -L "${path}" ] \
|
|
&& [ "$(sha256sum "${path}" | awk '{print $1}')" = "${expected_hash}" ] \
|
|
&& [ "$(stat -c '%a' "${path}")" = "${expected_mode}" ] \
|
|
&& [ "$(stat -c '%u' "${path}")" = "${expected_uid}" ] \
|
|
&& [ "$(stat -c '%g' "${path}")" = "${expected_gid}" ]
|
|
}
|
|
|
|
restore_exact_file() {
|
|
local source="$1" target="$2" expected_hash="$3" expected_mode="$4"
|
|
local expected_uid="$5" expected_gid="$6" tmp="$7"
|
|
local current_uid current_gid
|
|
|
|
bounded_cmd install -m "${expected_mode}" "${source}" "${tmp}" || return 1
|
|
current_uid="$(stat -c '%u' "${tmp}")" || return 1
|
|
current_gid="$(stat -c '%g' "${tmp}")" || return 1
|
|
if [ "${current_uid}" != "${expected_uid}" ] || [ "${current_gid}" != "${expected_gid}" ]; then
|
|
bounded_cmd sudo -n chown "${expected_uid}:${expected_gid}" "${tmp}" || return 1
|
|
fi
|
|
bounded_cmd mv -f "${tmp}" "${target}" || return 1
|
|
file_matches_metadata "${target}" "${expected_hash}" "${expected_mode}" \
|
|
"${expected_uid}" "${expected_gid}"
|
|
}
|
|
|
|
restore_managed_files() {
|
|
local tmp
|
|
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
|
tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}"
|
|
restore_exact_file "${RECEIPT_DIR}/prior-override.yaml" "${REMOTE_OVERRIDE}" \
|
|
"${PRIOR_OVERRIDE_HASH}" "${PRIOR_OVERRIDE_MODE}" \
|
|
"${PRIOR_OVERRIDE_UID}" "${PRIOR_OVERRIDE_GID}" "${tmp}" || return 1
|
|
else
|
|
bounded_cmd rm -f "${REMOTE_OVERRIDE}" || return 1
|
|
[ ! -e "${REMOTE_OVERRIDE}" ] || return 1
|
|
fi
|
|
|
|
if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then
|
|
tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}"
|
|
restore_exact_file "${RECEIPT_DIR}/prior-backup-disk.xml" "${REMOTE_CONFIG}" \
|
|
"${PRIOR_CONFIG_HASH}" "${PRIOR_CONFIG_MODE}" \
|
|
"${PRIOR_CONFIG_UID}" "${PRIOR_CONFIG_GID}" "${tmp}" || return 1
|
|
else
|
|
bounded_cmd rm -f "${REMOTE_CONFIG}" || return 1
|
|
[ ! -e "${REMOTE_CONFIG}" ] || return 1
|
|
fi
|
|
}
|
|
|
|
cleanup_owned_residue() {
|
|
local path
|
|
for path in \
|
|
"${REMOTE_OVERRIDE}.candidate.${RUN_ID}" \
|
|
"${REMOTE_CONFIG}.candidate.${RUN_ID}" \
|
|
"${REMOTE_OVERRIDE}.rollback.${RUN_ID}" \
|
|
"${REMOTE_CONFIG}.rollback.${RUN_ID}"; do
|
|
bounded_cmd rm -f "${path}" || return 1
|
|
[ ! -e "${path}" ] || return 1
|
|
done
|
|
if [ -e "${STAGE_DIR}" ]; then
|
|
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
|
|
bounded_cmd rm -f "${STAGE_OVERRIDE}" "${STAGE_CONFIG}" || return 1
|
|
bounded_cmd rmdir "${STAGE_DIR}" || return 1
|
|
fi
|
|
[ ! -e "${STAGE_DIR}" ] || return 1
|
|
}
|
|
|
|
rollback_deploy() {
|
|
local disk_count rollback_dir_terminal=retained
|
|
restore_managed_files || return 1
|
|
|
|
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
|
compose_managed_config || return 1
|
|
recreate_managed || return 1
|
|
else
|
|
compose_base_config || return 1
|
|
recreate_base || return 1
|
|
fi
|
|
wait_clickhouse || return 1
|
|
wait_api_and_listeners || return 1
|
|
|
|
disk_count="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1
|
|
[ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1
|
|
[ "$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
|
|
[ "$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
|
|
[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
|
|
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1
|
|
|
|
if [ "${HOST_DIR_CREATED}" -eq 1 ]; then
|
|
if bounded_cmd sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
|
|
rollback_dir_terminal=removed_empty
|
|
else
|
|
rollback_dir_terminal=retained_nonempty_no_delete
|
|
fi
|
|
fi
|
|
receipt rollback pass \
|
|
"prior_compose_state_restored_exact_hash_mode_uid_gid_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
|
|
}
|
|
|
|
on_exit() {
|
|
local rc=$? rollback_rc=0 cleanup_rc=0 terminal=failed
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
|
rollback_deploy
|
|
rollback_rc=$?
|
|
if [ "${rollback_rc}" -ne 0 ]; then
|
|
receipt rollback failed "prior_compose_state_restore_failed"
|
|
rc=91
|
|
fi
|
|
fi
|
|
cleanup_owned_residue
|
|
cleanup_rc=$?
|
|
if [ "${cleanup_rc}" -ne 0 ]; then
|
|
receipt cleanup failed "exact_stage_candidate_or_rollback_residue_cleanup_failed"
|
|
rc=92
|
|
else
|
|
receipt cleanup pass "exact_stage_candidate_and_rollback_residue_absence_verified"
|
|
fi
|
|
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ] && [ "${cleanup_rc}" -eq 0 ]; then
|
|
terminal=pass
|
|
fi
|
|
receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}"
|
|
exit "${rc}"
|
|
}
|
|
|
|
mkdir -p "${RECEIPT_ROOT}"
|
|
[ ! -e "${RECEIPT_DIR}" ]
|
|
mkdir -m 0700 "${RECEIPT_DIR}"
|
|
: > "${RECEIPT_LOG}"
|
|
trap on_exit EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
for lock_path in "${LOCK_FILE}" "${BACKUP_OPERATION_LOCK_FILE}" "${CANARY_LOCK_FILE}"; do
|
|
[ ! -L "${lock_path}" ] \
|
|
|| { receipt check failed operation_lock_symlink_unsafe; exit 75; }
|
|
if [ -e "${lock_path}" ]; then
|
|
[ -f "${lock_path}" ] \
|
|
|| { receipt check failed operation_lock_not_regular; exit 75; }
|
|
fi
|
|
done
|
|
exec 9>>"${LOCK_FILE}"
|
|
flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; }
|
|
exec 7>>"${CANARY_LOCK_FILE}"
|
|
flock -n 7 || { receipt check failed backup_canary_lock_held; exit 75; }
|
|
exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"
|
|
flock -n 8 || { receipt check failed backup_operation_lock_held; exit 75; }
|
|
|
|
for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss stat sudo timeout; do
|
|
command -v "${command_name}" >/dev/null 2>&1 \
|
|
|| { receipt check failed "required_command_missing_${command_name}"; exit 69; }
|
|
done
|
|
|
|
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
|
[ -f "${STAGE_OVERRIDE}" ] && [ ! -L "${STAGE_OVERRIDE}" ]
|
|
[ -f "${STAGE_CONFIG}" ] && [ ! -L "${STAGE_CONFIG}" ]
|
|
[ "$(sha256sum "${STAGE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
|
|
[ "$(sha256sum "${STAGE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
|
|
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
|
|
[ -d "${REMOTE_DIR}" ] && [ ! -L "${REMOTE_DIR}" ] && [ -w "${REMOTE_DIR}" ]
|
|
[ -d /backup ] && [ ! -L /backup ]
|
|
|
|
compose_base_config
|
|
compose_stage_config
|
|
compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
|
|
| grep -qx "${REMOTE_SERVICE}"
|
|
|
|
require_backup_idle() {
|
|
local phase="$1" active_native_backups process_rc
|
|
set +e
|
|
pgrep_cmd -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' \
|
|
>/dev/null 2>&1
|
|
process_rc=$?
|
|
set -e
|
|
case "${process_rc}" in
|
|
0)
|
|
receipt check failed "active_signoz_backup_or_restore_process_detected_${phase}"
|
|
return 75
|
|
;;
|
|
1) ;;
|
|
*)
|
|
receipt check failed "signoz_backup_process_probe_failed_${phase}_exit_${process_rc}"
|
|
return 69
|
|
;;
|
|
esac
|
|
active_native_backups="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" \
|
|
|| return 1
|
|
[ "${active_native_backups}" = 0 ] \
|
|
|| { receipt check failed "active_native_backup_or_restore_${phase}"; return 75; }
|
|
receipt check pass "backup_idle_${phase}_shared_operation_and_canary_locks_held"
|
|
}
|
|
|
|
require_backup_idle preflight
|
|
|
|
for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do
|
|
[ "$(docker_cmd inspect --format '{{.State.Running}}' "${container}")" = true ]
|
|
done
|
|
wait_clickhouse
|
|
wait_api_and_listeners
|
|
|
|
if [ -e "${REMOTE_OVERRIDE}" ]; then
|
|
[ -f "${REMOTE_OVERRIDE}" ] && [ ! -L "${REMOTE_OVERRIDE}" ]
|
|
PRIOR_OVERRIDE_PRESENT=1
|
|
fi
|
|
if [ -e "${REMOTE_CONFIG}" ]; then
|
|
[ -f "${REMOTE_CONFIG}" ] && [ ! -L "${REMOTE_CONFIG}" ]
|
|
PRIOR_CONFIG_PRESENT=1
|
|
fi
|
|
[ "${PRIOR_OVERRIDE_PRESENT}" -eq "${PRIOR_CONFIG_PRESENT}" ] \
|
|
|| { receipt check failed partial_managed_file_drift; exit 78; }
|
|
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
|
compose_managed_config
|
|
PRIOR_OVERRIDE_HASH="$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')"
|
|
PRIOR_OVERRIDE_MODE="$(stat -c '%a' "${REMOTE_OVERRIDE}")"
|
|
PRIOR_OVERRIDE_UID="$(stat -c '%u' "${REMOTE_OVERRIDE}")"
|
|
PRIOR_OVERRIDE_GID="$(stat -c '%g' "${REMOTE_OVERRIDE}")"
|
|
PRIOR_CONFIG_HASH="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
|
PRIOR_CONFIG_MODE="$(stat -c '%a' "${REMOTE_CONFIG}")"
|
|
PRIOR_CONFIG_UID="$(stat -c '%u' "${REMOTE_CONFIG}")"
|
|
PRIOR_CONFIG_GID="$(stat -c '%g' "${REMOTE_CONFIG}")"
|
|
bounded_cmd cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
|
|
bounded_cmd cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
|
|
[ "$(sha256sum "${RECEIPT_DIR}/prior-override.yaml" | awk '{print $1}')" = "${PRIOR_OVERRIDE_HASH}" ]
|
|
[ "$(sha256sum "${RECEIPT_DIR}/prior-backup-disk.xml" | awk '{print $1}')" = "${PRIOR_CONFIG_HASH}" ]
|
|
fi
|
|
|
|
PRIOR_DISK_COUNT="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
|
BEFORE_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
|
|
BEFORE_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
|
[ -n "${BEFORE_DATA_VOLUME}" ]
|
|
BEFORE_COLLECTOR_ID="$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
|
|
BEFORE_COLLECTOR_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
|
|
BEFORE_SIGNOZ_ID="$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
|
|
BEFORE_SIGNOZ_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
|
|
BEFORE_ZOOKEEPER_ID="$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
|
|
BEFORE_ZOOKEEPER_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"
|
|
|
|
receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid"
|
|
receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}"
|
|
receipt source_of_truth_diff pass \
|
|
"prior_override_${PRIOR_OVERRIDE_PRESENT}_hash_${PRIOR_OVERRIDE_HASH}_mode_${PRIOR_OVERRIDE_MODE}_uid_${PRIOR_OVERRIDE_UID}_gid_${PRIOR_OVERRIDE_GID}_prior_config_${PRIOR_CONFIG_PRESENT}_hash_${PRIOR_CONFIG_HASH}_mode_${PRIOR_CONFIG_MODE}_uid_${PRIOR_CONFIG_UID}_gid_${PRIOR_CONFIG_GID}_prior_disk_count_${PRIOR_DISK_COUNT}"
|
|
receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only"
|
|
receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback"
|
|
receipt check pass \
|
|
"compose_config_q_stage_hashes_runtime_preflight_dual_locks_backup_idle"
|
|
|
|
require_backup_idle pre_mutation
|
|
|
|
APPLY_STARTED=1
|
|
if [ -e "${HOST_BACKUP_DIR}" ]; then
|
|
[ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]
|
|
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
|
|
else
|
|
bounded_cmd sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
|
|
HOST_DIR_CREATED=1
|
|
fi
|
|
|
|
bounded_cmd install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
|
|
bounded_cmd install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
|
|
bounded_cmd mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
|
|
bounded_cmd mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
|
|
[ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
|
|
[ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
|
|
compose_managed_config
|
|
|
|
recreate_managed
|
|
wait_clickhouse
|
|
|
|
AFTER_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
|
|
AFTER_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
|
[ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ]
|
|
[ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ]
|
|
|
|
backup_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
|
config_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
|
[ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ]
|
|
[ "${config_mount}" = false ]
|
|
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
|
|
server_uid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
|
|
awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)"
|
|
server_gid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
|
|
awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)"
|
|
[ "${server_uid}" = 101 ]
|
|
[ "${server_gid}" = 101 ]
|
|
docker_cmd exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
|
|
'test -d /backups && test -w /backups'
|
|
|
|
disk_state="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
|
[ "${disk_state}" = "Local|0|0|0|1" ]
|
|
[ "$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
|
"EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ]
|
|
docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
|
|
| grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)'
|
|
|
|
[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
|
|
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
|
|
[ "$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
|
|
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
|
|
[ "$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
|
|
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
|
|
wait_api_and_listeners
|
|
|
|
receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged"
|
|
receipt post_verifier pass "disk_backups_local_rw_healthy_server_uid101_write_access_concurrency_disabled_collector_unchanged_listeners_4317_4318_8080_api_200"
|
|
receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_executed"
|
|
DEPLOY_VERIFIED=1
|
|
REMOTE_APPLY
|
|
|
|
LOCAL_APPLY_VERIFIED=1
|