feat(observability): close native SigNoz backup restore canary
This commit is contained in:
ogt
@@ -54,6 +54,9 @@ bash -n \
|
||||
scripts/backup/backup-langfuse.sh \
|
||||
scripts/backup/backup-monitoring.sh \
|
||||
scripts/backup/backup-signoz.sh \
|
||||
scripts/backup/clickhouse-native-backup.sh \
|
||||
scripts/backup/clickhouse-native-restore-drill.sh \
|
||||
scripts/backup/run-signoz-backup-canary.sh \
|
||||
scripts/backup/backup-open-webui.sh \
|
||||
scripts/backup/backup-clawbot.sh \
|
||||
scripts/backup/backup-configs.sh \
|
||||
@@ -81,6 +84,7 @@ python3 -m py_compile \
|
||||
scripts/ops/backup-alert-label-contract-check.py \
|
||||
scripts/ops/backup-alert-live-visibility-check.py \
|
||||
scripts/ops/recovery-scorecard-contract-check.py \
|
||||
scripts/backup/clickhouse-restore-inventory.py \
|
||||
scripts/ops/doc-secrets-sanity-check.py
|
||||
echo "Python 語法 OK"
|
||||
|
||||
|
||||
594
scripts/ops/deploy-signoz-clickhouse-backup-disk.sh
Executable file
594
scripts/ops/deploy-signoz-clickhouse-backup-disk.sh
Executable file
@@ -0,0 +1,594 @@
|
||||
#!/usr/bin/env bash
|
||||
# P0-OBS-002 - controlled deployment of the host110 ClickHouse native BACKUP disk.
|
||||
#
|
||||
# This script owns only an additive Compose override and one ClickHouse config
|
||||
# fragment. It never runs BACKUP/RESTORE, reads a dotenv/secret source, pulls an
|
||||
# image, or recreates another service.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml"
|
||||
LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
|
||||
|
||||
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
|
||||
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
||||
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
||||
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
||||
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
||||
REMOTE_PROJECT="signoz"
|
||||
REMOTE_SERVICE="clickhouse"
|
||||
CLICKHOUSE_CONTAINER="signoz-clickhouse"
|
||||
COLLECTOR_CONTAINER="signoz-otel-collector"
|
||||
SIGNOZ_CONTAINER="signoz"
|
||||
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
|
||||
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
||||
CONTAINER_BACKUP_DIR="/backups"
|
||||
CLICKHOUSE_DISK="backups"
|
||||
CLICKHOUSE_UID="101"
|
||||
CLICKHOUSE_GID="101"
|
||||
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
SAFE_HOME="/home/wooo"
|
||||
|
||||
SSH_OPTS=(
|
||||
-o BatchMode=yes
|
||||
-o ConnectTimeout=8
|
||||
-o ConnectionAttempts=1
|
||||
-o ServerAliveInterval=5
|
||||
-o ServerAliveCountMax=2
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: deploy-signoz-clickhouse-backup-disk.sh [--check|--apply]
|
||||
|
||||
Required environment:
|
||||
TRACE_ID Path-safe controlled-apply trace identifier
|
||||
RUN_ID Unique path-safe execution identifier
|
||||
WORK_ITEM_ID Path-safe work-item identifier
|
||||
|
||||
--check performs local source validation and remote read-only Compose/runtime
|
||||
checks. --apply atomically installs the two repo-owned artifacts, recreates only
|
||||
the ClickHouse service with --pull never, independently verifies runtime state,
|
||||
and restores the exact prior managed Compose state on failure.
|
||||
USAGE
|
||||
}
|
||||
|
||||
MODE="${1:---check}"
|
||||
case "${MODE}" in
|
||||
--check|--apply) ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) usage >&2; exit 64 ;;
|
||||
esac
|
||||
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
||||
|
||||
TRACE_ID="${TRACE_ID:-}"
|
||||
RUN_ID="${RUN_ID:-}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
||||
|
||||
valid_identity() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
||||
}
|
||||
|
||||
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
||||
valid_identity "${value}" || {
|
||||
printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
|
||||
exit 64
|
||||
}
|
||||
done
|
||||
|
||||
for command_name in python3 scp ssh; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 || {
|
||||
printf 'required local command missing: %s\n' "${command_name}" >&2
|
||||
exit 69
|
||||
}
|
||||
done
|
||||
|
||||
hash_file() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 "$1" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
validate_local_sources() {
|
||||
[ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ]
|
||||
[ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ]
|
||||
|
||||
python3 - "${LOCAL_CONFIG}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
root = ET.parse(path).getroot()
|
||||
assert root.tag == "clickhouse"
|
||||
disk = root.find("./storage_configuration/disks/backups")
|
||||
assert disk is not None
|
||||
assert disk.findtext("type") == "local"
|
||||
assert disk.findtext("path") == "/backups/"
|
||||
backups = root.find("./backups")
|
||||
assert backups is not None
|
||||
assert backups.findtext("allowed_disk") == "backups"
|
||||
assert backups.findtext("allowed_path") == "/backups/"
|
||||
assert backups.findtext("allow_concurrent_backups") == "false"
|
||||
assert backups.findtext("allow_concurrent_restores") == "false"
|
||||
PY
|
||||
|
||||
[ "$(grep -F -c ' clickhouse:' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
||||
[ "$(grep -F -c 'source: /backup/staging/signoz-clickhouse' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
||||
[ "$(grep -F -c 'target: /backups' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
||||
[ "$(grep -F -c 'target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
||||
! grep -Eq '^[[:space:]]*(image|build|command|environment):' "${LOCAL_OVERRIDE}"
|
||||
}
|
||||
|
||||
validate_local_sources
|
||||
OVERRIDE_SHA="$(hash_file "${LOCAL_OVERRIDE}")"
|
||||
CONFIG_SHA="$(hash_file "${LOCAL_CONFIG}")"
|
||||
|
||||
local_receipt() {
|
||||
local phase="$1" result="$2" detail="$3"
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
|
||||
}
|
||||
|
||||
remote_candidate_compose_check() {
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" \
|
||||
"env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
|
||||
< "${LOCAL_OVERRIDE}"
|
||||
}
|
||||
|
||||
remote_read_only_check() {
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
EXPECTED_OVERRIDE_SHA="$4"
|
||||
EXPECTED_CONFIG_SHA="$5"
|
||||
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
||||
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
||||
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
||||
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
||||
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
||||
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
SAFE_HOME="/home/wooo"
|
||||
|
||||
file_hash_or_absent() {
|
||||
if [ -f "$1" ] && [ ! -L "$1" ]; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
elif [ ! -e "$1" ]; then
|
||||
printf 'absent\n'
|
||||
else
|
||||
printf 'unsafe\n'
|
||||
fi
|
||||
}
|
||||
|
||||
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
|
||||
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
|
||||
docker inspect "${container}" >/dev/null
|
||||
done
|
||||
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
|
||||
config -q >/dev/null
|
||||
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
|
||||
config --services | grep -qx clickhouse
|
||||
|
||||
override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")"
|
||||
config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")"
|
||||
override_match=false
|
||||
config_match=false
|
||||
[ "${override_hash}" = "${EXPECTED_OVERRIDE_SHA}" ] && override_match=true
|
||||
[ "${config_hash}" = "${EXPECTED_CONFIG_SHA}" ] && config_match=true
|
||||
|
||||
host_dir=absent
|
||||
if [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]; then
|
||||
host_dir=present
|
||||
elif [ -e "${HOST_BACKUP_DIR}" ]; then
|
||||
host_dir=unsafe
|
||||
fi
|
||||
|
||||
disk_count="$(docker exec signoz-clickhouse clickhouse-client --query \
|
||||
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
||||
mount_count="$(docker inspect --format \
|
||||
'{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \
|
||||
signoz-clickhouse | tr -cd '1' | wc -c | xargs)"
|
||||
collector_running="$(docker inspect --format '{{.State.Running}}' signoz-otel-collector)"
|
||||
collector_restarts="$(docker inspect --format '{{.RestartCount}}' signoz-otel-collector)"
|
||||
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \
|
||||
"${config_match}" "${host_dir}" "${disk_count}" "${mount_count}" \
|
||||
"${collector_running}" "${collector_restarts}"
|
||||
REMOTE_CHECK
|
||||
}
|
||||
|
||||
local_receipt sensor_source pass "override_sha_${OVERRIDE_SHA}_config_sha_${CONFIG_SHA}"
|
||||
local_receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_disk_backups"
|
||||
remote_candidate_compose_check
|
||||
remote_read_only_check
|
||||
local_receipt check pass "compose_config_q_no_output_no_write"
|
||||
|
||||
if [ "${MODE}" = "--check" ]; then
|
||||
local_receipt terminal check_pass_no_write "source_and_remote_preflight_valid"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}"
|
||||
STAGE_CREATED=0
|
||||
|
||||
cleanup_stage() {
|
||||
if [ "${STAGE_CREATED}" -eq 1 ]; then
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
|
||||
set -u
|
||||
stage_dir="$1"
|
||||
rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml"
|
||||
rmdir "${stage_dir}" 2>/dev/null || true
|
||||
REMOTE_CLEANUP
|
||||
fi
|
||||
}
|
||||
trap cleanup_stage EXIT
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
|
||||
set -euo pipefail
|
||||
stage_dir="$1"
|
||||
[ ! -e "${stage_dir}" ]
|
||||
umask 077
|
||||
mkdir -m 0700 "${stage_dir}"
|
||||
REMOTE_STAGE
|
||||
STAGE_CREATED=1
|
||||
|
||||
scp -q "${SSH_OPTS[@]}" "${LOCAL_OVERRIDE}" \
|
||||
"${TARGET_HOST}:${STAGE_DIR}/override.yaml"
|
||||
scp -q "${SSH_OPTS[@]}" "${LOCAL_CONFIG}" \
|
||||
"${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml"
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
EXPECTED_OVERRIDE_SHA="$4"
|
||||
EXPECTED_CONFIG_SHA="$5"
|
||||
STAGE_DIR="$6"
|
||||
|
||||
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
||||
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
||||
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
||||
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
||||
STAGE_OVERRIDE="${STAGE_DIR}/override.yaml"
|
||||
STAGE_CONFIG="${STAGE_DIR}/backup_disk.xml"
|
||||
REMOTE_PROJECT="signoz"
|
||||
REMOTE_SERVICE="clickhouse"
|
||||
CLICKHOUSE_CONTAINER="signoz-clickhouse"
|
||||
COLLECTOR_CONTAINER="signoz-otel-collector"
|
||||
SIGNOZ_CONTAINER="signoz"
|
||||
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
|
||||
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
||||
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
SAFE_HOME="/home/wooo"
|
||||
LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock"
|
||||
|
||||
APPLY_STARTED=0
|
||||
DEPLOY_VERIFIED=0
|
||||
HOST_DIR_CREATED=0
|
||||
PRIOR_OVERRIDE_PRESENT=0
|
||||
PRIOR_CONFIG_PRESENT=0
|
||||
PRIOR_DISK_COUNT=0
|
||||
BEFORE_IMAGE_ID=""
|
||||
BEFORE_DATA_VOLUME=""
|
||||
BEFORE_COLLECTOR_ID=""
|
||||
BEFORE_COLLECTOR_RESTARTS=""
|
||||
BEFORE_SIGNOZ_ID=""
|
||||
BEFORE_SIGNOZ_RESTARTS=""
|
||||
BEFORE_ZOOKEEPER_ID=""
|
||||
BEFORE_ZOOKEEPER_RESTARTS=""
|
||||
|
||||
receipt() {
|
||||
local phase="$1" result="$2" detail="$3"
|
||||
local line
|
||||
line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
|
||||
printf '%s\n' "${line}"
|
||||
printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
|
||||
}
|
||||
|
||||
compose_base_config() {
|
||||
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
||||
-f "${REMOTE_BASE}" config -q >/dev/null 2>&1
|
||||
}
|
||||
|
||||
compose_managed_config() {
|
||||
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
||||
-f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" config -q >/dev/null 2>&1
|
||||
}
|
||||
|
||||
compose_stage_config() {
|
||||
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
||||
-f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config -q >/dev/null 2>&1
|
||||
}
|
||||
|
||||
recreate_managed() {
|
||||
timeout --signal=TERM --kill-after=30 300 \
|
||||
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
||||
-f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
|
||||
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
recreate_base() {
|
||||
timeout --signal=TERM --kill-after=30 300 \
|
||||
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
||||
-f "${REMOTE_BASE}" \
|
||||
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
wait_clickhouse() {
|
||||
local attempt running health
|
||||
for attempt in $(seq 1 36); do
|
||||
running="$(docker inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
|
||||
health="$(docker inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
|
||||
"${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
|
||||
if [ "${running}" = true ] && [ "${health}" = healthy ] \
|
||||
&& timeout --kill-after=5 10 docker exec "${CLICKHOUSE_CONTAINER}" \
|
||||
clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
listener_up() {
|
||||
local port="$1"
|
||||
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
||||
}
|
||||
|
||||
wait_api_and_listeners() {
|
||||
local attempt api_code
|
||||
for attempt in $(seq 1 30); do
|
||||
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}" 2>/dev/null || true)"
|
||||
if [ "${api_code}" = 200 ] && listener_up 4317 && listener_up 4318 && listener_up 8080; then
|
||||
return 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
restore_managed_files() {
|
||||
local tmp
|
||||
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
||||
tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}"
|
||||
install -m 0644 "${RECEIPT_DIR}/prior-override.yaml" "${tmp}"
|
||||
mv -f "${tmp}" "${REMOTE_OVERRIDE}"
|
||||
else
|
||||
rm -f "${REMOTE_OVERRIDE}"
|
||||
fi
|
||||
|
||||
if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then
|
||||
tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}"
|
||||
install -m 0644 "${RECEIPT_DIR}/prior-backup-disk.xml" "${tmp}"
|
||||
mv -f "${tmp}" "${REMOTE_CONFIG}"
|
||||
else
|
||||
rm -f "${REMOTE_CONFIG}"
|
||||
fi
|
||||
}
|
||||
|
||||
rollback_deploy() {
|
||||
local disk_count rollback_dir_terminal=retained
|
||||
restore_managed_files || return 1
|
||||
|
||||
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
||||
compose_managed_config || return 1
|
||||
recreate_managed || return 1
|
||||
else
|
||||
compose_base_config || return 1
|
||||
recreate_base || return 1
|
||||
fi
|
||||
wait_clickhouse || return 1
|
||||
wait_api_and_listeners || return 1
|
||||
|
||||
disk_count="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1
|
||||
[ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1
|
||||
[ "$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
|
||||
[ "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
|
||||
[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
|
||||
[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1
|
||||
|
||||
if [ "${HOST_DIR_CREATED}" -eq 1 ]; then
|
||||
if sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
|
||||
rollback_dir_terminal=removed_empty
|
||||
else
|
||||
rollback_dir_terminal=retained_nonempty_no_delete
|
||||
fi
|
||||
fi
|
||||
receipt rollback pass "prior_compose_state_restored_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
local rc=$? rollback_rc=0 terminal=failed
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
||||
rollback_deploy
|
||||
rollback_rc=$?
|
||||
if [ "${rollback_rc}" -ne 0 ]; then
|
||||
receipt rollback failed "prior_compose_state_restore_failed"
|
||||
rc=91
|
||||
fi
|
||||
fi
|
||||
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
|
||||
terminal=pass
|
||||
fi
|
||||
receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}"
|
||||
exit "${rc}"
|
||||
}
|
||||
|
||||
mkdir -p "${RECEIPT_ROOT}"
|
||||
[ ! -e "${RECEIPT_DIR}" ]
|
||||
mkdir -m 0700 "${RECEIPT_DIR}"
|
||||
: > "${RECEIPT_LOG}"
|
||||
trap on_exit EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
exec 9>>"${LOCK_FILE}"
|
||||
flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; }
|
||||
|
||||
for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss sudo timeout; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 \
|
||||
|| { receipt check failed "required_command_missing_${command_name}"; exit 69; }
|
||||
done
|
||||
|
||||
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
||||
[ -f "${STAGE_OVERRIDE}" ] && [ ! -L "${STAGE_OVERRIDE}" ]
|
||||
[ -f "${STAGE_CONFIG}" ] && [ ! -L "${STAGE_CONFIG}" ]
|
||||
[ "$(sha256sum "${STAGE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
|
||||
[ "$(sha256sum "${STAGE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
|
||||
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
|
||||
[ -d "${REMOTE_DIR}" ] && [ ! -L "${REMOTE_DIR}" ] && [ -w "${REMOTE_DIR}" ]
|
||||
[ -d /backup ] && [ ! -L /backup ]
|
||||
|
||||
compose_base_config
|
||||
compose_stage_config
|
||||
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
|
||||
-f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
|
||||
| grep -qx "${REMOTE_SERVICE}"
|
||||
|
||||
if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
||||
receipt check failed active_signoz_backup_detected
|
||||
exit 75
|
||||
fi
|
||||
active_native_backups="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
|
||||
[ "${active_native_backups}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
|
||||
|
||||
for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do
|
||||
[ "$(docker inspect --format '{{.State.Running}}' "${container}")" = true ]
|
||||
done
|
||||
wait_clickhouse
|
||||
wait_api_and_listeners
|
||||
|
||||
if [ -e "${REMOTE_OVERRIDE}" ]; then
|
||||
[ -f "${REMOTE_OVERRIDE}" ] && [ ! -L "${REMOTE_OVERRIDE}" ]
|
||||
PRIOR_OVERRIDE_PRESENT=1
|
||||
fi
|
||||
if [ -e "${REMOTE_CONFIG}" ]; then
|
||||
[ -f "${REMOTE_CONFIG}" ] && [ ! -L "${REMOTE_CONFIG}" ]
|
||||
PRIOR_CONFIG_PRESENT=1
|
||||
fi
|
||||
[ "${PRIOR_OVERRIDE_PRESENT}" -eq "${PRIOR_CONFIG_PRESENT}" ] \
|
||||
|| { receipt check failed partial_managed_file_drift; exit 78; }
|
||||
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
||||
compose_managed_config
|
||||
cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
|
||||
cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
|
||||
fi
|
||||
|
||||
PRIOR_DISK_COUNT="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
||||
BEFORE_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
BEFORE_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
[ -n "${BEFORE_DATA_VOLUME}" ]
|
||||
BEFORE_COLLECTOR_ID="$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
|
||||
BEFORE_COLLECTOR_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
|
||||
BEFORE_SIGNOZ_ID="$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
|
||||
BEFORE_SIGNOZ_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
|
||||
BEFORE_ZOOKEEPER_ID="$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
|
||||
BEFORE_ZOOKEEPER_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"
|
||||
|
||||
receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid"
|
||||
receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}"
|
||||
receipt source_of_truth_diff pass "prior_override_${PRIOR_OVERRIDE_PRESENT}_prior_config_${PRIOR_CONFIG_PRESENT}_prior_disk_count_${PRIOR_DISK_COUNT}"
|
||||
receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only"
|
||||
receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback"
|
||||
receipt check pass "compose_config_q_stage_hashes_runtime_preflight_backup_idle"
|
||||
|
||||
APPLY_STARTED=1
|
||||
if [ -e "${HOST_BACKUP_DIR}" ]; then
|
||||
[ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]
|
||||
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
|
||||
else
|
||||
sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
|
||||
HOST_DIR_CREATED=1
|
||||
fi
|
||||
|
||||
install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
|
||||
install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
|
||||
mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
|
||||
mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
|
||||
[ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
|
||||
[ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
|
||||
compose_managed_config
|
||||
|
||||
recreate_managed
|
||||
wait_clickhouse
|
||||
|
||||
AFTER_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
AFTER_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
[ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ]
|
||||
[ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ]
|
||||
|
||||
backup_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
config_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
[ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ]
|
||||
[ "${config_mount}" = false ]
|
||||
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
|
||||
server_uid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
|
||||
awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)"
|
||||
server_gid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
|
||||
awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)"
|
||||
[ "${server_uid}" = 101 ]
|
||||
[ "${server_gid}" = 101 ]
|
||||
docker exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
|
||||
'test -d /backups && test -w /backups'
|
||||
|
||||
disk_state="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
||||
[ "${disk_state}" = "Local|0|0|0|1" ]
|
||||
[ "$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ]
|
||||
docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
|
||||
| grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)'
|
||||
|
||||
[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
|
||||
[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
|
||||
[ "$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
|
||||
[ "$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
|
||||
[ "$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
|
||||
[ "$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
|
||||
wait_api_and_listeners
|
||||
|
||||
receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged"
|
||||
receipt post_verifier pass "disk_backups_local_rw_healthy_server_uid101_write_access_concurrency_disabled_collector_unchanged_listeners_4317_4318_8080_api_200"
|
||||
receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_executed"
|
||||
DEPLOY_VERIFIED=1
|
||||
REMOTE_APPLY
|
||||
|
||||
trap - EXIT
|
||||
cleanup_stage
|
||||
local_receipt terminal pass "remote_controlled_apply_and_postverify_complete"
|
||||
801
scripts/ops/deploy-signoz-native-backup-toolchain.sh
Executable file
801
scripts/ops/deploy-signoz-native-backup-toolchain.sh
Executable file
@@ -0,0 +1,801 @@
|
||||
#!/usr/bin/env bash
|
||||
# P0-OBS-002 - atomically deploy the host110 SigNoz native backup toolchain.
|
||||
#
|
||||
# This deployer owns seven repo-sourced files only. It does not run a backup or
|
||||
# restore, restart a container, pull an image, or inspect a secret-bearing
|
||||
# runtime source.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TARGET_HOST="wooo@192.168.0.110"
|
||||
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
SAFE_HOME="/home/wooo"
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
|
||||
REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"
|
||||
REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}"
|
||||
RECEIPT_ROOT="${SIGNOZ_TOOLCHAIN_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-native-backup-toolchain}"
|
||||
STAGE_ROOT="${SIGNOZ_TOOLCHAIN_STAGE_ROOT:-/tmp}"
|
||||
|
||||
LABELS=(
|
||||
backup-signoz.sh
|
||||
clickhouse-native-backup.sh
|
||||
clickhouse-native-restore-drill.sh
|
||||
clickhouse-restore-inventory.py
|
||||
run-signoz-backup-canary.sh
|
||||
backup_disk.xml
|
||||
isolated-cluster.xml
|
||||
)
|
||||
LOCAL_SOURCES=(
|
||||
"${ROOT_DIR}/scripts/backup/backup-signoz.sh"
|
||||
"${ROOT_DIR}/scripts/backup/clickhouse-native-backup.sh"
|
||||
"${ROOT_DIR}/scripts/backup/clickhouse-native-restore-drill.sh"
|
||||
"${ROOT_DIR}/scripts/backup/clickhouse-restore-inventory.py"
|
||||
"${ROOT_DIR}/scripts/backup/run-signoz-backup-canary.sh"
|
||||
"${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
|
||||
"${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
)
|
||||
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
||||
|
||||
SSH_OPTS=(
|
||||
-o BatchMode=yes
|
||||
-o ConnectTimeout=8
|
||||
-o ConnectionAttempts=1
|
||||
-o ServerAliveInterval=5
|
||||
-o ServerAliveCountMax=2
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: deploy-signoz-native-backup-toolchain.sh [--check|--apply]
|
||||
|
||||
Required environment:
|
||||
TRACE_ID Path-safe controlled-apply trace identifier
|
||||
RUN_ID Unique path-safe execution identifier
|
||||
WORK_ITEM_ID Path-safe work-item identifier
|
||||
|
||||
Optional safe absolute path environment:
|
||||
SIGNOZ_TOOLCHAIN_SCRIPT_DIR
|
||||
SIGNOZ_TOOLCHAIN_CONFIG_ROOT
|
||||
SIGNOZ_TOOLCHAIN_RECEIPT_ROOT
|
||||
SIGNOZ_TOOLCHAIN_STAGE_ROOT
|
||||
|
||||
--check validates local sources and performs a remote read-only target diff and
|
||||
runtime readiness check. --apply stages all seven sources, validates hashes and
|
||||
syntax, atomically replaces exact targets, and rolls every target back to its
|
||||
prior hash/mode/owner if any bounded verifier fails.
|
||||
|
||||
Flat restore-driver invocation must set:
|
||||
CLICKHOUSE_RESTORE_INVENTORY_HELPER=<script-dir>/clickhouse-restore-inventory.py
|
||||
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG=<config-root>/config.d/backup_disk.xml
|
||||
CLICKHOUSE_RESTORE_CLUSTER_CONFIG=<config-root>/restore-drill/config.d/isolated-cluster.xml
|
||||
USAGE
|
||||
}
|
||||
|
||||
MODE="${1:---check}"
|
||||
case "${MODE}" in
|
||||
--check|--apply) ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) usage >&2; exit 64 ;;
|
||||
esac
|
||||
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
||||
|
||||
TRACE_ID="${TRACE_ID:-}"
|
||||
RUN_ID="${RUN_ID:-}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
||||
|
||||
valid_identity() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
||||
}
|
||||
|
||||
safe_absolute_path() {
|
||||
local path="$1"
|
||||
[[ "${path}" == /* ]] || return 1
|
||||
[[ "${path}" != / && "${path}" != */ ]] || return 1
|
||||
[[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1
|
||||
case "/${path#/}/" in
|
||||
*'//'*|*'/../'*|*'/./'*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
paths_overlap() {
|
||||
local left="$1" right="$2"
|
||||
[[ "${left}" = "${right}" || "${left}" == "${right}"/* || "${right}" == "${left}"/* ]]
|
||||
}
|
||||
|
||||
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
||||
valid_identity "${value}" || {
|
||||
printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
|
||||
exit 64
|
||||
}
|
||||
done
|
||||
|
||||
for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
|
||||
safe_absolute_path "${path}" || {
|
||||
printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2
|
||||
exit 64
|
||||
}
|
||||
done
|
||||
case "${REMOTE_SCRIPT_DIR}" in /backup/*) ;; *) printf 'script dir must stay under /backup\n' >&2; exit 64 ;; esac
|
||||
case "${REMOTE_CONFIG_ROOT}" in /backup/*) ;; *) printf 'config root must stay under /backup\n' >&2; exit 64 ;; esac
|
||||
case "${RECEIPT_ROOT}" in /backup/*) ;; *) printf 'receipt root must stay under /backup\n' >&2; exit 64 ;; esac
|
||||
case "${STAGE_ROOT}" in /tmp|/tmp/*|/backup/deploy-staging|/backup/deploy-staging/*) ;;
|
||||
*) printf 'stage root must stay under /tmp or /backup/deploy-staging\n' >&2; exit 64 ;;
|
||||
esac
|
||||
if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \
|
||||
|| paths_overlap "${REMOTE_SCRIPT_DIR}" "${RECEIPT_ROOT}" \
|
||||
|| paths_overlap "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" \
|
||||
|| paths_overlap "${STAGE_ROOT}" "${REMOTE_SCRIPT_DIR}" \
|
||||
|| paths_overlap "${STAGE_ROOT}" "${REMOTE_CONFIG_ROOT}" \
|
||||
|| paths_overlap "${STAGE_ROOT}" "${RECEIPT_ROOT}"; then
|
||||
printf 'toolchain target, receipt, and stage paths must not overlap\n' >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
for command_name in bash python3 scp ssh; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 || {
|
||||
printf 'required local command missing: %s\n' "${command_name}" >&2
|
||||
exit 69
|
||||
}
|
||||
done
|
||||
|
||||
hash_file() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 "$1" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
validate_xml_pair() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
backup_path = Path(sys.argv[1])
|
||||
cluster_path = Path(sys.argv[2])
|
||||
backup = ET.parse(backup_path).getroot()
|
||||
cluster = ET.parse(cluster_path).getroot()
|
||||
assert backup.tag == "clickhouse"
|
||||
assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
|
||||
assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
|
||||
assert backup.findtext("./backups/allowed_disk") == "backups"
|
||||
assert backup.findtext("./backups/allowed_path") == "/backups/"
|
||||
assert backup.findtext("./backups/allow_concurrent_backups") == "false"
|
||||
assert backup.findtext("./backups/allow_concurrent_restores") == "false"
|
||||
assert cluster.tag == "clickhouse"
|
||||
assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
|
||||
assert cluster.findtext("./zookeeper/node/port") == "2181"
|
||||
assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
|
||||
assert cluster.findtext("./remote_servers/cluster/shard/replica/port") == "9000"
|
||||
assert cluster.findtext("./macros/shard") == "01"
|
||||
replica = cluster.find("./macros/replica")
|
||||
assert replica is not None
|
||||
assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
|
||||
PY
|
||||
}
|
||||
|
||||
validate_local_sources() {
|
||||
local index source
|
||||
for index in "${!LOCAL_SOURCES[@]}"; do
|
||||
source="${LOCAL_SOURCES[index]}"
|
||||
[ -f "${source}" ] && [ ! -L "${source}" ]
|
||||
[ -s "${source}" ]
|
||||
done
|
||||
|
||||
for index in 0 1 2 4; do
|
||||
bash -n "${LOCAL_SOURCES[index]}"
|
||||
done
|
||||
python3 - "${LOCAL_SOURCES[3]}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
||||
PY
|
||||
validate_xml_pair "${LOCAL_SOURCES[5]}" "${LOCAL_SOURCES[6]}"
|
||||
|
||||
python3 - "${LOCAL_SOURCES[2]}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
text = Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||
for token in (
|
||||
"CLICKHOUSE_RESTORE_INVENTORY_HELPER",
|
||||
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG",
|
||||
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG",
|
||||
):
|
||||
assert token in text
|
||||
PY
|
||||
}
|
||||
|
||||
validate_local_sources
|
||||
SOURCE_HASHES=()
|
||||
SOURCE_DETAIL=""
|
||||
for index in "${!LOCAL_SOURCES[@]}"; do
|
||||
source_hash="$(hash_file "${LOCAL_SOURCES[index]}")"
|
||||
SOURCE_HASHES+=("${source_hash}")
|
||||
SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_"
|
||||
done
|
||||
SOURCE_DETAIL="${SOURCE_DETAIL%_}"
|
||||
|
||||
CONFIG_DIR="${REMOTE_CONFIG_ROOT}/config.d"
|
||||
RESTORE_CONFIG_DIR="${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
||||
TARGET_PATHS=(
|
||||
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
||||
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
||||
"${CONFIG_DIR}/backup_disk.xml"
|
||||
"${RESTORE_CONFIG_DIR}/isolated-cluster.xml"
|
||||
)
|
||||
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-native-backup-toolchain.${RUN_ID}"
|
||||
|
||||
local_receipt() {
|
||||
local phase="$1" result="$2" detail="$3"
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
|
||||
}
|
||||
|
||||
remote_read_only_check() {
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \
|
||||
"${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
REMOTE_SCRIPT_DIR="$4"
|
||||
REMOTE_CONFIG_ROOT="$5"
|
||||
RECEIPT_ROOT="$6"
|
||||
STAGE_ROOT="$7"
|
||||
shift 7
|
||||
EXPECTED_HASHES=("$@")
|
||||
LABELS=(
|
||||
backup-signoz.sh
|
||||
clickhouse-native-backup.sh
|
||||
clickhouse-native-restore-drill.sh
|
||||
clickhouse-restore-inventory.py
|
||||
run-signoz-backup-canary.sh
|
||||
backup_disk.xml
|
||||
isolated-cluster.xml
|
||||
)
|
||||
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
||||
TARGETS=(
|
||||
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
||||
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
||||
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
|
||||
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
|
||||
)
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
|
||||
[ -d /backup ] && [ ! -L /backup ]
|
||||
for command_name in awk curl docker pgrep sha256sum ss stat; do
|
||||
command -v "${command_name}" >/dev/null 2>&1
|
||||
done
|
||||
|
||||
listener_up() {
|
||||
local port="$1"
|
||||
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
||||
}
|
||||
|
||||
validate_existing_ancestors() {
|
||||
local requested="$1" current="" component
|
||||
local -a components
|
||||
IFS='/' read -r -a components <<<"${requested#/}"
|
||||
for component in "${components[@]}"; do
|
||||
[ -n "${component}" ] || continue
|
||||
current="${current}/${component}"
|
||||
if [ -e "${current}" ] || [ -L "${current}" ]; then
|
||||
[ -d "${current}" ] && [ ! -L "${current}" ]
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
runtime_line() {
|
||||
local container="$1" value running health
|
||||
value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
|
||||
IFS='|' read -r _ _ _ running health <<<"${value}"
|
||||
[ "${running}" = true ]
|
||||
case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
||||
printf 'active SigNoz backup toolchain process detected\n' >&2
|
||||
exit 75
|
||||
fi
|
||||
validate_existing_ancestors "${REMOTE_SCRIPT_DIR}"
|
||||
validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/config.d"
|
||||
validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
||||
validate_existing_ancestors "${RECEIPT_ROOT}"
|
||||
validate_existing_ancestors "${STAGE_ROOT}"
|
||||
[ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ]
|
||||
active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
|
||||
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
|
||||
[ "${active_operations}" = 0 ]
|
||||
|
||||
runtime_summary=""
|
||||
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
|
||||
runtime_summary+="${container}_$(runtime_line "${container}")_"
|
||||
done
|
||||
for port in 4317 4318 8080; do listener_up "${port}"; done
|
||||
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
|
||||
[ "${api_code}" = 200 ]
|
||||
|
||||
matching=0
|
||||
drift=0
|
||||
absent=0
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
target="${TARGETS[index]}"
|
||||
if [ -e "${target}" ] || [ -L "${target}" ]; then
|
||||
[ -f "${target}" ] && [ ! -L "${target}" ]
|
||||
current_hash="$(sha256sum "${target}" | awk '{print $1}')"
|
||||
current_mode="$(stat -c '%a' "${target}")"
|
||||
if [ "${current_hash}" = "${EXPECTED_HASHES[index]}" ] && [ "0${current_mode}" = "${MODES[index]}" ]; then
|
||||
matching=$((matching + 1))
|
||||
else
|
||||
drift=$((drift + 1))
|
||||
fi
|
||||
else
|
||||
absent=$((absent + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"source_of_truth_diff","result":"pass","detail":"matching_%s_drift_%s_absent_%s_active_operations_0_api_200_ports_4317_4318_8080"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${matching}" "${drift}" "${absent}"
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"runtime_readiness","result":"pass","detail":"container_identity_started_at_restart_count_health_captured_%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${runtime_summary%_}"
|
||||
REMOTE_CHECK
|
||||
}
|
||||
|
||||
local_receipt sensor_source pass "${SOURCE_DETAIL}"
|
||||
local_receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
|
||||
remote_read_only_check
|
||||
local_receipt check pass "local_syntax_xml_python_compile_and_remote_read_only_preflight"
|
||||
|
||||
if [ "${MODE}" = --check ]; then
|
||||
local_receipt terminal check_pass_no_write "remote_target_unchanged_no_stage_no_candidate"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
STAGE_CREATED=0
|
||||
cleanup_stage() {
|
||||
if [ "${STAGE_CREATED}" -eq 1 ]; then
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
|
||||
set -u
|
||||
stage_dir="$1"
|
||||
if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
|
||||
rm -rf "${stage_dir}/pycache" "${stage_dir}/candidate-pycache"
|
||||
rm -f \
|
||||
"${stage_dir}/backup-signoz.sh" \
|
||||
"${stage_dir}/clickhouse-native-backup.sh" \
|
||||
"${stage_dir}/clickhouse-native-restore-drill.sh" \
|
||||
"${stage_dir}/clickhouse-restore-inventory.py" \
|
||||
"${stage_dir}/run-signoz-backup-canary.sh" \
|
||||
"${stage_dir}/backup_disk.xml" \
|
||||
"${stage_dir}/isolated-cluster.xml"
|
||||
rmdir "${stage_dir}" 2>/dev/null || true
|
||||
fi
|
||||
REMOTE_CLEANUP
|
||||
fi
|
||||
}
|
||||
trap cleanup_stage EXIT
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
|
||||
set -euo pipefail
|
||||
stage_dir="$1"
|
||||
[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
||||
stage_root="${stage_dir%/*}"
|
||||
[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ]
|
||||
umask 077
|
||||
mkdir -m 0700 "${stage_dir}"
|
||||
REMOTE_STAGE
|
||||
STAGE_CREATED=1
|
||||
|
||||
for index in "${!LOCAL_SOURCES[@]}"; do
|
||||
scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
|
||||
"${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"
|
||||
done
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \
|
||||
"${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
REMOTE_SCRIPT_DIR="$4"
|
||||
REMOTE_CONFIG_ROOT="$5"
|
||||
RECEIPT_ROOT="$6"
|
||||
STAGE_DIR="$7"
|
||||
shift 7
|
||||
EXPECTED_HASHES=("$@")
|
||||
LABELS=(
|
||||
backup-signoz.sh
|
||||
clickhouse-native-backup.sh
|
||||
clickhouse-native-restore-drill.sh
|
||||
clickhouse-restore-inventory.py
|
||||
run-signoz-backup-canary.sh
|
||||
backup_disk.xml
|
||||
isolated-cluster.xml
|
||||
)
|
||||
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
||||
STAGED=(
|
||||
"${STAGE_DIR}/backup-signoz.sh"
|
||||
"${STAGE_DIR}/clickhouse-native-backup.sh"
|
||||
"${STAGE_DIR}/clickhouse-native-restore-drill.sh"
|
||||
"${STAGE_DIR}/clickhouse-restore-inventory.py"
|
||||
"${STAGE_DIR}/run-signoz-backup-canary.sh"
|
||||
"${STAGE_DIR}/backup_disk.xml"
|
||||
"${STAGE_DIR}/isolated-cluster.xml"
|
||||
)
|
||||
TARGETS=(
|
||||
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
||||
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
||||
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
|
||||
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
|
||||
)
|
||||
TARGET_DIRS=(
|
||||
"${REMOTE_SCRIPT_DIR}"
|
||||
"${REMOTE_CONFIG_ROOT}/config.d"
|
||||
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
||||
)
|
||||
CONTAINERS=(signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1)
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
||||
PRIOR_DIR="${RECEIPT_DIR}/prior"
|
||||
PRIOR_MANIFEST="${RECEIPT_DIR}/prior-manifest.tsv"
|
||||
DEPLOYED_MANIFEST="${RECEIPT_DIR}/deployed-manifest.tsv"
|
||||
RUNTIME_BEFORE="${RECEIPT_DIR}/runtime-before.tsv"
|
||||
RUNTIME_AFTER="${RECEIPT_DIR}/runtime-after.tsv"
|
||||
LOCK_FILE="/tmp/awoooi-signoz-native-backup-toolchain.lock"
|
||||
|
||||
APPLY_STARTED=0
|
||||
DEPLOY_VERIFIED=0
|
||||
RUNTIME_BASELINE_CAPTURED=0
|
||||
LOCK_HELD=0
|
||||
REMOTE_UID="$(id -u)"
|
||||
REMOTE_GID="$(id -g)"
|
||||
CREATED_DIRS=()
|
||||
PRIOR_PRESENT=()
|
||||
PRIOR_MODES=()
|
||||
PRIOR_UIDS=()
|
||||
PRIOR_GIDS=()
|
||||
PRIOR_HASHES=()
|
||||
|
||||
for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 || {
|
||||
printf 'required remote command missing: %s\n' "${command_name}" >&2
|
||||
exit 69
|
||||
}
|
||||
done
|
||||
if [ "${REMOTE_UID}" -ne 0 ]; then
|
||||
command -v sudo >/dev/null 2>&1 || { printf 'required remote command missing: sudo\n' >&2; exit 69; }
|
||||
sudo -n true
|
||||
fi
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
|
||||
[ -d /backup ] && [ ! -L /backup ]
|
||||
|
||||
run_root() {
|
||||
if [ "${REMOTE_UID}" -eq 0 ]; then
|
||||
"$@"
|
||||
else
|
||||
sudo -n "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
root_hash() {
|
||||
if [ "${REMOTE_UID}" -eq 0 ]; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
else
|
||||
sudo -n sha256sum "$1" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_dir_tree() {
|
||||
local requested="$1" record_created="$2" current="" component
|
||||
local -a components
|
||||
IFS='/' read -r -a components <<<"${requested#/}"
|
||||
for component in "${components[@]}"; do
|
||||
[ -n "${component}" ] || continue
|
||||
current="${current}/${component}"
|
||||
if [ -e "${current}" ] || [ -L "${current}" ]; then
|
||||
[ -d "${current}" ] && [ ! -L "${current}" ]
|
||||
else
|
||||
run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0755 "${current}"
|
||||
if [ "${record_created}" = true ]; then
|
||||
CREATED_DIRS+=("${current}")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
ensure_dir_tree "${RECEIPT_ROOT}" false
|
||||
[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]
|
||||
run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0700 "${RECEIPT_DIR}"
|
||||
install -d -m 0700 "${PRIOR_DIR}"
|
||||
: > "${RECEIPT_LOG}"
|
||||
|
||||
receipt() {
|
||||
local phase="$1" result="$2" detail="$3" line
|
||||
line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
|
||||
printf '%s\n' "${line}"
|
||||
printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
|
||||
}
|
||||
|
||||
listener_up() {
|
||||
local port="$1"
|
||||
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
||||
}
|
||||
|
||||
runtime_snapshot() {
|
||||
local container value running health port api_code
|
||||
for container in "${CONTAINERS[@]}"; do
|
||||
value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
|
||||
IFS='|' read -r _ _ _ running health <<<"${value}"
|
||||
[ "${running}" = true ]
|
||||
case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
|
||||
printf 'container\t%s\t%s\n' "${container}" "${value}"
|
||||
done
|
||||
for port in 4317 4318 8080; do
|
||||
listener_up "${port}"
|
||||
printf 'listener\t%s\tup\n' "${port}"
|
||||
done
|
||||
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
|
||||
[ "${api_code}" = 200 ]
|
||||
printf 'api\thealth\t%s\n' "${api_code}"
|
||||
}
|
||||
|
||||
validate_xml_pair() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
backup_path = Path(sys.argv[1])
|
||||
cluster_path = Path(sys.argv[2])
|
||||
backup = ET.parse(backup_path).getroot()
|
||||
cluster = ET.parse(cluster_path).getroot()
|
||||
assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
|
||||
assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
|
||||
assert backup.findtext("./backups/allowed_disk") == "backups"
|
||||
assert backup.findtext("./backups/allowed_path") == "/backups/"
|
||||
assert backup.findtext("./backups/allow_concurrent_backups") == "false"
|
||||
assert backup.findtext("./backups/allow_concurrent_restores") == "false"
|
||||
assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
|
||||
assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
|
||||
assert cluster.findtext("./macros/shard") == "01"
|
||||
replica = cluster.find("./macros/replica")
|
||||
assert replica is not None
|
||||
assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
|
||||
PY
|
||||
}
|
||||
|
||||
cleanup_transients() {
|
||||
local index rc=0
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
run_root rm -f \
|
||||
"${TARGETS[index]}.candidate.${RUN_ID}" \
|
||||
"${TARGETS[index]}.rollback.${RUN_ID}" || rc=1
|
||||
done
|
||||
if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then
|
||||
rm -rf "${STAGE_DIR}/pycache" "${STAGE_DIR}/candidate-pycache" || rc=1
|
||||
rm -f "${STAGED[@]}" || rc=1
|
||||
rmdir "${STAGE_DIR}" 2>/dev/null || rc=1
|
||||
else
|
||||
rc=1
|
||||
fi
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
verify_residue_zero() {
|
||||
local index
|
||||
[ ! -e "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
[ ! -e "${TARGETS[index]}.candidate.${RUN_ID}" ] \
|
||||
&& [ ! -L "${TARGETS[index]}.candidate.${RUN_ID}" ] || return 1
|
||||
[ ! -e "${TARGETS[index]}.rollback.${RUN_ID}" ] \
|
||||
&& [ ! -L "${TARGETS[index]}.rollback.${RUN_ID}" ] || return 1
|
||||
done
|
||||
}
|
||||
|
||||
rollback_targets() (
|
||||
set -euo pipefail
|
||||
local index target rollback_candidate actual_mode actual_hash
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
target="${TARGETS[index]}"
|
||||
if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
|
||||
rollback_candidate="${target}.rollback.${RUN_ID}"
|
||||
run_root install \
|
||||
-o "${PRIOR_UIDS[index]}" -g "${PRIOR_GIDS[index]}" \
|
||||
-m "${PRIOR_MODES[index]}" \
|
||||
"${PRIOR_DIR}/${LABELS[index]}" "${rollback_candidate}"
|
||||
[ "$(root_hash "${rollback_candidate}")" = "${PRIOR_HASHES[index]}" ]
|
||||
run_root mv -f "${rollback_candidate}" "${target}"
|
||||
actual_mode="$(stat -c '%a' "${target}")"
|
||||
actual_hash="$(root_hash "${target}")"
|
||||
[ "${actual_mode}" = "${PRIOR_MODES[index]}" ]
|
||||
[ "${actual_hash}" = "${PRIOR_HASHES[index]}" ]
|
||||
else
|
||||
run_root rm -f "${target}"
|
||||
[ ! -e "${target}" ] && [ ! -L "${target}" ]
|
||||
fi
|
||||
done
|
||||
|
||||
for ((index=${#CREATED_DIRS[@]} - 1; index >= 0; index--)); do
|
||||
run_root rmdir "${CREATED_DIRS[index]}"
|
||||
done
|
||||
|
||||
if [ "${RUNTIME_BASELINE_CAPTURED}" -eq 1 ]; then
|
||||
runtime_snapshot > "${RECEIPT_DIR}/runtime-rollback.tsv"
|
||||
cmp -s "${RUNTIME_BEFORE}" "${RECEIPT_DIR}/runtime-rollback.tsv"
|
||||
fi
|
||||
receipt rollback pass "all_prior_files_modes_hashes_owners_and_runtime_restored"
|
||||
)
|
||||
|
||||
on_exit() {
|
||||
local rc=$? rollback_rc=0 cleanup_rc=0 residue_rc=0 terminal=failed
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
||||
rollback_targets
|
||||
rollback_rc=$?
|
||||
if [ "${rollback_rc}" -ne 0 ]; then
|
||||
receipt rollback failed "prior_state_restore_or_runtime_verifier_failed"
|
||||
rc=91
|
||||
fi
|
||||
elif [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 0 ]; then
|
||||
receipt rollback no_write "bounded_execution_not_started_targets_unchanged"
|
||||
fi
|
||||
cleanup_transients
|
||||
cleanup_rc=$?
|
||||
verify_residue_zero
|
||||
residue_rc=$?
|
||||
if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then
|
||||
rc=92
|
||||
fi
|
||||
if [ "${LOCK_HELD}" -eq 1 ]; then
|
||||
flock -u 9 >/dev/null 2>&1
|
||||
exec 9>&-
|
||||
rm -f "${LOCK_FILE}"
|
||||
fi
|
||||
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
|
||||
terminal=pass
|
||||
fi
|
||||
receipt terminal "${terminal}" "exit_${rc}_stage_candidate_residue_0_receipt_ref_${RECEIPT_LOG}"
|
||||
exit "${rc}"
|
||||
}
|
||||
|
||||
trap on_exit EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
exec 9>>"${LOCK_FILE}"
|
||||
flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; }
|
||||
LOCK_HELD=1
|
||||
|
||||
if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
||||
receipt check failed active_signoz_backup_toolchain_process
|
||||
exit 75
|
||||
fi
|
||||
active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
|
||||
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
|
||||
[ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
|
||||
|
||||
runtime_snapshot > "${RUNTIME_BEFORE}"
|
||||
RUNTIME_BASELINE_CAPTURED=1
|
||||
|
||||
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
||||
for index in "${!STAGED[@]}"; do
|
||||
[ -f "${STAGED[index]}" ] && [ ! -L "${STAGED[index]}" ]
|
||||
[ "$(sha256sum "${STAGED[index]}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
|
||||
done
|
||||
for index in 0 1 2 4; do bash -n "${STAGED[index]}"; done
|
||||
PYTHONPYCACHEPREFIX="${STAGE_DIR}/pycache" python3 -m py_compile "${STAGED[3]}"
|
||||
validate_xml_pair "${STAGED[5]}" "${STAGED[6]}"
|
||||
|
||||
: > "${PRIOR_MANIFEST}"
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
target="${TARGETS[index]}"
|
||||
[ ! -e "${target}.candidate.${RUN_ID}" ] && [ ! -L "${target}.candidate.${RUN_ID}" ]
|
||||
[ ! -e "${target}.rollback.${RUN_ID}" ] && [ ! -L "${target}.rollback.${RUN_ID}" ]
|
||||
if [ -e "${target}" ] || [ -L "${target}" ]; then
|
||||
[ -f "${target}" ] && [ ! -L "${target}" ]
|
||||
PRIOR_PRESENT[index]=1
|
||||
PRIOR_MODES[index]="$(stat -c '%a' "${target}")"
|
||||
PRIOR_UIDS[index]="$(stat -c '%u' "${target}")"
|
||||
PRIOR_GIDS[index]="$(stat -c '%g' "${target}")"
|
||||
PRIOR_HASHES[index]="$(root_hash "${target}")"
|
||||
run_root cp -p "${target}" "${PRIOR_DIR}/${LABELS[index]}"
|
||||
[ "$(root_hash "${PRIOR_DIR}/${LABELS[index]}")" = "${PRIOR_HASHES[index]}" ]
|
||||
else
|
||||
PRIOR_PRESENT[index]=0
|
||||
PRIOR_MODES[index]=absent
|
||||
PRIOR_UIDS[index]=absent
|
||||
PRIOR_GIDS[index]=absent
|
||||
PRIOR_HASHES[index]=absent
|
||||
fi
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"${LABELS[index]}" "${target}" "${PRIOR_PRESENT[index]}" \
|
||||
"${PRIOR_MODES[index]}" "${PRIOR_UIDS[index]}" "${PRIOR_GIDS[index]}" \
|
||||
"${PRIOR_HASHES[index]}" >> "${PRIOR_MANIFEST}"
|
||||
done
|
||||
|
||||
receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources"
|
||||
receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
|
||||
receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}"
|
||||
receipt ai_decision candidate "atomic_install_native_backup_toolchain_only"
|
||||
receipt risk_policy pass "risk_medium_bounded_files_only_no_runtime_execution_no_container_change_full_rollback"
|
||||
receipt check pass "runtime_baseline_active_operations_0_hash_bash_xml_py_compile"
|
||||
|
||||
APPLY_STARTED=1
|
||||
for directory in "${TARGET_DIRS[@]}"; do
|
||||
ensure_dir_tree "${directory}" true
|
||||
done
|
||||
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
target="${TARGETS[index]}"
|
||||
if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
|
||||
target_uid="${PRIOR_UIDS[index]}"
|
||||
target_gid="${PRIOR_GIDS[index]}"
|
||||
else
|
||||
target_uid="${REMOTE_UID}"
|
||||
target_gid="${REMOTE_GID}"
|
||||
fi
|
||||
run_root install -o "${target_uid}" -g "${target_gid}" -m "${MODES[index]}" \
|
||||
"${STAGED[index]}" "${target}.candidate.${RUN_ID}"
|
||||
[ "$(root_hash "${target}.candidate.${RUN_ID}")" = "${EXPECTED_HASHES[index]}" ]
|
||||
[ "0$(stat -c '%a' "${target}.candidate.${RUN_ID}")" = "${MODES[index]}" ]
|
||||
done
|
||||
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"
|
||||
done
|
||||
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
[ -f "${TARGETS[index]}" ] && [ ! -L "${TARGETS[index]}" ]
|
||||
[ "$(root_hash "${TARGETS[index]}")" = "${EXPECTED_HASHES[index]}" ]
|
||||
[ "0$(stat -c '%a' "${TARGETS[index]}")" = "${MODES[index]}" ]
|
||||
done
|
||||
for index in 0 1 2 4; do bash -n "${TARGETS[index]}"; done
|
||||
PYTHONPYCACHEPREFIX="${STAGE_DIR}/candidate-pycache" python3 -m py_compile "${TARGETS[3]}"
|
||||
validate_xml_pair "${TARGETS[5]}" "${TARGETS[6]}"
|
||||
|
||||
: > "${DEPLOYED_MANIFEST}"
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
printf '%s\t%s\t%s\t%s\n' \
|
||||
"${LABELS[index]}" "${TARGETS[index]}" "${MODES[index]}" \
|
||||
"${EXPECTED_HASHES[index]}" >> "${DEPLOYED_MANIFEST}"
|
||||
done
|
||||
|
||||
runtime_snapshot > "${RUNTIME_AFTER}"
|
||||
cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"
|
||||
receipt execution pass "seven_candidates_atomically_replaced_expected_hashes_and_modes"
|
||||
receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged"
|
||||
receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull"
|
||||
DEPLOY_VERIFIED=1
|
||||
REMOTE_APPLY
|
||||
|
||||
trap - EXIT
|
||||
cleanup_stage
|
||||
local_receipt terminal pass "remote_controlled_apply_postverify_and_stage_candidate_residue_0"
|
||||
218
scripts/ops/deploy-signoz-prometheus-route.sh
Executable file
218
scripts/ops/deploy-signoz-prometheus-route.sh
Executable file
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deploy only the canonical SigNoz blackbox route to the host110 Prometheus.
|
||||
# The full Prometheus file intentionally remains untouched because production
|
||||
# contains unrelated, independently governed scrape targets.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
|
||||
RUN_ID="${RUN_ID:?RUN_ID is required}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
|
||||
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
|
||||
PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
|
||||
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
|
||||
REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
|
||||
MODE="${1:-apply}"
|
||||
|
||||
case "${MODE}" in
|
||||
apply|--dry-run) ;;
|
||||
*) printf 'usage: %s [apply|--dry-run]\n' "$0" >&2; exit 64 ;;
|
||||
esac
|
||||
|
||||
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
||||
case "${value}" in
|
||||
*[!A-Za-z0-9._-]*|'')
|
||||
printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
|
||||
bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
||||
"${PROMETHEUS_URL}" "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
MODE="$4"
|
||||
PROMETHEUS_URL="$5"
|
||||
PROMETHEUS_CONTAINER="$6"
|
||||
REMOTE_CONFIG="$7"
|
||||
REMOTE_CANDIDATE="/tmp/awoooi-prometheus-signoz.${RUN_ID}.yml"
|
||||
RUNTIME_CANDIDATE="/etc/prometheus/.awoooi-prometheus-signoz.${RUN_ID}.yml"
|
||||
BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
|
||||
APPLY_STARTED=0
|
||||
DEPLOY_VERIFIED=0
|
||||
|
||||
cleanup_candidate() {
|
||||
rm -f "${REMOTE_CANDIDATE}"
|
||||
docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
|
||||
rm -f "${RUNTIME_CANDIDATE}" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
rollback_deploy() {
|
||||
cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}"
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
||||
printf 'rollback_verifier trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
local rc=$?
|
||||
trap - EXIT
|
||||
set +e
|
||||
cleanup_candidate
|
||||
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
|
||||
&& [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
||||
rollback_deploy
|
||||
fi
|
||||
exit "${rc}"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
test -f "${REMOTE_CONFIG}"
|
||||
test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
||||
|
||||
python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
source = Path(sys.argv[1])
|
||||
candidate = Path(sys.argv[2])
|
||||
text = source.read_text(encoding="utf-8")
|
||||
|
||||
canonical = " - http://192.168.0.110:8080/api/v1/health\n"
|
||||
legacy = " - 192.168.0.188:3301\n"
|
||||
anchor = " - http://192.168.0.110:3001\n"
|
||||
|
||||
|
||||
def block_bounds(payload: str, job: str) -> tuple[int, int]:
|
||||
marker = f' - job_name: "{job}"\n'
|
||||
start = payload.find(marker)
|
||||
if start < 0 or payload.find(marker, start + 1) >= 0:
|
||||
raise SystemExit(f"expected exactly one {job} job")
|
||||
end = payload.find("\n - job_name:", start + len(marker))
|
||||
return start, len(payload) if end < 0 else end + 1
|
||||
|
||||
|
||||
http_start, http_end = block_bounds(text, "blackbox-http")
|
||||
http_block = text[http_start:http_end]
|
||||
if http_block.count(canonical) > 1 or http_block.count(anchor) != 1:
|
||||
raise SystemExit("canonical HTTP target or insertion anchor is ambiguous")
|
||||
if canonical not in http_block:
|
||||
http_block = http_block.replace(anchor, anchor + canonical, 1)
|
||||
text = text[:http_start] + http_block + text[http_end:]
|
||||
|
||||
tcp_start, tcp_end = block_bounds(text, "blackbox-tcp")
|
||||
tcp_block = text[tcp_start:tcp_end]
|
||||
if tcp_block.count(legacy) > 1:
|
||||
raise SystemExit("legacy TCP target is ambiguous")
|
||||
tcp_block = tcp_block.replace(legacy, "", 1)
|
||||
text = text[:tcp_start] + tcp_block + text[tcp_end:]
|
||||
|
||||
http_start, http_end = block_bounds(text, "blackbox-http")
|
||||
tcp_start, tcp_end = block_bounds(text, "blackbox-tcp")
|
||||
if text[http_start:http_end].count(canonical) != 1:
|
||||
raise SystemExit("canonical HTTP target was not normalized exactly once")
|
||||
if legacy in text[tcp_start:tcp_end]:
|
||||
raise SystemExit("legacy TCP target remains after normalization")
|
||||
|
||||
candidate.write_text(text, encoding="utf-8")
|
||||
PY
|
||||
|
||||
docker cp "${REMOTE_CANDIDATE}" \
|
||||
"${PROMETHEUS_CONTAINER}:${RUNTIME_CANDIDATE}" >/dev/null
|
||||
docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${RUNTIME_CANDIDATE}"
|
||||
|
||||
SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
||||
CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
|
||||
printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
||||
"${SOURCE_SHA}" "${CANDIDATE_SHA}"
|
||||
|
||||
if [ "${MODE}" = "--dry-run" ]; then
|
||||
printf 'terminal trace_id=%s run_id=%s status=check_pass_no_write\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
|
||||
APPLY_STARTED=1
|
||||
cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
|
||||
|
||||
ACTIVE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
||||
RUNTIME_SHA="$(docker exec "${PROMETHEUS_CONTAINER}" \
|
||||
sha256sum /etc/prometheus/prometheus.yml | awk '{print $1}')"
|
||||
test "${ACTIVE_SHA}" = "${CANDIDATE_SHA}"
|
||||
test "${RUNTIME_SHA}" = "${CANDIDATE_SHA}"
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
||||
printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=target_first_config_apply active_sha=%s runtime_sha=%s ready=true backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${ACTIVE_SHA}" \
|
||||
"${RUNTIME_SHA}" "${BACKUP_CONFIG}"
|
||||
|
||||
target_state() {
|
||||
curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
|
||||
import json, sys
|
||||
payload = json.load(sys.stdin)
|
||||
targets = payload["data"]["activeTargets"]
|
||||
canonical = [t for t in targets if t.get("labels", {}).get("job") == "blackbox-http" and t.get("labels", {}).get("instance") == "http://192.168.0.110:8080/api/v1/health"]
|
||||
legacy = [t for t in targets if t.get("labels", {}).get("instance") == "192.168.0.188:3301"]
|
||||
if len(canonical) == 1:
|
||||
target = canonical[0]
|
||||
print("{}|{}|{}|{}".format(
|
||||
1,
|
||||
target.get("health", "unknown"),
|
||||
target.get("lastScrape", "never"),
|
||||
len(legacy),
|
||||
))
|
||||
else:
|
||||
print(f"{len(canonical)}|missing|never|{len(legacy)}")
|
||||
'
|
||||
}
|
||||
|
||||
FIRST_SCRAPE=""
|
||||
for attempt in $(seq 1 9); do
|
||||
IFS='|' read -r count health last_scrape legacy_count <<<"$(target_state)"
|
||||
printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s legacy_count=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
|
||||
"${last_scrape}" "${legacy_count}"
|
||||
if [ "${count}" = 1 ] && [ "${health}" = up ] \
|
||||
&& [ "${legacy_count}" = 0 ] && [ "${last_scrape}" != never ]; then
|
||||
FIRST_SCRAPE="${last_scrape}"
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
test -n "${FIRST_SCRAPE}"
|
||||
|
||||
sleep 16
|
||||
IFS='|' read -r count health SECOND_SCRAPE legacy_count <<<"$(target_state)"
|
||||
printf 'target_probe trace_id=%s run_id=%s sample=post_cooldown count=%s health=%s last_scrape=%s legacy_count=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${count}" "${health}" \
|
||||
"${SECOND_SCRAPE}" "${legacy_count}"
|
||||
test "${count}" = 1
|
||||
test "${health}" = up
|
||||
test "${legacy_count}" = 0
|
||||
test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
|
||||
|
||||
PROBE_VALUE="$(curl -fsS --get \
|
||||
--data-urlencode 'query=probe_success{job="blackbox-http",instance="http://192.168.0.110:8080/api/v1/health"}' \
|
||||
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
|
||||
import json, sys
|
||||
result = json.load(sys.stdin)["data"]["result"]
|
||||
print(result[0]["value"][1] if len(result) == 1 else "invalid")
|
||||
')"
|
||||
test "${PROBE_VALUE}" = 1
|
||||
|
||||
DEPLOY_VERIFIED=1
|
||||
printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=target_up_two_scrapes first=%s second=%s probe_success=%s legacy_target_count=0\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
|
||||
"${SECOND_SCRAPE}" "${PROBE_VALUE}"
|
||||
printf 'terminal trace_id=%s run_id=%s status=completed_target_first backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${BACKUP_CONFIG}"
|
||||
REMOTE
|
||||
231
scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
Normal file
231
scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
Normal file
@@ -0,0 +1,231 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
|
||||
OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
|
||||
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
|
||||
|
||||
|
||||
def executable_text() -> str:
|
||||
return "\n".join(
|
||||
line
|
||||
for line in DEPLOYER.read_text(encoding="utf-8").splitlines()
|
||||
if not line.lstrip().startswith("#")
|
||||
)
|
||||
|
||||
|
||||
def test_clickhouse_config_declares_one_allowed_local_backup_disk() -> None:
|
||||
root = ET.parse(CONFIG).getroot()
|
||||
assert root.tag == "clickhouse"
|
||||
|
||||
disks = root.findall("./storage_configuration/disks/*")
|
||||
assert [disk.tag for disk in disks] == ["backups"]
|
||||
assert disks[0].findtext("type") == "local"
|
||||
assert disks[0].findtext("path") == "/backups/"
|
||||
|
||||
backups = root.find("./backups")
|
||||
assert backups is not None
|
||||
assert backups.findtext("allowed_disk") == "backups"
|
||||
assert backups.findtext("allowed_path") == "/backups/"
|
||||
assert backups.findtext("allow_concurrent_backups") == "false"
|
||||
assert backups.findtext("allow_concurrent_restores") == "false"
|
||||
assert CONFIG.read_text(encoding="utf-8").count("<allowed_disk>") == 1
|
||||
|
||||
|
||||
def test_compose_override_is_additive_clickhouse_only_and_uses_dedicated_mounts() -> (
|
||||
None
|
||||
):
|
||||
text = OVERRIDE.read_text(encoding="utf-8")
|
||||
assert text.count(" clickhouse:") == 1
|
||||
assert "otel-collector:" not in text
|
||||
assert "signoz:" not in text
|
||||
assert "zookeeper:" not in text
|
||||
assert "source: /backup/staging/signoz-clickhouse" in text
|
||||
assert "target: /backups" in text
|
||||
assert (
|
||||
"source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml"
|
||||
) in text
|
||||
assert ("target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml") in text
|
||||
assert text.count("read_only: true") == 1
|
||||
assert text.count("read_only: false") == 1
|
||||
for forbidden in ("image:", "build:", "command:", "environment:"):
|
||||
assert forbidden not in text
|
||||
|
||||
|
||||
def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
|
||||
syntax = subprocess.run(
|
||||
["bash", "-n", str(DEPLOYER)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert syntax.returncode == 0, syntax.stderr
|
||||
|
||||
help_result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--help"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
||||
},
|
||||
)
|
||||
assert help_result.returncode == 0
|
||||
assert "--check" in help_result.stdout
|
||||
assert "--apply" in help_result.stdout
|
||||
|
||||
|
||||
def test_deployer_requires_all_three_controlled_apply_identifiers_before_ssh() -> None:
|
||||
env = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
||||
}
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "must be path-safe" in result.stderr
|
||||
assert "ssh:" not in result.stderr
|
||||
|
||||
|
||||
def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
check_terminal = text.index('if [ "${MODE}" = "--check" ]')
|
||||
stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
|
||||
first_scp = text.index("scp -q")
|
||||
|
||||
assert check_terminal < stage_assignment < first_scp
|
||||
assert "-f '${REMOTE_BASE}' -f - config -q" in text
|
||||
assert '< "${LOCAL_OVERRIDE}"' in text
|
||||
assert "check_pass_no_write" in text
|
||||
|
||||
|
||||
def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
executable = executable_text()
|
||||
|
||||
assert 'REMOTE_SERVICE="clickhouse"' in text
|
||||
assert "--env-file /dev/null" in text
|
||||
assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
|
||||
assert "timeout --signal=TERM --kill-after=30 300" in text
|
||||
assert "up -d --no-deps --pull never --force-recreate" in text
|
||||
assert '"${REMOTE_SERVICE}"' in text
|
||||
assert "docker compose down" not in executable
|
||||
assert "docker pull" not in executable
|
||||
assert "docker volume" not in executable
|
||||
assert "docker system prune" not in executable
|
||||
assert "github.com" not in text.lower()
|
||||
assert "ghcr.io" not in text.lower()
|
||||
|
||||
|
||||
def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "active_signoz_backup_detected" in text
|
||||
assert "active_native_backup_or_restore" in text
|
||||
assert "partial_managed_file_drift" in text
|
||||
assert "another_deploy_holds_lock" in text
|
||||
assert "backup-signoz[.]sh" in text
|
||||
assert "CREATING_BACKUP" in text
|
||||
assert "RESTORING" in text
|
||||
|
||||
|
||||
def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "BEFORE_IMAGE_ID" in text
|
||||
assert "AFTER_IMAGE_ID" in text
|
||||
assert "BEFORE_DATA_VOLUME" in text
|
||||
assert "AFTER_DATA_VOLUME" in text
|
||||
assert 'eq .Destination "/var/lib/clickhouse"' in text
|
||||
for prefix in ("COLLECTOR", "SIGNOZ", "ZOOKEEPER"):
|
||||
assert f"BEFORE_{prefix}_ID" in text
|
||||
assert f"BEFORE_{prefix}_RESTARTS" in text
|
||||
assert "image_and_data_volume_unchanged" in text
|
||||
|
||||
|
||||
def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "system.disks WHERE name='backups'" in text
|
||||
assert "Local|0|0|0|1" in text
|
||||
assert "EXISTS TABLE system.backups" in text
|
||||
assert "SHOW GRANTS" in text
|
||||
assert "BACKUP([,[:space:]]|$)" in text
|
||||
assert 'eq .Destination "/backups"' in text
|
||||
assert "101:101:750" in text
|
||||
assert "server_uid" in text
|
||||
assert "server_gid" in text
|
||||
assert "docker exec -u 101:101" in text
|
||||
assert "test -w /backups" in text
|
||||
assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
|
||||
assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
|
||||
identity_start = text.index('server_uid="')
|
||||
identity_block = text[
|
||||
identity_start : text.index("docker exec -u 101:101", identity_start)
|
||||
]
|
||||
assert "sh -c" not in identity_block
|
||||
assert "concurrency_disabled" in text
|
||||
for port in (4317, 4318, 8080):
|
||||
assert f"listener_up {port}" in text
|
||||
assert "api_code" in text
|
||||
assert "disk_backups_local_rw_healthy" in text
|
||||
|
||||
|
||||
def test_failed_apply_restores_prior_managed_compose_state() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "restore_managed_files" in text
|
||||
assert "rollback_deploy" in text
|
||||
assert "PRIOR_OVERRIDE_PRESENT" in text
|
||||
assert "PRIOR_CONFIG_PRESENT" in text
|
||||
assert "prior-override.yaml" in text
|
||||
assert "prior-backup-disk.xml" in text
|
||||
assert "prior_compose_state_restored" in text
|
||||
assert "retained_nonempty_no_delete" in text
|
||||
assert "rc=91" in text
|
||||
assert "trap on_exit EXIT" in text
|
||||
|
||||
|
||||
def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:
|
||||
executable = executable_text()
|
||||
lowered = executable.lower()
|
||||
assert ".config.env" not in lowered
|
||||
assert "config.env" not in lowered
|
||||
assert ".env" not in lowered
|
||||
assert "docker inspect --format '{{json .config.env}}'" not in lowered
|
||||
assert " backup all" not in lowered
|
||||
assert " restore all" not in lowered
|
||||
assert "clickhouse-client --query 'backup" not in lowered
|
||||
assert "clickhouse-client --query 'restore" not in lowered
|
||||
assert "no_backup_or_restore_executed" in executable
|
||||
|
||||
|
||||
def test_receipts_cover_controlled_apply_and_durable_terminal() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
for phase in (
|
||||
"sensor_source",
|
||||
"normalized_asset_identity",
|
||||
"source_of_truth_diff",
|
||||
"ai_decision",
|
||||
"risk_policy",
|
||||
"check",
|
||||
"execution",
|
||||
"post_verifier",
|
||||
"rollback",
|
||||
"closure_writeback",
|
||||
"terminal",
|
||||
):
|
||||
assert phase in text
|
||||
assert "receipts.jsonl" in text
|
||||
assert "override_sha_" in text
|
||||
assert "config_sha_" in text
|
||||
339
scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
Normal file
339
scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
Normal file
@@ -0,0 +1,339 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
|
||||
|
||||
|
||||
def deployer_text() -> str:
|
||||
return DEPLOYER.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def executable_text() -> str:
|
||||
return "\n".join(
|
||||
line
|
||||
for line in deployer_text().splitlines()
|
||||
if not line.lstrip().startswith("#")
|
||||
)
|
||||
|
||||
|
||||
def base_env() -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"TRACE_ID": "P0-OBS-002",
|
||||
"RUN_ID": "toolchain-test-run",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
}
|
||||
)
|
||||
return env
|
||||
|
||||
|
||||
def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
|
||||
syntax = subprocess.run(
|
||||
["bash", "-n", str(DEPLOYER)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert syntax.returncode == 0, syntax.stderr
|
||||
|
||||
help_result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--help"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
||||
},
|
||||
)
|
||||
assert help_result.returncode == 0
|
||||
assert "--check" in help_result.stdout
|
||||
assert "--apply" in help_result.stdout
|
||||
assert "CLICKHOUSE_RESTORE_INVENTORY_HELPER" in help_result.stdout
|
||||
assert "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" in help_result.stdout
|
||||
assert "CLICKHOUSE_RESTORE_CLUSTER_CONFIG" in help_result.stdout
|
||||
|
||||
|
||||
def test_deployer_requires_controlled_apply_ids_before_network_access() -> None:
|
||||
env = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
||||
}
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "must be path-safe" in result.stderr
|
||||
assert "ssh:" not in result.stderr
|
||||
|
||||
|
||||
def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> None:
|
||||
env = base_env()
|
||||
env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/scripts/../escape"
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "safe absolute paths" in result.stderr
|
||||
assert "ssh:" not in result.stderr
|
||||
|
||||
env = base_env()
|
||||
env["SIGNOZ_TOOLCHAIN_RECEIPT_ROOT"] = "/var/tmp/receipts"
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "receipt root must stay under /backup" in result.stderr
|
||||
|
||||
env = base_env()
|
||||
env["SIGNOZ_TOOLCHAIN_STAGE_ROOT"] = "/backup/deploy-staging"
|
||||
env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/deploy-staging/scripts"
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "must not overlap" in result.stderr
|
||||
|
||||
|
||||
def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
|
||||
text = deployer_text()
|
||||
assert 'TARGET_HOST="wooo@192.168.0.110"' in text
|
||||
assert "TARGET_HOST:-" not in text
|
||||
assert "TARGET_HOST=" not in text.replace('TARGET_HOST="wooo@192.168.0.110"', "", 1)
|
||||
|
||||
|
||||
def test_exact_seven_artifacts_and_target_modes_are_declared() -> None:
|
||||
text = deployer_text()
|
||||
for source in (
|
||||
"scripts/backup/backup-signoz.sh",
|
||||
"scripts/backup/clickhouse-native-backup.sh",
|
||||
"scripts/backup/clickhouse-native-restore-drill.sh",
|
||||
"scripts/backup/clickhouse-restore-inventory.py",
|
||||
"scripts/backup/run-signoz-backup-canary.sh",
|
||||
"ops/signoz/clickhouse/config.d/backup_disk.xml",
|
||||
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
):
|
||||
assert source in text
|
||||
|
||||
for target in (
|
||||
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh",
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh",
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh",
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py",
|
||||
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh",
|
||||
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml",
|
||||
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml",
|
||||
):
|
||||
assert target in text
|
||||
assert "MODES=(0755 0755 0755 0755 0755 0644 0644)" in text
|
||||
assert 'REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"' in text
|
||||
assert (
|
||||
'REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-'
|
||||
'/backup/config/signoz/clickhouse}"'
|
||||
) in text
|
||||
|
||||
|
||||
def test_local_and_remote_stage_validation_cover_hash_syntax_python_and_xml() -> None:
|
||||
text = deployer_text()
|
||||
assert "validate_local_sources" in text
|
||||
assert "SOURCE_HASHES" in text
|
||||
assert "bash -n" in text
|
||||
assert "python3 -m py_compile" in text
|
||||
assert "validate_xml_pair" in text
|
||||
for expected in (
|
||||
'"/backups/"',
|
||||
'"restore-zookeeper"',
|
||||
'"restore-clickhouse"',
|
||||
'"CLICKHOUSE_RESTORE_REPLICA"',
|
||||
):
|
||||
assert expected in text
|
||||
|
||||
|
||||
def test_check_mode_exits_before_remote_stage_or_scp() -> None:
|
||||
text = deployer_text()
|
||||
check_exit = text.index('if [ "${MODE}" = --check ]')
|
||||
stage_create = text.index("STAGE_CREATED=0")
|
||||
first_scp = text.index("scp -q")
|
||||
assert check_exit < stage_create < first_scp
|
||||
assert "check_pass_no_write" in text
|
||||
assert "remote_target_unchanged_no_stage_no_candidate" in text
|
||||
|
||||
remote_check = text[
|
||||
text.index("remote_read_only_check() {") : text.index(
|
||||
"local_receipt sensor_source", text.index("remote_read_only_check() {")
|
||||
)
|
||||
]
|
||||
for forbidden in ("mkdir ", "install ", "mv ", "rm ", "scp "):
|
||||
assert forbidden not in remote_check
|
||||
|
||||
|
||||
def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
log = tmp_path / "calls.log"
|
||||
ssh = fake_bin / "ssh"
|
||||
ssh.write_text(
|
||||
"#!/bin/sh\n"
|
||||
'printf "ssh %s\\n" "$*" >> "$FAKE_CALL_LOG"\n'
|
||||
"cat >/dev/null\n"
|
||||
'printf \'{"phase":"source_of_truth_diff","result":"pass"}\\n\'\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
scp = fake_bin / "scp"
|
||||
scp.write_text(
|
||||
'#!/bin/sh\nprintf "scp %s\\n" "$*" >> "$FAKE_CALL_LOG"\nexit 99\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
ssh.chmod(0o755)
|
||||
scp.chmod(0o755)
|
||||
|
||||
env = base_env()
|
||||
env["PATH"] = f"{fake_bin}:{env['PATH']}"
|
||||
env["FAKE_CALL_LOG"] = str(log)
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
calls = log.read_text(encoding="utf-8").splitlines()
|
||||
assert len(calls) == 1
|
||||
assert calls[0].startswith("ssh ")
|
||||
assert "wooo@192.168.0.110" in calls[0]
|
||||
assert "check_pass_no_write" in result.stdout
|
||||
|
||||
|
||||
def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
|
||||
text = deployer_text()
|
||||
for token in (
|
||||
"prior-manifest.tsv",
|
||||
"PRIOR_PRESENT",
|
||||
"PRIOR_MODES",
|
||||
"PRIOR_UIDS",
|
||||
"PRIOR_GIDS",
|
||||
"PRIOR_HASHES",
|
||||
"cp -p",
|
||||
"rollback_targets",
|
||||
"all_prior_files_modes_hashes_owners_and_runtime_restored",
|
||||
):
|
||||
assert token in text
|
||||
assert "trap on_exit EXIT" in text
|
||||
assert "rc=91" in text
|
||||
assert 'run_root rm -f "${target}"' in text
|
||||
assert 'run_root rmdir "${CREATED_DIRS[index]}"' in text
|
||||
assert "bounded_execution_not_started_targets_unchanged" in text
|
||||
|
||||
|
||||
def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
|
||||
text = deployer_text()
|
||||
assert '"${target}.candidate.${RUN_ID}"' in text
|
||||
assert (
|
||||
'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
|
||||
) in text
|
||||
assert "deployed-manifest.tsv" in text
|
||||
assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
|
||||
assert "stage_candidate_residue_0" in text
|
||||
assert "verify_residue_zero" in text
|
||||
|
||||
|
||||
def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
|
||||
text = deployer_text()
|
||||
for container in (
|
||||
"signoz-clickhouse",
|
||||
"signoz-otel-collector",
|
||||
"signoz",
|
||||
"signoz-zookeeper-1",
|
||||
):
|
||||
assert container in text
|
||||
for field in (
|
||||
".Id",
|
||||
".State.StartedAt",
|
||||
".RestartCount",
|
||||
".State.Running",
|
||||
'index .State "Health"',
|
||||
):
|
||||
assert field in text
|
||||
assert "runtime-before.tsv" in text
|
||||
assert "runtime-after.tsv" in text
|
||||
assert 'cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"' in text
|
||||
for port in (4317, 4318, 8080):
|
||||
assert "for port in 4317 4318 8080" in text
|
||||
assert str(port) in text
|
||||
assert "api_200_unchanged" in text
|
||||
|
||||
|
||||
def test_apply_fails_closed_while_backup_or_restore_is_active() -> None:
|
||||
text = deployer_text()
|
||||
assert "active_signoz_backup_toolchain_process" in text
|
||||
assert "active_native_backup_or_restore" in text
|
||||
assert "CREATING_BACKUP" in text
|
||||
assert "RESTORING" in text
|
||||
assert "another_toolchain_deploy_holds_lock" in text
|
||||
|
||||
|
||||
def test_deployer_never_executes_runtime_work_or_uses_forbidden_supply_chain() -> None:
|
||||
executable = executable_text().lower()
|
||||
for forbidden in (
|
||||
"docker restart",
|
||||
"docker compose",
|
||||
"docker pull",
|
||||
"docker run",
|
||||
"backup all",
|
||||
"restore all",
|
||||
"ansible-playbook",
|
||||
"github.com",
|
||||
"api.github.com",
|
||||
"raw.githubusercontent.com",
|
||||
"codeload.github.com",
|
||||
"ghcr.io",
|
||||
" gh ",
|
||||
):
|
||||
assert forbidden not in executable
|
||||
assert "no_backup_restore_restart_or_pull" in executable
|
||||
|
||||
|
||||
def test_receipts_cover_complete_controlled_apply_contract() -> None:
|
||||
text = deployer_text()
|
||||
for phase in (
|
||||
"sensor_source",
|
||||
"normalized_asset_identity",
|
||||
"source_of_truth_diff",
|
||||
"ai_decision",
|
||||
"risk_policy",
|
||||
"check",
|
||||
"execution",
|
||||
"post_verifier",
|
||||
"rollback",
|
||||
"closure_writeback",
|
||||
"terminal",
|
||||
):
|
||||
assert phase in text
|
||||
assert "receipts.jsonl" in text
|
||||
assert '"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"' in text
|
||||
assert "durable_manifests_and_receipts_ack" in text
|
||||
Reference in New Issue
Block a user