802 lines
28 KiB
Bash
Executable File
802 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# P0-OBS-002 - atomically deploy the host110 SigNoz native backup toolchain.
|
|
#
|
|
# This deployer owns seven repo-sourced files only. It does not run a backup or
|
|
# restore, restart a container, pull an image, or inspect a secret-bearing
|
|
# runtime source.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
TARGET_HOST="wooo@192.168.0.110"
|
|
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
SAFE_HOME="/home/wooo"
|
|
API_URL="http://127.0.0.1:8080/api/v1/health"
|
|
|
|
REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"
|
|
REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}"
|
|
RECEIPT_ROOT="${SIGNOZ_TOOLCHAIN_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-native-backup-toolchain}"
|
|
STAGE_ROOT="${SIGNOZ_TOOLCHAIN_STAGE_ROOT:-/tmp}"
|
|
|
|
LABELS=(
|
|
backup-signoz.sh
|
|
clickhouse-native-backup.sh
|
|
clickhouse-native-restore-drill.sh
|
|
clickhouse-restore-inventory.py
|
|
run-signoz-backup-canary.sh
|
|
backup_disk.xml
|
|
isolated-cluster.xml
|
|
)
|
|
LOCAL_SOURCES=(
|
|
"${ROOT_DIR}/scripts/backup/backup-signoz.sh"
|
|
"${ROOT_DIR}/scripts/backup/clickhouse-native-backup.sh"
|
|
"${ROOT_DIR}/scripts/backup/clickhouse-native-restore-drill.sh"
|
|
"${ROOT_DIR}/scripts/backup/clickhouse-restore-inventory.py"
|
|
"${ROOT_DIR}/scripts/backup/run-signoz-backup-canary.sh"
|
|
"${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
|
|
"${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
|
)
|
|
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
|
|
|
SSH_OPTS=(
|
|
-o BatchMode=yes
|
|
-o ConnectTimeout=8
|
|
-o ConnectionAttempts=1
|
|
-o ServerAliveInterval=5
|
|
-o ServerAliveCountMax=2
|
|
)
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: deploy-signoz-native-backup-toolchain.sh [--check|--apply]
|
|
|
|
Required environment:
|
|
TRACE_ID Path-safe controlled-apply trace identifier
|
|
RUN_ID Unique path-safe execution identifier
|
|
WORK_ITEM_ID Path-safe work-item identifier
|
|
|
|
Optional safe absolute path environment:
|
|
SIGNOZ_TOOLCHAIN_SCRIPT_DIR
|
|
SIGNOZ_TOOLCHAIN_CONFIG_ROOT
|
|
SIGNOZ_TOOLCHAIN_RECEIPT_ROOT
|
|
SIGNOZ_TOOLCHAIN_STAGE_ROOT
|
|
|
|
--check validates local sources and performs a remote read-only target diff and
|
|
runtime readiness check. --apply stages all seven sources, validates hashes and
|
|
syntax, atomically replaces exact targets, and rolls every target back to its
|
|
prior hash/mode/owner if any bounded verifier fails.
|
|
|
|
Flat restore-driver invocation must set:
|
|
CLICKHOUSE_RESTORE_INVENTORY_HELPER=<script-dir>/clickhouse-restore-inventory.py
|
|
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG=<config-root>/config.d/backup_disk.xml
|
|
CLICKHOUSE_RESTORE_CLUSTER_CONFIG=<config-root>/restore-drill/config.d/isolated-cluster.xml
|
|
USAGE
|
|
}
|
|
|
|
MODE="${1:---check}"
|
|
case "${MODE}" in
|
|
--check|--apply) ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) usage >&2; exit 64 ;;
|
|
esac
|
|
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
|
|
|
TRACE_ID="${TRACE_ID:-}"
|
|
RUN_ID="${RUN_ID:-}"
|
|
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
|
|
|
valid_identity() {
|
|
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
|
}
|
|
|
|
safe_absolute_path() {
|
|
local path="$1"
|
|
[[ "${path}" == /* ]] || return 1
|
|
[[ "${path}" != / && "${path}" != */ ]] || return 1
|
|
[[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1
|
|
case "/${path#/}/" in
|
|
*'//'*|*'/../'*|*'/./'*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
paths_overlap() {
|
|
local left="$1" right="$2"
|
|
[[ "${left}" = "${right}" || "${left}" == "${right}"/* || "${right}" == "${left}"/* ]]
|
|
}
|
|
|
|
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
|
valid_identity "${value}" || {
|
|
printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
|
|
exit 64
|
|
}
|
|
done
|
|
|
|
for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
|
|
safe_absolute_path "${path}" || {
|
|
printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2
|
|
exit 64
|
|
}
|
|
done
|
|
case "${REMOTE_SCRIPT_DIR}" in /backup/*) ;; *) printf 'script dir must stay under /backup\n' >&2; exit 64 ;; esac
|
|
case "${REMOTE_CONFIG_ROOT}" in /backup/*) ;; *) printf 'config root must stay under /backup\n' >&2; exit 64 ;; esac
|
|
case "${RECEIPT_ROOT}" in /backup/*) ;; *) printf 'receipt root must stay under /backup\n' >&2; exit 64 ;; esac
|
|
case "${STAGE_ROOT}" in /tmp|/tmp/*|/backup/deploy-staging|/backup/deploy-staging/*) ;;
|
|
*) printf 'stage root must stay under /tmp or /backup/deploy-staging\n' >&2; exit 64 ;;
|
|
esac
|
|
if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \
|
|
|| paths_overlap "${REMOTE_SCRIPT_DIR}" "${RECEIPT_ROOT}" \
|
|
|| paths_overlap "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" \
|
|
|| paths_overlap "${STAGE_ROOT}" "${REMOTE_SCRIPT_DIR}" \
|
|
|| paths_overlap "${STAGE_ROOT}" "${REMOTE_CONFIG_ROOT}" \
|
|
|| paths_overlap "${STAGE_ROOT}" "${RECEIPT_ROOT}"; then
|
|
printf 'toolchain target, receipt, and stage paths must not overlap\n' >&2
|
|
exit 64
|
|
fi
|
|
|
|
for command_name in bash python3 scp ssh; do
|
|
command -v "${command_name}" >/dev/null 2>&1 || {
|
|
printf 'required local command missing: %s\n' "${command_name}" >&2
|
|
exit 69
|
|
}
|
|
done
|
|
|
|
hash_file() {
|
|
if command -v sha256sum >/dev/null 2>&1; then
|
|
sha256sum "$1" | awk '{print $1}'
|
|
else
|
|
shasum -a 256 "$1" | awk '{print $1}'
|
|
fi
|
|
}
|
|
|
|
validate_xml_pair() {
|
|
python3 - "$1" "$2" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
backup_path = Path(sys.argv[1])
|
|
cluster_path = Path(sys.argv[2])
|
|
backup = ET.parse(backup_path).getroot()
|
|
cluster = ET.parse(cluster_path).getroot()
|
|
assert backup.tag == "clickhouse"
|
|
assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
|
|
assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
|
|
assert backup.findtext("./backups/allowed_disk") == "backups"
|
|
assert backup.findtext("./backups/allowed_path") == "/backups/"
|
|
assert backup.findtext("./backups/allow_concurrent_backups") == "false"
|
|
assert backup.findtext("./backups/allow_concurrent_restores") == "false"
|
|
assert cluster.tag == "clickhouse"
|
|
assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
|
|
assert cluster.findtext("./zookeeper/node/port") == "2181"
|
|
assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
|
|
assert cluster.findtext("./remote_servers/cluster/shard/replica/port") == "9000"
|
|
assert cluster.findtext("./macros/shard") == "01"
|
|
replica = cluster.find("./macros/replica")
|
|
assert replica is not None
|
|
assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
|
|
PY
|
|
}
|
|
|
|
validate_local_sources() {
|
|
local index source
|
|
for index in "${!LOCAL_SOURCES[@]}"; do
|
|
source="${LOCAL_SOURCES[index]}"
|
|
[ -f "${source}" ] && [ ! -L "${source}" ]
|
|
[ -s "${source}" ]
|
|
done
|
|
|
|
for index in 0 1 2 4; do
|
|
bash -n "${LOCAL_SOURCES[index]}"
|
|
done
|
|
python3 - "${LOCAL_SOURCES[3]}" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
path = Path(sys.argv[1])
|
|
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
|
PY
|
|
validate_xml_pair "${LOCAL_SOURCES[5]}" "${LOCAL_SOURCES[6]}"
|
|
|
|
python3 - "${LOCAL_SOURCES[2]}" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
text = Path(sys.argv[1]).read_text(encoding="utf-8")
|
|
for token in (
|
|
"CLICKHOUSE_RESTORE_INVENTORY_HELPER",
|
|
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG",
|
|
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG",
|
|
):
|
|
assert token in text
|
|
PY
|
|
}
|
|
|
|
validate_local_sources
|
|
SOURCE_HASHES=()
|
|
SOURCE_DETAIL=""
|
|
for index in "${!LOCAL_SOURCES[@]}"; do
|
|
source_hash="$(hash_file "${LOCAL_SOURCES[index]}")"
|
|
SOURCE_HASHES+=("${source_hash}")
|
|
SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_"
|
|
done
|
|
SOURCE_DETAIL="${SOURCE_DETAIL%_}"
|
|
|
|
CONFIG_DIR="${REMOTE_CONFIG_ROOT}/config.d"
|
|
RESTORE_CONFIG_DIR="${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
|
TARGET_PATHS=(
|
|
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
|
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
|
"${CONFIG_DIR}/backup_disk.xml"
|
|
"${RESTORE_CONFIG_DIR}/isolated-cluster.xml"
|
|
)
|
|
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-native-backup-toolchain.${RUN_ID}"
|
|
|
|
local_receipt() {
|
|
local phase="$1" result="$2" detail="$3"
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
|
|
}
|
|
|
|
remote_read_only_check() {
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
|
"${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \
|
|
"${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
REMOTE_SCRIPT_DIR="$4"
|
|
REMOTE_CONFIG_ROOT="$5"
|
|
RECEIPT_ROOT="$6"
|
|
STAGE_ROOT="$7"
|
|
shift 7
|
|
EXPECTED_HASHES=("$@")
|
|
LABELS=(
|
|
backup-signoz.sh
|
|
clickhouse-native-backup.sh
|
|
clickhouse-native-restore-drill.sh
|
|
clickhouse-restore-inventory.py
|
|
run-signoz-backup-canary.sh
|
|
backup_disk.xml
|
|
isolated-cluster.xml
|
|
)
|
|
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
|
TARGETS=(
|
|
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
|
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
|
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
|
|
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
|
|
)
|
|
API_URL="http://127.0.0.1:8080/api/v1/health"
|
|
|
|
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
|
|
[ -d /backup ] && [ ! -L /backup ]
|
|
for command_name in awk curl docker pgrep sha256sum ss stat; do
|
|
command -v "${command_name}" >/dev/null 2>&1
|
|
done
|
|
|
|
listener_up() {
|
|
local port="$1"
|
|
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
|
}
|
|
|
|
validate_existing_ancestors() {
|
|
local requested="$1" current="" component
|
|
local -a components
|
|
IFS='/' read -r -a components <<<"${requested#/}"
|
|
for component in "${components[@]}"; do
|
|
[ -n "${component}" ] || continue
|
|
current="${current}/${component}"
|
|
if [ -e "${current}" ] || [ -L "${current}" ]; then
|
|
[ -d "${current}" ] && [ ! -L "${current}" ]
|
|
else
|
|
break
|
|
fi
|
|
done
|
|
}
|
|
|
|
runtime_line() {
|
|
local container="$1" value running health
|
|
value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
|
|
IFS='|' read -r _ _ _ running health <<<"${value}"
|
|
[ "${running}" = true ]
|
|
case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
|
|
printf '%s' "${value}"
|
|
}
|
|
|
|
if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
|
printf 'active SigNoz backup toolchain process detected\n' >&2
|
|
exit 75
|
|
fi
|
|
validate_existing_ancestors "${REMOTE_SCRIPT_DIR}"
|
|
validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/config.d"
|
|
validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
|
validate_existing_ancestors "${RECEIPT_ROOT}"
|
|
validate_existing_ancestors "${STAGE_ROOT}"
|
|
[ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ]
|
|
active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
|
|
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
|
|
[ "${active_operations}" = 0 ]
|
|
|
|
runtime_summary=""
|
|
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
|
|
runtime_summary+="${container}_$(runtime_line "${container}")_"
|
|
done
|
|
for port in 4317 4318 8080; do listener_up "${port}"; done
|
|
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
|
|
[ "${api_code}" = 200 ]
|
|
|
|
matching=0
|
|
drift=0
|
|
absent=0
|
|
for index in "${!TARGETS[@]}"; do
|
|
target="${TARGETS[index]}"
|
|
if [ -e "${target}" ] || [ -L "${target}" ]; then
|
|
[ -f "${target}" ] && [ ! -L "${target}" ]
|
|
current_hash="$(sha256sum "${target}" | awk '{print $1}')"
|
|
current_mode="$(stat -c '%a' "${target}")"
|
|
if [ "${current_hash}" = "${EXPECTED_HASHES[index]}" ] && [ "0${current_mode}" = "${MODES[index]}" ]; then
|
|
matching=$((matching + 1))
|
|
else
|
|
drift=$((drift + 1))
|
|
fi
|
|
else
|
|
absent=$((absent + 1))
|
|
fi
|
|
done
|
|
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"source_of_truth_diff","result":"pass","detail":"matching_%s_drift_%s_absent_%s_active_operations_0_api_200_ports_4317_4318_8080"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${matching}" "${drift}" "${absent}"
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"runtime_readiness","result":"pass","detail":"container_identity_started_at_restart_count_health_captured_%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${runtime_summary%_}"
|
|
REMOTE_CHECK
|
|
}
|
|
|
|
local_receipt sensor_source pass "${SOURCE_DETAIL}"
|
|
local_receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
|
|
remote_read_only_check
|
|
local_receipt check pass "local_syntax_xml_python_compile_and_remote_read_only_preflight"
|
|
|
|
if [ "${MODE}" = --check ]; then
|
|
local_receipt terminal check_pass_no_write "remote_target_unchanged_no_stage_no_candidate"
|
|
exit 0
|
|
fi
|
|
|
|
STAGE_CREATED=0
|
|
cleanup_stage() {
|
|
if [ "${STAGE_CREATED}" -eq 1 ]; then
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
|
|
set -u
|
|
stage_dir="$1"
|
|
if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
|
|
rm -rf "${stage_dir}/pycache" "${stage_dir}/candidate-pycache"
|
|
rm -f \
|
|
"${stage_dir}/backup-signoz.sh" \
|
|
"${stage_dir}/clickhouse-native-backup.sh" \
|
|
"${stage_dir}/clickhouse-native-restore-drill.sh" \
|
|
"${stage_dir}/clickhouse-restore-inventory.py" \
|
|
"${stage_dir}/run-signoz-backup-canary.sh" \
|
|
"${stage_dir}/backup_disk.xml" \
|
|
"${stage_dir}/isolated-cluster.xml"
|
|
rmdir "${stage_dir}" 2>/dev/null || true
|
|
fi
|
|
REMOTE_CLEANUP
|
|
fi
|
|
}
|
|
trap cleanup_stage EXIT
|
|
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
|
|
set -euo pipefail
|
|
stage_dir="$1"
|
|
[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
|
stage_root="${stage_dir%/*}"
|
|
[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ]
|
|
umask 077
|
|
mkdir -m 0700 "${stage_dir}"
|
|
REMOTE_STAGE
|
|
STAGE_CREATED=1
|
|
|
|
for index in "${!LOCAL_SOURCES[@]}"; do
|
|
scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
|
|
"${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"
|
|
done
|
|
|
|
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
|
"${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \
|
|
"${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
REMOTE_SCRIPT_DIR="$4"
|
|
REMOTE_CONFIG_ROOT="$5"
|
|
RECEIPT_ROOT="$6"
|
|
STAGE_DIR="$7"
|
|
shift 7
|
|
EXPECTED_HASHES=("$@")
|
|
LABELS=(
|
|
backup-signoz.sh
|
|
clickhouse-native-backup.sh
|
|
clickhouse-native-restore-drill.sh
|
|
clickhouse-restore-inventory.py
|
|
run-signoz-backup-canary.sh
|
|
backup_disk.xml
|
|
isolated-cluster.xml
|
|
)
|
|
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
|
STAGED=(
|
|
"${STAGE_DIR}/backup-signoz.sh"
|
|
"${STAGE_DIR}/clickhouse-native-backup.sh"
|
|
"${STAGE_DIR}/clickhouse-native-restore-drill.sh"
|
|
"${STAGE_DIR}/clickhouse-restore-inventory.py"
|
|
"${STAGE_DIR}/run-signoz-backup-canary.sh"
|
|
"${STAGE_DIR}/backup_disk.xml"
|
|
"${STAGE_DIR}/isolated-cluster.xml"
|
|
)
|
|
TARGETS=(
|
|
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
|
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
|
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
|
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
|
|
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
|
|
)
|
|
TARGET_DIRS=(
|
|
"${REMOTE_SCRIPT_DIR}"
|
|
"${REMOTE_CONFIG_ROOT}/config.d"
|
|
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
|
)
|
|
CONTAINERS=(signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1)
|
|
API_URL="http://127.0.0.1:8080/api/v1/health"
|
|
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
|
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
|
PRIOR_DIR="${RECEIPT_DIR}/prior"
|
|
PRIOR_MANIFEST="${RECEIPT_DIR}/prior-manifest.tsv"
|
|
DEPLOYED_MANIFEST="${RECEIPT_DIR}/deployed-manifest.tsv"
|
|
RUNTIME_BEFORE="${RECEIPT_DIR}/runtime-before.tsv"
|
|
RUNTIME_AFTER="${RECEIPT_DIR}/runtime-after.tsv"
|
|
LOCK_FILE="/tmp/awoooi-signoz-native-backup-toolchain.lock"
|
|
|
|
APPLY_STARTED=0
|
|
DEPLOY_VERIFIED=0
|
|
RUNTIME_BASELINE_CAPTURED=0
|
|
LOCK_HELD=0
|
|
REMOTE_UID="$(id -u)"
|
|
REMOTE_GID="$(id -g)"
|
|
CREATED_DIRS=()
|
|
PRIOR_PRESENT=()
|
|
PRIOR_MODES=()
|
|
PRIOR_UIDS=()
|
|
PRIOR_GIDS=()
|
|
PRIOR_HASHES=()
|
|
|
|
for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat; do
|
|
command -v "${command_name}" >/dev/null 2>&1 || {
|
|
printf 'required remote command missing: %s\n' "${command_name}" >&2
|
|
exit 69
|
|
}
|
|
done
|
|
if [ "${REMOTE_UID}" -ne 0 ]; then
|
|
command -v sudo >/dev/null 2>&1 || { printf 'required remote command missing: sudo\n' >&2; exit 69; }
|
|
sudo -n true
|
|
fi
|
|
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
|
|
[ -d /backup ] && [ ! -L /backup ]
|
|
|
|
run_root() {
|
|
if [ "${REMOTE_UID}" -eq 0 ]; then
|
|
"$@"
|
|
else
|
|
sudo -n "$@"
|
|
fi
|
|
}
|
|
|
|
root_hash() {
|
|
if [ "${REMOTE_UID}" -eq 0 ]; then
|
|
sha256sum "$1" | awk '{print $1}'
|
|
else
|
|
sudo -n sha256sum "$1" | awk '{print $1}'
|
|
fi
|
|
}
|
|
|
|
ensure_dir_tree() {
|
|
local requested="$1" record_created="$2" current="" component
|
|
local -a components
|
|
IFS='/' read -r -a components <<<"${requested#/}"
|
|
for component in "${components[@]}"; do
|
|
[ -n "${component}" ] || continue
|
|
current="${current}/${component}"
|
|
if [ -e "${current}" ] || [ -L "${current}" ]; then
|
|
[ -d "${current}" ] && [ ! -L "${current}" ]
|
|
else
|
|
run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0755 "${current}"
|
|
if [ "${record_created}" = true ]; then
|
|
CREATED_DIRS+=("${current}")
|
|
fi
|
|
fi
|
|
done
|
|
}
|
|
|
|
ensure_dir_tree "${RECEIPT_ROOT}" false
|
|
[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]
|
|
run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0700 "${RECEIPT_DIR}"
|
|
install -d -m 0700 "${PRIOR_DIR}"
|
|
: > "${RECEIPT_LOG}"
|
|
|
|
receipt() {
|
|
local phase="$1" result="$2" detail="$3" line
|
|
line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
|
|
printf '%s\n' "${line}"
|
|
printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
|
|
}
|
|
|
|
listener_up() {
|
|
local port="$1"
|
|
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
|
}
|
|
|
|
runtime_snapshot() {
|
|
local container value running health port api_code
|
|
for container in "${CONTAINERS[@]}"; do
|
|
value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
|
|
IFS='|' read -r _ _ _ running health <<<"${value}"
|
|
[ "${running}" = true ]
|
|
case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
|
|
printf 'container\t%s\t%s\n' "${container}" "${value}"
|
|
done
|
|
for port in 4317 4318 8080; do
|
|
listener_up "${port}"
|
|
printf 'listener\t%s\tup\n' "${port}"
|
|
done
|
|
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
|
|
[ "${api_code}" = 200 ]
|
|
printf 'api\thealth\t%s\n' "${api_code}"
|
|
}
|
|
|
|
validate_xml_pair() {
|
|
python3 - "$1" "$2" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
backup_path = Path(sys.argv[1])
|
|
cluster_path = Path(sys.argv[2])
|
|
backup = ET.parse(backup_path).getroot()
|
|
cluster = ET.parse(cluster_path).getroot()
|
|
assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
|
|
assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
|
|
assert backup.findtext("./backups/allowed_disk") == "backups"
|
|
assert backup.findtext("./backups/allowed_path") == "/backups/"
|
|
assert backup.findtext("./backups/allow_concurrent_backups") == "false"
|
|
assert backup.findtext("./backups/allow_concurrent_restores") == "false"
|
|
assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
|
|
assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
|
|
assert cluster.findtext("./macros/shard") == "01"
|
|
replica = cluster.find("./macros/replica")
|
|
assert replica is not None
|
|
assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
|
|
PY
|
|
}
|
|
|
|
cleanup_transients() {
|
|
local index rc=0
|
|
for index in "${!TARGETS[@]}"; do
|
|
run_root rm -f \
|
|
"${TARGETS[index]}.candidate.${RUN_ID}" \
|
|
"${TARGETS[index]}.rollback.${RUN_ID}" || rc=1
|
|
done
|
|
if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then
|
|
rm -rf "${STAGE_DIR}/pycache" "${STAGE_DIR}/candidate-pycache" || rc=1
|
|
rm -f "${STAGED[@]}" || rc=1
|
|
rmdir "${STAGE_DIR}" 2>/dev/null || rc=1
|
|
else
|
|
rc=1
|
|
fi
|
|
return "${rc}"
|
|
}
|
|
|
|
verify_residue_zero() {
|
|
local index
|
|
[ ! -e "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
|
|
for index in "${!TARGETS[@]}"; do
|
|
[ ! -e "${TARGETS[index]}.candidate.${RUN_ID}" ] \
|
|
&& [ ! -L "${TARGETS[index]}.candidate.${RUN_ID}" ] || return 1
|
|
[ ! -e "${TARGETS[index]}.rollback.${RUN_ID}" ] \
|
|
&& [ ! -L "${TARGETS[index]}.rollback.${RUN_ID}" ] || return 1
|
|
done
|
|
}
|
|
|
|
rollback_targets() (
|
|
set -euo pipefail
|
|
local index target rollback_candidate actual_mode actual_hash
|
|
for index in "${!TARGETS[@]}"; do
|
|
target="${TARGETS[index]}"
|
|
if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
|
|
rollback_candidate="${target}.rollback.${RUN_ID}"
|
|
run_root install \
|
|
-o "${PRIOR_UIDS[index]}" -g "${PRIOR_GIDS[index]}" \
|
|
-m "${PRIOR_MODES[index]}" \
|
|
"${PRIOR_DIR}/${LABELS[index]}" "${rollback_candidate}"
|
|
[ "$(root_hash "${rollback_candidate}")" = "${PRIOR_HASHES[index]}" ]
|
|
run_root mv -f "${rollback_candidate}" "${target}"
|
|
actual_mode="$(stat -c '%a' "${target}")"
|
|
actual_hash="$(root_hash "${target}")"
|
|
[ "${actual_mode}" = "${PRIOR_MODES[index]}" ]
|
|
[ "${actual_hash}" = "${PRIOR_HASHES[index]}" ]
|
|
else
|
|
run_root rm -f "${target}"
|
|
[ ! -e "${target}" ] && [ ! -L "${target}" ]
|
|
fi
|
|
done
|
|
|
|
for ((index=${#CREATED_DIRS[@]} - 1; index >= 0; index--)); do
|
|
run_root rmdir "${CREATED_DIRS[index]}"
|
|
done
|
|
|
|
if [ "${RUNTIME_BASELINE_CAPTURED}" -eq 1 ]; then
|
|
runtime_snapshot > "${RECEIPT_DIR}/runtime-rollback.tsv"
|
|
cmp -s "${RUNTIME_BEFORE}" "${RECEIPT_DIR}/runtime-rollback.tsv"
|
|
fi
|
|
receipt rollback pass "all_prior_files_modes_hashes_owners_and_runtime_restored"
|
|
)
|
|
|
|
on_exit() {
|
|
local rc=$? rollback_rc=0 cleanup_rc=0 residue_rc=0 terminal=failed
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
|
rollback_targets
|
|
rollback_rc=$?
|
|
if [ "${rollback_rc}" -ne 0 ]; then
|
|
receipt rollback failed "prior_state_restore_or_runtime_verifier_failed"
|
|
rc=91
|
|
fi
|
|
elif [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 0 ]; then
|
|
receipt rollback no_write "bounded_execution_not_started_targets_unchanged"
|
|
fi
|
|
cleanup_transients
|
|
cleanup_rc=$?
|
|
verify_residue_zero
|
|
residue_rc=$?
|
|
if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then
|
|
rc=92
|
|
fi
|
|
if [ "${LOCK_HELD}" -eq 1 ]; then
|
|
flock -u 9 >/dev/null 2>&1
|
|
exec 9>&-
|
|
rm -f "${LOCK_FILE}"
|
|
fi
|
|
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
|
|
terminal=pass
|
|
fi
|
|
receipt terminal "${terminal}" "exit_${rc}_stage_candidate_residue_0_receipt_ref_${RECEIPT_LOG}"
|
|
exit "${rc}"
|
|
}
|
|
|
|
trap on_exit EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
exec 9>>"${LOCK_FILE}"
|
|
flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; }
|
|
LOCK_HELD=1
|
|
|
|
if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
|
receipt check failed active_signoz_backup_toolchain_process
|
|
exit 75
|
|
fi
|
|
active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
|
|
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
|
|
[ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
|
|
|
|
runtime_snapshot > "${RUNTIME_BEFORE}"
|
|
RUNTIME_BASELINE_CAPTURED=1
|
|
|
|
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
|
for index in "${!STAGED[@]}"; do
|
|
[ -f "${STAGED[index]}" ] && [ ! -L "${STAGED[index]}" ]
|
|
[ "$(sha256sum "${STAGED[index]}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
|
|
done
|
|
for index in 0 1 2 4; do bash -n "${STAGED[index]}"; done
|
|
PYTHONPYCACHEPREFIX="${STAGE_DIR}/pycache" python3 -m py_compile "${STAGED[3]}"
|
|
validate_xml_pair "${STAGED[5]}" "${STAGED[6]}"
|
|
|
|
: > "${PRIOR_MANIFEST}"
|
|
for index in "${!TARGETS[@]}"; do
|
|
target="${TARGETS[index]}"
|
|
[ ! -e "${target}.candidate.${RUN_ID}" ] && [ ! -L "${target}.candidate.${RUN_ID}" ]
|
|
[ ! -e "${target}.rollback.${RUN_ID}" ] && [ ! -L "${target}.rollback.${RUN_ID}" ]
|
|
if [ -e "${target}" ] || [ -L "${target}" ]; then
|
|
[ -f "${target}" ] && [ ! -L "${target}" ]
|
|
PRIOR_PRESENT[index]=1
|
|
PRIOR_MODES[index]="$(stat -c '%a' "${target}")"
|
|
PRIOR_UIDS[index]="$(stat -c '%u' "${target}")"
|
|
PRIOR_GIDS[index]="$(stat -c '%g' "${target}")"
|
|
PRIOR_HASHES[index]="$(root_hash "${target}")"
|
|
run_root cp -p "${target}" "${PRIOR_DIR}/${LABELS[index]}"
|
|
[ "$(root_hash "${PRIOR_DIR}/${LABELS[index]}")" = "${PRIOR_HASHES[index]}" ]
|
|
else
|
|
PRIOR_PRESENT[index]=0
|
|
PRIOR_MODES[index]=absent
|
|
PRIOR_UIDS[index]=absent
|
|
PRIOR_GIDS[index]=absent
|
|
PRIOR_HASHES[index]=absent
|
|
fi
|
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
|
"${LABELS[index]}" "${target}" "${PRIOR_PRESENT[index]}" \
|
|
"${PRIOR_MODES[index]}" "${PRIOR_UIDS[index]}" "${PRIOR_GIDS[index]}" \
|
|
"${PRIOR_HASHES[index]}" >> "${PRIOR_MANIFEST}"
|
|
done
|
|
|
|
receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources"
|
|
receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
|
|
receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}"
|
|
receipt ai_decision candidate "atomic_install_native_backup_toolchain_only"
|
|
receipt risk_policy pass "risk_medium_bounded_files_only_no_runtime_execution_no_container_change_full_rollback"
|
|
receipt check pass "runtime_baseline_active_operations_0_hash_bash_xml_py_compile"
|
|
|
|
APPLY_STARTED=1
|
|
for directory in "${TARGET_DIRS[@]}"; do
|
|
ensure_dir_tree "${directory}" true
|
|
done
|
|
|
|
for index in "${!TARGETS[@]}"; do
|
|
target="${TARGETS[index]}"
|
|
if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
|
|
target_uid="${PRIOR_UIDS[index]}"
|
|
target_gid="${PRIOR_GIDS[index]}"
|
|
else
|
|
target_uid="${REMOTE_UID}"
|
|
target_gid="${REMOTE_GID}"
|
|
fi
|
|
run_root install -o "${target_uid}" -g "${target_gid}" -m "${MODES[index]}" \
|
|
"${STAGED[index]}" "${target}.candidate.${RUN_ID}"
|
|
[ "$(root_hash "${target}.candidate.${RUN_ID}")" = "${EXPECTED_HASHES[index]}" ]
|
|
[ "0$(stat -c '%a' "${target}.candidate.${RUN_ID}")" = "${MODES[index]}" ]
|
|
done
|
|
|
|
for index in "${!TARGETS[@]}"; do
|
|
run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"
|
|
done
|
|
|
|
for index in "${!TARGETS[@]}"; do
|
|
[ -f "${TARGETS[index]}" ] && [ ! -L "${TARGETS[index]}" ]
|
|
[ "$(root_hash "${TARGETS[index]}")" = "${EXPECTED_HASHES[index]}" ]
|
|
[ "0$(stat -c '%a' "${TARGETS[index]}")" = "${MODES[index]}" ]
|
|
done
|
|
for index in 0 1 2 4; do bash -n "${TARGETS[index]}"; done
|
|
PYTHONPYCACHEPREFIX="${STAGE_DIR}/candidate-pycache" python3 -m py_compile "${TARGETS[3]}"
|
|
validate_xml_pair "${TARGETS[5]}" "${TARGETS[6]}"
|
|
|
|
: > "${DEPLOYED_MANIFEST}"
|
|
for index in "${!TARGETS[@]}"; do
|
|
printf '%s\t%s\t%s\t%s\n' \
|
|
"${LABELS[index]}" "${TARGETS[index]}" "${MODES[index]}" \
|
|
"${EXPECTED_HASHES[index]}" >> "${DEPLOYED_MANIFEST}"
|
|
done
|
|
|
|
runtime_snapshot > "${RUNTIME_AFTER}"
|
|
cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"
|
|
receipt execution pass "seven_candidates_atomically_replaced_expected_hashes_and_modes"
|
|
receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged"
|
|
receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull"
|
|
DEPLOY_VERIFIED=1
|
|
REMOTE_APPLY
|
|
|
|
trap - EXIT
|
|
cleanup_stage
|
|
local_receipt terminal pass "remote_controlled_apply_postverify_and_stage_candidate_residue_0"
|