feat(observability): add safe SigNoz metadata backup lane
This commit is contained in:
ogt
505
scripts/ops/deploy-signoz-metadata-toolchain.sh
Executable file
505
scripts/ops/deploy-signoz-metadata-toolchain.sh
Executable file
@@ -0,0 +1,505 @@
|
||||
#!/usr/bin/env bash
|
||||
# P0-OBS-002 - deploy an immutable, inactive SigNoz metadata toolchain to host110.
|
||||
#
|
||||
# The deployer never reads credentials, SQLite, or Docker volumes. It does not
|
||||
# change an active pointer or invoke an authenticated metadata export. Failed
|
||||
# candidates are moved into the run receipt directory rather than deleted.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TARGET_HOST="${SIGNOZ_METADATA_TARGET_HOST:-wooo@192.168.0.110}"
|
||||
REMOTE_ROOT="${SIGNOZ_METADATA_REMOTE_ROOT:-/backup/toolchains/signoz-metadata}"
|
||||
RECEIPT_ROOT="${SIGNOZ_METADATA_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-metadata-toolchain}"
|
||||
STAGE_ROOT="${SIGNOZ_METADATA_STAGE_ROOT:-/tmp}"
|
||||
SSH_TIMEOUT_SECONDS="${SIGNOZ_METADATA_SSH_TIMEOUT_SECONDS:-120}"
|
||||
SCP_TIMEOUT_SECONDS="${SIGNOZ_METADATA_SCP_TIMEOUT_SECONDS:-120}"
|
||||
REMOTE_TIMEOUT_SECONDS="${SIGNOZ_METADATA_REMOTE_TIMEOUT_SECONDS:-60}"
|
||||
KILL_AFTER_SECONDS="${SIGNOZ_METADATA_KILL_AFTER_SECONDS:-10}"
|
||||
LOCAL_PYTHON_BIN="${SIGNOZ_METADATA_LOCAL_PYTHON_BIN:-python3.11}"
|
||||
|
||||
LABELS=(
|
||||
metadata-export-policy.json
|
||||
signoz_metadata_contract.py
|
||||
signoz-metadata-export.py
|
||||
verify-signoz-metadata-export.py
|
||||
signoz-metadata-restore-drill.py
|
||||
)
|
||||
LOCAL_SOURCES=(
|
||||
"${ROOT_DIR}/config/signoz/metadata-export-policy.json"
|
||||
"${ROOT_DIR}/scripts/backup/signoz_metadata_contract.py"
|
||||
"${ROOT_DIR}/scripts/backup/signoz-metadata-export.py"
|
||||
"${ROOT_DIR}/scripts/backup/verify-signoz-metadata-export.py"
|
||||
"${ROOT_DIR}/scripts/backup/signoz-metadata-restore-drill.py"
|
||||
)
|
||||
MODES=(0644 0644 0755 0755 0755)
|
||||
SSH_OPTIONS=(
|
||||
-o BatchMode=yes
|
||||
-o ConnectTimeout=8
|
||||
-o ConnectionAttempts=1
|
||||
-o ServerAliveInterval=5
|
||||
-o ServerAliveCountMax=2
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: deploy-signoz-metadata-toolchain.sh [--check|--apply]
|
||||
|
||||
Required environment:
|
||||
TRACE_ID Path-safe controlled-apply trace identifier
|
||||
RUN_ID Unique path-safe execution identifier
|
||||
WORK_ITEM_ID Must be P0-OBS-002
|
||||
|
||||
--check validates the five local sources and performs a no-write host110 diff.
|
||||
--apply creates one immutable exact-hash directory. It does not create or
|
||||
change an active pointer and does not run an authenticated metadata export.
|
||||
USAGE
|
||||
}
|
||||
|
||||
MODE="${1:---check}"
|
||||
case "${MODE}" in
|
||||
--check|--apply) ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) usage >&2; exit 64 ;;
|
||||
esac
|
||||
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
||||
|
||||
TRACE_ID="${TRACE_ID:-}"
|
||||
RUN_ID="${RUN_ID:-}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
||||
|
||||
valid_identity() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
||||
}
|
||||
|
||||
valid_positive_integer() {
|
||||
[[ "$1" =~ ^[1-9][0-9]*$ ]]
|
||||
}
|
||||
|
||||
safe_absolute_path() {
|
||||
local path="$1"
|
||||
[[ "${path}" == /* ]] || return 1
|
||||
[[ "${path}" != / && "${path}" != */ ]] || return 1
|
||||
[[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1
|
||||
case "/${path#/}/" in
|
||||
*'//'*|*'/../'*|*'/./'*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
valid_identity "${TRACE_ID}" || { echo "invalid TRACE_ID" >&2; exit 64; }
|
||||
valid_identity "${RUN_ID}" || { echo "invalid RUN_ID" >&2; exit 64; }
|
||||
[ "${WORK_ITEM_ID}" = "P0-OBS-002" ] || {
|
||||
echo "WORK_ITEM_ID must be P0-OBS-002" >&2
|
||||
exit 64
|
||||
}
|
||||
for value in "${SSH_TIMEOUT_SECONDS}" "${SCP_TIMEOUT_SECONDS}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" "${KILL_AFTER_SECONDS}"; do
|
||||
valid_positive_integer "${value}" || { echo "invalid timeout" >&2; exit 64; }
|
||||
done
|
||||
for path in "${REMOTE_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
|
||||
safe_absolute_path "${path}" || { echo "unsafe remote path" >&2; exit 64; }
|
||||
done
|
||||
|
||||
bounded_ssh() {
|
||||
timeout --kill-after="${KILL_AFTER_SECONDS}" "${SSH_TIMEOUT_SECONDS}" ssh "$@"
|
||||
}
|
||||
|
||||
bounded_scp() {
|
||||
timeout --kill-after="${KILL_AFTER_SECONDS}" "${SCP_TIMEOUT_SECONDS}" scp "$@"
|
||||
}
|
||||
|
||||
hash_file() {
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
}
|
||||
|
||||
emit() {
|
||||
local phase="$1" terminal="$2" detail="$3"
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-metadata-toolchain","phase":"%s","terminal":"%s","detail":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${terminal}" "${detail}"
|
||||
}
|
||||
|
||||
validate_local_sources() {
|
||||
local source
|
||||
command -v "${LOCAL_PYTHON_BIN}" >/dev/null 2>&1
|
||||
"${LOCAL_PYTHON_BIN}" -c 'import sys; assert sys.version_info >= (3, 10)'
|
||||
for source in "${LOCAL_SOURCES[@]}"; do
|
||||
[ -f "${source}" ] && [ ! -L "${source}" ] && [ -s "${source}" ]
|
||||
done
|
||||
"${LOCAL_PYTHON_BIN}" - "${LOCAL_SOURCES[@]:1}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
for value in sys.argv[1:]:
|
||||
path = Path(value)
|
||||
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
||||
PY
|
||||
PYTHONPATH="${ROOT_DIR}/scripts/backup" "${LOCAL_PYTHON_BIN}" - \
|
||||
"${LOCAL_SOURCES[0]}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from signoz_metadata_contract import load_policy
|
||||
|
||||
policy = load_policy(Path(sys.argv[1]))
|
||||
assert policy["work_item_id"] == "P0-OBS-002"
|
||||
assert policy["source_runtime"]["raw_database_access_allowed"] is False
|
||||
assert policy["source_runtime"]["raw_volume_access_allowed"] is False
|
||||
assert policy["completion_contract"]["full_sqlite_completion_claim_allowed"] is False
|
||||
PY
|
||||
}
|
||||
|
||||
validate_local_sources
|
||||
SOURCE_HASHES=()
|
||||
SOURCE_DETAIL=""
|
||||
for index in "${!LOCAL_SOURCES[@]}"; do
|
||||
source_hash="$(hash_file "${LOCAL_SOURCES[index]}")"
|
||||
SOURCE_HASHES+=("${source_hash}")
|
||||
SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_"
|
||||
done
|
||||
SOURCE_DETAIL="${SOURCE_DETAIL%_}"
|
||||
BUNDLE_ID="$(printf '%s\n' "${SOURCE_HASHES[@]}" | sha256sum | awk '{print $1}')"
|
||||
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
||||
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-metadata-toolchain.${RUN_ID}"
|
||||
|
||||
emit sensor_source pass "local_exact_sources_5"
|
||||
emit normalized_asset_identity pass "bundle_${BUNDLE_ID}"
|
||||
emit ai_decision pass "candidate_additive_immutable_inactive_source_deploy"
|
||||
emit risk_policy_decision pass "medium_no_active_pointer_no_secret_no_raw_sqlite"
|
||||
|
||||
remote_check() {
|
||||
bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" bash -s -- \
|
||||
"${REMOTE_ROOT}" "${BUNDLE_ID}" "${REMOTE_TIMEOUT_SECONDS}" \
|
||||
"${KILL_AFTER_SECONDS}" "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
|
||||
set -euo pipefail
|
||||
|
||||
REMOTE_ROOT="$1"
|
||||
BUNDLE_ID="$2"
|
||||
REMOTE_TIMEOUT_SECONDS="$3"
|
||||
KILL_AFTER_SECONDS="$4"
|
||||
shift 4
|
||||
EXPECTED_HASHES=("$@")
|
||||
LABELS=(
|
||||
metadata-export-policy.json
|
||||
signoz_metadata_contract.py
|
||||
signoz-metadata-export.py
|
||||
verify-signoz-metadata-export.py
|
||||
signoz-metadata-restore-drill.py
|
||||
)
|
||||
MODES=(0644 0644 0755 0755 0755)
|
||||
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
||||
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 5 ]
|
||||
[ -d /backup ] && [ ! -L /backup ]
|
||||
[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ]
|
||||
for command_name in curl docker pgrep python3 sha256sum stat timeout; do
|
||||
command -v "${command_name}" >/dev/null 2>&1
|
||||
done
|
||||
python3 -c 'import sys; assert sys.version_info >= (3, 10)'
|
||||
|
||||
container_state="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
|
||||
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
|
||||
api_status="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
|
||||
http://127.0.0.1:8080/api/v1/health)"
|
||||
[ "${api_status}" = "200" ]
|
||||
|
||||
active_operations="$(
|
||||
(pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \
|
||||
| wc -l | tr -d ' '
|
||||
)"
|
||||
[ "${active_operations}" = "0" ]
|
||||
|
||||
state=absent
|
||||
drift=0
|
||||
if [ -e "${TARGET_DIR}" ] || [ -L "${TARGET_DIR}" ]; then
|
||||
[ -d "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ] || drift=$((drift + 1))
|
||||
for index in "${!LABELS[@]}"; do
|
||||
target="${TARGET_DIR}/${LABELS[index]}"
|
||||
if [ ! -f "${target}" ] || [ -L "${target}" ]; then
|
||||
drift=$((drift + 1))
|
||||
continue
|
||||
fi
|
||||
[ "$(sha256sum "${target}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ] || drift=$((drift + 1))
|
||||
[ "0$(stat -c '%a' "${target}")" = "${MODES[index]}" ] || drift=$((drift + 1))
|
||||
done
|
||||
state=exact
|
||||
[ "${drift}" -eq 0 ] || state=drift
|
||||
fi
|
||||
|
||||
candidate_count=0
|
||||
for candidate in "${REMOTE_ROOT}"/."${BUNDLE_ID}".candidate.*; do
|
||||
[ -e "${candidate}" ] || [ -L "${candidate}" ] || continue
|
||||
candidate_count=$((candidate_count + 1))
|
||||
done
|
||||
[ "${candidate_count}" -eq 0 ]
|
||||
[ "${drift}" -eq 0 ]
|
||||
printf 'REMOTE_CHECK terminal=check_pass_no_write state=%s drift=%s candidate_count=%s api=%s active_operations=%s container_state_sha256=%s\n' \
|
||||
"${state}" "${drift}" "${candidate_count}" "${api_status}" "${active_operations}" \
|
||||
"$(printf '%s' "${container_state}" | sha256sum | awk '{print $1}')"
|
||||
REMOTE_CHECK
|
||||
}
|
||||
|
||||
quarantine_transport_stage() {
|
||||
bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" sudo -n bash -s -- \
|
||||
"${STAGE_DIR}" "${RECEIPT_ROOT}" "${RUN_ID}" <<'REMOTE_QUARANTINE'
|
||||
set -euo pipefail
|
||||
stage_dir="$1"
|
||||
receipt_root="$2"
|
||||
run_id="$3"
|
||||
receipt_dir="${receipt_root}/${run_id}"
|
||||
if [ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
||||
mkdir -p "${receipt_root}"
|
||||
if [ ! -e "${receipt_dir}" ] && [ ! -L "${receipt_dir}" ]; then
|
||||
mkdir -m 0700 "${receipt_dir}"
|
||||
fi
|
||||
[ -d "${receipt_dir}" ] && [ ! -L "${receipt_dir}" ]
|
||||
[ ! -e "${receipt_dir}/transport-stage" ] && [ ! -L "${receipt_dir}/transport-stage" ]
|
||||
mv "${stage_dir}" "${receipt_dir}/transport-stage"
|
||||
REMOTE_QUARANTINE
|
||||
}
|
||||
|
||||
if ! REMOTE_CHECK_OUTPUT="$(remote_check)"; then
|
||||
emit source_of_truth_diff failed "remote_read_only_check_failed"
|
||||
emit terminal failed_no_write "remote_check_failed"
|
||||
exit 1
|
||||
fi
|
||||
case "${REMOTE_CHECK_OUTPUT}" in
|
||||
*"terminal=check_pass_no_write state=absent "*) REMOTE_STATE=absent ;;
|
||||
*"terminal=check_pass_no_write state=exact "*) REMOTE_STATE=exact ;;
|
||||
*)
|
||||
emit source_of_truth_diff failed "remote_check_receipt_invalid"
|
||||
emit terminal failed_no_write "remote_check_receipt_invalid"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
emit source_of_truth_diff pass "remote_state_${REMOTE_STATE}_drift_0"
|
||||
|
||||
if [ "${MODE}" = "--check" ]; then
|
||||
emit check_mode check_pass_no_write "bundle_${BUNDLE_ID}_remote_state_${REMOTE_STATE}"
|
||||
emit rollback no_write "active_pointer_unchanged"
|
||||
emit terminal check_pass_no_write "source_ready_remote_state_${REMOTE_STATE}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${REMOTE_STATE}" = "exact" ]; then
|
||||
emit check_mode pass "immutable_bundle_already_exact"
|
||||
emit bounded_execution idempotent_no_write "bundle_${BUNDLE_ID}"
|
||||
emit independent_post_verifier pass "remote_exact_hash_mode_api_identity"
|
||||
emit rollback no_write "active_pointer_absent"
|
||||
emit learning_writeback partial_degraded "runtime_export_not_executed"
|
||||
emit terminal pass_source_deployed_inactive "idempotent_runtime_export_pending"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" bash -s -- \
|
||||
"${STAGE_DIR}" <<'REMOTE_STAGE'; then
|
||||
set -euo pipefail
|
||||
stage_dir="$1"
|
||||
[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
||||
stage_root="${stage_dir%/*}"
|
||||
[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ]
|
||||
umask 077
|
||||
mkdir -m 0700 "${stage_dir}"
|
||||
REMOTE_STAGE
|
||||
emit bounded_execution failed "transport_stage_create_failed"
|
||||
emit terminal failed_no_active_pointer_write "stage_create_failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for index in "${!LOCAL_SOURCES[@]}"; do
|
||||
if ! bounded_scp -q "${SSH_OPTIONS[@]}" "${LOCAL_SOURCES[index]}" \
|
||||
"${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"; then
|
||||
quarantine_transport_stage >/dev/null 2>&1 || true
|
||||
emit bounded_execution failed "transport_failed_index_${index}"
|
||||
emit terminal failed_no_active_pointer_write "transport_stage_quarantine_attempted"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
if ! REMOTE_APPLY_OUTPUT="$(bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" \
|
||||
sudo -n bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${REMOTE_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" "${BUNDLE_ID}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" "${KILL_AFTER_SECONDS}" \
|
||||
"${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
REMOTE_ROOT="$4"
|
||||
RECEIPT_ROOT="$5"
|
||||
STAGE_DIR="$6"
|
||||
BUNDLE_ID="$7"
|
||||
REMOTE_TIMEOUT_SECONDS="$8"
|
||||
KILL_AFTER_SECONDS="$9"
|
||||
shift 9
|
||||
EXPECTED_HASHES=("$@")
|
||||
LABELS=(
|
||||
metadata-export-policy.json
|
||||
signoz_metadata_contract.py
|
||||
signoz-metadata-export.py
|
||||
verify-signoz-metadata-export.py
|
||||
signoz-metadata-restore-drill.py
|
||||
)
|
||||
MODES=(0644 0644 0755 0755 0755)
|
||||
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
||||
CANDIDATE_DIR="${REMOTE_ROOT}/.${BUNDLE_ID}.candidate.${RUN_ID}"
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
||||
DEPLOYED=0
|
||||
TERMINAL=failed
|
||||
|
||||
emit_remote() {
|
||||
local phase="$1" terminal="$2" detail="$3"
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-metadata-toolchain","phase":"%s","terminal":"%s","detail":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${terminal}" "${detail}" | tee -a "${RECEIPT_LOG}"
|
||||
}
|
||||
|
||||
finalize() {
|
||||
local rc=$? quarantine_terminal=not_required
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
if [ -d "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]; then
|
||||
if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then
|
||||
mv "${STAGE_DIR}" "${RECEIPT_DIR}/transport-stage"
|
||||
fi
|
||||
if [ "${rc}" -ne 0 ]; then
|
||||
quarantine_terminal=inactive_candidates_preserved
|
||||
if [ -d "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]; then
|
||||
mv "${CANDIDATE_DIR}" "${RECEIPT_DIR}/quarantine-candidate"
|
||||
fi
|
||||
if [ "${DEPLOYED}" -eq 1 ] && [ -d "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]; then
|
||||
mv "${TARGET_DIR}" "${RECEIPT_DIR}/quarantine-target"
|
||||
fi
|
||||
fi
|
||||
emit_remote rollback "${quarantine_terminal}" "active_pointer_never_created"
|
||||
emit_remote terminal "${TERMINAL}" "exit_${rc}_runtime_export_not_executed"
|
||||
fi
|
||||
exit "${rc}"
|
||||
}
|
||||
trap finalize EXIT HUP INT TERM
|
||||
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 5 ]
|
||||
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
||||
[ ! -e "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]
|
||||
[ ! -e "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]
|
||||
[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ]
|
||||
[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]
|
||||
python3 -c 'import sys; assert sys.version_info >= (3, 10)'
|
||||
umask 077
|
||||
mkdir -p "${REMOTE_ROOT}" "${RECEIPT_ROOT}"
|
||||
mkdir -m 0700 "${RECEIPT_DIR}" "${CANDIDATE_DIR}"
|
||||
: > "${RECEIPT_LOG}"
|
||||
chmod 0600 "${RECEIPT_LOG}"
|
||||
|
||||
exec 8>>/tmp/awoooi-signoz-backup-operation.lock
|
||||
flock -n 8
|
||||
active_operations="$(
|
||||
(pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \
|
||||
| wc -l | tr -d ' '
|
||||
)"
|
||||
[ "${active_operations}" = "0" ]
|
||||
|
||||
runtime_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
|
||||
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
|
||||
api_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
|
||||
http://127.0.0.1:8080/api/v1/health)"
|
||||
[ "${api_before}" = "200" ]
|
||||
emit_remote sensor_source pass "staged_sources_5_runtime_api_200"
|
||||
|
||||
for index in "${!LABELS[@]}"; do
|
||||
staged="${STAGE_DIR}/${LABELS[index]}"
|
||||
[ -f "${staged}" ] && [ ! -L "${staged}" ]
|
||||
[ "$(sha256sum "${staged}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
|
||||
install -o root -g root -m "${MODES[index]}" "${staged}" \
|
||||
"${CANDIDATE_DIR}/${LABELS[index]}"
|
||||
done
|
||||
|
||||
python3 - "${CANDIDATE_DIR}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
root = Path(sys.argv[1])
|
||||
for name in (
|
||||
"signoz_metadata_contract.py",
|
||||
"signoz-metadata-export.py",
|
||||
"verify-signoz-metadata-export.py",
|
||||
"signoz-metadata-restore-drill.py",
|
||||
):
|
||||
path = root / name
|
||||
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
||||
PY
|
||||
PYTHONPATH="${CANDIDATE_DIR}" python3 "${CANDIDATE_DIR}/signoz-metadata-export.py" \
|
||||
--check --base-url https://signoz.invalid \
|
||||
--output-dir "${RECEIPT_DIR}/offline-output-must-not-exist" \
|
||||
--policy "${CANDIDATE_DIR}/metadata-export-policy.json" \
|
||||
--trace-id "${TRACE_ID}" --run-id "${RUN_ID}" \
|
||||
--work-item-id "${WORK_ITEM_ID}" >/dev/null
|
||||
[ ! -e "${RECEIPT_DIR}/offline-output-must-not-exist" ]
|
||||
emit_remote check_mode pass "candidate_hash_mode_compile_policy_check"
|
||||
|
||||
mv "${CANDIDATE_DIR}" "${TARGET_DIR}"
|
||||
chmod 0750 "${TARGET_DIR}"
|
||||
DEPLOYED=1
|
||||
|
||||
for index in "${!LABELS[@]}"; do
|
||||
target="${TARGET_DIR}/${LABELS[index]}"
|
||||
[ -f "${target}" ] && [ ! -L "${target}" ]
|
||||
[ "$(sha256sum "${target}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
|
||||
[ "0$(stat -c '%a' "${target}")" = "${MODES[index]}" ]
|
||||
printf '%s\t%s\t%s\n' "${LABELS[index]}" "${EXPECTED_HASHES[index]}" "${MODES[index]}" \
|
||||
>> "${RECEIPT_DIR}/deployed-manifest.tsv"
|
||||
done
|
||||
chmod 0600 "${RECEIPT_DIR}/deployed-manifest.tsv"
|
||||
runtime_after="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
|
||||
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
|
||||
api_after="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
|
||||
http://127.0.0.1:8080/api/v1/health)"
|
||||
[ "${runtime_after}" = "${runtime_before}" ]
|
||||
[ "${api_after}" = "200" ]
|
||||
emit_remote bounded_execution pass "immutable_bundle_deployed_active_pointer_0"
|
||||
emit_remote independent_post_verifier pass "hash_mode_compile_api_identity_unchanged"
|
||||
emit_remote learning_writeback partial_degraded "authenticated_export_restore_pending"
|
||||
TERMINAL=pass_source_deployed_inactive
|
||||
REMOTE_APPLY
|
||||
)"; then
|
||||
quarantine_transport_stage >/dev/null 2>&1 || true
|
||||
emit bounded_execution failed "remote_apply_failed_inactive_candidate_quarantined"
|
||||
emit terminal failed_no_active_pointer_write "runtime_export_not_executed"
|
||||
exit 1
|
||||
fi
|
||||
case "${REMOTE_APPLY_OUTPUT}" in
|
||||
*'"phase":"terminal","terminal":"pass_source_deployed_inactive"'*) ;;
|
||||
*)
|
||||
emit independent_post_verifier failed "remote_apply_terminal_invalid"
|
||||
emit terminal failed_no_active_pointer_write "runtime_export_not_executed"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
if ! POSTCHECK_OUTPUT="$(remote_check)"; then
|
||||
emit independent_post_verifier failed "remote_postcheck_failed"
|
||||
emit terminal failed_source_deploy_unverified "active_pointer_0"
|
||||
exit 1
|
||||
fi
|
||||
case "${POSTCHECK_OUTPUT}" in
|
||||
*"terminal=check_pass_no_write state=exact "*) ;;
|
||||
*)
|
||||
emit independent_post_verifier failed "remote_postcheck_receipt_invalid"
|
||||
emit terminal failed_source_deploy_unverified "active_pointer_0"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
emit check_mode pass "precheck_absent"
|
||||
emit bounded_execution pass "immutable_bundle_${BUNDLE_ID}_active_pointer_0"
|
||||
emit independent_post_verifier pass "postcheck_exact_hash_mode_api_identity"
|
||||
emit rollback not_required "inactive_additive_bundle"
|
||||
emit learning_writeback partial_degraded "runtime_export_restore_zero_residue_pending"
|
||||
emit terminal pass_source_deployed_inactive "runtime_export_not_executed"
|
||||
179
scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py
Normal file
179
scripts/ops/tests/test_signoz_metadata_toolchain_deploy.py
Normal file
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
DEPLOYER = ROOT / "scripts" / "ops" / "deploy-signoz-metadata-toolchain.sh"
|
||||
|
||||
|
||||
def _write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def fake_environment(tmp_path: Path, *, mode: str) -> dict[str, str]:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
event_log = tmp_path / "events.log"
|
||||
ssh_count = tmp_path / "ssh.count"
|
||||
event_log.touch()
|
||||
ssh_count.write_text("0\n", encoding="utf-8")
|
||||
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""#!/bin/bash
|
||||
set -eu
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do shift; done
|
||||
shift
|
||||
exec "$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "ssh",
|
||||
"""#!/bin/bash
|
||||
set -eu
|
||||
count="$(cat "${TEST_SSH_COUNT:?}")"
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "$count" > "${TEST_SSH_COUNT:?}"
|
||||
printf 'ssh %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
cat >/dev/null || true
|
||||
case "${FAKE_SSH_MODE:?}" in
|
||||
check_absent)
|
||||
printf 'REMOTE_CHECK terminal=check_pass_no_write state=absent drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
|
||||
;;
|
||||
check_drift)
|
||||
exit 3
|
||||
;;
|
||||
apply)
|
||||
case "$count" in
|
||||
1)
|
||||
printf 'REMOTE_CHECK terminal=check_pass_no_write state=absent drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
|
||||
;;
|
||||
2) ;;
|
||||
3)
|
||||
printf '{"phase":"terminal","terminal":"pass_source_deployed_inactive"}\n'
|
||||
;;
|
||||
4)
|
||||
printf 'REMOTE_CHECK terminal=check_pass_no_write state=exact drift=0 candidate_count=0 api=200 active_operations=0 container_state_sha256=%064d\n' 0
|
||||
;;
|
||||
*) exit 90 ;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "scp",
|
||||
"""#!/bin/bash
|
||||
set -eu
|
||||
printf 'scp %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
""",
|
||||
)
|
||||
return {
|
||||
**os.environ,
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TEST_EVENT_LOG": str(event_log),
|
||||
"TEST_SSH_COUNT": str(ssh_count),
|
||||
"FAKE_SSH_MODE": mode,
|
||||
"TRACE_ID": "trace-P0-OBS-002-metadata-deploy",
|
||||
"RUN_ID": f"run-{mode}",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
}
|
||||
|
||||
|
||||
def run_deployer(
|
||||
tmp_path: Path, *, mode: str, apply: bool
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["bash", str(DEPLOYER), "--apply" if apply else "--check"],
|
||||
env=fake_environment(tmp_path, mode=mode),
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def receipt_rows(stdout: str) -> list[dict[str, object]]:
|
||||
return [json.loads(line) for line in stdout.splitlines() if line.startswith("{")]
|
||||
|
||||
|
||||
def test_help_documents_inactive_immutable_contract() -> None:
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--help"],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
assert "immutable exact-hash directory" in result.stdout
|
||||
assert "does not create or" in result.stdout
|
||||
assert "active pointer" in result.stdout
|
||||
|
||||
|
||||
def test_check_mode_is_no_write_and_ready_when_bundle_absent(tmp_path: Path) -> None:
|
||||
result = run_deployer(tmp_path, mode="check_absent", apply=False)
|
||||
assert result.returncode == 0, result.stderr
|
||||
rows = receipt_rows(result.stdout)
|
||||
assert rows[-1]["terminal"] == "check_pass_no_write"
|
||||
assert rows[-1]["detail"].endswith("remote_state_absent")
|
||||
events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines()
|
||||
assert len([line for line in events if line.startswith("ssh ")]) == 1
|
||||
assert not [line for line in events if line.startswith("scp ")]
|
||||
|
||||
|
||||
def test_check_mode_fails_closed_on_remote_drift(tmp_path: Path) -> None:
|
||||
result = run_deployer(tmp_path, mode="check_drift", apply=False)
|
||||
assert result.returncode == 1
|
||||
rows = receipt_rows(result.stdout)
|
||||
assert rows[-1]["terminal"] == "failed_no_write"
|
||||
|
||||
|
||||
def test_apply_uses_five_transfers_and_exact_postcheck(tmp_path: Path) -> None:
|
||||
result = run_deployer(tmp_path, mode="apply", apply=True)
|
||||
assert result.returncode == 0, result.stderr
|
||||
rows = receipt_rows(result.stdout)
|
||||
assert rows[-1]["terminal"] == "pass_source_deployed_inactive"
|
||||
assert rows[-1]["detail"] == "runtime_export_not_executed"
|
||||
events = (tmp_path / "events.log").read_text(encoding="utf-8").splitlines()
|
||||
assert len([line for line in events if line.startswith("scp ")]) == 5
|
||||
assert len([line for line in events if line.startswith("ssh ")]) == 4
|
||||
|
||||
|
||||
def test_deployer_has_no_active_pointer_or_destructive_fallback() -> None:
|
||||
source = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert '"${REMOTE_ROOT}/current"' in source
|
||||
assert "active_pointer_0" in source
|
||||
assert "pass_source_deployed_inactive" in source
|
||||
assert "quarantine-candidate" in source
|
||||
assert "quarantine-target" in source
|
||||
assert "docker stop" not in source
|
||||
assert "docker start" not in source
|
||||
assert "docker restart" not in source
|
||||
assert "sqlite3" not in source
|
||||
assert "github.com" not in source.casefold()
|
||||
assert "curl |" not in source
|
||||
assert "curl -fsSL" not in source
|
||||
assert "\nrm " not in source
|
||||
assert "ln -s" not in source
|
||||
|
||||
|
||||
def test_exact_five_file_supply_chain_and_modes_are_fixed() -> None:
|
||||
source = DEPLOYER.read_text(encoding="utf-8")
|
||||
for path in (
|
||||
"config/signoz/metadata-export-policy.json",
|
||||
"scripts/backup/signoz_metadata_contract.py",
|
||||
"scripts/backup/signoz-metadata-export.py",
|
||||
"scripts/backup/verify-signoz-metadata-export.py",
|
||||
"scripts/backup/signoz-metadata-restore-drill.py",
|
||||
):
|
||||
assert path in source
|
||||
assert "MODES=(0644 0644 0755 0755 0755)" in source
|
||||
assert '[ "${#EXPECTED_HASHES[@]}" -eq 5 ]' in source
|
||||
Reference in New Issue
Block a user