517 lines
19 KiB
Bash
Executable File
517 lines
19 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# P0-OBS-002 - deploy an immutable, inactive SigNoz metadata toolchain to host110.
|
|
#
|
|
# The deployer never reads credentials, SQLite, or Docker volumes. It does not
|
|
# change an active pointer or invoke an authenticated metadata export. Failed
|
|
# candidates are moved into the run receipt directory rather than deleted.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
TARGET_HOST="${SIGNOZ_METADATA_TARGET_HOST:-wooo@192.168.0.110}"
|
|
REMOTE_ROOT="${SIGNOZ_METADATA_REMOTE_ROOT:-/backup/toolchains/signoz-metadata}"
|
|
RECEIPT_ROOT="${SIGNOZ_METADATA_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-metadata-toolchain}"
|
|
STAGE_ROOT="${SIGNOZ_METADATA_STAGE_ROOT:-/tmp}"
|
|
SSH_TIMEOUT_SECONDS="${SIGNOZ_METADATA_SSH_TIMEOUT_SECONDS:-120}"
|
|
SCP_TIMEOUT_SECONDS="${SIGNOZ_METADATA_SCP_TIMEOUT_SECONDS:-120}"
|
|
REMOTE_TIMEOUT_SECONDS="${SIGNOZ_METADATA_REMOTE_TIMEOUT_SECONDS:-60}"
|
|
KILL_AFTER_SECONDS="${SIGNOZ_METADATA_KILL_AFTER_SECONDS:-10}"
|
|
LOCAL_PYTHON_BIN="${SIGNOZ_METADATA_LOCAL_PYTHON_BIN:-python3.11}"
|
|
|
|
LABELS=(
|
|
metadata-export-policy.json
|
|
signoz_metadata_contract.py
|
|
signoz-metadata-export.py
|
|
verify-signoz-metadata-export.py
|
|
signoz-metadata-restore-drill.py
|
|
)
|
|
LOCAL_SOURCES=(
|
|
"${ROOT_DIR}/config/signoz/metadata-export-policy.json"
|
|
"${ROOT_DIR}/scripts/backup/signoz_metadata_contract.py"
|
|
"${ROOT_DIR}/scripts/backup/signoz-metadata-export.py"
|
|
"${ROOT_DIR}/scripts/backup/verify-signoz-metadata-export.py"
|
|
"${ROOT_DIR}/scripts/backup/signoz-metadata-restore-drill.py"
|
|
)
|
|
MODES=(0644 0644 0755 0755 0755)
|
|
SSH_OPTIONS=(
|
|
-o BatchMode=yes
|
|
-o ConnectTimeout=8
|
|
-o ConnectionAttempts=1
|
|
-o ServerAliveInterval=5
|
|
-o ServerAliveCountMax=2
|
|
)
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: deploy-signoz-metadata-toolchain.sh [--check|--apply]
|
|
|
|
Required environment:
|
|
TRACE_ID Path-safe controlled-apply trace identifier
|
|
RUN_ID Unique path-safe execution identifier
|
|
WORK_ITEM_ID Must be P0-OBS-002
|
|
|
|
--check validates the five local sources and performs a no-write host110 diff.
|
|
--apply creates one immutable exact-hash directory. It does not create or
|
|
change an active pointer and does not run an authenticated metadata export.
|
|
USAGE
|
|
}
|
|
|
|
MODE="${1:---check}"
|
|
case "${MODE}" in
|
|
--check|--apply) ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) usage >&2; exit 64 ;;
|
|
esac
|
|
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
|
|
|
TRACE_ID="${TRACE_ID:-}"
|
|
RUN_ID="${RUN_ID:-}"
|
|
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
|
|
|
valid_identity() {
|
|
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
|
}
|
|
|
|
valid_positive_integer() {
|
|
[[ "$1" =~ ^[1-9][0-9]*$ ]]
|
|
}
|
|
|
|
safe_absolute_path() {
|
|
local path="$1"
|
|
[[ "${path}" == /* ]] || return 1
|
|
[[ "${path}" != / && "${path}" != */ ]] || return 1
|
|
[[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1
|
|
case "/${path#/}/" in
|
|
*'//'*|*'/../'*|*'/./'*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
valid_identity "${TRACE_ID}" || { echo "invalid TRACE_ID" >&2; exit 64; }
|
|
valid_identity "${RUN_ID}" || { echo "invalid RUN_ID" >&2; exit 64; }
|
|
[ "${WORK_ITEM_ID}" = "P0-OBS-002" ] || {
|
|
echo "WORK_ITEM_ID must be P0-OBS-002" >&2
|
|
exit 64
|
|
}
|
|
for value in "${SSH_TIMEOUT_SECONDS}" "${SCP_TIMEOUT_SECONDS}" \
|
|
"${REMOTE_TIMEOUT_SECONDS}" "${KILL_AFTER_SECONDS}"; do
|
|
valid_positive_integer "${value}" || { echo "invalid timeout" >&2; exit 64; }
|
|
done
|
|
for path in "${REMOTE_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
|
|
safe_absolute_path "${path}" || { echo "unsafe remote path" >&2; exit 64; }
|
|
done
|
|
|
|
bounded_ssh() {
|
|
timeout --kill-after="${KILL_AFTER_SECONDS}" "${SSH_TIMEOUT_SECONDS}" ssh "$@"
|
|
}
|
|
|
|
bounded_scp() {
|
|
timeout --kill-after="${KILL_AFTER_SECONDS}" "${SCP_TIMEOUT_SECONDS}" scp "$@"
|
|
}
|
|
|
|
hash_file() {
|
|
sha256sum "$1" | awk '{print $1}'
|
|
}
|
|
|
|
emit() {
|
|
local phase="$1" terminal="$2" detail="$3"
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-metadata-toolchain","phase":"%s","terminal":"%s","detail":"%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${terminal}" "${detail}"
|
|
}
|
|
|
|
validate_local_sources() {
|
|
local source
|
|
command -v "${LOCAL_PYTHON_BIN}" >/dev/null 2>&1
|
|
"${LOCAL_PYTHON_BIN}" -c 'import sys; assert sys.version_info >= (3, 10)'
|
|
for source in "${LOCAL_SOURCES[@]}"; do
|
|
[ -f "${source}" ] && [ ! -L "${source}" ] && [ -s "${source}" ]
|
|
done
|
|
"${LOCAL_PYTHON_BIN}" - "${LOCAL_SOURCES[@]:1}" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
for value in sys.argv[1:]:
|
|
path = Path(value)
|
|
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
|
PY
|
|
PYTHONPATH="${ROOT_DIR}/scripts/backup" "${LOCAL_PYTHON_BIN}" - \
|
|
"${LOCAL_SOURCES[0]}" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
from signoz_metadata_contract import load_policy
|
|
|
|
policy = load_policy(Path(sys.argv[1]))
|
|
assert policy["work_item_id"] == "P0-OBS-002"
|
|
assert policy["source_runtime"]["raw_database_access_allowed"] is False
|
|
assert policy["source_runtime"]["raw_volume_access_allowed"] is False
|
|
assert policy["completion_contract"]["full_sqlite_completion_claim_allowed"] is False
|
|
PY
|
|
}
|
|
|
|
validate_local_sources
|
|
SOURCE_HASHES=()
|
|
SOURCE_DETAIL=""
|
|
for index in "${!LOCAL_SOURCES[@]}"; do
|
|
source_hash="$(hash_file "${LOCAL_SOURCES[index]}")"
|
|
SOURCE_HASHES+=("${source_hash}")
|
|
SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_"
|
|
done
|
|
SOURCE_DETAIL="${SOURCE_DETAIL%_}"
|
|
BUNDLE_ID="$(printf '%s\n' "${SOURCE_HASHES[@]}" | sha256sum | awk '{print $1}')"
|
|
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
|
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-metadata-toolchain.${RUN_ID}"
|
|
|
|
emit sensor_source pass "local_exact_sources_5"
|
|
emit normalized_asset_identity pass "bundle_${BUNDLE_ID}"
|
|
emit ai_decision pass "candidate_additive_immutable_inactive_source_deploy"
|
|
emit risk_policy_decision pass "medium_no_active_pointer_no_secret_no_raw_sqlite"
|
|
|
|
remote_check() {
|
|
# The immutable root is deliberately 0700/root-owned. Run this bounded
|
|
# verifier as root so it can distinguish exact from absent without relaxing
|
|
# directory permissions. REMOTE_CHECK contains only stat/hash/process/API
|
|
# reads and performs no writes.
|
|
bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" sudo -n bash -s -- \
|
|
"${REMOTE_ROOT}" "${BUNDLE_ID}" "${REMOTE_TIMEOUT_SECONDS}" \
|
|
"${KILL_AFTER_SECONDS}" "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
|
|
set -euo pipefail
|
|
|
|
REMOTE_ROOT="$1"
|
|
BUNDLE_ID="$2"
|
|
REMOTE_TIMEOUT_SECONDS="$3"
|
|
KILL_AFTER_SECONDS="$4"
|
|
shift 4
|
|
EXPECTED_HASHES=("$@")
|
|
LABELS=(
|
|
metadata-export-policy.json
|
|
signoz_metadata_contract.py
|
|
signoz-metadata-export.py
|
|
verify-signoz-metadata-export.py
|
|
signoz-metadata-restore-drill.py
|
|
)
|
|
MODES=(0644 0644 0755 0755 0755)
|
|
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
|
|
|
[ "${#EXPECTED_HASHES[@]}" -eq 5 ]
|
|
[ -d /backup ] && [ ! -L /backup ]
|
|
[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ]
|
|
for command_name in curl docker pgrep python3 sha256sum stat timeout; do
|
|
command -v "${command_name}" >/dev/null 2>&1
|
|
done
|
|
python3 -c 'import sys; assert sys.version_info >= (3, 10)'
|
|
|
|
container_state="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
|
|
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
|
|
api_status="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
|
|
http://127.0.0.1:8080/api/v1/health)"
|
|
[ "${api_status}" = "200" ]
|
|
|
|
active_operations="$(
|
|
(pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \
|
|
| wc -l | tr -d ' '
|
|
)"
|
|
[ "${active_operations}" = "0" ]
|
|
|
|
state=absent
|
|
drift=0
|
|
if [ -e "${TARGET_DIR}" ] || [ -L "${TARGET_DIR}" ]; then
|
|
[ -d "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ] || drift=$((drift + 1))
|
|
for index in "${!LABELS[@]}"; do
|
|
target="${TARGET_DIR}/${LABELS[index]}"
|
|
if [ ! -f "${target}" ] || [ -L "${target}" ]; then
|
|
drift=$((drift + 1))
|
|
continue
|
|
fi
|
|
[ "$(sha256sum "${target}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ] || drift=$((drift + 1))
|
|
[ "0$(stat -c '%a' "${target}")" = "${MODES[index]}" ] || drift=$((drift + 1))
|
|
done
|
|
state=exact
|
|
[ "${drift}" -eq 0 ] || state=drift
|
|
fi
|
|
|
|
candidate_count=0
|
|
for candidate in "${REMOTE_ROOT}"/."${BUNDLE_ID}".candidate.*; do
|
|
[ -e "${candidate}" ] || [ -L "${candidate}" ] || continue
|
|
candidate_count=$((candidate_count + 1))
|
|
done
|
|
[ "${candidate_count}" -eq 0 ]
|
|
[ "${drift}" -eq 0 ]
|
|
printf 'REMOTE_CHECK terminal=check_pass_no_write state=%s drift=%s candidate_count=%s api=%s active_operations=%s container_state_sha256=%s\n' \
|
|
"${state}" "${drift}" "${candidate_count}" "${api_status}" "${active_operations}" \
|
|
"$(printf '%s' "${container_state}" | sha256sum | awk '{print $1}')"
|
|
REMOTE_CHECK
|
|
}
|
|
|
|
quarantine_transport_stage() {
|
|
bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" sudo -n bash -s -- \
|
|
"${STAGE_DIR}" "${RECEIPT_ROOT}" "${RUN_ID}" <<'REMOTE_QUARANTINE'
|
|
set -euo pipefail
|
|
stage_dir="$1"
|
|
receipt_root="$2"
|
|
run_id="$3"
|
|
receipt_dir="${receipt_root}/${run_id}"
|
|
if [ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
|
|
exit 0
|
|
fi
|
|
[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
|
mkdir -p "${receipt_root}"
|
|
if [ ! -e "${receipt_dir}" ] && [ ! -L "${receipt_dir}" ]; then
|
|
mkdir -m 0700 "${receipt_dir}"
|
|
fi
|
|
[ -d "${receipt_dir}" ] && [ ! -L "${receipt_dir}" ]
|
|
[ ! -e "${receipt_dir}/transport-stage" ] && [ ! -L "${receipt_dir}/transport-stage" ]
|
|
mv "${stage_dir}" "${receipt_dir}/transport-stage"
|
|
REMOTE_QUARANTINE
|
|
}
|
|
|
|
if ! REMOTE_CHECK_OUTPUT="$(remote_check)"; then
|
|
emit source_of_truth_diff failed "remote_read_only_check_failed"
|
|
emit terminal failed_no_write "remote_check_failed"
|
|
exit 1
|
|
fi
|
|
case "${REMOTE_CHECK_OUTPUT}" in
|
|
*"terminal=check_pass_no_write state=absent "*) REMOTE_STATE=absent ;;
|
|
*"terminal=check_pass_no_write state=exact "*) REMOTE_STATE=exact ;;
|
|
*)
|
|
emit source_of_truth_diff failed "remote_check_receipt_invalid"
|
|
emit terminal failed_no_write "remote_check_receipt_invalid"
|
|
exit 1
|
|
;;
|
|
esac
|
|
emit source_of_truth_diff pass "remote_state_${REMOTE_STATE}_drift_0"
|
|
|
|
if [ "${MODE}" = "--check" ]; then
|
|
emit check_mode check_pass_no_write "bundle_${BUNDLE_ID}_remote_state_${REMOTE_STATE}"
|
|
emit rollback no_write "active_pointer_unchanged"
|
|
emit terminal check_pass_no_write "source_ready_remote_state_${REMOTE_STATE}"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "${REMOTE_STATE}" = "exact" ]; then
|
|
emit check_mode pass "immutable_bundle_already_exact"
|
|
emit bounded_execution idempotent_no_write "bundle_${BUNDLE_ID}"
|
|
emit independent_post_verifier pass "remote_exact_hash_mode_api_identity"
|
|
emit rollback no_write "active_pointer_absent"
|
|
emit learning_writeback partial_degraded "runtime_export_not_executed"
|
|
emit terminal pass_source_deployed_inactive "idempotent_runtime_export_pending"
|
|
exit 0
|
|
fi
|
|
|
|
if ! bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" bash -s -- \
|
|
"${STAGE_DIR}" <<'REMOTE_STAGE'; then
|
|
set -euo pipefail
|
|
stage_dir="$1"
|
|
[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
|
stage_root="${stage_dir%/*}"
|
|
[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ]
|
|
umask 077
|
|
mkdir -m 0700 "${stage_dir}"
|
|
REMOTE_STAGE
|
|
emit bounded_execution failed "transport_stage_create_failed"
|
|
emit terminal failed_no_active_pointer_write "stage_create_failed"
|
|
exit 1
|
|
fi
|
|
|
|
for index in "${!LOCAL_SOURCES[@]}"; do
|
|
if ! bounded_scp -q "${SSH_OPTIONS[@]}" "${LOCAL_SOURCES[index]}" \
|
|
"${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"; then
|
|
quarantine_transport_stage >/dev/null 2>&1 || true
|
|
emit bounded_execution failed "transport_failed_index_${index}"
|
|
emit terminal failed_no_active_pointer_write "transport_stage_quarantine_attempted"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if ! REMOTE_APPLY_OUTPUT="$(bounded_ssh "${SSH_OPTIONS[@]}" "${TARGET_HOST}" \
|
|
sudo -n bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
|
"${REMOTE_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" "${BUNDLE_ID}" \
|
|
"${REMOTE_TIMEOUT_SECONDS}" "${KILL_AFTER_SECONDS}" \
|
|
"${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
REMOTE_ROOT="$4"
|
|
RECEIPT_ROOT="$5"
|
|
STAGE_DIR="$6"
|
|
BUNDLE_ID="$7"
|
|
REMOTE_TIMEOUT_SECONDS="$8"
|
|
KILL_AFTER_SECONDS="$9"
|
|
shift 9
|
|
EXPECTED_HASHES=("$@")
|
|
LABELS=(
|
|
metadata-export-policy.json
|
|
signoz_metadata_contract.py
|
|
signoz-metadata-export.py
|
|
verify-signoz-metadata-export.py
|
|
signoz-metadata-restore-drill.py
|
|
)
|
|
MODES=(0644 0644 0755 0755 0755)
|
|
TARGET_DIR="${REMOTE_ROOT}/${BUNDLE_ID}"
|
|
CANDIDATE_DIR="${REMOTE_ROOT}/.${BUNDLE_ID}.candidate.${RUN_ID}"
|
|
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
|
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
|
DEPLOYED=0
|
|
TERMINAL=failed
|
|
|
|
emit_remote() {
|
|
local phase="$1" terminal="$2" detail="$3"
|
|
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-metadata-toolchain","phase":"%s","terminal":"%s","detail":"%s"}\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${terminal}" "${detail}" | tee -a "${RECEIPT_LOG}"
|
|
}
|
|
|
|
finalize() {
|
|
local rc=$? quarantine_terminal=not_required
|
|
trap - EXIT HUP INT TERM
|
|
set +e
|
|
if [ -d "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]; then
|
|
if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then
|
|
mv "${STAGE_DIR}" "${RECEIPT_DIR}/transport-stage"
|
|
fi
|
|
if [ "${rc}" -ne 0 ]; then
|
|
quarantine_terminal=inactive_candidates_preserved
|
|
if [ -d "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]; then
|
|
mv "${CANDIDATE_DIR}" "${RECEIPT_DIR}/quarantine-candidate"
|
|
fi
|
|
if [ "${DEPLOYED}" -eq 1 ] && [ -d "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]; then
|
|
mv "${TARGET_DIR}" "${RECEIPT_DIR}/quarantine-target"
|
|
fi
|
|
fi
|
|
emit_remote rollback "${quarantine_terminal}" "active_pointer_never_created"
|
|
emit_remote terminal "${TERMINAL}" "exit_${rc}_runtime_export_not_executed"
|
|
fi
|
|
exit "${rc}"
|
|
}
|
|
trap finalize EXIT HUP INT TERM
|
|
|
|
[ "${#EXPECTED_HASHES[@]}" -eq 5 ]
|
|
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
|
[ ! -e "${TARGET_DIR}" ] && [ ! -L "${TARGET_DIR}" ]
|
|
[ ! -e "${CANDIDATE_DIR}" ] && [ ! -L "${CANDIDATE_DIR}" ]
|
|
[ ! -e "${REMOTE_ROOT}/current" ] && [ ! -L "${REMOTE_ROOT}/current" ]
|
|
[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]
|
|
python3 -c 'import sys; assert sys.version_info >= (3, 10)'
|
|
umask 077
|
|
mkdir -p "${REMOTE_ROOT}" "${RECEIPT_ROOT}"
|
|
mkdir -m 0700 "${RECEIPT_DIR}" "${CANDIDATE_DIR}"
|
|
: > "${RECEIPT_LOG}"
|
|
chmod 0600 "${RECEIPT_LOG}"
|
|
|
|
OPERATION_LOCK=/tmp/awoooi-signoz-backup-operation.lock
|
|
[ -f "${OPERATION_LOCK}" ] && [ ! -L "${OPERATION_LOCK}" ]
|
|
# The shared lock is intentionally owned by the unprivileged backup account.
|
|
# Open the existing inode read-only so Linux fs.protected_regular=2 cannot
|
|
# reject a root O_CREAT/O_APPEND open in sticky /tmp. flock(2) still provides
|
|
# the same exclusive inter-process lock without changing file contents,
|
|
# ownership, or mode. Missing/unsafe lock state fails closed.
|
|
exec 8<"${OPERATION_LOCK}"
|
|
flock -n 8
|
|
active_operations="$(
|
|
(pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' || true) \
|
|
| wc -l | tr -d ' '
|
|
)"
|
|
[ "${active_operations}" = "0" ]
|
|
|
|
runtime_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
|
|
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
|
|
api_before="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
|
|
http://127.0.0.1:8080/api/v1/health)"
|
|
[ "${api_before}" = "200" ]
|
|
emit_remote sensor_source pass "staged_sources_5_runtime_api_200"
|
|
|
|
for index in "${!LABELS[@]}"; do
|
|
staged="${STAGE_DIR}/${LABELS[index]}"
|
|
[ -f "${staged}" ] && [ ! -L "${staged}" ]
|
|
[ "$(sha256sum "${staged}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
|
|
install -o root -g root -m "${MODES[index]}" "${staged}" \
|
|
"${CANDIDATE_DIR}/${LABELS[index]}"
|
|
done
|
|
|
|
python3 - "${CANDIDATE_DIR}" <<'PY'
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
root = Path(sys.argv[1])
|
|
for name in (
|
|
"signoz_metadata_contract.py",
|
|
"signoz-metadata-export.py",
|
|
"verify-signoz-metadata-export.py",
|
|
"signoz-metadata-restore-drill.py",
|
|
):
|
|
path = root / name
|
|
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
|
PY
|
|
PYTHONPATH="${CANDIDATE_DIR}" python3 "${CANDIDATE_DIR}/signoz-metadata-export.py" \
|
|
--check --base-url https://signoz.invalid \
|
|
--output-dir "${RECEIPT_DIR}/offline-output-must-not-exist" \
|
|
--policy "${CANDIDATE_DIR}/metadata-export-policy.json" \
|
|
--trace-id "${TRACE_ID}" --run-id "${RUN_ID}" \
|
|
--work-item-id "${WORK_ITEM_ID}" >/dev/null
|
|
[ ! -e "${RECEIPT_DIR}/offline-output-must-not-exist" ]
|
|
emit_remote check_mode pass "candidate_hash_mode_compile_policy_check"
|
|
|
|
mv "${CANDIDATE_DIR}" "${TARGET_DIR}"
|
|
chmod 0750 "${TARGET_DIR}"
|
|
DEPLOYED=1
|
|
|
|
for index in "${!LABELS[@]}"; do
|
|
target="${TARGET_DIR}/${LABELS[index]}"
|
|
[ -f "${target}" ] && [ ! -L "${target}" ]
|
|
[ "$(sha256sum "${target}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
|
|
[ "0$(stat -c '%a' "${target}")" = "${MODES[index]}" ]
|
|
printf '%s\t%s\t%s\n' "${LABELS[index]}" "${EXPECTED_HASHES[index]}" "${MODES[index]}" \
|
|
>> "${RECEIPT_DIR}/deployed-manifest.tsv"
|
|
done
|
|
chmod 0600 "${RECEIPT_DIR}/deployed-manifest.tsv"
|
|
runtime_after="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${REMOTE_TIMEOUT_SECONDS}" docker inspect --format \
|
|
'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' signoz)"
|
|
api_after="$(timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
|
"${REMOTE_TIMEOUT_SECONDS}" curl -sS -o /dev/null -w '%{http_code}' \
|
|
http://127.0.0.1:8080/api/v1/health)"
|
|
[ "${runtime_after}" = "${runtime_before}" ]
|
|
[ "${api_after}" = "200" ]
|
|
emit_remote bounded_execution pass "immutable_bundle_deployed_active_pointer_0"
|
|
emit_remote independent_post_verifier pass "hash_mode_compile_api_identity_unchanged"
|
|
emit_remote learning_writeback partial_degraded "authenticated_export_restore_pending"
|
|
TERMINAL=pass_source_deployed_inactive
|
|
REMOTE_APPLY
|
|
)"; then
|
|
quarantine_transport_stage >/dev/null 2>&1 || true
|
|
emit bounded_execution failed "remote_apply_failed_inactive_candidate_quarantined"
|
|
emit terminal failed_no_active_pointer_write "runtime_export_not_executed"
|
|
exit 1
|
|
fi
|
|
case "${REMOTE_APPLY_OUTPUT}" in
|
|
*'"phase":"terminal","terminal":"pass_source_deployed_inactive"'*) ;;
|
|
*)
|
|
emit independent_post_verifier failed "remote_apply_terminal_invalid"
|
|
emit terminal failed_no_active_pointer_write "runtime_export_not_executed"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if ! POSTCHECK_OUTPUT="$(remote_check)"; then
|
|
emit independent_post_verifier failed "remote_postcheck_failed"
|
|
emit terminal failed_source_deploy_unverified "active_pointer_0"
|
|
exit 1
|
|
fi
|
|
case "${POSTCHECK_OUTPUT}" in
|
|
*"terminal=check_pass_no_write state=exact "*) ;;
|
|
*)
|
|
emit independent_post_verifier failed "remote_postcheck_receipt_invalid"
|
|
emit terminal failed_source_deploy_unverified "active_pointer_0"
|
|
exit 1
|
|
;;
|
|
esac
|
|
emit check_mode pass "precheck_absent"
|
|
emit bounded_execution pass "immutable_bundle_${BUNDLE_ID}_active_pointer_0"
|
|
emit independent_post_verifier pass "postcheck_exact_hash_mode_api_identity"
|
|
emit rollback not_required "inactive_additive_bundle"
|
|
emit learning_writeback partial_degraded "runtime_export_restore_zero_residue_pending"
|
|
emit terminal pass_source_deployed_inactive "runtime_export_not_executed"
|