fix(observability): harden native backup controlled apply

This commit is contained in:
ogt
2026-07-15 09:43:40 +08:00
parent 322e812d93
commit 032be5f400
12 changed files with 1849 additions and 190 deletions

View File

@@ -181,6 +181,8 @@ runtime_observations:
clickhouse_native_backup_disk_status: deployed_verified
clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4
clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4
clickhouse_native_current_source_status: pending_toolchain_deploy_and_runtime_canary
clickhouse_native_current_source_deployed: false
clickhouse_native_restore_terminal: verified
clickhouse_native_restic_snapshot_id: 574e2a1a
clickhouse_native_offsite_status: pending

View File

@@ -347,6 +347,8 @@ controlled_applies:
risk: high
completion_scope: clickhouse_native_only
source:
evidence_scope: historical_production_receipt
deployment_status: deployed_verified
backup_orchestrator_sha256: 2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520
native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a
isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7
@@ -361,6 +363,36 @@ controlled_applies:
ruff: pass
ruff_format: pass
yaml_xml_parse: pass
current_source:
observed_at: "2026-07-15T09:40:50+08:00"
evidence_scope: committed_current_source
status: pending_toolchain_deploy_and_runtime_canary
deployment_status: pending
current_source_deployed: false
runtime_verifier_terminal: pending
historical_production_receipt_preserved: true
expected_file_count: 7
hashes:
backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2
native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a
isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7
inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863
canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886
backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
drift_from_historical_production_receipt:
- backup_orchestrator_sha256
- canary_wrapper_sha256
- inventory_verifier_sha256
source_validation:
backup_restore_deployer_tests_passed: 92
canonical_preflight_tests_passed: 23
bash_syntax: pass
ruff: pass
ruff_format: pass
yaml_parse: pass
diff_check: pass
independent_reaudit: no_commit_blocker
toolchain_controlled_apply:
check_run_id: P0-OBS-002-native-toolchain-20260715T001124Z-check5
check_terminal: pass
@@ -609,6 +641,15 @@ pending_verifiers:
- bind_inventory_hash_operation_id_artifact_hash_and_run_identity
- resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered
- regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits
clickhouse_native_current_source_toolchain:
status: pending_toolchain_deploy_and_runtime_canary
current_source_deployed: false
runtime_verifier_terminal: pending
historical_production_receipt_preserved: true
required_preconditions:
- controlled_toolchain_apply_with_current_source_hash_readback
- same_run_native_backup_and_isolated_restore_canary
- independent_runtime_and_cleanup_verifier
scope_boundaries:
grpc_bridge_rollback_indicated: false
@@ -629,6 +670,7 @@ terminal:
- signoz_backup_offsite_dr_pending
- clickhouse_native_failed_artifact_retention_pending
- clickhouse_native_resume_submitted_inventory_pending
- clickhouse_native_current_source_toolchain_pending
- service_registry_runtime_mirror_reconciliation_pending
- github_freeze_legacy_asset_retirement_pending
- monitoring_generator_duplicate_awoooi_api_identity_pending

View File

@@ -15,14 +15,19 @@ source "$(dirname "$0")/common.sh"
SERVICE="signoz"
LOCAL_REPO="${BACKUP_BASE}/signoz"
DUMP_DIR="/tmp/signoz-backup-$$"
TMP_ROOT="${SIGNOZ_BACKUP_TMP_ROOT:-/tmp}"
DUMP_DIR=""
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}"
COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}"
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}"
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"
# Destructive Restic forget/prune is a separate critical break-glass workflow.
# The regular and canary backup paths are backup-only and fail closed if a
# caller attempts to re-enable inline retention cleanup.
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-1}"
OPERATION_LOCK_FILE="${SIGNOZ_BACKUP_OPERATION_LOCK_FILE:-/tmp/awoooi-signoz-backup-operation.lock}"
RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}"
RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}"
RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}"
@@ -40,7 +45,8 @@ COLLECTOR_STARTED_AT_BEFORE=""
COLLECTOR_RESTARTS_BEFORE=""
COLLECTOR_HEALTH_BEFORE=""
COLLECTOR_OBSERVER_PID=""
COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
COLLECTOR_OBSERVATIONS=""
WORKSPACE_CREATED=0
CLEANUP_COMPLETE=0
FAILURE_NOTIFIED=0
@@ -126,13 +132,24 @@ cleanup() {
local exit_code=$?
if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
return "${exit_code}"
exit "${exit_code}"
fi
CLEANUP_COMPLETE=1
trap '' HUP INT TERM
trap - EXIT HUP INT TERM
set +e
stop_collector_observer
rm -rf -- "${DUMP_DIR}"
return "${exit_code}"
if [ "${WORKSPACE_CREATED}" -eq 1 ]; then
if [ -n "${DUMP_DIR}" ] && [ -d "${DUMP_DIR}" ] \
&& [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
&& [[ "${DUMP_DIR}" == "${TMP_ROOT%/}"/signoz-backup.* ]]; then
rm -rf -- "${DUMP_DIR}" || exit_code=90
[ ! -e "${DUMP_DIR}" ] || exit_code=90
else
log_error "拒絕清理不符合本 run identity 的 SigNoz workspace"
exit_code=90
fi
fi
exit "${exit_code}"
}
on_signal() {
@@ -140,6 +157,28 @@ on_signal() {
exit "$((128 + signal_number))"
}
create_private_workspace() {
[[ "${TMP_ROOT}" == /* ]] \
&& [[ ! "${TMP_ROOT}" =~ (^|/)\.\.?(/|$) ]] \
&& [[ "${TMP_ROOT}" != *$'\n'* ]] \
&& [ -d "${TMP_ROOT}" ] && [ -w "${TMP_ROOT}" ] \
&& [ ! -L "${TMP_ROOT}" ] || {
log_error "SigNoz workspace root 必須是安全、可寫、非 symlink 的絕對目錄"
return 1
}
DUMP_DIR="$(mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX")" || return 1
WORKSPACE_CREATED=1
[ -d "${DUMP_DIR}" ] && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
&& [ -r "${DUMP_DIR}" ] && [ -w "${DUMP_DIR}" ] && [ -x "${DUMP_DIR}" ] || {
log_error "mktemp 未建立本 run 擁有的 private SigNoz workspace"
return 1
}
chmod 700 "${DUMP_DIR}" || return 1
COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
log_info "建立 SigNoz private workspace: ${DUMP_DIR}"
}
run_native_clickhouse_backup() {
local mode="$1"
@@ -184,6 +223,32 @@ write_restic_receipt() {
mv "${receipt_tmp}" "${receipt_file}"
}
acquire_operation_lock() {
case "${OPERATION_LOCK_FILE}" in
/*) ;;
*)
log_error "SigNoz operation lock path 必須是絕對路徑"
return 1
;;
esac
[ ! -L "${OPERATION_LOCK_FILE}" ] || {
log_error "SigNoz operation lock 不得是 symlink"
return 1
}
if [ -e "${OPERATION_LOCK_FILE}" ]; then
[ -f "${OPERATION_LOCK_FILE}" ] || {
log_error "SigNoz operation lock 必須是 regular file"
return 1
}
fi
exec 8>>"${OPERATION_LOCK_FILE}"
flock -n 8 || {
log_error "另一個 SigNoz backup/deploy operation 正在執行"
return 1
}
log_info "取得 SigNoz backup/deploy 共用 operation lock"
}
main() {
local start_time end_time duration snapshot_id snapshot_json tags
local initial_observation initial_running final_id final_running
@@ -210,7 +275,18 @@ main() {
exit 1
}
done
mkdir -p "${DUMP_DIR}"
[ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ] || {
notify_failure_once "inline Restic forget/prune 已停用;僅允許獨立 critical break-glass workflow"
exit 1
}
acquire_operation_lock || {
notify_failure_once "SigNoz backup/deploy operation lock 取得失敗"
exit 1
}
create_private_workspace || {
notify_failure_once "建立 private SigNoz backup workspace 失敗"
exit 1
}
# Check-mode must pass before the backup window. It writes only durable
# receipts and does not submit BACKUP or create a server artifact.
@@ -305,11 +381,7 @@ main() {
}
log_success "Restic 備份完成: ${snapshot_id}"
if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
else
cleanup_old_backups "${LOCAL_REPO}"
fi
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
# Final readback is separate from the continuity observer receipt.
initial_observation="$(collector_identity_state)" || {

View File

@@ -12,6 +12,12 @@ set -euo pipefail
COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}"
BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}"
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-/backup/scripts/clickhouse-native-restore-drill.sh}"
CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-/backup/scripts/clickhouse-restore-inventory.py}"
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-/backup/config/signoz/clickhouse/config.d/backup_disk.xml}"
CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml}"
CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}"
CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-/backup/logs/clickhouse-native-restore-drill}"
MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}"
MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}"
MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}"
@@ -21,6 +27,7 @@ LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}"
CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}"
TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}"
DOCKER_TIMEOUT_SECONDS="${SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS:-15}"
LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}"
MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}"
STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}"
@@ -94,6 +101,307 @@ require_command() {
command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
}
run_collector_docker() {
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${DOCKER_TIMEOUT_SECONDS}" docker "$@"
}
require_native_restore_contract() {
local label path
for label in \
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK \
CLICKHOUSE_RESTORE_INVENTORY_HELPER \
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG \
CLICKHOUSE_RESTORE_CLUSTER_CONFIG; do
path="${!label}"
case "${path}" in
/*) ;;
*) fail_closed "native_restore_contract_${label}_path_not_absolute" ;;
esac
[ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \
|| fail_closed "native_restore_contract_${label}_missing_unreadable_or_symlink"
done
[ -x "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" ] \
|| fail_closed "native_restore_contract_post_verify_hook_not_executable"
[ -x "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" ] \
|| fail_closed "native_restore_contract_inventory_helper_not_executable"
}
acquire_canary_lock() {
[ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe"
if [ -e "${LOCK_FILE}" ]; then
[ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file"
fi
exec 9>>"${LOCK_FILE}"
flock -n 9 || fail_closed "another_signoz_canary_holds_lock"
}
require_native_receipt_contract() {
local label path run_dir
for label in CLICKHOUSE_NATIVE_RECEIPT_ROOT CLICKHOUSE_RESTORE_RECEIPT_ROOT; do
path="${!label}"
case "${path}" in
/*) ;;
*) fail_closed "native_receipt_contract_${label}_path_not_absolute" ;;
esac
[[ "${path}" != *$'\n'* ]] \
|| fail_closed "native_receipt_contract_${label}_path_contains_newline"
[ ! -L "${path}" ] \
|| fail_closed "native_receipt_contract_${label}_root_symlink_unsafe"
if [ -e "${path}" ]; then
[ -d "${path}" ] && [ -r "${path}" ] && [ -w "${path}" ] \
|| fail_closed "native_receipt_contract_${label}_root_not_readable_writable_directory"
fi
run_dir="${path}/${RUN_ID}"
[ ! -e "${run_dir}" ] && [ ! -L "${run_dir}" ] \
|| fail_closed "native_receipt_contract_${label}_stale_same_run_path_present"
done
[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" != "${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" ] \
|| fail_closed "native_and_restore_receipt_roots_must_be_distinct"
}
verify_same_run_native_restore_receipts() {
local native_receipts restore_receipts restore_status
native_receipts="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl"
restore_receipts="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl"
restore_status="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/status.json"
for path in "${native_receipts}" "${restore_receipts}" "${restore_status}"; do
[ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \
|| fail_closed "same_run_native_restore_receipt_missing_unreadable_or_symlink"
done
python3 - "${native_receipts}" "${restore_receipts}" "${restore_status}" \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" <<'PY' \
|| fail_closed "same_run_native_restore_receipt_validation_failed"
import json
import os
import sys
def reject(message: str) -> None:
print(f"receipt_verifier:{message}", file=sys.stderr)
raise SystemExit(1)
native_path, restore_path, status_path, trace_id, run_id, work_item_id = sys.argv[1:]
identity = {
"trace_id": trace_id,
"run_id": run_id,
"work_item_id": work_item_id,
}
def read_jsonl(path: str, label: str) -> list[dict[str, object]]:
if os.path.islink(path) or not os.path.isfile(path):
reject(f"{label}_unsafe")
records: list[dict[str, object]] = []
try:
with open(path, encoding="utf-8") as source:
for line_number, raw_line in enumerate(source, start=1):
if not raw_line.strip():
continue
value = json.loads(raw_line)
if not isinstance(value, dict):
reject(f"{label}_line_{line_number}_not_object")
for key, expected in identity.items():
if value.get(key) != expected:
reject(f"{label}_line_{line_number}_{key}_mismatch")
invocation_id = value.get("invocation_id")
if not isinstance(invocation_id, str) or not invocation_id:
reject(f"{label}_line_{line_number}_invocation_id_invalid")
records.append(value)
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
reject(f"{label}_malformed_{type(exc).__name__}")
if not records:
reject(f"{label}_empty")
return records
def mode_records(
records: list[dict[str, object]], mode: str, label: str
) -> list[dict[str, object]]:
selected = [
record
for record in records
if str(record.get("invocation_id", "")).startswith(f"{mode}-")
]
if not selected:
reject(f"{label}_{mode}_invocation_missing")
return selected
def require_exact(
records: list[dict[str, object]],
*,
label: str,
stage: str,
terminal: str,
detail_prefix: str | None = None,
) -> None:
matches = [
record
for record in records
if record.get("stage") == stage
and record.get("terminal") == terminal
and (
detail_prefix is None
or str(record.get("detail", "")).startswith(detail_prefix)
)
]
if len(matches) != 1:
reject(f"{label}_{stage}_{terminal}_count_{len(matches)}")
native_records = read_jsonl(native_path, "native_receipts")
native_check = mode_records(native_records, "check", "native")
native_apply = mode_records(native_records, "apply", "native")
require_exact(
native_check,
label="native_check",
stage="terminal",
terminal="check_pass_no_write",
)
require_exact(
native_apply,
label="native_apply",
stage="terminal",
terminal="pass",
)
require_exact(
native_apply,
label="native_apply",
stage="independent_verifier",
terminal="pass",
)
require_exact(
native_apply,
label="native_apply_hook_check",
stage="command",
terminal="pass",
detail_prefix="step_post_verify_hook_check_exit_0_",
)
require_exact(
native_apply,
label="native_apply_hook",
stage="command",
terminal="pass",
detail_prefix="step_post_verify_hook_exit_0_",
)
restore_records = read_jsonl(restore_path, "restore_receipts")
restore_check = mode_records(restore_records, "check", "restore")
restore_apply = mode_records(restore_records, "apply", "restore")
require_exact(
restore_check,
label="restore_check",
stage="terminal",
terminal="check_pass_no_runtime_write",
)
require_exact(
restore_apply,
label="restore_apply",
stage="terminal",
terminal="verified",
)
require_exact(
restore_apply,
label="restore_apply",
stage="independent_post_verifier",
terminal="pass",
)
require_exact(
restore_apply,
label="restore_apply",
stage="cleanup",
terminal="pass",
)
if os.path.islink(status_path) or not os.path.isfile(status_path):
reject("restore_status_unsafe")
try:
with open(status_path, encoding="utf-8") as source:
status = json.load(source)
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
reject(f"restore_status_malformed_{type(exc).__name__}")
if not isinstance(status, dict):
reject("restore_status_not_object")
for key, expected in identity.items():
if status.get(key) != expected:
reject(f"restore_status_{key}_mismatch")
expected_strings = {
"schema_version": "clickhouse_native_restore_drill_status_v1",
"mode": "apply",
"terminal": "verified",
"restore_terminal": "RESTORED",
"network_mode": "docker_internal_only",
"staging_network_mode": "none",
}
for key, expected in expected_strings.items():
if not isinstance(status.get(key), str) or status.get(key) != expected:
reject(f"restore_status_{key}_invalid")
for key, expected in (("exit_code", 0), ("manifest_parity", 1)):
if type(status.get(key)) is not int or status.get(key) != expected:
reject(f"restore_status_{key}_invalid")
for key in (
"check_pass",
"apply_pass",
"cleanup_verified",
"artifact_staging_verified",
"clickhouse_artifact_readback_verified",
"ephemeral_runtime_created",
"isolated_keeper",
):
if status.get(key) is not True:
reject(f"restore_status_{key}_not_true")
for key in (
"production_network_attached",
"production_restore_performed",
"allow_non_empty_tables",
):
if status.get(key) is not False:
reject(f"restore_status_{key}_not_false")
def positive_integer(key: str) -> int:
value = status.get(key)
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
reject(f"restore_status_{key}_not_positive_integer")
return value
source_database_count = positive_integer("source_database_count")
restored_database_count = positive_integer("restored_database_count")
if source_database_count != restored_database_count:
reject("restore_status_database_count_mismatch")
source_table_count = positive_integer("source_table_count")
restored_table_count = positive_integer("restored_table_count")
if source_table_count != restored_table_count:
reject("restore_status_table_count_mismatch")
positive_integer("critical_nonzero_count")
check_count = status.get("check_table_count")
check_pass_count = status.get("check_table_pass_count")
if (
isinstance(check_count, bool)
or not isinstance(check_count, int)
or isinstance(check_pass_count, bool)
or not isinstance(check_pass_count, int)
or check_count <= 0
or check_count != check_pass_count
):
reject("restore_status_check_table_counts_invalid")
PY
emit_receipt "native_restore_receipt_verifier" "pass" \
"same_run_machine_readable_native_check_apply_hook_and_isolated_restore_status_verified"
}
require_monitor_fragment() {
local fragment="$1"
local count
@@ -138,7 +446,7 @@ require_monitor_function_fragment() {
collector_running_state() {
local running
if ! running="$(docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
return 1
fi
case "${running}" in
@@ -149,7 +457,7 @@ collector_running_state() {
}
collector_runtime_metadata() {
docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
"${COLLECTOR_NAME}" 2>/dev/null
}
@@ -247,13 +555,15 @@ cleanup() {
if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then
overall_terminal="check_pass_no_write"
elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then
overall_terminal="pass"
overall_terminal="partial_degraded"
fi
emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}"
if [ "${overall_terminal}" = "pass" ]; then
emit_receipt "closure_writeback" "pass" \
"no_incident_notification_or_learning_delta_local_durable_receipt_ack"
if [ "${overall_terminal}" = "partial_degraded" ]; then
emit_receipt "runtime_operation" "pass" \
"clickhouse_native_backup_and_isolated_restore_runtime_verifier_pass"
emit_receipt "closure_writeback" "partial_degraded" \
"local_durable_receipt_ack_incident_not_applicable_telegram_not_sent_km_rag_writeback_pending_offsite_false_sqlite_application_consistent_export_pending"
elif [ "${overall_terminal}" = "check_pass_no_write" ]; then
emit_receipt "closure_writeback" "not_applicable" \
"check_mode_no_runtime_change_local_durable_receipt_ack"
@@ -261,7 +571,8 @@ cleanup() {
emit_receipt "closure_writeback" "deferred" \
"failed_terminal_requires_followup_no_completion_claim"
fi
emit_receipt "terminal" "${overall_terminal}" "backup_canary_${overall_terminal}"
emit_receipt "terminal" "${overall_terminal}" \
"backup_canary_${overall_terminal}_clickhouse_native_scope_only_offsite_false_sqlite_pending"
printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \
"${lease_terminal}" "${exit_code}"
@@ -521,7 +832,7 @@ main() {
local container_health_before collector_metadata_before
local monitor_log_identity="" monitor_log_size=""
local source_diff_detail ai_decision_detail
local backup_status
local backup_status native_restore_contract_detail="not_applicable"
setup_receipts
@@ -532,11 +843,17 @@ main() {
*) fail_closed "unsupported_mode_use_check_or_apply" ;;
esac
for command_name in awk date docker env flock grep kill pgrep sha256sum sleep ss timeout tr; do
# Hold the canary lock before inspecting or hashing any mutable backup or
# restore toolchain path. This closes the check/apply substitution window.
require_command flock
acquire_canary_lock
for command_name in awk date docker env grep kill pgrep python3 sha256sum sleep ss timeout tr; do
require_command "${command_name}"
done
valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout"
valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after"
valid_positive_integer "${DOCKER_TIMEOUT_SECONDS}" || fail_closed "invalid_docker_timeout"
valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval"
case "${EXPECT_COLLECTOR_STOP}" in
0|1) ;;
@@ -545,7 +862,11 @@ main() {
[ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \
|| fail_closed "backup_script_missing_unexecutable_or_symlink"
if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
require_native_restore_contract
require_native_receipt_contract
native_restore_contract_detail="hook_$(sha256sum "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" | awk '{print $1}')_inventory_$(sha256sum "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" | awk '{print $1}')_disk_config_$(sha256sum "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" | awk '{print $1}')_cluster_config_$(sha256sum "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" | awk '{print $1}')"
else
for command_name in cp crontab id mktemp mv rm stat tail; do
require_command "${command_name}"
done
@@ -555,12 +876,6 @@ main() {
verify_monitor_contract
fi
[ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe"
if [ -e "${LOCK_FILE}" ]; then
[ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file"
fi
exec 9>>"${LOCK_FILE}"
flock -n 9 || fail_closed "another_signoz_canary_holds_lock"
if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
fail_closed "another_signoz_backup_is_running"
fi
@@ -591,18 +906,18 @@ main() {
PRIOR_LEASE_PRESENT=0
PRIOR_LEASE_HASH="not_applicable"
emit_receipt "sensor_source" "pass" \
"online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
source_diff_detail="online_native_backup_monitor_and_cooldown_contract_not_applicable"
ai_decision_detail="run_online_native_backup_without_collector_or_monitor_mutation"
"online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_${native_restore_contract_detail}_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
source_diff_detail="online_native_backup_isolated_restore_contract_bound_monitor_and_cooldown_contract_not_applicable"
ai_decision_detail="run_online_native_backup_with_mandatory_isolated_restore_verifier_without_collector_or_monitor_mutation"
fi
emit_receipt "normalized_asset_identity" "pass" \
"asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker"
emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}"
emit_receipt "ai_decision" "candidate" "${ai_decision_detail}"
emit_receipt "risk_policy" "pass" \
"risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_independent_verifier"
"risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_mandatory_isolated_restore_independent_verifier"
emit_receipt "check" "pass" \
"mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent"
"mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_${native_restore_contract_detail}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent"
if [ "${MODE}" = "check" ]; then
CHECK_VERIFIED=1
@@ -622,6 +937,12 @@ main() {
set +e
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \
env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" \
CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" \
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" \
CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" \
CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" \
CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" \
BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \
> "${BACKUP_OUTPUT}" 2>&1 &
BACKUP_PID=$!
@@ -645,6 +966,7 @@ main() {
else
grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \
|| fail_closed "native_backup_success_terminal_missing"
verify_same_run_native_restore_receipts
fi
if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then

View File

@@ -25,7 +25,9 @@ def _run_backup(
native_apply_status: int = 0,
continuity_mode: str = "stable",
restic_backup_status: int = 0,
skip_retention: bool = False,
skip_retention: bool = True,
operation_lock_status: int = 0,
workspace_root_symlink: bool = False,
) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
harness = tmp_path / "backup"
fake_bin = tmp_path / "bin"
@@ -33,6 +35,7 @@ def _run_backup(
event_log = tmp_path / "events.log"
collector_id = tmp_path / "collector.id"
collector_state = tmp_path / "collector.state"
workspace_root = tmp_path / "workspace-root"
harness.mkdir()
fake_bin.mkdir()
(backup_base / "signoz" / "data").mkdir(parents=True)
@@ -43,6 +46,12 @@ def _run_backup(
"true\n" if initial_running else "false\n",
encoding="utf-8",
)
if workspace_root_symlink:
workspace_target = tmp_path / "workspace-target"
workspace_target.mkdir()
workspace_root.symlink_to(workspace_target, target_is_directory=True)
else:
workspace_root.mkdir()
(harness / "common.sh").write_text(
"""\
@@ -139,6 +148,15 @@ case " $* " in
printf '[{"short_id":"native123"}]\\n'
;;
esac
""",
)
_write_executable(
fake_bin / "flock",
"""\
#!/bin/bash
set -eu
printf 'flock %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
exit "${FAKE_OPERATION_LOCK_STATUS:-0}"
""",
)
@@ -161,6 +179,9 @@ esac
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
"TIMEOUT_KILL_AFTER_SECONDS": "1",
"BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0",
"SIGNOZ_BACKUP_OPERATION_LOCK_FILE": str(tmp_path / "operation.lock"),
"FAKE_OPERATION_LOCK_STATUS": str(operation_lock_status),
"SIGNOZ_BACKUP_TMP_ROOT": str(workspace_root),
}
process = subprocess.Popen(
["bash", str(harness / SCRIPT.name)],
@@ -177,7 +198,13 @@ esac
stderr,
)
events = event_log.read_text(encoding="utf-8").splitlines()
return result, events, Path(f"/tmp/signoz-backup-{process.pid}")
workspace_events = _events(events, "info 建立 SigNoz private workspace: ")
dump_dir = (
Path(workspace_events[-1].split(": ", 1)[1])
if workspace_events
else tmp_path / "workspace-not-created"
)
return result, events, dump_dir
def _events(events: list[str], prefix: str) -> list[str]:
@@ -201,6 +228,12 @@ def test_source_sunsets_raw_volumes_and_collector_mutation() -> None:
assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
assert 'notify_clawbot "success"' not in source
assert "no-downtime continuity verifier" in source
assert "awoooi-signoz-backup-operation.lock" in source
assert "flock -n 8" in source
assert "cleanup_old_backups" not in source
assert 'mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX"' in source
assert "/tmp/signoz-backup-$$" not in source
assert "WORKSPACE_CREATED=1" in source
def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
@@ -233,6 +266,32 @@ def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
assert not dump_dir.exists()
def test_operation_lock_contention_fails_before_native_or_restic(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, operation_lock_status=1)
assert result.returncode != 0
assert _events(events, "flock -n 8")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert len(_events(events, "notify failed signoz ")) == 1
assert not dump_dir.exists()
def test_symlink_workspace_root_fails_before_native_or_restic(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, workspace_root_symlink=True)
assert result.returncode != 0
assert _events(events, "flock -n 8")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert any("workspace root" in event for event in events)
assert not dump_dir.exists()
def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None:
result, events, dump_dir = _run_backup(tmp_path, native_check_status=19)
@@ -307,6 +366,20 @@ def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None:
assert not _events(events, "retention ")
def test_inline_destructive_retention_request_fails_before_backup(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, skip_retention=False)
assert result.returncode != 0
assert not _events(events, "flock ")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert not _events(events, "retention ")
assert any("critical break-glass workflow" in event for event in events)
assert not dump_dir.exists()
def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None:
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")

View File

@@ -7,6 +7,8 @@ import subprocess
import time
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[3]
WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh"
@@ -115,6 +117,9 @@ while [[ "${1:-}" == --kill-after=* ]]; do
done
[ "$#" -gt 1 ]
shift
if [ "${FAKE_DOCKER_TIMEOUT:-0}" = "1" ] && [ "${1:-}" = "docker" ]; then
exit 124
fi
exec "$@"
""",
)
@@ -122,6 +127,9 @@ exec "$@"
fake_bin / "flock",
"""\
#!/bin/bash
if [ -n "${TEST_FLOCK_MARKER:-}" ]; then
printf 'held\n' > "${TEST_FLOCK_MARKER}"
fi
exit 0
""",
)
@@ -171,14 +179,103 @@ print(f"{digest} {sys.argv[1]}")
)
def _restore_hook_source() -> str:
return """\
#!/bin/bash
set -eu
[ "$#" -eq 1 ]
case "$1" in
--check) mode="check" ;;
--apply) mode="apply" ;;
*) exit 64 ;;
esac
receipt_dir="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:?}/${RUN_ID:?}"
mkdir -p "$receipt_dir"
receipt_log="$receipt_dir/receipts.jsonl"
identity_run="$RUN_ID"
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "forged_identity" ]; then
identity_run="stale-$RUN_ID"
fi
invocation_id="${mode}-fake-restore-$$"
emit() {
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","stage":"%s","terminal":"%s","detail":"fake"}\n' \
"$TRACE_ID" "$identity_run" "$WORK_ITEM_ID" "$invocation_id" "$1" "$2" \
>> "$receipt_log"
}
if [ "$mode" = "check" ]; then
emit check check_pass_no_runtime_write
emit cleanup pass
emit terminal check_pass_no_runtime_write
exit 0
fi
emit check pass
emit independent_post_verifier pass
emit cleanup pass
emit terminal verified
if [ "${FAKE_RECEIPT_FAULT:-valid}" != "missing_status" ]; then
cat > "$receipt_dir/status.json" <<EOF
{
"schema_version":"clickhouse_native_restore_drill_status_v1",
"trace_id":"$TRACE_ID",
"run_id":"$identity_run",
"work_item_id":"$WORK_ITEM_ID",
"mode":"apply",
"terminal":"verified",
"exit_code":0,
"network_mode":"docker_internal_only",
"staging_network_mode":"none",
"production_network_attached":false,
"production_restore_performed":false,
"allow_non_empty_tables":false,
"isolated_keeper":true,
"artifact_staging_verified":true,
"clickhouse_artifact_readback_verified":true,
"ephemeral_runtime_created":true,
"check_pass":true,
"apply_pass":true,
"restore_terminal":"RESTORED",
"manifest_parity":1,
"source_database_count":6,
"restored_database_count":6,
"source_table_count":100,
"restored_table_count":100,
"check_table_count":44,
"check_table_pass_count":44,
"critical_nonzero_count":4,
"cleanup_verified":true
}
EOF
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "symlink_status" ]; then
mv "$receipt_dir/status.json" "$receipt_dir/status-target.json"
ln -s "status-target.json" "$receipt_dir/status.json"
fi
fi
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "malformed_restore_receipt" ]; then
printf '{malformed-json\n' >> "$receipt_log"
fi
"""
def _backup_source() -> str:
return """\
#!/bin/bash
set -eu
[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80
[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ]
[ "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}" = "${TEST_RESTORE_HOOK:?}" ] || exit 82
[ "${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-}" = "${TEST_INVENTORY_HELPER:?}" ] || exit 83
[ "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-}" = "${TEST_BACKUP_DISK_CONFIG:?}" ] || exit 84
[ "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-}" = "${TEST_CLUSTER_CONFIG:?}" ] || exit 85
[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-}" = "${TEST_NATIVE_RECEIPT_ROOT:?}" ] || exit 86
[ "${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-}" = "${TEST_RESTORE_RECEIPT_ROOT:?}" ] || exit 87
printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
> "${TEST_BACKUP_IDENTITIES:?}"
printf '%s\t%s\t%s\t%s\n' \
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" \
"$CLICKHOUSE_RESTORE_INVENTORY_HELPER" \
"$CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" \
"$CLICKHOUSE_RESTORE_CLUSTER_CONFIG" \
> "${TEST_RESTORE_CONTRACT:?}"
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}"
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
@@ -204,6 +301,23 @@ if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
else
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
native_dir="$CLICKHOUSE_NATIVE_RECEIPT_ROOT/$RUN_ID"
mkdir -p "$native_dir"
native_receipts="$native_dir/receipts.jsonl"
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"check-fake-native","stage":"terminal","terminal":"check_pass_no_write","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
if [ "${FAKE_RECEIPT_FAULT:-valid}" != "no_hook" ]; then
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --check
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_check_exit_0_stdout_fake_stderr_fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --apply
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_exit_0_stdout_fake_stderr_fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
fi
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"independent_verifier","terminal":"pass","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"terminal","terminal":"pass","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
fi
if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then
printf '4317\n' > "${TEST_LISTENERS:?}"
@@ -229,13 +343,30 @@ def _run_canary(
duplicate_cron: bool = False,
drop_listener: bool = False,
expect_collector_stop: bool = True,
native_restore_contract: str = "valid",
native_receipt_state: str = "valid",
docker_timeout: bool = False,
) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]:
if native_receipt_state not in {
"valid",
"no_hook",
"forged_identity",
"missing_status",
"symlink_status",
"malformed_restore_receipt",
}:
raise ValueError(f"unsupported native receipt state: {native_receipt_state}")
fake_bin = tmp_path / "bin"
cooldown_dir = tmp_path / "cooldown"
receipt_root = tmp_path / "receipts"
native_receipt_root = tmp_path / "native-receipts"
restore_receipt_root = tmp_path / "restore-receipts"
fake_bin.mkdir()
cooldown_dir.mkdir()
receipt_root.mkdir()
native_receipt_root.mkdir()
restore_receipt_root.mkdir()
monitor_script = tmp_path / "docker-health-monitor.sh"
monitor_log = tmp_path / "monitor.log"
@@ -244,6 +375,12 @@ def _run_canary(
listeners = tmp_path / "listeners"
identities = tmp_path / "backup-identities.tsv"
lease_observed = tmp_path / "lease-observed.txt"
restore_contract_observed = tmp_path / "restore-contract.tsv"
flock_marker = tmp_path / "canary-lock-held.txt"
restore_hook = tmp_path / "clickhouse-native-restore-drill.sh"
inventory_helper = tmp_path / "clickhouse-restore-inventory.py"
backup_disk_config = tmp_path / "backup_disk.xml"
cluster_config = tmp_path / "isolated-cluster.xml"
cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown"
run_id = f"canary-{tmp_path.name}"
@@ -252,6 +389,19 @@ def _run_canary(
_monitor_source(cooldown_dir, omit_contract=omit_monitor_contract),
)
_write_executable(backup_script, _backup_source())
if native_restore_contract == "symlink":
restore_hook_target = tmp_path / "restore-hook-target.sh"
_write_executable(restore_hook_target, _restore_hook_source())
restore_hook.symlink_to(restore_hook_target)
elif native_restore_contract == "valid":
_write_executable(restore_hook, _restore_hook_source())
elif native_restore_contract != "missing":
raise ValueError(
f"unsupported native restore contract: {native_restore_contract}"
)
_write_executable(inventory_helper, "#!/bin/sh\nexit 0\n")
backup_disk_config.write_text("<clickhouse/>\n", encoding="utf-8")
cluster_config.write_text("<clickhouse/>\n", encoding="utf-8")
_install_fake_commands(fake_bin)
monitor_log.write_text("[test] monitor seed\n", encoding="utf-8")
collector_state.write_text("true\n", encoding="utf-8")
@@ -274,10 +424,17 @@ def _run_canary(
"SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"),
"SIGNOZ_CANARY_TIMEOUT_SECONDS": "5",
"SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2",
"SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS": "2",
"SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2",
"SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds),
"SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05",
"SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"),
"CLICKHOUSE_NATIVE_POST_VERIFY_HOOK": str(restore_hook),
"CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(inventory_helper),
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_disk_config),
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config),
"CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(native_receipt_root),
"CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(restore_receipt_root),
"TEST_MONITOR_SCRIPT": str(monitor_script),
"TEST_MONITOR_LOG": str(monitor_log),
"TEST_COLLECTOR_STATE": str(collector_state),
@@ -285,11 +442,21 @@ def _run_canary(
"TEST_BACKUP_IDENTITIES": str(identities),
"TEST_COOLDOWN_FILE": str(cooldown_file),
"TEST_LEASE_OBSERVED": str(lease_observed),
"TEST_RESTORE_CONTRACT": str(restore_contract_observed),
"TEST_RESTORE_HOOK": str(restore_hook),
"TEST_INVENTORY_HELPER": str(inventory_helper),
"TEST_BACKUP_DISK_CONFIG": str(backup_disk_config),
"TEST_CLUSTER_CONFIG": str(cluster_config),
"TEST_NATIVE_RECEIPT_ROOT": str(native_receipt_root),
"TEST_RESTORE_RECEIPT_ROOT": str(restore_receipt_root),
"TEST_FLOCK_MARKER": str(flock_marker),
"FAKE_MONITOR_MODE": monitor_mode,
"FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds),
"FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0",
"FAKE_DROP_LISTENER": "1" if drop_listener else "0",
"FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0",
"FAKE_RECEIPT_FAULT": native_receipt_state,
"FAKE_DOCKER_TIMEOUT": "1" if docker_timeout else "0",
}
started_at = int(time.time())
result = subprocess.run(
@@ -308,6 +475,13 @@ def _run_canary(
"lease_observed": lease_observed,
"collector_state": collector_state,
"listeners": listeners,
"restore_contract": restore_contract_observed,
"restore_hook": restore_hook,
"flock_marker": flock_marker,
"native_receipts": native_receipt_root / run_id / "receipts.jsonl",
"restore_receipts": restore_receipt_root / run_id / "receipts.jsonl",
"restore_status": restore_receipt_root / run_id / "status.json",
"backup_output": receipt_root / run_id / "backup-output.log",
}
return result, paths, str(started_at)
@@ -337,7 +511,9 @@ def test_existing_lease_is_atomically_replaced_and_exactly_restored(
assert identities[2] == "P0-OBS-002\n"
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "runtime_operation") == "pass"
assert _terminal(receipts, "closure_writeback") == "partial_degraded"
assert _terminal(receipts, "terminal") == "partial_degraded"
metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8"))
assert metadata["trace_id"] == "trace-P0-OBS-002"
assert metadata["prior_present"] == 1
@@ -368,7 +544,7 @@ def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None:
assert not paths["cooldown"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
@@ -380,13 +556,30 @@ def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
assert not paths["lease_observed"].exists()
assert paths["collector_state"].read_text(encoding="utf-8") == "true\n"
assert paths["restore_contract"].read_text(encoding="utf-8").split("\t") == [
str(paths["restore_hook"]),
str(tmp_path / "clickhouse-restore-inventory.py"),
str(tmp_path / "backup_disk.xml"),
f"{tmp_path / 'isolated-cluster.xml'}\n",
]
receipts = _receipts(paths["receipts"])
assert paths["native_receipts"].is_file()
assert paths["restore_receipts"].is_file()
restore_status = json.loads(paths["restore_status"].read_text(encoding="utf-8"))
assert restore_status["terminal"] == "verified"
assert restore_status["check_table_count"] == 44
assert restore_status["check_table_pass_count"] == 44
assert any(
item["stage"] == "native_restore_receipt_verifier"
and item["terminal"] == "pass"
for item in receipts
)
assert any(
item["stage"] == "lease" and item["terminal"] == "not_required"
for item in receipts
)
assert _terminal(receipts, "rollback") == "no_write"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
@@ -406,7 +599,7 @@ def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
item for item in receipts if item["stage"] == "source_of_truth_diff"
][-1]
assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"]
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None:
@@ -496,6 +689,34 @@ def test_wrapper_deployment_and_static_contracts_are_owned() -> None:
assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper
assert "monitor_auto_repair_contaminated_canary" in wrapper
assert "collector_post_backup_runtime_failed" in wrapper
assert "/backup/scripts/clickhouse-native-restore-drill.sh" in wrapper
assert "/backup/scripts/clickhouse-restore-inventory.py" in wrapper
assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in wrapper
assert (
"/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
in wrapper
)
assert (
'CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}"'
in wrapper
)
assert (
'CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}"' in wrapper
)
assert (
'CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}"'
in wrapper
)
main_source = wrapper[wrapper.index("main() {") :]
assert main_source.index("acquire_canary_lock") < main_source.index(
"require_native_restore_contract"
)
assert "same_run_native_restore_receipt_validation_failed" in wrapper
assert '"check_table_count"' in wrapper
assert "mandatory_isolated_restore_verifier" in wrapper
assert "SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS" in wrapper
assert 'emit_receipt "runtime_operation" "pass"' in wrapper
assert 'emit_receipt "closure_writeback" "partial_degraded"' in wrapper
assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8")
assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text(
encoding="utf-8"
@@ -518,6 +739,80 @@ def test_check_mode_proves_contract_without_lease_or_backup_write(
assert _terminal(check_receipts, "terminal") == "check_pass_no_write"
@pytest.mark.parametrize("contract_state", ["missing", "symlink"])
def test_native_check_fails_closed_for_unsafe_restore_contract(
tmp_path: Path,
contract_state: str,
) -> None:
result, paths, _ = _run_canary(
tmp_path,
mode="check",
expect_collector_stop=False,
native_restore_contract=contract_state,
)
assert result.returncode != 0
assert "native_restore_contract_CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" in result.stderr
assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n"
assert not paths["identities"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "terminal") == "failed"
@pytest.mark.parametrize(
"receipt_state",
[
"no_hook",
"forged_identity",
"missing_status",
"symlink_status",
"malformed_restore_receipt",
],
)
def test_native_apply_rejects_missing_or_forged_machine_receipts(
tmp_path: Path,
receipt_state: str,
) -> None:
result, paths, _ = _run_canary(
tmp_path,
expect_collector_stop=False,
native_receipt_state=receipt_state,
)
assert result.returncode != 0
assert "========== SigNoz ClickHouse native 備份完成 (1s) ==========" in paths[
"backup_output"
].read_text(encoding="utf-8")
assert (
"same_run_native_restore_receipt_validation_failed" in result.stderr
or "same_run_native_restore_receipt_missing_unreadable_or_symlink"
in result.stderr
)
assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n"
receipts = _receipts(paths["receipts"])
assert not any(
item["stage"] == "native_restore_receipt_verifier"
and item["terminal"] == "pass"
for item in receipts
)
assert _terminal(receipts, "terminal") == "failed"
def test_docker_metadata_timeout_fails_before_backup(tmp_path: Path) -> None:
result, paths, _ = _run_canary(
tmp_path,
mode="check",
expect_collector_stop=False,
docker_timeout=True,
)
assert result.returncode != 0
assert "collector_runtime_metadata_unreadable" in result.stderr
assert not paths["identities"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "terminal") == "failed"
def test_wrapper_is_valid_bash() -> None:
result = subprocess.run(
["bash", "-n", str(WRAPPER)],

View File

@@ -11,7 +11,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml"
LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
TARGET_HOST="wooo@192.168.0.110"
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
@@ -31,6 +31,12 @@ RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
API_URL="http://127.0.0.1:8080/api/v1/health"
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SAFE_HOME="/home/wooo"
LOCAL_KILL_AFTER_SECONDS=15
REMOTE_CHECK_TIMEOUT_SECONDS=180
REMOTE_STAGE_TIMEOUT_SECONDS=60
REMOTE_TRANSFER_TIMEOUT_SECONDS=60
REMOTE_APPLY_TIMEOUT_SECONDS=3600
REMOTE_CLEANUP_TIMEOUT_SECONDS=60
SSH_OPTS=(
-o BatchMode=yes
@@ -79,7 +85,7 @@ for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
}
done
for command_name in python3 scp ssh; do
for command_name in python3 scp ssh timeout; do
command -v "${command_name}" >/dev/null 2>&1 || {
printf 'required local command missing: %s\n' "${command_name}" >&2
exit 69
@@ -94,6 +100,18 @@ hash_file() {
fi
}
bounded_ssh() {
local timeout_seconds="$1"
shift
timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
"${timeout_seconds}" ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" "$@"
}
bounded_scp() {
timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
"${REMOTE_TRANSFER_TIMEOUT_SECONDS}" scp -q "${SSH_OPTS[@]}" "$@"
}
validate_local_sources() {
[ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ]
[ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ]
@@ -136,13 +154,13 @@ local_receipt() {
}
remote_candidate_compose_check() {
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" \
"env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" \
"timeout --signal=TERM --kill-after=15 120 env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
< "${LOCAL_OVERRIDE}"
}
remote_read_only_check() {
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" bash -s -- \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK'
set -euo pipefail
@@ -159,6 +177,24 @@ REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SAFE_HOME="/home/wooo"
COMMAND_TIMEOUT_SECONDS=30
COMPOSE_TIMEOUT_SECONDS=120
KILL_AFTER_SECONDS=15
for command_name in docker timeout; do
command -v "${command_name}" >/dev/null 2>&1
done
docker_cmd() {
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
"${COMMAND_TIMEOUT_SECONDS}" docker "$@"
}
compose_cmd() {
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
"${COMPOSE_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p signoz "$@"
}
file_hash_or_absent() {
if [ -f "$1" ] && [ ! -L "$1" ]; then
@@ -172,14 +208,10 @@ file_hash_or_absent() {
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
docker inspect "${container}" >/dev/null
docker_cmd inspect "${container}" >/dev/null
done
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
config -q >/dev/null
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
config --services | grep -qx clickhouse
compose_cmd -f "${REMOTE_BASE}" config -q >/dev/null
compose_cmd -f "${REMOTE_BASE}" config --services | grep -qx clickhouse
override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")"
config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")"
@@ -195,13 +227,13 @@ elif [ -e "${HOST_BACKUP_DIR}" ]; then
host_dir=unsafe
fi
disk_count="$(docker exec signoz-clickhouse clickhouse-client --query \
disk_count="$(docker_cmd exec signoz-clickhouse clickhouse-client --query \
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
mount_count="$(docker inspect --format \
mount_count="$(docker_cmd inspect --format \
'{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \
signoz-clickhouse | tr -cd '1' | wc -c | xargs)"
collector_running="$(docker inspect --format '{{.State.Running}}' signoz-otel-collector)"
collector_restarts="$(docker inspect --format '{{.RestartCount}}' signoz-otel-collector)"
collector_running="$(docker_cmd inspect --format '{{.State.Running}}' signoz-otel-collector)"
collector_restarts="$(docker_cmd inspect --format '{{.RestartCount}}' signoz-otel-collector)"
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \
@@ -223,20 +255,58 @@ fi
STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}"
STAGE_CREATED=0
LOCAL_APPLY_VERIFIED=0
cleanup_stage() {
if [ "${STAGE_CREATED}" -eq 1 ]; then
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
set -u
local cleanup_rc
[ "${STAGE_CREATED}" -eq 1 ] || return 0
bounded_ssh "${REMOTE_CLEANUP_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1
set -euo pipefail
stage_dir="$1"
if [ ! -e "${stage_dir}" ]; then
exit 0
fi
[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml"
rmdir "${stage_dir}" 2>/dev/null || true
rmdir "${stage_dir}"
[ ! -e "${stage_dir}" ]
REMOTE_CLEANUP
fi
cleanup_rc=$?
[ "${cleanup_rc}" -eq 0 ] || return "${cleanup_rc}"
STAGE_CREATED=0
}
trap cleanup_stage EXIT
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
on_local_exit() {
local rc=$?
local cleanup_rc=0 cleanup_terminal=not_created terminal=failed
trap - EXIT HUP INT TERM
set +e
if [ "${STAGE_CREATED}" -eq 1 ]; then
cleanup_stage
cleanup_rc=$?
if [ "${cleanup_rc}" -eq 0 ]; then
cleanup_terminal=pass_absence_verified
else
cleanup_terminal=failed_or_residue_present
rc=92
fi
fi
local_receipt cleanup "$([ "${cleanup_rc}" -eq 0 ] && printf pass || printf failed)" \
"remote_stage_${cleanup_terminal}"
if [ "${rc}" -eq 0 ] && [ "${LOCAL_APPLY_VERIFIED}" -eq 1 ]; then
terminal=pass
fi
local_receipt terminal "${terminal}" \
"remote_controlled_apply_${terminal}_stage_${cleanup_terminal}_exit_${rc}"
exit "${rc}"
}
trap on_local_exit EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
bounded_ssh "${REMOTE_STAGE_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
set -euo pipefail
stage_dir="$1"
[ ! -e "${stage_dir}" ]
@@ -245,12 +315,12 @@ mkdir -m 0700 "${stage_dir}"
REMOTE_STAGE
STAGE_CREATED=1
scp -q "${SSH_OPTS[@]}" "${LOCAL_OVERRIDE}" \
bounded_scp "${LOCAL_OVERRIDE}" \
"${TARGET_HOST}:${STAGE_DIR}/override.yaml"
scp -q "${SSH_OPTS[@]}" "${LOCAL_CONFIG}" \
bounded_scp "${LOCAL_CONFIG}" \
"${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml"
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
bounded_ssh "${REMOTE_APPLY_TIMEOUT_SECONDS}" bash -s -- \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY'
set -euo pipefail
@@ -282,12 +352,26 @@ API_URL="http://127.0.0.1:8080/api/v1/health"
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SAFE_HOME="/home/wooo"
LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock"
BACKUP_OPERATION_LOCK_FILE="/tmp/awoooi-signoz-backup-operation.lock"
CANARY_LOCK_FILE="/tmp/awoooi-signoz-backup-canary.lock"
COMMAND_TIMEOUT_SECONDS=30
COMPOSE_CONFIG_TIMEOUT_SECONDS=120
COMPOSE_APPLY_TIMEOUT_SECONDS=300
KILL_AFTER_SECONDS=15
APPLY_STARTED=0
DEPLOY_VERIFIED=0
HOST_DIR_CREATED=0
PRIOR_OVERRIDE_PRESENT=0
PRIOR_CONFIG_PRESENT=0
PRIOR_OVERRIDE_HASH="absent"
PRIOR_OVERRIDE_MODE="absent"
PRIOR_OVERRIDE_UID="absent"
PRIOR_OVERRIDE_GID="absent"
PRIOR_CONFIG_HASH="absent"
PRIOR_CONFIG_MODE="absent"
PRIOR_CONFIG_UID="absent"
PRIOR_CONFIG_GID="absent"
PRIOR_DISK_COUNT=0
BEFORE_IMAGE_ID=""
BEFORE_DATA_VOLUME=""
@@ -307,38 +391,60 @@ receipt() {
printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
}
docker_cmd() {
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
"${COMMAND_TIMEOUT_SECONDS}" docker "$@"
}
bounded_cmd() {
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
"${COMMAND_TIMEOUT_SECONDS}" "$@"
}
compose_config_cmd() {
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
"${COMPOSE_CONFIG_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
}
compose_apply_cmd() {
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
"${COMPOSE_APPLY_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
}
pgrep_cmd() {
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
"${COMMAND_TIMEOUT_SECONDS}" pgrep "$@"
}
ss_cmd() {
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
"${COMMAND_TIMEOUT_SECONDS}" ss "$@"
}
compose_base_config() {
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-f "${REMOTE_BASE}" config -q >/dev/null 2>&1
compose_config_cmd -f "${REMOTE_BASE}" config -q >/dev/null 2>&1
}
compose_managed_config() {
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" config -q >/dev/null 2>&1
compose_config_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
config -q >/dev/null 2>&1
}
compose_stage_config() {
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config -q >/dev/null 2>&1
compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" \
config -q >/dev/null 2>&1
}
recreate_managed() {
timeout --signal=TERM --kill-after=30 300 \
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
compose_apply_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
>/dev/null 2>&1
}
recreate_base() {
timeout --signal=TERM --kill-after=30 300 \
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-f "${REMOTE_BASE}" \
compose_apply_cmd -f "${REMOTE_BASE}" \
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
>/dev/null 2>&1
}
@@ -346,11 +452,11 @@ recreate_base() {
wait_clickhouse() {
local attempt running health
for attempt in $(seq 1 36); do
running="$(docker inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
health="$(docker inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
running="$(docker_cmd inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
health="$(docker_cmd inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
"${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
if [ "${running}" = true ] && [ "${health}" = healthy ] \
&& timeout --kill-after=5 10 docker exec "${CLICKHOUSE_CONTAINER}" \
&& docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then
return 0
fi
@@ -361,7 +467,7 @@ wait_clickhouse() {
listener_up() {
local port="$1"
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
ss_cmd -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
}
wait_api_and_listeners() {
@@ -376,25 +482,72 @@ wait_api_and_listeners() {
return 1
}
file_matches_metadata() {
local path="$1" expected_hash="$2" expected_mode="$3" expected_uid="$4" expected_gid="$5"
[ -f "${path}" ] && [ ! -L "${path}" ] \
&& [ "$(sha256sum "${path}" | awk '{print $1}')" = "${expected_hash}" ] \
&& [ "$(stat -c '%a' "${path}")" = "${expected_mode}" ] \
&& [ "$(stat -c '%u' "${path}")" = "${expected_uid}" ] \
&& [ "$(stat -c '%g' "${path}")" = "${expected_gid}" ]
}
restore_exact_file() {
local source="$1" target="$2" expected_hash="$3" expected_mode="$4"
local expected_uid="$5" expected_gid="$6" tmp="$7"
local current_uid current_gid
bounded_cmd install -m "${expected_mode}" "${source}" "${tmp}" || return 1
current_uid="$(stat -c '%u' "${tmp}")" || return 1
current_gid="$(stat -c '%g' "${tmp}")" || return 1
if [ "${current_uid}" != "${expected_uid}" ] || [ "${current_gid}" != "${expected_gid}" ]; then
bounded_cmd sudo -n chown "${expected_uid}:${expected_gid}" "${tmp}" || return 1
fi
bounded_cmd mv -f "${tmp}" "${target}" || return 1
file_matches_metadata "${target}" "${expected_hash}" "${expected_mode}" \
"${expected_uid}" "${expected_gid}"
}
restore_managed_files() {
local tmp
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}"
install -m 0644 "${RECEIPT_DIR}/prior-override.yaml" "${tmp}"
mv -f "${tmp}" "${REMOTE_OVERRIDE}"
restore_exact_file "${RECEIPT_DIR}/prior-override.yaml" "${REMOTE_OVERRIDE}" \
"${PRIOR_OVERRIDE_HASH}" "${PRIOR_OVERRIDE_MODE}" \
"${PRIOR_OVERRIDE_UID}" "${PRIOR_OVERRIDE_GID}" "${tmp}" || return 1
else
rm -f "${REMOTE_OVERRIDE}"
bounded_cmd rm -f "${REMOTE_OVERRIDE}" || return 1
[ ! -e "${REMOTE_OVERRIDE}" ] || return 1
fi
if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then
tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}"
install -m 0644 "${RECEIPT_DIR}/prior-backup-disk.xml" "${tmp}"
mv -f "${tmp}" "${REMOTE_CONFIG}"
restore_exact_file "${RECEIPT_DIR}/prior-backup-disk.xml" "${REMOTE_CONFIG}" \
"${PRIOR_CONFIG_HASH}" "${PRIOR_CONFIG_MODE}" \
"${PRIOR_CONFIG_UID}" "${PRIOR_CONFIG_GID}" "${tmp}" || return 1
else
rm -f "${REMOTE_CONFIG}"
bounded_cmd rm -f "${REMOTE_CONFIG}" || return 1
[ ! -e "${REMOTE_CONFIG}" ] || return 1
fi
}
cleanup_owned_residue() {
local path
for path in \
"${REMOTE_OVERRIDE}.candidate.${RUN_ID}" \
"${REMOTE_CONFIG}.candidate.${RUN_ID}" \
"${REMOTE_OVERRIDE}.rollback.${RUN_ID}" \
"${REMOTE_CONFIG}.rollback.${RUN_ID}"; do
bounded_cmd rm -f "${path}" || return 1
[ ! -e "${path}" ] || return 1
done
if [ -e "${STAGE_DIR}" ]; then
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
bounded_cmd rm -f "${STAGE_OVERRIDE}" "${STAGE_CONFIG}" || return 1
bounded_cmd rmdir "${STAGE_DIR}" || return 1
fi
[ ! -e "${STAGE_DIR}" ] || return 1
}
rollback_deploy() {
local disk_count rollback_dir_terminal=retained
restore_managed_files || return 1
@@ -409,26 +562,27 @@ rollback_deploy() {
wait_clickhouse || return 1
wait_api_and_listeners || return 1
disk_count="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
disk_count="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1
[ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1
[ "$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
[ "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1
[ "$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
[ "$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1
if [ "${HOST_DIR_CREATED}" -eq 1 ]; then
if sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
if bounded_cmd sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
rollback_dir_terminal=removed_empty
else
rollback_dir_terminal=retained_nonempty_no_delete
fi
fi
receipt rollback pass "prior_compose_state_restored_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
receipt rollback pass \
"prior_compose_state_restored_exact_hash_mode_uid_gid_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
}
on_exit() {
local rc=$? rollback_rc=0 terminal=failed
local rc=$? rollback_rc=0 cleanup_rc=0 terminal=failed
trap - EXIT HUP INT TERM
set +e
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
@@ -439,7 +593,15 @@ on_exit() {
rc=91
fi
fi
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
cleanup_owned_residue
cleanup_rc=$?
if [ "${cleanup_rc}" -ne 0 ]; then
receipt cleanup failed "exact_stage_candidate_or_rollback_residue_cleanup_failed"
rc=92
else
receipt cleanup pass "exact_stage_candidate_and_rollback_residue_absence_verified"
fi
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ] && [ "${cleanup_rc}" -eq 0 ]; then
terminal=pass
fi
receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}"
@@ -455,10 +617,22 @@ trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM
for lock_path in "${LOCK_FILE}" "${BACKUP_OPERATION_LOCK_FILE}" "${CANARY_LOCK_FILE}"; do
[ ! -L "${lock_path}" ] \
|| { receipt check failed operation_lock_symlink_unsafe; exit 75; }
if [ -e "${lock_path}" ]; then
[ -f "${lock_path}" ] \
|| { receipt check failed operation_lock_not_regular; exit 75; }
fi
done
exec 9>>"${LOCK_FILE}"
flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; }
exec 7>>"${CANARY_LOCK_FILE}"
flock -n 7 || { receipt check failed backup_canary_lock_held; exit 75; }
exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"
flock -n 8 || { receipt check failed backup_operation_lock_held; exit 75; }
for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss sudo timeout; do
for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss stat sudo timeout; do
command -v "${command_name}" >/dev/null 2>&1 \
|| { receipt check failed "required_command_missing_${command_name}"; exit 69; }
done
@@ -474,21 +648,39 @@ done
compose_base_config
compose_stage_config
env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
-f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
| grep -qx "${REMOTE_SERVICE}"
if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
receipt check failed active_signoz_backup_detected
exit 75
fi
active_native_backups="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
[ "${active_native_backups}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
require_backup_idle() {
local phase="$1" active_native_backups process_rc
set +e
pgrep_cmd -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' \
>/dev/null 2>&1
process_rc=$?
set -e
case "${process_rc}" in
0)
receipt check failed "active_signoz_backup_or_restore_process_detected_${phase}"
return 75
;;
1) ;;
*)
receipt check failed "signoz_backup_process_probe_failed_${phase}_exit_${process_rc}"
return 69
;;
esac
active_native_backups="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" \
|| return 1
[ "${active_native_backups}" = 0 ] \
|| { receipt check failed "active_native_backup_or_restore_${phase}"; return 75; }
receipt check pass "backup_idle_${phase}_shared_operation_and_canary_locks_held"
}
require_backup_idle preflight
for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do
[ "$(docker inspect --format '{{.State.Running}}' "${container}")" = true ]
[ "$(docker_cmd inspect --format '{{.State.Running}}' "${container}")" = true ]
done
wait_clickhouse
wait_api_and_listeners
@@ -505,42 +697,56 @@ fi
|| { receipt check failed partial_managed_file_drift; exit 78; }
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
compose_managed_config
cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
PRIOR_OVERRIDE_HASH="$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')"
PRIOR_OVERRIDE_MODE="$(stat -c '%a' "${REMOTE_OVERRIDE}")"
PRIOR_OVERRIDE_UID="$(stat -c '%u' "${REMOTE_OVERRIDE}")"
PRIOR_OVERRIDE_GID="$(stat -c '%g' "${REMOTE_OVERRIDE}")"
PRIOR_CONFIG_HASH="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
PRIOR_CONFIG_MODE="$(stat -c '%a' "${REMOTE_CONFIG}")"
PRIOR_CONFIG_UID="$(stat -c '%u' "${REMOTE_CONFIG}")"
PRIOR_CONFIG_GID="$(stat -c '%g' "${REMOTE_CONFIG}")"
bounded_cmd cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
bounded_cmd cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
[ "$(sha256sum "${RECEIPT_DIR}/prior-override.yaml" | awk '{print $1}')" = "${PRIOR_OVERRIDE_HASH}" ]
[ "$(sha256sum "${RECEIPT_DIR}/prior-backup-disk.xml" | awk '{print $1}')" = "${PRIOR_CONFIG_HASH}" ]
fi
PRIOR_DISK_COUNT="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
PRIOR_DISK_COUNT="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
BEFORE_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
BEFORE_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
BEFORE_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
BEFORE_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
[ -n "${BEFORE_DATA_VOLUME}" ]
BEFORE_COLLECTOR_ID="$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
BEFORE_COLLECTOR_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
BEFORE_SIGNOZ_ID="$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
BEFORE_SIGNOZ_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
BEFORE_ZOOKEEPER_ID="$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
BEFORE_ZOOKEEPER_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"
BEFORE_COLLECTOR_ID="$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
BEFORE_COLLECTOR_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
BEFORE_SIGNOZ_ID="$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
BEFORE_SIGNOZ_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
BEFORE_ZOOKEEPER_ID="$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
BEFORE_ZOOKEEPER_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"
receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid"
receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}"
receipt source_of_truth_diff pass "prior_override_${PRIOR_OVERRIDE_PRESENT}_prior_config_${PRIOR_CONFIG_PRESENT}_prior_disk_count_${PRIOR_DISK_COUNT}"
receipt source_of_truth_diff pass \
"prior_override_${PRIOR_OVERRIDE_PRESENT}_hash_${PRIOR_OVERRIDE_HASH}_mode_${PRIOR_OVERRIDE_MODE}_uid_${PRIOR_OVERRIDE_UID}_gid_${PRIOR_OVERRIDE_GID}_prior_config_${PRIOR_CONFIG_PRESENT}_hash_${PRIOR_CONFIG_HASH}_mode_${PRIOR_CONFIG_MODE}_uid_${PRIOR_CONFIG_UID}_gid_${PRIOR_CONFIG_GID}_prior_disk_count_${PRIOR_DISK_COUNT}"
receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only"
receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback"
receipt check pass "compose_config_q_stage_hashes_runtime_preflight_backup_idle"
receipt check pass \
"compose_config_q_stage_hashes_runtime_preflight_dual_locks_backup_idle"
require_backup_idle pre_mutation
APPLY_STARTED=1
if [ -e "${HOST_BACKUP_DIR}" ]; then
[ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
else
sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
bounded_cmd sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
HOST_DIR_CREATED=1
fi
install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
bounded_cmd install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
bounded_cmd install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
bounded_cmd mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
bounded_cmd mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
[ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
[ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
compose_managed_config
@@ -548,39 +754,39 @@ compose_managed_config
recreate_managed
wait_clickhouse
AFTER_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
AFTER_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
AFTER_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
AFTER_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
[ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ]
[ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ]
backup_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
config_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
backup_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
config_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
[ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ]
[ "${config_mount}" = false ]
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
server_uid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
server_uid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)"
server_gid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
server_gid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)"
[ "${server_uid}" = 101 ]
[ "${server_gid}" = 101 ]
docker exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
docker_cmd exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
'test -d /backups && test -w /backups'
disk_state="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
disk_state="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
"SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
[ "${disk_state}" = "Local|0|0|0|1" ]
[ "$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
[ "$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
"EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ]
docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
| grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)'
[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
[ "$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
[ "$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
[ "$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
[ "$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
[ "$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
[ "$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
wait_api_and_listeners
receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged"
@@ -589,6 +795,4 @@ receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_execute
DEPLOY_VERIFIED=1
REMOTE_APPLY
trap - EXIT
cleanup_stage
local_receipt terminal pass "remote_controlled_apply_and_postverify_complete"
LOCAL_APPLY_VERIFIED=1

View File

@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# P0-OBS-002 - atomically deploy the host110 SigNoz native backup toolchain.
# P0-OBS-002 - bounded controlled deploy of the host110 SigNoz native backup toolchain.
#
# This deployer owns seven repo-sourced files only. It does not run a backup or
# restore, restart a container, pull an image, or inspect a secret-bearing
@@ -12,6 +12,13 @@ TARGET_HOST="wooo@192.168.0.110"
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
SAFE_HOME="/home/wooo"
API_URL="http://127.0.0.1:8080/api/v1/health"
CANARY_LOCK="/tmp/awoooi-signoz-backup-canary.lock"
OPERATION_LOCK="/tmp/awoooi-signoz-backup-operation.lock"
SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"
SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"
DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"
TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"
REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"
REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}"
@@ -61,10 +68,18 @@ Optional safe absolute path environment:
SIGNOZ_TOOLCHAIN_RECEIPT_ROOT
SIGNOZ_TOOLCHAIN_STAGE_ROOT
Optional bounded-command timeout environment (positive integer seconds):
SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS
SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS
SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS
SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS
--check validates local sources and performs a remote read-only target diff and
runtime readiness check. --apply stages all seven sources, validates hashes and
syntax, atomically replaces exact targets, and rolls every target back to its
prior hash/mode/owner if any bounded verifier fails.
syntax, uses a same-filesystem atomic replacement for each exact target, and
rolls every target back to its prior hash/mode/owner if any bounded verifier
fails. SSH, SCP, and Docker calls are bounded, and apply holds both the SigNoz
canary lock and backup operation lock through post-verification or rollback.
Flat restore-driver invocation must set:
CLICKHOUSE_RESTORE_INVENTORY_HELPER=<script-dir>/clickhouse-restore-inventory.py
@@ -89,6 +104,10 @@ valid_identity() {
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
}
valid_positive_integer() {
[[ "$1" =~ ^[1-9][0-9]*$ ]]
}
safe_absolute_path() {
local path="$1"
[[ "${path}" == /* ]] || return 1
@@ -111,6 +130,24 @@ for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
}
done
for value in \
"${SSH_COMMAND_TIMEOUT_SECONDS}" \
"${SCP_COMMAND_TIMEOUT_SECONDS}" \
"${DOCKER_COMMAND_TIMEOUT_SECONDS}" \
"${TIMEOUT_KILL_AFTER_SECONDS}"; do
valid_positive_integer "${value}" || {
printf 'toolchain command timeouts must be positive integer seconds\n' >&2
exit 64
}
done
[ "${SSH_COMMAND_TIMEOUT_SECONDS}" -le 1200 ] \
&& [ "${SCP_COMMAND_TIMEOUT_SECONDS}" -le 600 ] \
&& [ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" -le 300 ] \
&& [ "${TIMEOUT_KILL_AFTER_SECONDS}" -le 60 ] || {
printf 'toolchain command timeout exceeds bounded maximum\n' >&2
exit 64
}
for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
safe_absolute_path "${path}" || {
printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2
@@ -133,13 +170,25 @@ if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \
exit 64
fi
for command_name in bash python3 scp ssh; do
for command_name in bash python3 scp ssh timeout; do
command -v "${command_name}" >/dev/null 2>&1 || {
printf 'required local command missing: %s\n' "${command_name}" >&2
exit 69
}
done
bounded_ssh() {
timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${SSH_COMMAND_TIMEOUT_SECONDS}" ssh "$@"
}
bounded_scp() {
timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${SCP_COMMAND_TIMEOUT_SECONDS}" scp "$@"
}
hash_file() {
if command -v sha256sum >/dev/null 2>&1; then
sha256sum "$1" | awk '{print $1}'
@@ -241,9 +290,10 @@ local_receipt() {
}
remote_read_only_check() {
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \
"${DOCKER_COMMAND_TIMEOUT_SECONDS}" "${TIMEOUT_KILL_AFTER_SECONDS}" \
"${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
set -euo pipefail
@@ -254,7 +304,9 @@ REMOTE_SCRIPT_DIR="$4"
REMOTE_CONFIG_ROOT="$5"
RECEIPT_ROOT="$6"
STAGE_ROOT="$7"
shift 7
DOCKER_COMMAND_TIMEOUT_SECONDS="$8"
TIMEOUT_KILL_AFTER_SECONDS="$9"
shift 9
EXPECTED_HASHES=("$@")
LABELS=(
backup-signoz.sh
@@ -279,10 +331,18 @@ API_URL="http://127.0.0.1:8080/api/v1/health"
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
[ -d /backup ] && [ ! -L /backup ]
for command_name in awk curl docker pgrep sha256sum ss stat; do
for command_name in awk curl docker pgrep sha256sum ss stat timeout; do
command -v "${command_name}" >/dev/null 2>&1
done
[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]
[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]]
bounded_docker() {
timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@"
}
listener_up() {
local port="$1"
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
@@ -305,7 +365,7 @@ validate_existing_ancestors() {
runtime_line() {
local container="$1" value running health
value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
IFS='|' read -r _ _ _ running health <<<"${value}"
[ "${running}" = true ]
case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
@@ -322,7 +382,7 @@ validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
validate_existing_ancestors "${RECEIPT_ROOT}"
validate_existing_ancestors "${STAGE_ROOT}"
[ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ]
active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
[ "${active_operations}" = 0 ]
@@ -373,7 +433,7 @@ fi
STAGE_CREATED=0
cleanup_stage() {
if [ "${STAGE_CREATED}" -eq 1 ]; then
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
set -u
stage_dir="$1"
if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
@@ -393,7 +453,7 @@ REMOTE_CLEANUP
}
trap cleanup_stage EXIT
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
set -euo pipefail
stage_dir="$1"
[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
@@ -405,13 +465,15 @@ REMOTE_STAGE
STAGE_CREATED=1
for index in "${!LOCAL_SOURCES[@]}"; do
scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
bounded_scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
"${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"
done
ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
"${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \
"${CANARY_LOCK}" "${OPERATION_LOCK}" "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \
"${TIMEOUT_KILL_AFTER_SECONDS}" \
"${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
set -euo pipefail
@@ -422,7 +484,11 @@ REMOTE_SCRIPT_DIR="$4"
REMOTE_CONFIG_ROOT="$5"
RECEIPT_ROOT="$6"
STAGE_DIR="$7"
shift 7
CANARY_LOCK="$8"
OPERATION_LOCK="$9"
DOCKER_COMMAND_TIMEOUT_SECONDS="${10}"
TIMEOUT_KILL_AFTER_SECONDS="${11}"
shift 11
EXPECTED_HASHES=("$@")
LABELS=(
backup-signoz.sh
@@ -472,6 +538,8 @@ APPLY_STARTED=0
DEPLOY_VERIFIED=0
RUNTIME_BASELINE_CAPTURED=0
LOCK_HELD=0
CANARY_LOCK_HELD=0
OPERATION_LOCK_HELD=0
REMOTE_UID="$(id -u)"
REMOTE_GID="$(id -g)"
CREATED_DIRS=()
@@ -481,7 +549,7 @@ PRIOR_UIDS=()
PRIOR_GIDS=()
PRIOR_HASHES=()
for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat; do
for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat timeout; do
command -v "${command_name}" >/dev/null 2>&1 || {
printf 'required remote command missing: %s\n' "${command_name}" >&2
exit 69
@@ -493,6 +561,10 @@ if [ "${REMOTE_UID}" -ne 0 ]; then
fi
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
[ -d /backup ] && [ ! -L /backup ]
[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]
[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]]
[ "${CANARY_LOCK}" = "/tmp/awoooi-signoz-backup-canary.lock" ]
[ "${OPERATION_LOCK}" = "/tmp/awoooi-signoz-backup-operation.lock" ]
run_root() {
if [ "${REMOTE_UID}" -eq 0 ]; then
@@ -502,6 +574,11 @@ run_root() {
fi
}
bounded_docker() {
timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@"
}
root_hash() {
if [ "${REMOTE_UID}" -eq 0 ]; then
sha256sum "$1" | awk '{print $1}'
@@ -550,7 +627,7 @@ listener_up() {
runtime_snapshot() {
local container value running health port api_code
for container in "${CONTAINERS[@]}"; do
value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
IFS='|' read -r _ _ _ running health <<<"${value}"
[ "${running}" = true ]
case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
@@ -673,6 +750,14 @@ on_exit() {
if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then
rc=92
fi
if [ "${OPERATION_LOCK_HELD}" -eq 1 ]; then
flock -u 7 >/dev/null 2>&1
exec 7>&-
fi
if [ "${CANARY_LOCK_HELD}" -eq 1 ]; then
flock -u 8 >/dev/null 2>&1
exec 8>&-
fi
if [ "${LOCK_HELD}" -eq 1 ]; then
flock -u 9 >/dev/null 2>&1
exec 9>&-
@@ -694,11 +779,33 @@ exec 9>>"${LOCK_FILE}"
flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; }
LOCK_HELD=1
[ ! -L "${CANARY_LOCK}" ] \
|| { receipt check failed canary_lock_symlink_unsafe; exit 75; }
if [ -e "${CANARY_LOCK}" ]; then
[ -f "${CANARY_LOCK}" ] \
|| { receipt check failed canary_lock_not_regular_file; exit 75; }
fi
exec 8>>"${CANARY_LOCK}"
flock -n 8 \
|| { receipt check failed another_signoz_canary_holds_canary_lock; exit 75; }
CANARY_LOCK_HELD=1
[ ! -L "${OPERATION_LOCK}" ] \
|| { receipt check failed backup_operation_lock_symlink_unsafe; exit 75; }
if [ -e "${OPERATION_LOCK}" ]; then
[ -f "${OPERATION_LOCK}" ] \
|| { receipt check failed backup_operation_lock_not_regular_file; exit 75; }
fi
exec 7>>"${OPERATION_LOCK}"
flock -n 7 \
|| { receipt check failed another_signoz_backup_holds_operation_lock; exit 75; }
OPERATION_LOCK_HELD=1
if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
receipt check failed active_signoz_backup_toolchain_process
exit 75
fi
active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
[ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
@@ -744,9 +851,9 @@ done
receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources"
receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}"
receipt ai_decision candidate "atomic_install_native_backup_toolchain_only"
receipt risk_policy pass "risk_medium_bounded_files_only_no_runtime_execution_no_container_change_full_rollback"
receipt check pass "runtime_baseline_active_operations_0_hash_bash_xml_py_compile"
receipt ai_decision candidate "same_filesystem_candidate_replace_native_backup_toolchain_only"
receipt risk_policy pass "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_backup_operation_locks_no_runtime_execution_no_container_change_full_rollback"
receipt check pass "dual_signoz_canary_and_backup_operation_locks_held_runtime_baseline_active_operations_0_hash_bash_xml_py_compile"
APPLY_STARTED=1
for directory in "${TARGET_DIRS[@]}"; do
@@ -790,7 +897,7 @@ done
runtime_snapshot > "${RUNTIME_AFTER}"
cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"
receipt execution pass "seven_candidates_atomically_replaced_expected_hashes_and_modes"
receipt execution pass "seven_same_filesystem_candidates_replaced_with_full_rollback_expected_hashes_and_modes"
receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged"
receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull"
DEPLOY_VERIFIED=1

View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import os
from pathlib import Path
import re
import subprocess
import xml.etree.ElementTree as ET
@@ -10,6 +11,12 @@ ROOT = Path(__file__).resolve().parents[3]
CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
BACKUP_SCRIPT = ROOT / "scripts/backup/backup-signoz.sh"
def write_executable(path: Path, text: str) -> None:
path.write_text(text, encoding="utf-8")
path.chmod(0o755)
def executable_text() -> str:
@@ -105,9 +112,9 @@ def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> No
text = DEPLOYER.read_text(encoding="utf-8")
check_terminal = text.index('if [ "${MODE}" = "--check" ]')
stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
first_scp = text.index("scp -q")
first_transfer = text.index('bounded_scp "${LOCAL_OVERRIDE}"')
assert check_terminal < stage_assignment < first_scp
assert check_terminal < stage_assignment < first_transfer
assert "-f '${REMOTE_BASE}' -f - config -q" in text
assert '< "${LOCAL_OVERRIDE}"' in text
assert "check_pass_no_write" in text
@@ -120,7 +127,12 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
assert 'REMOTE_SERVICE="clickhouse"' in text
assert "--env-file /dev/null" in text
assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
assert "timeout --signal=TERM --kill-after=30 300" in text
assert "bounded_ssh" in text
assert "bounded_scp" in text
assert "docker_cmd" in text
assert "compose_config_cmd" in text
assert "compose_apply_cmd" in text
assert "REMOTE_APPLY_TIMEOUT_SECONDS=3600" in text
assert "up -d --no-deps --pull never --force-recreate" in text
assert '"${REMOTE_SERVICE}"' in text
assert "docker compose down" not in executable
@@ -133,15 +145,135 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
text = DEPLOYER.read_text(encoding="utf-8")
assert "active_signoz_backup_detected" in text
assert "active_signoz_backup_or_restore_process_detected" in text
assert "active_native_backup_or_restore" in text
assert "signoz_backup_process_probe_failed" in text
assert "partial_managed_file_drift" in text
assert "another_deploy_holds_lock" in text
assert "backup-signoz[.]sh" in text
for process in (
"backup-signoz",
"clickhouse-native-backup",
"clickhouse-native-restore-drill",
"run-signoz-backup-canary",
):
assert process in text
probe_block = text[
text.index("require_backup_idle()") : text.index(
"require_backup_idle preflight"
)
]
assert "process_rc=$?" in probe_block
assert 'case "${process_rc}" in' in probe_block
assert "1) ;;" in probe_block
assert "return 69" in probe_block
assert "CREATING_BACKUP" in text
assert "RESTORING" in text
def test_target_host_is_fixed_and_ignores_ambient_override(tmp_path: Path) -> None:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
ssh_log = tmp_path / "ssh.log"
write_executable(
fake_bin / "ssh",
"""#!/bin/bash
printf '%s\\n' "$*" >> "${FAKE_SSH_LOG:?}"
cat >/dev/null
""",
)
write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
env = os.environ.copy()
env.update(
{
"PATH": f"{fake_bin}:{env['PATH']}",
"TRACE_ID": "P0-OBS-002-test",
"RUN_ID": "fixed-host-test",
"WORK_ITEM_ID": "P0-OBS-002",
"TARGET_HOST": "attacker@example.invalid",
"FAKE_SSH_LOG": str(ssh_log),
}
)
result = subprocess.run(
["bash", str(DEPLOYER), "--check"],
check=False,
capture_output=True,
text=True,
env=env,
)
assert result.returncode == 0, result.stderr
invocations = ssh_log.read_text(encoding="utf-8").splitlines()
assert len(invocations) == 2
assert all("wooo@192.168.0.110" in line for line in invocations)
assert all("attacker@example.invalid" not in line for line in invocations)
def test_all_transport_and_remote_docker_operations_are_bounded() -> None:
text = DEPLOYER.read_text(encoding="utf-8")
executable = executable_text()
assert executable.count(' ssh "${SSH_OPTS[@]}"') == 1
assert executable.count(' scp -q "${SSH_OPTS[@]}"') == 1
assert not re.search(r"\bdocker (inspect|exec)\b", executable)
assert 'timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}"' in text
assert 'docker_cmd exec "${CLICKHOUSE_CONTAINER}"' in text
assert "ss_cmd -H -lnt" in text
for mutation in (
"bounded_cmd cp -p",
"bounded_cmd install -m",
"bounded_cmd mv -f",
"bounded_cmd sudo -n install",
"bounded_cmd rm -f",
):
assert mutation in text
def test_stage_cleanup_failure_changes_apply_terminal_to_failed(tmp_path: Path) -> None:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
ssh_count = tmp_path / "ssh-count"
write_executable(
fake_bin / "ssh",
"""#!/bin/bash
count=0
if [ -f "${FAKE_SSH_COUNT:?}" ]; then
count="$(cat "${FAKE_SSH_COUNT}")"
fi
count=$((count + 1))
printf '%s\\n' "${count}" > "${FAKE_SSH_COUNT}"
cat >/dev/null
if [ "${count}" -eq 5 ]; then
exit 1
fi
""",
)
write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
env = os.environ.copy()
env.update(
{
"PATH": f"{fake_bin}:{env['PATH']}",
"TRACE_ID": "P0-OBS-002-test",
"RUN_ID": "cleanup-failure-test",
"WORK_ITEM_ID": "P0-OBS-002",
"FAKE_SSH_COUNT": str(ssh_count),
}
)
result = subprocess.run(
["bash", str(DEPLOYER), "--apply"],
check=False,
capture_output=True,
text=True,
env=env,
)
assert result.returncode == 92
assert '"phase":"cleanup","result":"failed"' in result.stdout
assert '"phase":"terminal","result":"failed"' in result.stdout
assert "failed_or_residue_present" in result.stdout
def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
text = DEPLOYER.read_text(encoding="utf-8")
assert "BEFORE_IMAGE_ID" in text
@@ -166,13 +298,13 @@ def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
assert "101:101:750" in text
assert "server_uid" in text
assert "server_gid" in text
assert "docker exec -u 101:101" in text
assert "docker_cmd exec -u 101:101" in text
assert "test -w /backups" in text
assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
identity_start = text.index('server_uid="')
identity_block = text[
identity_start : text.index("docker exec -u 101:101", identity_start)
identity_start : text.index("docker_cmd exec -u 101:101", identity_start)
]
assert "sh -c" not in identity_block
assert "concurrency_disabled" in text
@@ -194,6 +326,48 @@ def test_failed_apply_restores_prior_managed_compose_state() -> None:
assert "retained_nonempty_no_delete" in text
assert "rc=91" in text
assert "trap on_exit EXIT" in text
for token in (
"PRIOR_OVERRIDE_HASH",
"PRIOR_OVERRIDE_MODE",
"PRIOR_OVERRIDE_UID",
"PRIOR_OVERRIDE_GID",
"PRIOR_CONFIG_HASH",
"PRIOR_CONFIG_MODE",
"PRIOR_CONFIG_UID",
"PRIOR_CONFIG_GID",
"file_matches_metadata",
"restore_exact_file",
"prior_compose_state_restored_exact_hash_mode_uid_gid",
):
assert token in text
assert "exact_stage_candidate_and_rollback_residue_absence_verified" in text
def test_deployer_and_backup_execution_share_operation_lock_contract() -> None:
deployer = DEPLOYER.read_text(encoding="utf-8")
backup = BACKUP_SCRIPT.read_text(encoding="utf-8")
operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
assert operation_lock in deployer
assert operation_lock in backup
assert canary_lock in deployer
assert 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' in deployer
assert 'exec 7>>"${CANARY_LOCK_FILE}"' in deployer
assert "flock -n 8 || { receipt check failed backup_operation_lock_held" in deployer
assert "flock -n 7 || { receipt check failed backup_canary_lock_held" in deployer
assert 'exec 8>>"${OPERATION_LOCK_FILE}"' in backup
assert "require_backup_idle pre_mutation" in deployer
assert deployer.index('exec 7>>"${CANARY_LOCK_FILE}"') < deployer.index(
'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"'
)
assert deployer.index('exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"') < deployer.index(
"require_backup_idle pre_mutation"
)
remote_cleanup_call = deployer.rindex(" cleanup_owned_residue")
assert remote_cleanup_call < deployer.index(
'receipt terminal "${terminal}"', remote_cleanup_call
)
def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:

View File

@@ -3,10 +3,13 @@ from __future__ import annotations
import os
from pathlib import Path
import subprocess
import time
ROOT = Path(__file__).resolve().parents[3]
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
BACKUP = ROOT / "scripts/backup/backup-signoz.sh"
CANARY = ROOT / "scripts/backup/run-signoz-backup-canary.sh"
def deployer_text() -> str:
@@ -119,6 +122,33 @@ def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> Non
assert "must not overlap" in result.stderr
def test_timeout_overrides_fail_closed_before_network_access() -> None:
for key, value, expected in (
(
"SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS",
"0",
"positive integer seconds",
),
(
"SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS",
"301",
"exceeds bounded maximum",
),
):
env = base_env()
env[key] = value
result = subprocess.run(
["bash", str(DEPLOYER), "--check"],
check=False,
capture_output=True,
text=True,
env=env,
)
assert result.returncode == 64
assert expected in result.stderr
assert "ssh:" not in result.stderr
def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
text = deployer_text()
assert 'TARGET_HOST="wooo@192.168.0.110"' in text
@@ -229,6 +259,70 @@ def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
assert "check_pass_no_write" in result.stdout
def test_hung_ssh_is_bounded_and_cannot_claim_check_pass(tmp_path: Path) -> None:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
ssh = fake_bin / "ssh"
ssh.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8")
scp = fake_bin / "scp"
scp.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8")
ssh.chmod(0o755)
scp.chmod(0o755)
env = base_env()
env["PATH"] = f"{fake_bin}:{env['PATH']}"
env["SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS"] = "1"
env["SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS"] = "1"
started = time.monotonic()
result = subprocess.run(
["bash", str(DEPLOYER), "--check"],
check=False,
capture_output=True,
text=True,
env=env,
timeout=5,
)
elapsed = time.monotonic() - started
assert result.returncode == 124
assert elapsed < 4
assert "check_pass_no_write" not in result.stdout
def test_all_ssh_scp_and_docker_calls_use_bounded_wrappers() -> None:
text = deployer_text()
executable = executable_text()
assert (
'SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"'
in text
)
assert (
'SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"'
in text
)
assert (
'DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"'
in text
)
assert (
'TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"'
in text
)
assert "bounded_ssh()" in text
assert "bounded_scp()" in text
assert text.count('ssh "$@"') == 1
assert text.count('scp "$@"') == 1
assert executable.count("bounded_docker()") == 2
assert "$(docker inspect" not in executable
assert "$(docker exec" not in executable
assert "\n docker inspect" not in executable
assert "\n docker exec" not in executable
assert executable.count("bounded_docker inspect") == 2
assert executable.count("bounded_docker exec") == 2
assert "timeout --signal=TERM" in executable
def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
text = deployer_text()
for token in (
@@ -250,18 +344,60 @@ def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
assert "bounded_execution_not_started_targets_unchanged" in text
def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
def test_apply_uses_same_filesystem_candidates_and_accurate_replace_receipt() -> None:
text = deployer_text()
assert '"${target}.candidate.${RUN_ID}"' in text
assert (
'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
) in text
assert "deployed-manifest.tsv" in text
assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
assert (
"seven_same_filesystem_candidates_replaced_with_full_rollback_"
"expected_hashes_and_modes"
) in text
assert "seven_candidates_atomically_replaced" not in text
assert "atomically replaces exact targets" not in text
assert "stage_candidate_residue_0" in text
assert "verify_residue_zero" in text
def test_apply_holds_dual_backup_locks_through_postverify_or_rollback() -> None:
text = deployer_text()
backup_text = BACKUP.read_text(encoding="utf-8")
canary_text = CANARY.read_text(encoding="utf-8")
canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
assert f'CANARY_LOCK="{canary_lock}"' in text
assert f'OPERATION_LOCK="{operation_lock}"' in text
assert canary_lock in canary_text
assert operation_lock in backup_text
canary_acquire = text.index('exec 8>>"${CANARY_LOCK}"')
operation_acquire = text.index('exec 7>>"${OPERATION_LOCK}"')
active_check = text.index("if pgrep -af", operation_acquire)
apply_started = text.index("APPLY_STARTED=1", operation_acquire)
target_replace = text.index(
'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}"', apply_started
)
assert canary_acquire < operation_acquire < active_check < apply_started
assert apply_started < target_replace
on_exit = text[text.index("on_exit() {") : text.index("trap on_exit EXIT")]
rollback = on_exit.index("rollback_targets")
release_operation = on_exit.index("flock -u 7")
release_canary = on_exit.index("flock -u 8")
assert rollback < release_operation < release_canary
assert 'rm -f "${OPERATION_LOCK}"' not in text
assert 'rm -f "${CANARY_LOCK}"' not in text
assert "another_signoz_backup_holds_operation_lock" in text
assert "another_signoz_canary_holds_canary_lock" in text
assert (
"risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_"
"backup_operation_locks_"
"no_runtime_execution_no_container_change_full_rollback"
) in text
def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
text = deployer_text()
for container in (

View File

@@ -9,6 +9,7 @@ only from the committed post-closure receipt.
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any
@@ -30,10 +31,45 @@ EXPECTED_POST_CLOSURE_BLOCKERS = [
"signoz_backup_offsite_dr_pending",
"clickhouse_native_failed_artifact_retention_pending",
"clickhouse_native_resume_submitted_inventory_pending",
"clickhouse_native_current_source_toolchain_pending",
"service_registry_runtime_mirror_reconciliation_pending",
"github_freeze_legacy_asset_retirement_pending",
"monitoring_generator_duplicate_awoooi_api_identity_pending",
]
TOOLCHAIN_SOURCE_PATHS = {
"backup_orchestrator_sha256": "scripts/backup/backup-signoz.sh",
"native_backup_sha256": "scripts/backup/clickhouse-native-backup.sh",
"isolated_restore_sha256": "scripts/backup/clickhouse-native-restore-drill.sh",
"inventory_verifier_sha256": "scripts/backup/clickhouse-restore-inventory.py",
"canary_wrapper_sha256": "scripts/backup/run-signoz-backup-canary.sh",
"backup_disk_config_sha256": "ops/signoz/clickhouse/config.d/backup_disk.xml",
"isolated_cluster_config_sha256": (
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
),
}
HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES = {
"backup_orchestrator_sha256": (
"2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520"
),
"native_backup_sha256": (
"754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a"
),
"isolated_restore_sha256": (
"57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7"
),
"inventory_verifier_sha256": (
"79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c"
),
"canary_wrapper_sha256": (
"64d23d9f7a64d1e4e1347b98d45e50e26eadea1741d0e28a5e2e5d74b22322d8"
),
"backup_disk_config_sha256": (
"c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9"
),
"isolated_cluster_config_sha256": (
"e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b"
),
}
def _load_yaml(path: Path) -> Any:
@@ -45,6 +81,27 @@ def _read(root: Path, relative: str) -> str:
return (root / relative).read_text(encoding="utf-8")
def _sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _current_toolchain_source_hashes(root: Path) -> dict[str, str]:
hashes: dict[str, str] = {}
for field, relative in TOOLCHAIN_SOURCE_PATHS.items():
source = root / relative
if not source.is_file() or source.is_symlink():
return {}
try:
hashes[field] = _sha256_file(source)
except OSError:
return {}
return hashes
def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
contract = _load_yaml(contract_path)
post_closure = contract.get("runtime_observations", {}).get(
@@ -425,17 +482,61 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
native_restic = native_apply4.get("restic", {})
native_post = native_apply4.get("independent_post_verifier", {})
toolchain_apply = native_canary.get("toolchain_controlled_apply", {})
historical_native_source = native_canary.get("source", {})
current_native_source = native_canary.get("current_source", {})
historical_source_hashes = {
field: historical_native_source.get(field) for field in TOOLCHAIN_SOURCE_PATHS
}
declared_current_hashes = current_native_source.get("hashes", {})
actual_current_hashes = _current_toolchain_source_hashes(root)
actual_historical_drift = sorted(
field
for field in TOOLCHAIN_SOURCE_PATHS
if actual_current_hashes.get(field) != historical_source_hashes.get(field)
)
declared_historical_drift = current_native_source.get(
"drift_from_historical_production_receipt", []
)
checks["native_backup_historical_source_receipt_preserved"] = (
historical_native_source.get("evidence_scope")
== "historical_production_receipt"
and historical_native_source.get("deployment_status") == "deployed_verified"
and historical_source_hashes == HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES
)
checks["native_backup_current_source_hashes_bound"] = (
current_native_source.get("evidence_scope") == "committed_current_source"
and current_native_source.get("expected_file_count")
== len(TOOLCHAIN_SOURCE_PATHS)
and set(declared_current_hashes) == set(TOOLCHAIN_SOURCE_PATHS)
and declared_current_hashes == actual_current_hashes
and declared_historical_drift == actual_historical_drift
)
pending_source_state = (
current_native_source.get("status")
== "pending_toolchain_deploy_and_runtime_canary"
and current_native_source.get("deployment_status") == "pending"
and current_native_source.get("current_source_deployed") is False
and current_native_source.get("runtime_verifier_terminal") == "pending"
)
deployed_source_state = (
current_native_source.get("status") == "deployed_verified"
and current_native_source.get("deployment_status") == "deployed"
and current_native_source.get("current_source_deployed") is True
and current_native_source.get("runtime_verifier_terminal") == "pass"
and current_native_source.get("deployed_source_hashes") == actual_current_hashes
and bool(current_native_source.get("toolchain_apply_run_id"))
and bool(current_native_source.get("runtime_canary_run_id"))
)
checks["native_backup_current_source_deployment_state_explicit"] = (
current_native_source.get("historical_production_receipt_preserved") is True
and (pending_source_state or deployed_source_state)
)
checks["native_backup_restore_canary_closed"] = (
native_canary.get("completion_scope") == "clickhouse_native_only"
and native_canary.get("source", {}).get("native_backup_sha256")
== "754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a"
and native_canary.get("source", {}).get("isolated_restore_sha256")
== "57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7"
and native_canary.get("source", {}).get("post_canary_focused_tests_passed")
== 94
and native_canary.get("source", {}).get("post_format_focused_tests_passed")
== 94
and native_canary.get("source", {}).get("ruff_format") == "pass"
and checks["native_backup_historical_source_receipt_preserved"]
and historical_native_source.get("post_canary_focused_tests_passed") == 94
and historical_native_source.get("post_format_focused_tests_passed") == 94
and historical_native_source.get("ruff_format") == "pass"
and toolchain_apply.get("apply_run_id")
== "P0-OBS-002-native-toolchain-20260715T001252Z-apply4"
and toolchain_apply.get("apply_terminal") == "pass"
@@ -568,6 +669,9 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
resume_inventory = pending_verifiers.get(
"clickhouse_native_resume_submitted_inventory", {}
)
current_source_toolchain = pending_verifiers.get(
"clickhouse_native_current_source_toolchain", {}
)
checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = (
native_migration.get("prior_run_id") == retry2.get("run_id")
and native_migration.get("run_id") == native_apply4.get("run_id")
@@ -595,6 +699,14 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
== "pending_durable_submitted_inventory_contract"
and resume_inventory.get("same_run_resume_safe") is False
and len(resume_inventory.get("required_preconditions", [])) == 4
and current_source_toolchain.get("status")
== current_native_source.get("status")
== "pending_toolchain_deploy_and_runtime_canary"
and current_source_toolchain.get("current_source_deployed") is False
and current_source_toolchain.get("runtime_verifier_terminal") == "pending"
and current_source_toolchain.get("historical_production_receipt_preserved")
is True
and len(current_source_toolchain.get("required_preconditions", [])) == 3
)
checks["post_closure_contract_mirror_consistent"] = (
@@ -621,6 +733,10 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
== toolchain_apply.get("apply_run_id")
and post_closure.get("clickhouse_native_backup_toolchain_postcheck_run_id")
== toolchain_apply.get("postcheck_run_id")
and post_closure.get("clickhouse_native_current_source_status")
== current_native_source.get("status")
and post_closure.get("clickhouse_native_current_source_deployed")
== current_native_source.get("current_source_deployed")
and post_closure.get("clickhouse_native_restore_terminal") == "verified"
and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a"
and post_closure.get("clickhouse_native_offsite_status") == "pending"
@@ -684,10 +800,21 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
"phase": "wave_a_dual_allow",
"source_ready": source_ready,
"runtime_status": terminal.get("status", "unknown"),
"backup_toolchain_current_source_status": current_native_source.get(
"status", "unknown"
),
"backup_toolchain_current_source_deployed": current_native_source.get(
"current_source_deployed", False
),
"backup_toolchain_runtime_verifier_terminal": current_native_source.get(
"runtime_verifier_terminal", "unknown"
),
"backup_toolchain_source_drift_fields": actual_historical_drift,
"active_route": "incumbent",
"writes_performed": 0,
"checks": checks,
"blockers": blockers,
"post_closure_blockers": list(regression_terminal.get("blockers", [])),
"terminal": "pass" if source_ready else "failed",
}

View File

@@ -1,5 +1,7 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import os
import subprocess
@@ -25,6 +27,16 @@ GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml"
POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml"
def _load_preflight_module():
spec = importlib.util.spec_from_file_location(
"signoz_canonical_route_preflight_under_test", PREFLIGHT
)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
assert (
@@ -86,11 +98,28 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
assert payload["writes_performed"] == 0
assert payload["terminal"] == "pass"
assert payload["runtime_status"] == "partial_degraded"
assert (
payload["backup_toolchain_current_source_status"]
== "pending_toolchain_deploy_and_runtime_canary"
)
assert payload["backup_toolchain_current_source_deployed"] is False
assert payload["backup_toolchain_runtime_verifier_terminal"] == "pending"
assert payload["backup_toolchain_source_drift_fields"] == [
"backup_orchestrator_sha256",
"canary_wrapper_sha256",
"inventory_verifier_sha256",
]
assert payload["active_route"] == "incumbent"
assert payload["checks"] and all(payload["checks"].values())
assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True
assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True
assert payload["checks"]["post_closure_schema_v2"] is True
assert payload["checks"]["native_backup_current_source_hashes_bound"] is True
assert (
payload["checks"]["native_backup_current_source_deployment_state_explicit"]
is True
)
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
assert payload["checks"]["post_release_600s_verifier_consistent"] is True
assert (
payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"]
@@ -621,6 +650,69 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No
assert canary["terminal"] == "production_closed_clickhouse_native_scope"
def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt() -> (
None
):
module = _load_preflight_module()
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"]
historical = canary["source"]
current = canary["current_source"]
historical_hashes = {
field: historical[field] for field in module.TOOLCHAIN_SOURCE_PATHS
}
assert historical["evidence_scope"] == "historical_production_receipt"
assert historical["deployment_status"] == "deployed_verified"
assert historical_hashes == module.HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES
assert historical["inventory_verifier_sha256"] == (
"79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c"
)
current_hashes = {
field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest()
for field, relative in module.TOOLCHAIN_SOURCE_PATHS.items()
}
assert current["hashes"] == current_hashes
assert current["hashes"]["inventory_verifier_sha256"] == (
"c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863"
)
assert current["status"] == "pending_toolchain_deploy_and_runtime_canary"
assert current["deployment_status"] == "pending"
assert current["current_source_deployed"] is False
assert current["runtime_verifier_terminal"] == "pending"
assert current["historical_production_receipt_preserved"] is True
assert current["drift_from_historical_production_receipt"] == [
"backup_orchestrator_sha256",
"canary_wrapper_sha256",
"inventory_verifier_sha256",
]
def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale(
monkeypatch,
) -> None:
module = _load_preflight_module()
real_sha256_file = module._sha256_file
def stale_inventory_hash(path: Path) -> str:
if path.name == "clickhouse-restore-inventory.py":
return "0" * 64
return real_sha256_file(path)
monkeypatch.setattr(module, "_sha256_file", stale_inventory_hash)
payload = module.evaluate(ROOT, CONTRACT)
assert payload["source_ready"] is False
assert payload["terminal"] == "failed"
assert payload["checks"]["native_backup_current_source_hashes_bound"] is False
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
assert (
payload["backup_toolchain_current_source_status"]
== "pending_toolchain_deploy_and_runtime_canary"
)
def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None:
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
verifier = receipt["post_release_runtime_verifier"]
@@ -667,6 +759,14 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() ->
]
assert resume["same_run_resume_safe"] is False
assert len(resume["required_preconditions"]) == 4
current_source = receipt["pending_verifiers"][
"clickhouse_native_current_source_toolchain"
]
assert current_source["status"] == "pending_toolchain_deploy_and_runtime_canary"
assert current_source["current_source_deployed"] is False
assert current_source["runtime_verifier_terminal"] == "pending"
assert current_source["historical_production_receipt_preserved"] is True
assert len(current_source["required_preconditions"]) == 3
assets = receipt["asset_reconciliation"]
assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001"
assert assets["source_service_count"] == 27
@@ -697,6 +797,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() ->
"signoz_backup_offsite_dr_pending",
"clickhouse_native_failed_artifact_retention_pending",
"clickhouse_native_resume_submitted_inventory_pending",
"clickhouse_native_current_source_toolchain_pending",
"service_registry_runtime_mirror_reconciliation_pending",
"github_freeze_legacy_asset_retirement_pending",
"monitoring_generator_duplicate_awoooi_api_identity_pending",
@@ -737,6 +838,10 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N
assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == (
"P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4"
)
assert post_closure["clickhouse_native_current_source_status"] == (
"pending_toolchain_deploy_and_runtime_canary"
)
assert post_closure["clickhouse_native_current_source_deployed"] is False
assert post_closure["clickhouse_native_restore_terminal"] == "verified"
assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a"
assert post_closure["clickhouse_native_offsite_status"] == "pending"