wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(observability): harden native backup controlled apply

Browse Source
This commit is contained in:
ogt
2026-07-15 09:43:40 +08:00
parent 322e812d93
commit 032be5f400
12 changed files with 1849 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
scripts/backup/backup-signoz.sh
View File

@@ -15,14 +15,19 @@ source "$(dirname "$0")/common.sh"
SERVICE="signoz"
LOCAL_REPO="${BACKUP_BASE}/signoz"
DUMP_DIR="/tmp/signoz-backup-$$"
TMP_ROOT="${SIGNOZ_BACKUP_TMP_ROOT:-/tmp}"
DUMP_DIR=""
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}"
COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}"
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}"
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"
# Destructive Restic forget/prune is a separate critical break-glass workflow.
# The regular and canary backup paths are backup-only and fail closed if a
# caller attempts to re-enable inline retention cleanup.
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-1}"
OPERATION_LOCK_FILE="${SIGNOZ_BACKUP_OPERATION_LOCK_FILE:-/tmp/awoooi-signoz-backup-operation.lock}"
RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}"
RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}"
RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}"
@@ -40,7 +45,8 @@ COLLECTOR_STARTED_AT_BEFORE=""
COLLECTOR_RESTARTS_BEFORE=""
COLLECTOR_HEALTH_BEFORE=""
COLLECTOR_OBSERVER_PID=""
COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
COLLECTOR_OBSERVATIONS=""
WORKSPACE_CREATED=0
CLEANUP_COMPLETE=0
FAILURE_NOTIFIED=0
@@ -126,13 +132,24 @@ cleanup() {
local exit_code=$?
if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
return "${exit_code}"
exit "${exit_code}"
fi
CLEANUP_COMPLETE=1
trap '' HUP INT TERM
trap - EXIT HUP INT TERM
set +e
stop_collector_observer
rm -rf -- "${DUMP_DIR}"
return "${exit_code}"
if [ "${WORKSPACE_CREATED}" -eq 1 ]; then
if [ -n "${DUMP_DIR}" ] && [ -d "${DUMP_DIR}" ] \
&& [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
&& [[ "${DUMP_DIR}" == "${TMP_ROOT%/}"/signoz-backup.* ]]; then
rm -rf -- "${DUMP_DIR}" || exit_code=90
[ ! -e "${DUMP_DIR}" ] || exit_code=90
else
log_error "拒絕清理不符合本 run identity 的 SigNoz workspace"
exit_code=90
fi
fi
exit "${exit_code}"
}
on_signal() {
@@ -140,6 +157,28 @@ on_signal() {
exit "$((128 + signal_number))"
}
create_private_workspace() {
[[ "${TMP_ROOT}" == /* ]] \
&& [[ ! "${TMP_ROOT}" =~ (^|/)\.\.?(/|$) ]] \
&& [[ "${TMP_ROOT}" != *$'\n'* ]] \
&& [ -d "${TMP_ROOT}" ] && [ -w "${TMP_ROOT}" ] \
&& [ ! -L "${TMP_ROOT}" ] || {
log_error "SigNoz workspace root 必須是安全、可寫、非 symlink 的絕對目錄"
return 1
}
DUMP_DIR="$(mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX")" || return 1
WORKSPACE_CREATED=1
[ -d "${DUMP_DIR}" ] && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
&& [ -r "${DUMP_DIR}" ] && [ -w "${DUMP_DIR}" ] && [ -x "${DUMP_DIR}" ] || {
log_error "mktemp 未建立本 run 擁有的 private SigNoz workspace"
return 1
}
chmod 700 "${DUMP_DIR}" || return 1
COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
log_info "建立 SigNoz private workspace: ${DUMP_DIR}"
}
run_native_clickhouse_backup() {
local mode="$1"
@@ -184,6 +223,32 @@ write_restic_receipt() {
mv "${receipt_tmp}" "${receipt_file}"
}
acquire_operation_lock() {
case "${OPERATION_LOCK_FILE}" in
/*) ;;
*)
log_error "SigNoz operation lock path 必須是絕對路徑"
return 1
;;
esac
[ ! -L "${OPERATION_LOCK_FILE}" ] || {
log_error "SigNoz operation lock 不得是 symlink"
return 1
}
if [ -e "${OPERATION_LOCK_FILE}" ]; then
[ -f "${OPERATION_LOCK_FILE}" ] || {
log_error "SigNoz operation lock 必須是 regular file"
return 1
}
fi
exec 8>>"${OPERATION_LOCK_FILE}"
flock -n 8 || {
log_error "另一個 SigNoz backup/deploy operation 正在執行"
return 1
}
log_info "取得 SigNoz backup/deploy 共用 operation lock"
}
main() {
local start_time end_time duration snapshot_id snapshot_json tags
local initial_observation initial_running final_id final_running
@@ -210,7 +275,18 @@ main() {
exit 1
}
done
mkdir -p "${DUMP_DIR}"
[ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ] || {
notify_failure_once "inline Restic forget/prune 已停用；僅允許獨立 critical break-glass workflow"
exit 1
}
acquire_operation_lock || {
notify_failure_once "SigNoz backup/deploy operation lock 取得失敗"
exit 1
}
create_private_workspace || {
notify_failure_once "建立 private SigNoz backup workspace 失敗"
exit 1
}
# Check-mode must pass before the backup window. It writes only durable
# receipts and does not submit BACKUP or create a server artifact.
@@ -305,11 +381,7 @@ main() {
}
log_success "Restic 備份完成: ${snapshot_id}"
if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
else
cleanup_old_backups "${LOCAL_REPO}"
fi
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
# Final readback is separate from the continuity observer receipt.
initial_observation="$(collector_identity_state)" || {

364
scripts/backup/run-signoz-backup-canary.sh
View File

@@ -12,6 +12,12 @@ set -euo pipefail
COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}"
BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}"
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-/backup/scripts/clickhouse-native-restore-drill.sh}"
CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-/backup/scripts/clickhouse-restore-inventory.py}"
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-/backup/config/signoz/clickhouse/config.d/backup_disk.xml}"
CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml}"
CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}"
CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-/backup/logs/clickhouse-native-restore-drill}"
MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}"
MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}"
MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}"
@@ -21,6 +27,7 @@ LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}"
CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}"
TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}"
DOCKER_TIMEOUT_SECONDS="${SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS:-15}"
LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}"
MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}"
STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}"
@@ -94,6 +101,307 @@ require_command() {
command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
}
run_collector_docker() {
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
"${DOCKER_TIMEOUT_SECONDS}" docker "$@"
}
require_native_restore_contract() {
local label path
for label in \
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK \
CLICKHOUSE_RESTORE_INVENTORY_HELPER \
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG \
CLICKHOUSE_RESTORE_CLUSTER_CONFIG; do
path="${!label}"
case "${path}" in
/*) ;;
*) fail_closed "native_restore_contract_${label}_path_not_absolute" ;;
esac
[ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \
|| fail_closed "native_restore_contract_${label}_missing_unreadable_or_symlink"
done
[ -x "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" ] \
|| fail_closed "native_restore_contract_post_verify_hook_not_executable"
[ -x "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" ] \
|| fail_closed "native_restore_contract_inventory_helper_not_executable"
}
acquire_canary_lock() {
[ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe"
if [ -e "${LOCK_FILE}" ]; then
[ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file"
fi
exec 9>>"${LOCK_FILE}"
flock -n 9 || fail_closed "another_signoz_canary_holds_lock"
}
require_native_receipt_contract() {
local label path run_dir
for label in CLICKHOUSE_NATIVE_RECEIPT_ROOT CLICKHOUSE_RESTORE_RECEIPT_ROOT; do
path="${!label}"
case "${path}" in
/*) ;;
*) fail_closed "native_receipt_contract_${label}_path_not_absolute" ;;
esac
[[ "${path}" != *$'\n'* ]] \
|| fail_closed "native_receipt_contract_${label}_path_contains_newline"
[ ! -L "${path}" ] \
|| fail_closed "native_receipt_contract_${label}_root_symlink_unsafe"
if [ -e "${path}" ]; then
[ -d "${path}" ] && [ -r "${path}" ] && [ -w "${path}" ] \
|| fail_closed "native_receipt_contract_${label}_root_not_readable_writable_directory"
fi
run_dir="${path}/${RUN_ID}"
[ ! -e "${run_dir}" ] && [ ! -L "${run_dir}" ] \
|| fail_closed "native_receipt_contract_${label}_stale_same_run_path_present"
done
[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" != "${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" ] \
|| fail_closed "native_and_restore_receipt_roots_must_be_distinct"
}
verify_same_run_native_restore_receipts() {
local native_receipts restore_receipts restore_status
native_receipts="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl"
restore_receipts="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl"
restore_status="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/status.json"
for path in "${native_receipts}" "${restore_receipts}" "${restore_status}"; do
[ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \
|| fail_closed "same_run_native_restore_receipt_missing_unreadable_or_symlink"
done
python3 - "${native_receipts}" "${restore_receipts}" "${restore_status}" \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" <<'PY' \
|| fail_closed "same_run_native_restore_receipt_validation_failed"
import json
import os
import sys
def reject(message: str) -> None:
print(f"receipt_verifier:{message}", file=sys.stderr)
raise SystemExit(1)
native_path, restore_path, status_path, trace_id, run_id, work_item_id = sys.argv[1:]
identity = {
"trace_id": trace_id,
"run_id": run_id,
"work_item_id": work_item_id,
}
def read_jsonl(path: str, label: str) -> list[dict[str, object]]:
if os.path.islink(path) or not os.path.isfile(path):
reject(f"{label}_unsafe")
records: list[dict[str, object]] = []
try:
with open(path, encoding="utf-8") as source:
for line_number, raw_line in enumerate(source, start=1):
if not raw_line.strip():
continue
value = json.loads(raw_line)
if not isinstance(value, dict):
reject(f"{label}_line_{line_number}_not_object")
for key, expected in identity.items():
if value.get(key) != expected:
reject(f"{label}_line_{line_number}_{key}_mismatch")
invocation_id = value.get("invocation_id")
if not isinstance(invocation_id, str) or not invocation_id:
reject(f"{label}_line_{line_number}_invocation_id_invalid")
records.append(value)
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
reject(f"{label}_malformed_{type(exc).__name__}")
if not records:
reject(f"{label}_empty")
return records
def mode_records(
records: list[dict[str, object]], mode: str, label: str
) -> list[dict[str, object]]:
selected = [
record
for record in records
if str(record.get("invocation_id", "")).startswith(f"{mode}-")
]
if not selected:
reject(f"{label}_{mode}_invocation_missing")
return selected
def require_exact(
records: list[dict[str, object]],
*,
label: str,
stage: str,
terminal: str,
detail_prefix: str | None = None,
) -> None:
matches = [
record
for record in records
if record.get("stage") == stage
and record.get("terminal") == terminal
and (
detail_prefix is None
or str(record.get("detail", "")).startswith(detail_prefix)
)
]
if len(matches) != 1:
reject(f"{label}_{stage}_{terminal}_count_{len(matches)}")
native_records = read_jsonl(native_path, "native_receipts")
native_check = mode_records(native_records, "check", "native")
native_apply = mode_records(native_records, "apply", "native")
require_exact(
native_check,
label="native_check",
stage="terminal",
terminal="check_pass_no_write",
)
require_exact(
native_apply,
label="native_apply",
stage="terminal",
terminal="pass",
)
require_exact(
native_apply,
label="native_apply",
stage="independent_verifier",
terminal="pass",
)
require_exact(
native_apply,
label="native_apply_hook_check",
stage="command",
terminal="pass",
detail_prefix="step_post_verify_hook_check_exit_0_",
)
require_exact(
native_apply,
label="native_apply_hook",
stage="command",
terminal="pass",
detail_prefix="step_post_verify_hook_exit_0_",
)
restore_records = read_jsonl(restore_path, "restore_receipts")
restore_check = mode_records(restore_records, "check", "restore")
restore_apply = mode_records(restore_records, "apply", "restore")
require_exact(
restore_check,
label="restore_check",
stage="terminal",
terminal="check_pass_no_runtime_write",
)
require_exact(
restore_apply,
label="restore_apply",
stage="terminal",
terminal="verified",
)
require_exact(
restore_apply,
label="restore_apply",
stage="independent_post_verifier",
terminal="pass",
)
require_exact(
restore_apply,
label="restore_apply",
stage="cleanup",
terminal="pass",
)
if os.path.islink(status_path) or not os.path.isfile(status_path):
reject("restore_status_unsafe")
try:
with open(status_path, encoding="utf-8") as source:
status = json.load(source)
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
reject(f"restore_status_malformed_{type(exc).__name__}")
if not isinstance(status, dict):
reject("restore_status_not_object")
for key, expected in identity.items():
if status.get(key) != expected:
reject(f"restore_status_{key}_mismatch")
expected_strings = {
"schema_version": "clickhouse_native_restore_drill_status_v1",
"mode": "apply",
"terminal": "verified",
"restore_terminal": "RESTORED",
"network_mode": "docker_internal_only",
"staging_network_mode": "none",
}
for key, expected in expected_strings.items():
if not isinstance(status.get(key), str) or status.get(key) != expected:
reject(f"restore_status_{key}_invalid")
for key, expected in (("exit_code", 0), ("manifest_parity", 1)):
if type(status.get(key)) is not int or status.get(key) != expected:
reject(f"restore_status_{key}_invalid")
for key in (
"check_pass",
"apply_pass",
"cleanup_verified",
"artifact_staging_verified",
"clickhouse_artifact_readback_verified",
"ephemeral_runtime_created",
"isolated_keeper",
):
if status.get(key) is not True:
reject(f"restore_status_{key}_not_true")
for key in (
"production_network_attached",
"production_restore_performed",
"allow_non_empty_tables",
):
if status.get(key) is not False:
reject(f"restore_status_{key}_not_false")
def positive_integer(key: str) -> int:
value = status.get(key)
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
reject(f"restore_status_{key}_not_positive_integer")
return value
source_database_count = positive_integer("source_database_count")
restored_database_count = positive_integer("restored_database_count")
if source_database_count != restored_database_count:
reject("restore_status_database_count_mismatch")
source_table_count = positive_integer("source_table_count")
restored_table_count = positive_integer("restored_table_count")
if source_table_count != restored_table_count:
reject("restore_status_table_count_mismatch")
positive_integer("critical_nonzero_count")
check_count = status.get("check_table_count")
check_pass_count = status.get("check_table_pass_count")
if (
isinstance(check_count, bool)
or not isinstance(check_count, int)
or isinstance(check_pass_count, bool)
or not isinstance(check_pass_count, int)
or check_count <= 0
or check_count != check_pass_count
):
reject("restore_status_check_table_counts_invalid")
PY
emit_receipt "native_restore_receipt_verifier" "pass" \
"same_run_machine_readable_native_check_apply_hook_and_isolated_restore_status_verified"
}
require_monitor_fragment() {
local fragment="$1"
local count
@@ -138,7 +446,7 @@ require_monitor_function_fragment() {
collector_running_state() {
local running
if ! running="$(docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
return 1
fi
case "${running}" in
@@ -149,7 +457,7 @@ collector_running_state() {
}
collector_runtime_metadata() {
docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
"${COLLECTOR_NAME}" 2>/dev/null
}
@@ -247,13 +555,15 @@ cleanup() {
if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then
overall_terminal="check_pass_no_write"
elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then
overall_terminal="pass"
overall_terminal="partial_degraded"
fi
emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}"
if [ "${overall_terminal}" = "pass" ]; then
emit_receipt "closure_writeback" "pass" \
"no_incident_notification_or_learning_delta_local_durable_receipt_ack"
if [ "${overall_terminal}" = "partial_degraded" ]; then
emit_receipt "runtime_operation" "pass" \
"clickhouse_native_backup_and_isolated_restore_runtime_verifier_pass"
emit_receipt "closure_writeback" "partial_degraded" \
"local_durable_receipt_ack_incident_not_applicable_telegram_not_sent_km_rag_writeback_pending_offsite_false_sqlite_application_consistent_export_pending"
elif [ "${overall_terminal}" = "check_pass_no_write" ]; then
emit_receipt "closure_writeback" "not_applicable" \
"check_mode_no_runtime_change_local_durable_receipt_ack"
@@ -261,7 +571,8 @@ cleanup() {
emit_receipt "closure_writeback" "deferred" \
"failed_terminal_requires_followup_no_completion_claim"
fi
emit_receipt "terminal" "${overall_terminal}" "backup_canary_${overall_terminal}"
emit_receipt "terminal" "${overall_terminal}" \
"backup_canary_${overall_terminal}_clickhouse_native_scope_only_offsite_false_sqlite_pending"
printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \
"${lease_terminal}" "${exit_code}"
@@ -521,7 +832,7 @@ main() {
local container_health_before collector_metadata_before
local monitor_log_identity="" monitor_log_size=""
local source_diff_detail ai_decision_detail
local backup_status
local backup_status native_restore_contract_detail="not_applicable"
setup_receipts
@@ -532,11 +843,17 @@ main() {
*) fail_closed "unsupported_mode_use_check_or_apply" ;;
esac
for command_name in awk date docker env flock grep kill pgrep sha256sum sleep ss timeout tr; do
# Hold the canary lock before inspecting or hashing any mutable backup or
# restore toolchain path. This closes the check/apply substitution window.
require_command flock
acquire_canary_lock
for command_name in awk date docker env grep kill pgrep python3 sha256sum sleep ss timeout tr; do
require_command "${command_name}"
done
valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout"
valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after"
valid_positive_integer "${DOCKER_TIMEOUT_SECONDS}" || fail_closed "invalid_docker_timeout"
valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval"
case "${EXPECT_COLLECTOR_STOP}" in
0|1) ;;
@@ -545,7 +862,11 @@ main() {
[ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \
|| fail_closed "backup_script_missing_unexecutable_or_symlink"
if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
require_native_restore_contract
require_native_receipt_contract
native_restore_contract_detail="hook_$(sha256sum "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" | awk '{print $1}')_inventory_$(sha256sum "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" | awk '{print $1}')_disk_config_$(sha256sum "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" | awk '{print $1}')_cluster_config_$(sha256sum "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" | awk '{print $1}')"
else
for command_name in cp crontab id mktemp mv rm stat tail; do
require_command "${command_name}"
done
@@ -555,12 +876,6 @@ main() {
verify_monitor_contract
fi
[ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe"
if [ -e "${LOCK_FILE}" ]; then
[ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file"
fi
exec 9>>"${LOCK_FILE}"
flock -n 9 || fail_closed "another_signoz_canary_holds_lock"
if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
fail_closed "another_signoz_backup_is_running"
fi
@@ -591,18 +906,18 @@ main() {
PRIOR_LEASE_PRESENT=0
PRIOR_LEASE_HASH="not_applicable"
emit_receipt "sensor_source" "pass" \
"online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
source_diff_detail="online_native_backup_monitor_and_cooldown_contract_not_applicable"
ai_decision_detail="run_online_native_backup_without_collector_or_monitor_mutation"
"online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_${native_restore_contract_detail}_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
source_diff_detail="online_native_backup_isolated_restore_contract_bound_monitor_and_cooldown_contract_not_applicable"
ai_decision_detail="run_online_native_backup_with_mandatory_isolated_restore_verifier_without_collector_or_monitor_mutation"
fi
emit_receipt "normalized_asset_identity" "pass" \
"asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker"
emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}"
emit_receipt "ai_decision" "candidate" "${ai_decision_detail}"
emit_receipt "risk_policy" "pass" \
"risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_independent_verifier"
"risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_mandatory_isolated_restore_independent_verifier"
emit_receipt "check" "pass" \
"mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent"
"mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_${native_restore_contract_detail}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent"
if [ "${MODE}" = "check" ]; then
CHECK_VERIFIED=1
@@ -622,6 +937,12 @@ main() {
set +e
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \
env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" \
CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" \
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" \
CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" \
CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" \
CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" \
BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \
> "${BACKUP_OUTPUT}" 2>&1 &
BACKUP_PID=$!
@@ -645,6 +966,7 @@ main() {
else
grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \
|| fail_closed "native_backup_success_terminal_missing"
verify_same_run_native_restore_receipts
fi
if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then

77
scripts/backup/tests/test_backup_signoz_collector_restore.py
View File

@@ -25,7 +25,9 @@ def _run_backup(
native_apply_status: int = 0,
continuity_mode: str = "stable",
restic_backup_status: int = 0,
skip_retention: bool = False,
skip_retention: bool = True,
operation_lock_status: int = 0,
workspace_root_symlink: bool = False,
) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
harness = tmp_path / "backup"
fake_bin = tmp_path / "bin"
@@ -33,6 +35,7 @@ def _run_backup(
event_log = tmp_path / "events.log"
collector_id = tmp_path / "collector.id"
collector_state = tmp_path / "collector.state"
workspace_root = tmp_path / "workspace-root"
harness.mkdir()
fake_bin.mkdir()
(backup_base / "signoz" / "data").mkdir(parents=True)
@@ -43,6 +46,12 @@ def _run_backup(
"true\n" if initial_running else "false\n",
encoding="utf-8",
)
if workspace_root_symlink:
workspace_target = tmp_path / "workspace-target"
workspace_target.mkdir()
workspace_root.symlink_to(workspace_target, target_is_directory=True)
else:
workspace_root.mkdir()
(harness / "common.sh").write_text(
"""\
@@ -139,6 +148,15 @@ case " $* " in
printf '[{"short_id":"native123"}]\\n'
;;
esac
""",
)
_write_executable(
fake_bin / "flock",
"""\
#!/bin/bash
set -eu
printf 'flock %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
exit "${FAKE_OPERATION_LOCK_STATUS:-0}"
""",
)
@@ -161,6 +179,9 @@ esac
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
"TIMEOUT_KILL_AFTER_SECONDS": "1",
"BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0",
"SIGNOZ_BACKUP_OPERATION_LOCK_FILE": str(tmp_path / "operation.lock"),
"FAKE_OPERATION_LOCK_STATUS": str(operation_lock_status),
"SIGNOZ_BACKUP_TMP_ROOT": str(workspace_root),
}
process = subprocess.Popen(
["bash", str(harness / SCRIPT.name)],
@@ -177,7 +198,13 @@ esac
stderr,
)
events = event_log.read_text(encoding="utf-8").splitlines()
return result, events, Path(f"/tmp/signoz-backup-{process.pid}")
workspace_events = _events(events, "info 建立 SigNoz private workspace: ")
dump_dir = (
Path(workspace_events[-1].split(": ", 1)[1])
if workspace_events
else tmp_path / "workspace-not-created"
)
return result, events, dump_dir
def _events(events: list[str], prefix: str) -> list[str]:
@@ -201,6 +228,12 @@ def test_source_sunsets_raw_volumes_and_collector_mutation() -> None:
assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
assert 'notify_clawbot "success"' not in source
assert "no-downtime continuity verifier" in source
assert "awoooi-signoz-backup-operation.lock" in source
assert "flock -n 8" in source
assert "cleanup_old_backups" not in source
assert 'mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX"' in source
assert "/tmp/signoz-backup-$$" not in source
assert "WORKSPACE_CREATED=1" in source
def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
@@ -233,6 +266,32 @@ def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
assert not dump_dir.exists()
def test_operation_lock_contention_fails_before_native_or_restic(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, operation_lock_status=1)
assert result.returncode != 0
assert _events(events, "flock -n 8")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert len(_events(events, "notify failed signoz ")) == 1
assert not dump_dir.exists()
def test_symlink_workspace_root_fails_before_native_or_restic(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, workspace_root_symlink=True)
assert result.returncode != 0
assert _events(events, "flock -n 8")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert any("workspace root" in event for event in events)
assert not dump_dir.exists()
def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None:
result, events, dump_dir = _run_backup(tmp_path, native_check_status=19)
@@ -307,6 +366,20 @@ def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None:
assert not _events(events, "retention ")
def test_inline_destructive_retention_request_fails_before_backup(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, skip_retention=False)
assert result.returncode != 0
assert not _events(events, "flock ")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert not _events(events, "retention ")
assert any("critical break-glass workflow" in event for event in events)
assert not dump_dir.exists()
def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None:
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")

303
scripts/backup/tests/test_signoz_backup_canary_wrapper.py
View File

@@ -7,6 +7,8 @@ import subprocess
import time
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[3]
WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh"
@@ -115,6 +117,9 @@ while [[ "${1:-}" == --kill-after=* ]]; do
done
[ "$#" -gt 1 ]
shift
if [ "${FAKE_DOCKER_TIMEOUT:-0}" = "1" ] && [ "${1:-}" = "docker" ]; then
exit 124
fi
exec "$@"
""",
)
@@ -122,6 +127,9 @@ exec "$@"
fake_bin / "flock",
"""\
#!/bin/bash
if [ -n "${TEST_FLOCK_MARKER:-}" ]; then
printf 'held\n' > "${TEST_FLOCK_MARKER}"
fi
exit 0
""",
)
@@ -171,14 +179,103 @@ print(f"{digest} {sys.argv[1]}")
)
def _restore_hook_source() -> str:
return """\
#!/bin/bash
set -eu
[ "$#" -eq 1 ]
case "$1" in
--check) mode="check" ;;
--apply) mode="apply" ;;
*) exit 64 ;;
esac
receipt_dir="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:?}/${RUN_ID:?}"
mkdir -p "$receipt_dir"
receipt_log="$receipt_dir/receipts.jsonl"
identity_run="$RUN_ID"
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "forged_identity" ]; then
identity_run="stale-$RUN_ID"
fi
invocation_id="${mode}-fake-restore-$$"
emit() {
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","stage":"%s","terminal":"%s","detail":"fake"}\n' \
"$TRACE_ID" "$identity_run" "$WORK_ITEM_ID" "$invocation_id" "$1" "$2" \
>> "$receipt_log"
}
if [ "$mode" = "check" ]; then
emit check check_pass_no_runtime_write
emit cleanup pass
emit terminal check_pass_no_runtime_write
exit 0
fi
emit check pass
emit independent_post_verifier pass
emit cleanup pass
emit terminal verified
if [ "${FAKE_RECEIPT_FAULT:-valid}" != "missing_status" ]; then
cat > "$receipt_dir/status.json" <<EOF
{
"schema_version":"clickhouse_native_restore_drill_status_v1",
"trace_id":"$TRACE_ID",
"run_id":"$identity_run",
"work_item_id":"$WORK_ITEM_ID",
"mode":"apply",
"terminal":"verified",
"exit_code":0,
"network_mode":"docker_internal_only",
"staging_network_mode":"none",
"production_network_attached":false,
"production_restore_performed":false,
"allow_non_empty_tables":false,
"isolated_keeper":true,
"artifact_staging_verified":true,
"clickhouse_artifact_readback_verified":true,
"ephemeral_runtime_created":true,
"check_pass":true,
"apply_pass":true,
"restore_terminal":"RESTORED",
"manifest_parity":1,
"source_database_count":6,
"restored_database_count":6,
"source_table_count":100,
"restored_table_count":100,
"check_table_count":44,
"check_table_pass_count":44,
"critical_nonzero_count":4,
"cleanup_verified":true
}
EOF
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "symlink_status" ]; then
mv "$receipt_dir/status.json" "$receipt_dir/status-target.json"
ln -s "status-target.json" "$receipt_dir/status.json"
fi
fi
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "malformed_restore_receipt" ]; then
printf '{malformed-json\n' >> "$receipt_log"
fi
"""
def _backup_source() -> str:
return """\
#!/bin/bash
set -eu
[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80
[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ]
[ "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}" = "${TEST_RESTORE_HOOK:?}" ] || exit 82
[ "${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-}" = "${TEST_INVENTORY_HELPER:?}" ] || exit 83
[ "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-}" = "${TEST_BACKUP_DISK_CONFIG:?}" ] || exit 84
[ "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-}" = "${TEST_CLUSTER_CONFIG:?}" ] || exit 85
[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-}" = "${TEST_NATIVE_RECEIPT_ROOT:?}" ] || exit 86
[ "${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-}" = "${TEST_RESTORE_RECEIPT_ROOT:?}" ] || exit 87
printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
> "${TEST_BACKUP_IDENTITIES:?}"
printf '%s\t%s\t%s\t%s\n' \
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" \
"$CLICKHOUSE_RESTORE_INVENTORY_HELPER" \
"$CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" \
"$CLICKHOUSE_RESTORE_CLUSTER_CONFIG" \
> "${TEST_RESTORE_CONTRACT:?}"
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}"
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
@@ -204,6 +301,23 @@ if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
else
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
native_dir="$CLICKHOUSE_NATIVE_RECEIPT_ROOT/$RUN_ID"
mkdir -p "$native_dir"
native_receipts="$native_dir/receipts.jsonl"
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"check-fake-native","stage":"terminal","terminal":"check_pass_no_write","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
if [ "${FAKE_RECEIPT_FAULT:-valid}" != "no_hook" ]; then
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --check
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_check_exit_0_stdout_fake_stderr_fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --apply
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_exit_0_stdout_fake_stderr_fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
fi
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"independent_verifier","terminal":"pass","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"terminal","terminal":"pass","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
fi
if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then
printf '4317\n' > "${TEST_LISTENERS:?}"
@@ -229,13 +343,30 @@ def _run_canary(
duplicate_cron: bool = False,
drop_listener: bool = False,
expect_collector_stop: bool = True,
native_restore_contract: str = "valid",
native_receipt_state: str = "valid",
docker_timeout: bool = False,
) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]:
if native_receipt_state not in {
"valid",
"no_hook",
"forged_identity",
"missing_status",
"symlink_status",
"malformed_restore_receipt",
}:
raise ValueError(f"unsupported native receipt state: {native_receipt_state}")
fake_bin = tmp_path / "bin"
cooldown_dir = tmp_path / "cooldown"
receipt_root = tmp_path / "receipts"
native_receipt_root = tmp_path / "native-receipts"
restore_receipt_root = tmp_path / "restore-receipts"
fake_bin.mkdir()
cooldown_dir.mkdir()
receipt_root.mkdir()
native_receipt_root.mkdir()
restore_receipt_root.mkdir()
monitor_script = tmp_path / "docker-health-monitor.sh"
monitor_log = tmp_path / "monitor.log"
@@ -244,6 +375,12 @@ def _run_canary(
listeners = tmp_path / "listeners"
identities = tmp_path / "backup-identities.tsv"
lease_observed = tmp_path / "lease-observed.txt"
restore_contract_observed = tmp_path / "restore-contract.tsv"
flock_marker = tmp_path / "canary-lock-held.txt"
restore_hook = tmp_path / "clickhouse-native-restore-drill.sh"
inventory_helper = tmp_path / "clickhouse-restore-inventory.py"
backup_disk_config = tmp_path / "backup_disk.xml"
cluster_config = tmp_path / "isolated-cluster.xml"
cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown"
run_id = f"canary-{tmp_path.name}"
@@ -252,6 +389,19 @@ def _run_canary(
_monitor_source(cooldown_dir, omit_contract=omit_monitor_contract),
)
_write_executable(backup_script, _backup_source())
if native_restore_contract == "symlink":
restore_hook_target = tmp_path / "restore-hook-target.sh"
_write_executable(restore_hook_target, _restore_hook_source())
restore_hook.symlink_to(restore_hook_target)
elif native_restore_contract == "valid":
_write_executable(restore_hook, _restore_hook_source())
elif native_restore_contract != "missing":
raise ValueError(
f"unsupported native restore contract: {native_restore_contract}"
)
_write_executable(inventory_helper, "#!/bin/sh\nexit 0\n")
backup_disk_config.write_text("<clickhouse/>\n", encoding="utf-8")
cluster_config.write_text("<clickhouse/>\n", encoding="utf-8")
_install_fake_commands(fake_bin)
monitor_log.write_text("[test] monitor seed\n", encoding="utf-8")
collector_state.write_text("true\n", encoding="utf-8")
@@ -274,10 +424,17 @@ def _run_canary(
"SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"),
"SIGNOZ_CANARY_TIMEOUT_SECONDS": "5",
"SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2",
"SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS": "2",
"SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2",
"SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds),
"SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05",
"SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"),
"CLICKHOUSE_NATIVE_POST_VERIFY_HOOK": str(restore_hook),
"CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(inventory_helper),
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_disk_config),
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config),
"CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(native_receipt_root),
"CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(restore_receipt_root),
"TEST_MONITOR_SCRIPT": str(monitor_script),
"TEST_MONITOR_LOG": str(monitor_log),
"TEST_COLLECTOR_STATE": str(collector_state),
@@ -285,11 +442,21 @@ def _run_canary(
"TEST_BACKUP_IDENTITIES": str(identities),
"TEST_COOLDOWN_FILE": str(cooldown_file),
"TEST_LEASE_OBSERVED": str(lease_observed),
"TEST_RESTORE_CONTRACT": str(restore_contract_observed),
"TEST_RESTORE_HOOK": str(restore_hook),
"TEST_INVENTORY_HELPER": str(inventory_helper),
"TEST_BACKUP_DISK_CONFIG": str(backup_disk_config),
"TEST_CLUSTER_CONFIG": str(cluster_config),
"TEST_NATIVE_RECEIPT_ROOT": str(native_receipt_root),
"TEST_RESTORE_RECEIPT_ROOT": str(restore_receipt_root),
"TEST_FLOCK_MARKER": str(flock_marker),
"FAKE_MONITOR_MODE": monitor_mode,
"FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds),
"FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0",
"FAKE_DROP_LISTENER": "1" if drop_listener else "0",
"FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0",
"FAKE_RECEIPT_FAULT": native_receipt_state,
"FAKE_DOCKER_TIMEOUT": "1" if docker_timeout else "0",
}
started_at = int(time.time())
result = subprocess.run(
@@ -308,6 +475,13 @@ def _run_canary(
"lease_observed": lease_observed,
"collector_state": collector_state,
"listeners": listeners,
"restore_contract": restore_contract_observed,
"restore_hook": restore_hook,
"flock_marker": flock_marker,
"native_receipts": native_receipt_root / run_id / "receipts.jsonl",
"restore_receipts": restore_receipt_root / run_id / "receipts.jsonl",
"restore_status": restore_receipt_root / run_id / "status.json",
"backup_output": receipt_root / run_id / "backup-output.log",
}
return result, paths, str(started_at)
@@ -337,7 +511,9 @@ def test_existing_lease_is_atomically_replaced_and_exactly_restored(
assert identities[2] == "P0-OBS-002\n"
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "runtime_operation") == "pass"
assert _terminal(receipts, "closure_writeback") == "partial_degraded"
assert _terminal(receipts, "terminal") == "partial_degraded"
metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8"))
assert metadata["trace_id"] == "trace-P0-OBS-002"
assert metadata["prior_present"] == 1
@@ -368,7 +544,7 @@ def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None:
assert not paths["cooldown"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
@@ -380,13 +556,30 @@ def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
assert not paths["lease_observed"].exists()
assert paths["collector_state"].read_text(encoding="utf-8") == "true\n"
assert paths["restore_contract"].read_text(encoding="utf-8").split("\t") == [
str(paths["restore_hook"]),
str(tmp_path / "clickhouse-restore-inventory.py"),
str(tmp_path / "backup_disk.xml"),
f"{tmp_path / 'isolated-cluster.xml'}\n",
]
receipts = _receipts(paths["receipts"])
assert paths["native_receipts"].is_file()
assert paths["restore_receipts"].is_file()
restore_status = json.loads(paths["restore_status"].read_text(encoding="utf-8"))
assert restore_status["terminal"] == "verified"
assert restore_status["check_table_count"] == 44
assert restore_status["check_table_pass_count"] == 44
assert any(
item["stage"] == "native_restore_receipt_verifier"
and item["terminal"] == "pass"
for item in receipts
)
assert any(
item["stage"] == "lease" and item["terminal"] == "not_required"
for item in receipts
)
assert _terminal(receipts, "rollback") == "no_write"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
@@ -406,7 +599,7 @@ def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
item for item in receipts if item["stage"] == "source_of_truth_diff"
][-1]
assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"]
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None:
@@ -496,6 +689,34 @@ def test_wrapper_deployment_and_static_contracts_are_owned() -> None:
assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper
assert "monitor_auto_repair_contaminated_canary" in wrapper
assert "collector_post_backup_runtime_failed" in wrapper
assert "/backup/scripts/clickhouse-native-restore-drill.sh" in wrapper
assert "/backup/scripts/clickhouse-restore-inventory.py" in wrapper
assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in wrapper
assert (
"/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
in wrapper
)
assert (
'CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}"'
in wrapper
)
assert (
'CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}"' in wrapper
)
assert (
'CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}"'
in wrapper
)
main_source = wrapper[wrapper.index("main() {") :]
assert main_source.index("acquire_canary_lock") < main_source.index(
"require_native_restore_contract"
)
assert "same_run_native_restore_receipt_validation_failed" in wrapper
assert '"check_table_count"' in wrapper
assert "mandatory_isolated_restore_verifier" in wrapper
assert "SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS" in wrapper
assert 'emit_receipt "runtime_operation" "pass"' in wrapper
assert 'emit_receipt "closure_writeback" "partial_degraded"' in wrapper
assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8")
assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text(
encoding="utf-8"
@@ -518,6 +739,80 @@ def test_check_mode_proves_contract_without_lease_or_backup_write(
assert _terminal(check_receipts, "terminal") == "check_pass_no_write"
@pytest.mark.parametrize("contract_state", ["missing", "symlink"])
def test_native_check_fails_closed_for_unsafe_restore_contract(
tmp_path: Path,
contract_state: str,
) -> None:
result, paths, _ = _run_canary(
tmp_path,
mode="check",
expect_collector_stop=False,
native_restore_contract=contract_state,
)
assert result.returncode != 0
assert "native_restore_contract_CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" in result.stderr
assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n"
assert not paths["identities"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "terminal") == "failed"
@pytest.mark.parametrize(
"receipt_state",
[
"no_hook",
"forged_identity",
"missing_status",
"symlink_status",
"malformed_restore_receipt",
],
)
def test_native_apply_rejects_missing_or_forged_machine_receipts(
tmp_path: Path,
receipt_state: str,
) -> None:
result, paths, _ = _run_canary(
tmp_path,
expect_collector_stop=False,
native_receipt_state=receipt_state,
)
assert result.returncode != 0
assert "========== SigNoz ClickHouse native 備份完成 (1s) ==========" in paths[
"backup_output"
].read_text(encoding="utf-8")
assert (
"same_run_native_restore_receipt_validation_failed" in result.stderr
or "same_run_native_restore_receipt_missing_unreadable_or_symlink"
in result.stderr
)
assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n"
receipts = _receipts(paths["receipts"])
assert not any(
item["stage"] == "native_restore_receipt_verifier"
and item["terminal"] == "pass"
for item in receipts
)
assert _terminal(receipts, "terminal") == "failed"
def test_docker_metadata_timeout_fails_before_backup(tmp_path: Path) -> None:
result, paths, _ = _run_canary(
tmp_path,
mode="check",
expect_collector_stop=False,
docker_timeout=True,
)
assert result.returncode != 0
assert "collector_runtime_metadata_unreadable" in result.stderr
assert not paths["identities"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "terminal") == "failed"
def test_wrapper_is_valid_bash() -> None:
result = subprocess.run(
["bash", "-n", str(WRAPPER)],