From 032be5f40012e12ad6681866278b0eacd44ce20e Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 09:43:40 +0800 Subject: [PATCH] fix(observability): harden native backup controlled apply --- ...rganization-canonical-route-migration.yaml | 2 + .../p0-obs-002-post-closure-regression.yaml | 42 ++ scripts/backup/backup-signoz.sh | 98 +++- scripts/backup/run-signoz-backup-canary.sh | 364 ++++++++++++++- .../test_backup_signoz_collector_restore.py | 77 +++- .../test_signoz_backup_canary_wrapper.py | 303 ++++++++++++- .../deploy-signoz-clickhouse-backup-disk.sh | 426 +++++++++++++----- .../deploy-signoz-native-backup-toolchain.sh | 149 +++++- ...st_signoz_clickhouse_backup_disk_deploy.py | 188 +++++++- ...t_signoz_native_backup_toolchain_deploy.py | 140 +++++- .../signoz-canonical-route-preflight.py | 145 +++++- .../test_signoz_canonical_route_migration.py | 105 +++++ 12 files changed, 1849 insertions(+), 190 deletions(-) diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index 3a94d61c3..afda9b611 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -181,6 +181,8 @@ runtime_observations: clickhouse_native_backup_disk_status: deployed_verified clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4 clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4 + clickhouse_native_current_source_status: pending_toolchain_deploy_and_runtime_canary + clickhouse_native_current_source_deployed: false clickhouse_native_restore_terminal: verified clickhouse_native_restic_snapshot_id: 574e2a1a clickhouse_native_offsite_status: pending diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 0bb28a860..73f015236 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -347,6 +347,8 @@ controlled_applies: risk: high completion_scope: clickhouse_native_only source: + evidence_scope: historical_production_receipt + deployment_status: deployed_verified backup_orchestrator_sha256: 2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520 native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7 @@ -361,6 +363,36 @@ controlled_applies: ruff: pass ruff_format: pass yaml_xml_parse: pass + current_source: + observed_at: "2026-07-15T09:40:50+08:00" + evidence_scope: committed_current_source + status: pending_toolchain_deploy_and_runtime_canary + deployment_status: pending + current_source_deployed: false + runtime_verifier_terminal: pending + historical_production_receipt_preserved: true + expected_file_count: 7 + hashes: + backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2 + native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a + isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7 + inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863 + canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886 + backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9 + isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b + drift_from_historical_production_receipt: + - backup_orchestrator_sha256 + - canary_wrapper_sha256 + - inventory_verifier_sha256 + source_validation: + backup_restore_deployer_tests_passed: 92 + canonical_preflight_tests_passed: 23 + bash_syntax: pass + ruff: pass + ruff_format: pass + yaml_parse: pass + diff_check: pass + independent_reaudit: no_commit_blocker toolchain_controlled_apply: check_run_id: P0-OBS-002-native-toolchain-20260715T001124Z-check5 check_terminal: pass @@ -609,6 +641,15 @@ pending_verifiers: - bind_inventory_hash_operation_id_artifact_hash_and_run_identity - resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered - regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits + clickhouse_native_current_source_toolchain: + status: pending_toolchain_deploy_and_runtime_canary + current_source_deployed: false + runtime_verifier_terminal: pending + historical_production_receipt_preserved: true + required_preconditions: + - controlled_toolchain_apply_with_current_source_hash_readback + - same_run_native_backup_and_isolated_restore_canary + - independent_runtime_and_cleanup_verifier scope_boundaries: grpc_bridge_rollback_indicated: false @@ -629,6 +670,7 @@ terminal: - signoz_backup_offsite_dr_pending - clickhouse_native_failed_artifact_retention_pending - clickhouse_native_resume_submitted_inventory_pending + - clickhouse_native_current_source_toolchain_pending - service_registry_runtime_mirror_reconciliation_pending - github_freeze_legacy_asset_retirement_pending - monitoring_generator_duplicate_awoooi_api_identity_pending diff --git a/scripts/backup/backup-signoz.sh b/scripts/backup/backup-signoz.sh index 406a9ceab..6b4127ef9 100755 --- a/scripts/backup/backup-signoz.sh +++ b/scripts/backup/backup-signoz.sh @@ -15,14 +15,19 @@ source "$(dirname "$0")/common.sh" SERVICE="signoz" LOCAL_REPO="${BACKUP_BASE}/signoz" -DUMP_DIR="/tmp/signoz-backup-$$" +TMP_ROOT="${SIGNOZ_BACKUP_TMP_ROOT:-/tmp}" +DUMP_DIR="" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}" COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}" COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}" COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}" TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}" -BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}" +# Destructive Restic forget/prune is a separate critical break-glass workflow. +# The regular and canary backup paths are backup-only and fail closed if a +# caller attempts to re-enable inline retention cleanup. +BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-1}" +OPERATION_LOCK_FILE="${SIGNOZ_BACKUP_OPERATION_LOCK_FILE:-/tmp/awoooi-signoz-backup-operation.lock}" RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}" RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}" RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}" @@ -40,7 +45,8 @@ COLLECTOR_STARTED_AT_BEFORE="" COLLECTOR_RESTARTS_BEFORE="" COLLECTOR_HEALTH_BEFORE="" COLLECTOR_OBSERVER_PID="" -COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv" +COLLECTOR_OBSERVATIONS="" +WORKSPACE_CREATED=0 CLEANUP_COMPLETE=0 FAILURE_NOTIFIED=0 @@ -126,13 +132,24 @@ cleanup() { local exit_code=$? if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then - return "${exit_code}" + exit "${exit_code}" fi CLEANUP_COMPLETE=1 - trap '' HUP INT TERM + trap - EXIT HUP INT TERM + set +e stop_collector_observer - rm -rf -- "${DUMP_DIR}" - return "${exit_code}" + if [ "${WORKSPACE_CREATED}" -eq 1 ]; then + if [ -n "${DUMP_DIR}" ] && [ -d "${DUMP_DIR}" ] \ + && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \ + && [[ "${DUMP_DIR}" == "${TMP_ROOT%/}"/signoz-backup.* ]]; then + rm -rf -- "${DUMP_DIR}" || exit_code=90 + [ ! -e "${DUMP_DIR}" ] || exit_code=90 + else + log_error "拒絕清理不符合本 run identity 的 SigNoz workspace" + exit_code=90 + fi + fi + exit "${exit_code}" } on_signal() { @@ -140,6 +157,28 @@ on_signal() { exit "$((128 + signal_number))" } +create_private_workspace() { + [[ "${TMP_ROOT}" == /* ]] \ + && [[ ! "${TMP_ROOT}" =~ (^|/)\.\.?(/|$) ]] \ + && [[ "${TMP_ROOT}" != *$'\n'* ]] \ + && [ -d "${TMP_ROOT}" ] && [ -w "${TMP_ROOT}" ] \ + && [ ! -L "${TMP_ROOT}" ] || { + log_error "SigNoz workspace root 必須是安全、可寫、非 symlink 的絕對目錄" + return 1 + } + + DUMP_DIR="$(mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX")" || return 1 + WORKSPACE_CREATED=1 + [ -d "${DUMP_DIR}" ] && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \ + && [ -r "${DUMP_DIR}" ] && [ -w "${DUMP_DIR}" ] && [ -x "${DUMP_DIR}" ] || { + log_error "mktemp 未建立本 run 擁有的 private SigNoz workspace" + return 1 + } + chmod 700 "${DUMP_DIR}" || return 1 + COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv" + log_info "建立 SigNoz private workspace: ${DUMP_DIR}" +} + run_native_clickhouse_backup() { local mode="$1" @@ -184,6 +223,32 @@ write_restic_receipt() { mv "${receipt_tmp}" "${receipt_file}" } +acquire_operation_lock() { + case "${OPERATION_LOCK_FILE}" in + /*) ;; + *) + log_error "SigNoz operation lock path 必須是絕對路徑" + return 1 + ;; + esac + [ ! -L "${OPERATION_LOCK_FILE}" ] || { + log_error "SigNoz operation lock 不得是 symlink" + return 1 + } + if [ -e "${OPERATION_LOCK_FILE}" ]; then + [ -f "${OPERATION_LOCK_FILE}" ] || { + log_error "SigNoz operation lock 必須是 regular file" + return 1 + } + fi + exec 8>>"${OPERATION_LOCK_FILE}" + flock -n 8 || { + log_error "另一個 SigNoz backup/deploy operation 正在執行" + return 1 + } + log_info "取得 SigNoz backup/deploy 共用 operation lock" +} + main() { local start_time end_time duration snapshot_id snapshot_json tags local initial_observation initial_running final_id final_running @@ -210,7 +275,18 @@ main() { exit 1 } done - mkdir -p "${DUMP_DIR}" + [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ] || { + notify_failure_once "inline Restic forget/prune 已停用;僅允許獨立 critical break-glass workflow" + exit 1 + } + acquire_operation_lock || { + notify_failure_once "SigNoz backup/deploy operation lock 取得失敗" + exit 1 + } + create_private_workspace || { + notify_failure_once "建立 private SigNoz backup workspace 失敗" + exit 1 + } # Check-mode must pass before the backup window. It writes only durable # receipts and does not submit BACKUP or create a server artifact. @@ -305,11 +381,7 @@ main() { } log_success "Restic 備份完成: ${snapshot_id}" - if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then - log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)" - else - cleanup_old_backups "${LOCAL_REPO}" - fi + log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)" # Final readback is separate from the continuity observer receipt. initial_observation="$(collector_identity_state)" || { diff --git a/scripts/backup/run-signoz-backup-canary.sh b/scripts/backup/run-signoz-backup-canary.sh index c6bdddfe4..15ac97425 100755 --- a/scripts/backup/run-signoz-backup-canary.sh +++ b/scripts/backup/run-signoz-backup-canary.sh @@ -12,6 +12,12 @@ set -euo pipefail COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}" BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}" +CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-/backup/scripts/clickhouse-native-restore-drill.sh}" +CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-/backup/scripts/clickhouse-restore-inventory.py}" +CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-/backup/config/signoz/clickhouse/config.d/backup_disk.xml}" +CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml}" +CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}" +CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-/backup/logs/clickhouse-native-restore-drill}" MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}" MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}" MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}" @@ -21,6 +27,7 @@ LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}" CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}" TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}" +DOCKER_TIMEOUT_SECONDS="${SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS:-15}" LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}" MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}" STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}" @@ -94,6 +101,307 @@ require_command() { command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}" } +run_collector_docker() { + timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${DOCKER_TIMEOUT_SECONDS}" docker "$@" +} + +require_native_restore_contract() { + local label path + + for label in \ + CLICKHOUSE_NATIVE_POST_VERIFY_HOOK \ + CLICKHOUSE_RESTORE_INVENTORY_HELPER \ + CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG \ + CLICKHOUSE_RESTORE_CLUSTER_CONFIG; do + path="${!label}" + case "${path}" in + /*) ;; + *) fail_closed "native_restore_contract_${label}_path_not_absolute" ;; + esac + [ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \ + || fail_closed "native_restore_contract_${label}_missing_unreadable_or_symlink" + done + + [ -x "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" ] \ + || fail_closed "native_restore_contract_post_verify_hook_not_executable" + [ -x "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" ] \ + || fail_closed "native_restore_contract_inventory_helper_not_executable" +} + +acquire_canary_lock() { + [ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe" + if [ -e "${LOCK_FILE}" ]; then + [ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file" + fi + exec 9>>"${LOCK_FILE}" + flock -n 9 || fail_closed "another_signoz_canary_holds_lock" +} + +require_native_receipt_contract() { + local label path run_dir + + for label in CLICKHOUSE_NATIVE_RECEIPT_ROOT CLICKHOUSE_RESTORE_RECEIPT_ROOT; do + path="${!label}" + case "${path}" in + /*) ;; + *) fail_closed "native_receipt_contract_${label}_path_not_absolute" ;; + esac + [[ "${path}" != *$'\n'* ]] \ + || fail_closed "native_receipt_contract_${label}_path_contains_newline" + [ ! -L "${path}" ] \ + || fail_closed "native_receipt_contract_${label}_root_symlink_unsafe" + if [ -e "${path}" ]; then + [ -d "${path}" ] && [ -r "${path}" ] && [ -w "${path}" ] \ + || fail_closed "native_receipt_contract_${label}_root_not_readable_writable_directory" + fi + run_dir="${path}/${RUN_ID}" + [ ! -e "${run_dir}" ] && [ ! -L "${run_dir}" ] \ + || fail_closed "native_receipt_contract_${label}_stale_same_run_path_present" + done + + [ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" != "${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" ] \ + || fail_closed "native_and_restore_receipt_roots_must_be_distinct" +} + +verify_same_run_native_restore_receipts() { + local native_receipts restore_receipts restore_status + + native_receipts="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl" + restore_receipts="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl" + restore_status="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/status.json" + + for path in "${native_receipts}" "${restore_receipts}" "${restore_status}"; do + [ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \ + || fail_closed "same_run_native_restore_receipt_missing_unreadable_or_symlink" + done + + python3 - "${native_receipts}" "${restore_receipts}" "${restore_status}" \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" <<'PY' \ + || fail_closed "same_run_native_restore_receipt_validation_failed" +import json +import os +import sys + + +def reject(message: str) -> None: + print(f"receipt_verifier:{message}", file=sys.stderr) + raise SystemExit(1) + + +native_path, restore_path, status_path, trace_id, run_id, work_item_id = sys.argv[1:] +identity = { + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": work_item_id, +} + + +def read_jsonl(path: str, label: str) -> list[dict[str, object]]: + if os.path.islink(path) or not os.path.isfile(path): + reject(f"{label}_unsafe") + records: list[dict[str, object]] = [] + try: + with open(path, encoding="utf-8") as source: + for line_number, raw_line in enumerate(source, start=1): + if not raw_line.strip(): + continue + value = json.loads(raw_line) + if not isinstance(value, dict): + reject(f"{label}_line_{line_number}_not_object") + for key, expected in identity.items(): + if value.get(key) != expected: + reject(f"{label}_line_{line_number}_{key}_mismatch") + invocation_id = value.get("invocation_id") + if not isinstance(invocation_id, str) or not invocation_id: + reject(f"{label}_line_{line_number}_invocation_id_invalid") + records.append(value) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + reject(f"{label}_malformed_{type(exc).__name__}") + if not records: + reject(f"{label}_empty") + return records + + +def mode_records( + records: list[dict[str, object]], mode: str, label: str +) -> list[dict[str, object]]: + selected = [ + record + for record in records + if str(record.get("invocation_id", "")).startswith(f"{mode}-") + ] + if not selected: + reject(f"{label}_{mode}_invocation_missing") + return selected + + +def require_exact( + records: list[dict[str, object]], + *, + label: str, + stage: str, + terminal: str, + detail_prefix: str | None = None, +) -> None: + matches = [ + record + for record in records + if record.get("stage") == stage + and record.get("terminal") == terminal + and ( + detail_prefix is None + or str(record.get("detail", "")).startswith(detail_prefix) + ) + ] + if len(matches) != 1: + reject(f"{label}_{stage}_{terminal}_count_{len(matches)}") + + +native_records = read_jsonl(native_path, "native_receipts") +native_check = mode_records(native_records, "check", "native") +native_apply = mode_records(native_records, "apply", "native") +require_exact( + native_check, + label="native_check", + stage="terminal", + terminal="check_pass_no_write", +) +require_exact( + native_apply, + label="native_apply", + stage="terminal", + terminal="pass", +) +require_exact( + native_apply, + label="native_apply", + stage="independent_verifier", + terminal="pass", +) +require_exact( + native_apply, + label="native_apply_hook_check", + stage="command", + terminal="pass", + detail_prefix="step_post_verify_hook_check_exit_0_", +) +require_exact( + native_apply, + label="native_apply_hook", + stage="command", + terminal="pass", + detail_prefix="step_post_verify_hook_exit_0_", +) + +restore_records = read_jsonl(restore_path, "restore_receipts") +restore_check = mode_records(restore_records, "check", "restore") +restore_apply = mode_records(restore_records, "apply", "restore") +require_exact( + restore_check, + label="restore_check", + stage="terminal", + terminal="check_pass_no_runtime_write", +) +require_exact( + restore_apply, + label="restore_apply", + stage="terminal", + terminal="verified", +) +require_exact( + restore_apply, + label="restore_apply", + stage="independent_post_verifier", + terminal="pass", +) +require_exact( + restore_apply, + label="restore_apply", + stage="cleanup", + terminal="pass", +) + +if os.path.islink(status_path) or not os.path.isfile(status_path): + reject("restore_status_unsafe") +try: + with open(status_path, encoding="utf-8") as source: + status = json.load(source) +except (OSError, UnicodeError, json.JSONDecodeError) as exc: + reject(f"restore_status_malformed_{type(exc).__name__}") +if not isinstance(status, dict): + reject("restore_status_not_object") +for key, expected in identity.items(): + if status.get(key) != expected: + reject(f"restore_status_{key}_mismatch") + +expected_strings = { + "schema_version": "clickhouse_native_restore_drill_status_v1", + "mode": "apply", + "terminal": "verified", + "restore_terminal": "RESTORED", + "network_mode": "docker_internal_only", + "staging_network_mode": "none", +} +for key, expected in expected_strings.items(): + if not isinstance(status.get(key), str) or status.get(key) != expected: + reject(f"restore_status_{key}_invalid") +for key, expected in (("exit_code", 0), ("manifest_parity", 1)): + if type(status.get(key)) is not int or status.get(key) != expected: + reject(f"restore_status_{key}_invalid") +for key in ( + "check_pass", + "apply_pass", + "cleanup_verified", + "artifact_staging_verified", + "clickhouse_artifact_readback_verified", + "ephemeral_runtime_created", + "isolated_keeper", +): + if status.get(key) is not True: + reject(f"restore_status_{key}_not_true") +for key in ( + "production_network_attached", + "production_restore_performed", + "allow_non_empty_tables", +): + if status.get(key) is not False: + reject(f"restore_status_{key}_not_false") + + +def positive_integer(key: str) -> int: + value = status.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + reject(f"restore_status_{key}_not_positive_integer") + return value + + +source_database_count = positive_integer("source_database_count") +restored_database_count = positive_integer("restored_database_count") +if source_database_count != restored_database_count: + reject("restore_status_database_count_mismatch") +source_table_count = positive_integer("source_table_count") +restored_table_count = positive_integer("restored_table_count") +if source_table_count != restored_table_count: + reject("restore_status_table_count_mismatch") +positive_integer("critical_nonzero_count") +check_count = status.get("check_table_count") +check_pass_count = status.get("check_table_pass_count") +if ( + isinstance(check_count, bool) + or not isinstance(check_count, int) + or isinstance(check_pass_count, bool) + or not isinstance(check_pass_count, int) + or check_count <= 0 + or check_count != check_pass_count +): + reject("restore_status_check_table_counts_invalid") +PY + + emit_receipt "native_restore_receipt_verifier" "pass" \ + "same_run_machine_readable_native_check_apply_hook_and_isolated_restore_status_verified" +} + require_monitor_fragment() { local fragment="$1" local count @@ -138,7 +446,7 @@ require_monitor_function_fragment() { collector_running_state() { local running - if ! running="$(docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then + if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then return 1 fi case "${running}" in @@ -149,7 +457,7 @@ collector_running_state() { } collector_runtime_metadata() { - docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \ + run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \ "${COLLECTOR_NAME}" 2>/dev/null } @@ -247,13 +555,15 @@ cleanup() { if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then overall_terminal="check_pass_no_write" elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then - overall_terminal="pass" + overall_terminal="partial_degraded" fi emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}" - if [ "${overall_terminal}" = "pass" ]; then - emit_receipt "closure_writeback" "pass" \ - "no_incident_notification_or_learning_delta_local_durable_receipt_ack" + if [ "${overall_terminal}" = "partial_degraded" ]; then + emit_receipt "runtime_operation" "pass" \ + "clickhouse_native_backup_and_isolated_restore_runtime_verifier_pass" + emit_receipt "closure_writeback" "partial_degraded" \ + "local_durable_receipt_ack_incident_not_applicable_telegram_not_sent_km_rag_writeback_pending_offsite_false_sqlite_application_consistent_export_pending" elif [ "${overall_terminal}" = "check_pass_no_write" ]; then emit_receipt "closure_writeback" "not_applicable" \ "check_mode_no_runtime_change_local_durable_receipt_ack" @@ -261,7 +571,8 @@ cleanup() { emit_receipt "closure_writeback" "deferred" \ "failed_terminal_requires_followup_no_completion_claim" fi - emit_receipt "terminal" "${overall_terminal}" "backup_canary_${overall_terminal}" + emit_receipt "terminal" "${overall_terminal}" \ + "backup_canary_${overall_terminal}_clickhouse_native_scope_only_offsite_false_sqlite_pending" printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \ "${lease_terminal}" "${exit_code}" @@ -521,7 +832,7 @@ main() { local container_health_before collector_metadata_before local monitor_log_identity="" monitor_log_size="" local source_diff_detail ai_decision_detail - local backup_status + local backup_status native_restore_contract_detail="not_applicable" setup_receipts @@ -532,11 +843,17 @@ main() { *) fail_closed "unsupported_mode_use_check_or_apply" ;; esac - for command_name in awk date docker env flock grep kill pgrep sha256sum sleep ss timeout tr; do + # Hold the canary lock before inspecting or hashing any mutable backup or + # restore toolchain path. This closes the check/apply substitution window. + require_command flock + acquire_canary_lock + + for command_name in awk date docker env grep kill pgrep python3 sha256sum sleep ss timeout tr; do require_command "${command_name}" done valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout" valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after" + valid_positive_integer "${DOCKER_TIMEOUT_SECONDS}" || fail_closed "invalid_docker_timeout" valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval" case "${EXPECT_COLLECTOR_STOP}" in 0|1) ;; @@ -545,7 +862,11 @@ main() { [ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \ || fail_closed "backup_script_missing_unexecutable_or_symlink" - if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then + if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then + require_native_restore_contract + require_native_receipt_contract + native_restore_contract_detail="hook_$(sha256sum "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" | awk '{print $1}')_inventory_$(sha256sum "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" | awk '{print $1}')_disk_config_$(sha256sum "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" | awk '{print $1}')_cluster_config_$(sha256sum "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" | awk '{print $1}')" + else for command_name in cp crontab id mktemp mv rm stat tail; do require_command "${command_name}" done @@ -555,12 +876,6 @@ main() { verify_monitor_contract fi - [ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe" - if [ -e "${LOCK_FILE}" ]; then - [ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file" - fi - exec 9>>"${LOCK_FILE}" - flock -n 9 || fail_closed "another_signoz_canary_holds_lock" if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then fail_closed "another_signoz_backup_is_running" fi @@ -591,18 +906,18 @@ main() { PRIOR_LEASE_PRESENT=0 PRIOR_LEASE_HASH="not_applicable" emit_receipt "sensor_source" "pass" \ - "online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}" - source_diff_detail="online_native_backup_monitor_and_cooldown_contract_not_applicable" - ai_decision_detail="run_online_native_backup_without_collector_or_monitor_mutation" + "online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_${native_restore_contract_detail}_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}" + source_diff_detail="online_native_backup_isolated_restore_contract_bound_monitor_and_cooldown_contract_not_applicable" + ai_decision_detail="run_online_native_backup_with_mandatory_isolated_restore_verifier_without_collector_or_monitor_mutation" fi emit_receipt "normalized_asset_identity" "pass" \ "asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker" emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}" emit_receipt "ai_decision" "candidate" "${ai_decision_detail}" emit_receipt "risk_policy" "pass" \ - "risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_independent_verifier" + "risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_mandatory_isolated_restore_independent_verifier" emit_receipt "check" "pass" \ - "mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent" + "mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_${native_restore_contract_detail}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent" if [ "${MODE}" = "check" ]; then CHECK_VERIFIED=1 @@ -622,6 +937,12 @@ main() { set +e timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \ env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ + CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" \ + CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" \ + CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" \ + CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" \ + CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" \ + CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" \ BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \ > "${BACKUP_OUTPUT}" 2>&1 & BACKUP_PID=$! @@ -645,6 +966,7 @@ main() { else grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \ || fail_closed "native_backup_success_terminal_missing" + verify_same_run_native_restore_receipts fi if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then diff --git a/scripts/backup/tests/test_backup_signoz_collector_restore.py b/scripts/backup/tests/test_backup_signoz_collector_restore.py index 71113b96c..52a7b64f5 100644 --- a/scripts/backup/tests/test_backup_signoz_collector_restore.py +++ b/scripts/backup/tests/test_backup_signoz_collector_restore.py @@ -25,7 +25,9 @@ def _run_backup( native_apply_status: int = 0, continuity_mode: str = "stable", restic_backup_status: int = 0, - skip_retention: bool = False, + skip_retention: bool = True, + operation_lock_status: int = 0, + workspace_root_symlink: bool = False, ) -> tuple[subprocess.CompletedProcess[str], list[str], Path]: harness = tmp_path / "backup" fake_bin = tmp_path / "bin" @@ -33,6 +35,7 @@ def _run_backup( event_log = tmp_path / "events.log" collector_id = tmp_path / "collector.id" collector_state = tmp_path / "collector.state" + workspace_root = tmp_path / "workspace-root" harness.mkdir() fake_bin.mkdir() (backup_base / "signoz" / "data").mkdir(parents=True) @@ -43,6 +46,12 @@ def _run_backup( "true\n" if initial_running else "false\n", encoding="utf-8", ) + if workspace_root_symlink: + workspace_target = tmp_path / "workspace-target" + workspace_target.mkdir() + workspace_root.symlink_to(workspace_target, target_is_directory=True) + else: + workspace_root.mkdir() (harness / "common.sh").write_text( """\ @@ -139,6 +148,15 @@ case " $* " in printf '[{"short_id":"native123"}]\\n' ;; esac +""", + ) + _write_executable( + fake_bin / "flock", + """\ +#!/bin/bash +set -eu +printf 'flock %s\n' "$*" >> "${TEST_EVENT_LOG:?}" +exit "${FAKE_OPERATION_LOCK_STATUS:-0}" """, ) @@ -161,6 +179,9 @@ esac "COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1", "TIMEOUT_KILL_AFTER_SECONDS": "1", "BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0", + "SIGNOZ_BACKUP_OPERATION_LOCK_FILE": str(tmp_path / "operation.lock"), + "FAKE_OPERATION_LOCK_STATUS": str(operation_lock_status), + "SIGNOZ_BACKUP_TMP_ROOT": str(workspace_root), } process = subprocess.Popen( ["bash", str(harness / SCRIPT.name)], @@ -177,7 +198,13 @@ esac stderr, ) events = event_log.read_text(encoding="utf-8").splitlines() - return result, events, Path(f"/tmp/signoz-backup-{process.pid}") + workspace_events = _events(events, "info 建立 SigNoz private workspace: ") + dump_dir = ( + Path(workspace_events[-1].split(": ", 1)[1]) + if workspace_events + else tmp_path / "workspace-not-created" + ) + return result, events, dump_dir def _events(events: list[str], prefix: str) -> list[str]: @@ -201,6 +228,12 @@ def test_source_sunsets_raw_volumes_and_collector_mutation() -> None: assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source assert 'notify_clawbot "success"' not in source assert "no-downtime continuity verifier" in source + assert "awoooi-signoz-backup-operation.lock" in source + assert "flock -n 8" in source + assert "cleanup_old_backups" not in source + assert 'mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX"' in source + assert "/tmp/signoz-backup-$$" not in source + assert "WORKSPACE_CREATED=1" in source def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime( @@ -233,6 +266,32 @@ def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime( assert not dump_dir.exists() +def test_operation_lock_contention_fails_before_native_or_restic( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup(tmp_path, operation_lock_status=1) + + assert result.returncode != 0 + assert _events(events, "flock -n 8") + assert not _events(events, "native ") + assert not _events(events, "restic ") + assert len(_events(events, "notify failed signoz ")) == 1 + assert not dump_dir.exists() + + +def test_symlink_workspace_root_fails_before_native_or_restic( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup(tmp_path, workspace_root_symlink=True) + + assert result.returncode != 0 + assert _events(events, "flock -n 8") + assert not _events(events, "native ") + assert not _events(events, "restic ") + assert any("workspace root" in event for event in events) + assert not dump_dir.exists() + + def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None: result, events, dump_dir = _run_backup(tmp_path, native_check_status=19) @@ -307,6 +366,20 @@ def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None: assert not _events(events, "retention ") +def test_inline_destructive_retention_request_fails_before_backup( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup(tmp_path, skip_retention=False) + + assert result.returncode != 0 + assert not _events(events, "flock ") + assert not _events(events, "native ") + assert not _events(events, "restic ") + assert not _events(events, "retention ") + assert any("critical break-glass workflow" in event for event in events) + assert not dump_dir.exists() + + def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None: playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8") diff --git a/scripts/backup/tests/test_signoz_backup_canary_wrapper.py b/scripts/backup/tests/test_signoz_backup_canary_wrapper.py index f2ba902af..c7629e394 100644 --- a/scripts/backup/tests/test_signoz_backup_canary_wrapper.py +++ b/scripts/backup/tests/test_signoz_backup_canary_wrapper.py @@ -7,6 +7,8 @@ import subprocess import time from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[3] WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh" @@ -115,6 +117,9 @@ while [[ "${1:-}" == --kill-after=* ]]; do done [ "$#" -gt 1 ] shift +if [ "${FAKE_DOCKER_TIMEOUT:-0}" = "1" ] && [ "${1:-}" = "docker" ]; then + exit 124 +fi exec "$@" """, ) @@ -122,6 +127,9 @@ exec "$@" fake_bin / "flock", """\ #!/bin/bash +if [ -n "${TEST_FLOCK_MARKER:-}" ]; then + printf 'held\n' > "${TEST_FLOCK_MARKER}" +fi exit 0 """, ) @@ -171,14 +179,103 @@ print(f"{digest} {sys.argv[1]}") ) +def _restore_hook_source() -> str: + return """\ +#!/bin/bash +set -eu +[ "$#" -eq 1 ] +case "$1" in + --check) mode="check" ;; + --apply) mode="apply" ;; + *) exit 64 ;; +esac +receipt_dir="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:?}/${RUN_ID:?}" +mkdir -p "$receipt_dir" +receipt_log="$receipt_dir/receipts.jsonl" +identity_run="$RUN_ID" +if [ "${FAKE_RECEIPT_FAULT:-valid}" = "forged_identity" ]; then + identity_run="stale-$RUN_ID" +fi +invocation_id="${mode}-fake-restore-$$" +emit() { + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","stage":"%s","terminal":"%s","detail":"fake"}\n' \ + "$TRACE_ID" "$identity_run" "$WORK_ITEM_ID" "$invocation_id" "$1" "$2" \ + >> "$receipt_log" +} +if [ "$mode" = "check" ]; then + emit check check_pass_no_runtime_write + emit cleanup pass + emit terminal check_pass_no_runtime_write + exit 0 +fi +emit check pass +emit independent_post_verifier pass +emit cleanup pass +emit terminal verified +if [ "${FAKE_RECEIPT_FAULT:-valid}" != "missing_status" ]; then + cat > "$receipt_dir/status.json" <> "$receipt_log" +fi +""" + + def _backup_source() -> str: return """\ #!/bin/bash set -eu [ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80 [ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ] +[ "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}" = "${TEST_RESTORE_HOOK:?}" ] || exit 82 +[ "${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-}" = "${TEST_INVENTORY_HELPER:?}" ] || exit 83 +[ "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-}" = "${TEST_BACKUP_DISK_CONFIG:?}" ] || exit 84 +[ "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-}" = "${TEST_CLUSTER_CONFIG:?}" ] || exit 85 +[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-}" = "${TEST_NATIVE_RECEIPT_ROOT:?}" ] || exit 86 +[ "${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-}" = "${TEST_RESTORE_RECEIPT_ROOT:?}" ] || exit 87 printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \ > "${TEST_BACKUP_IDENTITIES:?}" +printf '%s\t%s\t%s\t%s\n' \ + "$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" \ + "$CLICKHOUSE_RESTORE_INVENTORY_HELPER" \ + "$CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" \ + "$CLICKHOUSE_RESTORE_CLUSTER_CONFIG" \ + > "${TEST_RESTORE_CONTRACT:?}" if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}" printf 'false\n' > "${TEST_COLLECTOR_STATE:?}" @@ -204,6 +301,23 @@ if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then printf 'true\n' > "${TEST_COLLECTOR_STATE:?}" else sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}" + native_dir="$CLICKHOUSE_NATIVE_RECEIPT_ROOT/$RUN_ID" + mkdir -p "$native_dir" + native_receipts="$native_dir/receipts.jsonl" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"check-fake-native","stage":"terminal","terminal":"check_pass_no_write","detail":"fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" + if [ "${FAKE_RECEIPT_FAULT:-valid}" != "no_hook" ]; then + "$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --check + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_check_exit_0_stdout_fake_stderr_fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" + "$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --apply + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_exit_0_stdout_fake_stderr_fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" + fi + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"independent_verifier","terminal":"pass","detail":"fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"terminal","terminal":"pass","detail":"fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" fi if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then printf '4317\n' > "${TEST_LISTENERS:?}" @@ -229,13 +343,30 @@ def _run_canary( duplicate_cron: bool = False, drop_listener: bool = False, expect_collector_stop: bool = True, + native_restore_contract: str = "valid", + native_receipt_state: str = "valid", + docker_timeout: bool = False, ) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]: + if native_receipt_state not in { + "valid", + "no_hook", + "forged_identity", + "missing_status", + "symlink_status", + "malformed_restore_receipt", + }: + raise ValueError(f"unsupported native receipt state: {native_receipt_state}") + fake_bin = tmp_path / "bin" cooldown_dir = tmp_path / "cooldown" receipt_root = tmp_path / "receipts" + native_receipt_root = tmp_path / "native-receipts" + restore_receipt_root = tmp_path / "restore-receipts" fake_bin.mkdir() cooldown_dir.mkdir() receipt_root.mkdir() + native_receipt_root.mkdir() + restore_receipt_root.mkdir() monitor_script = tmp_path / "docker-health-monitor.sh" monitor_log = tmp_path / "monitor.log" @@ -244,6 +375,12 @@ def _run_canary( listeners = tmp_path / "listeners" identities = tmp_path / "backup-identities.tsv" lease_observed = tmp_path / "lease-observed.txt" + restore_contract_observed = tmp_path / "restore-contract.tsv" + flock_marker = tmp_path / "canary-lock-held.txt" + restore_hook = tmp_path / "clickhouse-native-restore-drill.sh" + inventory_helper = tmp_path / "clickhouse-restore-inventory.py" + backup_disk_config = tmp_path / "backup_disk.xml" + cluster_config = tmp_path / "isolated-cluster.xml" cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown" run_id = f"canary-{tmp_path.name}" @@ -252,6 +389,19 @@ def _run_canary( _monitor_source(cooldown_dir, omit_contract=omit_monitor_contract), ) _write_executable(backup_script, _backup_source()) + if native_restore_contract == "symlink": + restore_hook_target = tmp_path / "restore-hook-target.sh" + _write_executable(restore_hook_target, _restore_hook_source()) + restore_hook.symlink_to(restore_hook_target) + elif native_restore_contract == "valid": + _write_executable(restore_hook, _restore_hook_source()) + elif native_restore_contract != "missing": + raise ValueError( + f"unsupported native restore contract: {native_restore_contract}" + ) + _write_executable(inventory_helper, "#!/bin/sh\nexit 0\n") + backup_disk_config.write_text("\n", encoding="utf-8") + cluster_config.write_text("\n", encoding="utf-8") _install_fake_commands(fake_bin) monitor_log.write_text("[test] monitor seed\n", encoding="utf-8") collector_state.write_text("true\n", encoding="utf-8") @@ -274,10 +424,17 @@ def _run_canary( "SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"), "SIGNOZ_CANARY_TIMEOUT_SECONDS": "5", "SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2", + "SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS": "2", "SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2", "SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds), "SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05", "SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"), + "CLICKHOUSE_NATIVE_POST_VERIFY_HOOK": str(restore_hook), + "CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(inventory_helper), + "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_disk_config), + "CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config), + "CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(native_receipt_root), + "CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(restore_receipt_root), "TEST_MONITOR_SCRIPT": str(monitor_script), "TEST_MONITOR_LOG": str(monitor_log), "TEST_COLLECTOR_STATE": str(collector_state), @@ -285,11 +442,21 @@ def _run_canary( "TEST_BACKUP_IDENTITIES": str(identities), "TEST_COOLDOWN_FILE": str(cooldown_file), "TEST_LEASE_OBSERVED": str(lease_observed), + "TEST_RESTORE_CONTRACT": str(restore_contract_observed), + "TEST_RESTORE_HOOK": str(restore_hook), + "TEST_INVENTORY_HELPER": str(inventory_helper), + "TEST_BACKUP_DISK_CONFIG": str(backup_disk_config), + "TEST_CLUSTER_CONFIG": str(cluster_config), + "TEST_NATIVE_RECEIPT_ROOT": str(native_receipt_root), + "TEST_RESTORE_RECEIPT_ROOT": str(restore_receipt_root), + "TEST_FLOCK_MARKER": str(flock_marker), "FAKE_MONITOR_MODE": monitor_mode, "FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds), "FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0", "FAKE_DROP_LISTENER": "1" if drop_listener else "0", "FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0", + "FAKE_RECEIPT_FAULT": native_receipt_state, + "FAKE_DOCKER_TIMEOUT": "1" if docker_timeout else "0", } started_at = int(time.time()) result = subprocess.run( @@ -308,6 +475,13 @@ def _run_canary( "lease_observed": lease_observed, "collector_state": collector_state, "listeners": listeners, + "restore_contract": restore_contract_observed, + "restore_hook": restore_hook, + "flock_marker": flock_marker, + "native_receipts": native_receipt_root / run_id / "receipts.jsonl", + "restore_receipts": restore_receipt_root / run_id / "receipts.jsonl", + "restore_status": restore_receipt_root / run_id / "status.json", + "backup_output": receipt_root / run_id / "backup-output.log", } return result, paths, str(started_at) @@ -337,7 +511,9 @@ def test_existing_lease_is_atomically_replaced_and_exactly_restored( assert identities[2] == "P0-OBS-002\n" receipts = _receipts(paths["receipts"]) assert _terminal(receipts, "rollback") == "restored" - assert _terminal(receipts, "terminal") == "pass" + assert _terminal(receipts, "runtime_operation") == "pass" + assert _terminal(receipts, "closure_writeback") == "partial_degraded" + assert _terminal(receipts, "terminal") == "partial_degraded" metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8")) assert metadata["trace_id"] == "trace-P0-OBS-002" assert metadata["prior_present"] == 1 @@ -368,7 +544,7 @@ def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None: assert not paths["cooldown"].exists() receipts = _receipts(paths["receipts"]) assert _terminal(receipts, "rollback") == "restored" - assert _terminal(receipts, "terminal") == "pass" + assert _terminal(receipts, "terminal") == "partial_degraded" def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease( @@ -380,13 +556,30 @@ def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease( assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" assert not paths["lease_observed"].exists() assert paths["collector_state"].read_text(encoding="utf-8") == "true\n" + assert paths["restore_contract"].read_text(encoding="utf-8").split("\t") == [ + str(paths["restore_hook"]), + str(tmp_path / "clickhouse-restore-inventory.py"), + str(tmp_path / "backup_disk.xml"), + f"{tmp_path / 'isolated-cluster.xml'}\n", + ] receipts = _receipts(paths["receipts"]) + assert paths["native_receipts"].is_file() + assert paths["restore_receipts"].is_file() + restore_status = json.loads(paths["restore_status"].read_text(encoding="utf-8")) + assert restore_status["terminal"] == "verified" + assert restore_status["check_table_count"] == 44 + assert restore_status["check_table_pass_count"] == 44 + assert any( + item["stage"] == "native_restore_receipt_verifier" + and item["terminal"] == "pass" + for item in receipts + ) assert any( item["stage"] == "lease" and item["terminal"] == "not_required" for item in receipts ) assert _terminal(receipts, "rollback") == "no_write" - assert _terminal(receipts, "terminal") == "pass" + assert _terminal(receipts, "terminal") == "partial_degraded" def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract( @@ -406,7 +599,7 @@ def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract( item for item in receipts if item["stage"] == "source_of_truth_diff" ][-1] assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"] - assert _terminal(receipts, "terminal") == "pass" + assert _terminal(receipts, "terminal") == "partial_degraded" def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None: @@ -496,6 +689,34 @@ def test_wrapper_deployment_and_static_contracts_are_owned() -> None: assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper assert "monitor_auto_repair_contaminated_canary" in wrapper assert "collector_post_backup_runtime_failed" in wrapper + assert "/backup/scripts/clickhouse-native-restore-drill.sh" in wrapper + assert "/backup/scripts/clickhouse-restore-inventory.py" in wrapper + assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in wrapper + assert ( + "/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + in wrapper + ) + assert ( + 'CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}"' + in wrapper + ) + assert ( + 'CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}"' in wrapper + ) + assert ( + 'CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}"' + in wrapper + ) + main_source = wrapper[wrapper.index("main() {") :] + assert main_source.index("acquire_canary_lock") < main_source.index( + "require_native_restore_contract" + ) + assert "same_run_native_restore_receipt_validation_failed" in wrapper + assert '"check_table_count"' in wrapper + assert "mandatory_isolated_restore_verifier" in wrapper + assert "SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS" in wrapper + assert 'emit_receipt "runtime_operation" "pass"' in wrapper + assert 'emit_receipt "closure_writeback" "partial_degraded"' in wrapper assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8") assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text( encoding="utf-8" @@ -518,6 +739,80 @@ def test_check_mode_proves_contract_without_lease_or_backup_write( assert _terminal(check_receipts, "terminal") == "check_pass_no_write" +@pytest.mark.parametrize("contract_state", ["missing", "symlink"]) +def test_native_check_fails_closed_for_unsafe_restore_contract( + tmp_path: Path, + contract_state: str, +) -> None: + result, paths, _ = _run_canary( + tmp_path, + mode="check", + expect_collector_stop=False, + native_restore_contract=contract_state, + ) + + assert result.returncode != 0 + assert "native_restore_contract_CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" in result.stderr + assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n" + assert not paths["identities"].exists() + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "terminal") == "failed" + + +@pytest.mark.parametrize( + "receipt_state", + [ + "no_hook", + "forged_identity", + "missing_status", + "symlink_status", + "malformed_restore_receipt", + ], +) +def test_native_apply_rejects_missing_or_forged_machine_receipts( + tmp_path: Path, + receipt_state: str, +) -> None: + result, paths, _ = _run_canary( + tmp_path, + expect_collector_stop=False, + native_receipt_state=receipt_state, + ) + + assert result.returncode != 0 + assert "========== SigNoz ClickHouse native 備份完成 (1s) ==========" in paths[ + "backup_output" + ].read_text(encoding="utf-8") + assert ( + "same_run_native_restore_receipt_validation_failed" in result.stderr + or "same_run_native_restore_receipt_missing_unreadable_or_symlink" + in result.stderr + ) + assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n" + receipts = _receipts(paths["receipts"]) + assert not any( + item["stage"] == "native_restore_receipt_verifier" + and item["terminal"] == "pass" + for item in receipts + ) + assert _terminal(receipts, "terminal") == "failed" + + +def test_docker_metadata_timeout_fails_before_backup(tmp_path: Path) -> None: + result, paths, _ = _run_canary( + tmp_path, + mode="check", + expect_collector_stop=False, + docker_timeout=True, + ) + + assert result.returncode != 0 + assert "collector_runtime_metadata_unreadable" in result.stderr + assert not paths["identities"].exists() + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "terminal") == "failed" + + def test_wrapper_is_valid_bash() -> None: result = subprocess.run( ["bash", "-n", str(WRAPPER)], diff --git a/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh index 22a79e705..da4e27be0 100755 --- a/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh +++ b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh @@ -11,7 +11,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml" LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml" -TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}" +TARGET_HOST="wooo@192.168.0.110" REMOTE_DIR="/home/wooo/signoz/deploy/docker" REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml" REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml" @@ -31,6 +31,12 @@ RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk" API_URL="http://127.0.0.1:8080/api/v1/health" SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" SAFE_HOME="/home/wooo" +LOCAL_KILL_AFTER_SECONDS=15 +REMOTE_CHECK_TIMEOUT_SECONDS=180 +REMOTE_STAGE_TIMEOUT_SECONDS=60 +REMOTE_TRANSFER_TIMEOUT_SECONDS=60 +REMOTE_APPLY_TIMEOUT_SECONDS=3600 +REMOTE_CLEANUP_TIMEOUT_SECONDS=60 SSH_OPTS=( -o BatchMode=yes @@ -79,7 +85,7 @@ for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do } done -for command_name in python3 scp ssh; do +for command_name in python3 scp ssh timeout; do command -v "${command_name}" >/dev/null 2>&1 || { printf 'required local command missing: %s\n' "${command_name}" >&2 exit 69 @@ -94,6 +100,18 @@ hash_file() { fi } +bounded_ssh() { + local timeout_seconds="$1" + shift + timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \ + "${timeout_seconds}" ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" "$@" +} + +bounded_scp() { + timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \ + "${REMOTE_TRANSFER_TIMEOUT_SECONDS}" scp -q "${SSH_OPTS[@]}" "$@" +} + validate_local_sources() { [ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ] [ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ] @@ -136,13 +154,13 @@ local_receipt() { } remote_candidate_compose_check() { - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" \ - "env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \ + bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" \ + "timeout --signal=TERM --kill-after=15 120 env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \ < "${LOCAL_OVERRIDE}" } remote_read_only_check() { - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ + bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" bash -s -- \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ "${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK' set -euo pipefail @@ -159,6 +177,24 @@ REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml" HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse" SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" SAFE_HOME="/home/wooo" +COMMAND_TIMEOUT_SECONDS=30 +COMPOSE_TIMEOUT_SECONDS=120 +KILL_AFTER_SECONDS=15 + +for command_name in docker timeout; do + command -v "${command_name}" >/dev/null 2>&1 +done + +docker_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker "$@" +} + +compose_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMPOSE_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p signoz "$@" +} file_hash_or_absent() { if [ -f "$1" ] && [ ! -L "$1" ]; then @@ -172,14 +208,10 @@ file_hash_or_absent() { [ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ] for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do - docker inspect "${container}" >/dev/null + docker_cmd inspect "${container}" >/dev/null done -env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \ - config -q >/dev/null -env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \ - config --services | grep -qx clickhouse +compose_cmd -f "${REMOTE_BASE}" config -q >/dev/null +compose_cmd -f "${REMOTE_BASE}" config --services | grep -qx clickhouse override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")" config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")" @@ -195,13 +227,13 @@ elif [ -e "${HOST_BACKUP_DIR}" ]; then host_dir=unsafe fi -disk_count="$(docker exec signoz-clickhouse clickhouse-client --query \ +disk_count="$(docker_cmd exec signoz-clickhouse clickhouse-client --query \ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" -mount_count="$(docker inspect --format \ +mount_count="$(docker_cmd inspect --format \ '{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \ signoz-clickhouse | tr -cd '1' | wc -c | xargs)" -collector_running="$(docker inspect --format '{{.State.Running}}' signoz-otel-collector)" -collector_restarts="$(docker inspect --format '{{.RestartCount}}' signoz-otel-collector)" +collector_running="$(docker_cmd inspect --format '{{.State.Running}}' signoz-otel-collector)" +collector_restarts="$(docker_cmd inspect --format '{{.RestartCount}}' signoz-otel-collector)" printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \ @@ -223,20 +255,58 @@ fi STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}" STAGE_CREATED=0 +LOCAL_APPLY_VERIFIED=0 cleanup_stage() { - if [ "${STAGE_CREATED}" -eq 1 ]; then - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true -set -u + local cleanup_rc + [ "${STAGE_CREATED}" -eq 1 ] || return 0 + bounded_ssh "${REMOTE_CLEANUP_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 +set -euo pipefail stage_dir="$1" +if [ ! -e "${stage_dir}" ]; then + exit 0 +fi +[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ] rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml" -rmdir "${stage_dir}" 2>/dev/null || true +rmdir "${stage_dir}" +[ ! -e "${stage_dir}" ] REMOTE_CLEANUP - fi + cleanup_rc=$? + [ "${cleanup_rc}" -eq 0 ] || return "${cleanup_rc}" + STAGE_CREATED=0 } -trap cleanup_stage EXIT -ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' +on_local_exit() { + local rc=$? + local cleanup_rc=0 cleanup_terminal=not_created terminal=failed + trap - EXIT HUP INT TERM + set +e + if [ "${STAGE_CREATED}" -eq 1 ]; then + cleanup_stage + cleanup_rc=$? + if [ "${cleanup_rc}" -eq 0 ]; then + cleanup_terminal=pass_absence_verified + else + cleanup_terminal=failed_or_residue_present + rc=92 + fi + fi + local_receipt cleanup "$([ "${cleanup_rc}" -eq 0 ] && printf pass || printf failed)" \ + "remote_stage_${cleanup_terminal}" + if [ "${rc}" -eq 0 ] && [ "${LOCAL_APPLY_VERIFIED}" -eq 1 ]; then + terminal=pass + fi + local_receipt terminal "${terminal}" \ + "remote_controlled_apply_${terminal}_stage_${cleanup_terminal}_exit_${rc}" + exit "${rc}" +} + +trap on_local_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +bounded_ssh "${REMOTE_STAGE_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' set -euo pipefail stage_dir="$1" [ ! -e "${stage_dir}" ] @@ -245,12 +315,12 @@ mkdir -m 0700 "${stage_dir}" REMOTE_STAGE STAGE_CREATED=1 -scp -q "${SSH_OPTS[@]}" "${LOCAL_OVERRIDE}" \ +bounded_scp "${LOCAL_OVERRIDE}" \ "${TARGET_HOST}:${STAGE_DIR}/override.yaml" -scp -q "${SSH_OPTS[@]}" "${LOCAL_CONFIG}" \ +bounded_scp "${LOCAL_CONFIG}" \ "${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml" -ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ +bounded_ssh "${REMOTE_APPLY_TIMEOUT_SECONDS}" bash -s -- \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ "${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY' set -euo pipefail @@ -282,12 +352,26 @@ API_URL="http://127.0.0.1:8080/api/v1/health" SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" SAFE_HOME="/home/wooo" LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock" +BACKUP_OPERATION_LOCK_FILE="/tmp/awoooi-signoz-backup-operation.lock" +CANARY_LOCK_FILE="/tmp/awoooi-signoz-backup-canary.lock" +COMMAND_TIMEOUT_SECONDS=30 +COMPOSE_CONFIG_TIMEOUT_SECONDS=120 +COMPOSE_APPLY_TIMEOUT_SECONDS=300 +KILL_AFTER_SECONDS=15 APPLY_STARTED=0 DEPLOY_VERIFIED=0 HOST_DIR_CREATED=0 PRIOR_OVERRIDE_PRESENT=0 PRIOR_CONFIG_PRESENT=0 +PRIOR_OVERRIDE_HASH="absent" +PRIOR_OVERRIDE_MODE="absent" +PRIOR_OVERRIDE_UID="absent" +PRIOR_OVERRIDE_GID="absent" +PRIOR_CONFIG_HASH="absent" +PRIOR_CONFIG_MODE="absent" +PRIOR_CONFIG_UID="absent" +PRIOR_CONFIG_GID="absent" PRIOR_DISK_COUNT=0 BEFORE_IMAGE_ID="" BEFORE_DATA_VOLUME="" @@ -307,38 +391,60 @@ receipt() { printf '%s\n' "${line}" >> "${RECEIPT_LOG}" } +docker_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker "$@" +} + +bounded_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" "$@" +} + +compose_config_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMPOSE_CONFIG_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@" +} + +compose_apply_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMPOSE_APPLY_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@" +} + +pgrep_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" pgrep "$@" +} + +ss_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" ss "$@" +} + compose_base_config() { - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" config -q >/dev/null 2>&1 + compose_config_cmd -f "${REMOTE_BASE}" config -q >/dev/null 2>&1 } compose_managed_config() { - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" config -q >/dev/null 2>&1 + compose_config_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \ + config -q >/dev/null 2>&1 } compose_stage_config() { - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config -q >/dev/null 2>&1 + compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" \ + config -q >/dev/null 2>&1 } recreate_managed() { - timeout --signal=TERM --kill-after=30 300 \ - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \ + compose_apply_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \ up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \ >/dev/null 2>&1 } recreate_base() { - timeout --signal=TERM --kill-after=30 300 \ - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" \ + compose_apply_cmd -f "${REMOTE_BASE}" \ up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \ >/dev/null 2>&1 } @@ -346,11 +452,11 @@ recreate_base() { wait_clickhouse() { local attempt running health for attempt in $(seq 1 36); do - running="$(docker inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)" - health="$(docker inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \ + running="$(docker_cmd inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)" + health="$(docker_cmd inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \ "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)" if [ "${running}" = true ] && [ "${health}" = healthy ] \ - && timeout --kill-after=5 10 docker exec "${CLICKHOUSE_CONTAINER}" \ + && docker_cmd exec "${CLICKHOUSE_CONTAINER}" \ clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then return 0 fi @@ -361,7 +467,7 @@ wait_clickhouse() { listener_up() { local port="$1" - ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' + ss_cmd -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' } wait_api_and_listeners() { @@ -376,25 +482,72 @@ wait_api_and_listeners() { return 1 } +file_matches_metadata() { + local path="$1" expected_hash="$2" expected_mode="$3" expected_uid="$4" expected_gid="$5" + [ -f "${path}" ] && [ ! -L "${path}" ] \ + && [ "$(sha256sum "${path}" | awk '{print $1}')" = "${expected_hash}" ] \ + && [ "$(stat -c '%a' "${path}")" = "${expected_mode}" ] \ + && [ "$(stat -c '%u' "${path}")" = "${expected_uid}" ] \ + && [ "$(stat -c '%g' "${path}")" = "${expected_gid}" ] +} + +restore_exact_file() { + local source="$1" target="$2" expected_hash="$3" expected_mode="$4" + local expected_uid="$5" expected_gid="$6" tmp="$7" + local current_uid current_gid + + bounded_cmd install -m "${expected_mode}" "${source}" "${tmp}" || return 1 + current_uid="$(stat -c '%u' "${tmp}")" || return 1 + current_gid="$(stat -c '%g' "${tmp}")" || return 1 + if [ "${current_uid}" != "${expected_uid}" ] || [ "${current_gid}" != "${expected_gid}" ]; then + bounded_cmd sudo -n chown "${expected_uid}:${expected_gid}" "${tmp}" || return 1 + fi + bounded_cmd mv -f "${tmp}" "${target}" || return 1 + file_matches_metadata "${target}" "${expected_hash}" "${expected_mode}" \ + "${expected_uid}" "${expected_gid}" +} + restore_managed_files() { local tmp if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}" - install -m 0644 "${RECEIPT_DIR}/prior-override.yaml" "${tmp}" - mv -f "${tmp}" "${REMOTE_OVERRIDE}" + restore_exact_file "${RECEIPT_DIR}/prior-override.yaml" "${REMOTE_OVERRIDE}" \ + "${PRIOR_OVERRIDE_HASH}" "${PRIOR_OVERRIDE_MODE}" \ + "${PRIOR_OVERRIDE_UID}" "${PRIOR_OVERRIDE_GID}" "${tmp}" || return 1 else - rm -f "${REMOTE_OVERRIDE}" + bounded_cmd rm -f "${REMOTE_OVERRIDE}" || return 1 + [ ! -e "${REMOTE_OVERRIDE}" ] || return 1 fi if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}" - install -m 0644 "${RECEIPT_DIR}/prior-backup-disk.xml" "${tmp}" - mv -f "${tmp}" "${REMOTE_CONFIG}" + restore_exact_file "${RECEIPT_DIR}/prior-backup-disk.xml" "${REMOTE_CONFIG}" \ + "${PRIOR_CONFIG_HASH}" "${PRIOR_CONFIG_MODE}" \ + "${PRIOR_CONFIG_UID}" "${PRIOR_CONFIG_GID}" "${tmp}" || return 1 else - rm -f "${REMOTE_CONFIG}" + bounded_cmd rm -f "${REMOTE_CONFIG}" || return 1 + [ ! -e "${REMOTE_CONFIG}" ] || return 1 fi } +cleanup_owned_residue() { + local path + for path in \ + "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" \ + "${REMOTE_CONFIG}.candidate.${RUN_ID}" \ + "${REMOTE_OVERRIDE}.rollback.${RUN_ID}" \ + "${REMOTE_CONFIG}.rollback.${RUN_ID}"; do + bounded_cmd rm -f "${path}" || return 1 + [ ! -e "${path}" ] || return 1 + done + if [ -e "${STAGE_DIR}" ]; then + [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1 + bounded_cmd rm -f "${STAGE_OVERRIDE}" "${STAGE_CONFIG}" || return 1 + bounded_cmd rmdir "${STAGE_DIR}" || return 1 + fi + [ ! -e "${STAGE_DIR}" ] || return 1 +} + rollback_deploy() { local disk_count rollback_dir_terminal=retained restore_managed_files || return 1 @@ -409,26 +562,27 @@ rollback_deploy() { wait_clickhouse || return 1 wait_api_and_listeners || return 1 - disk_count="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + disk_count="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1 [ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1 - [ "$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1 - [ "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1 - [ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1 - [ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1 + [ "$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1 + [ "$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1 + [ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1 + [ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1 if [ "${HOST_DIR_CREATED}" -eq 1 ]; then - if sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then + if bounded_cmd sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then rollback_dir_terminal=removed_empty else rollback_dir_terminal=retained_nonempty_no_delete fi fi - receipt rollback pass "prior_compose_state_restored_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}" + receipt rollback pass \ + "prior_compose_state_restored_exact_hash_mode_uid_gid_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}" } on_exit() { - local rc=$? rollback_rc=0 terminal=failed + local rc=$? rollback_rc=0 cleanup_rc=0 terminal=failed trap - EXIT HUP INT TERM set +e if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then @@ -439,7 +593,15 @@ on_exit() { rc=91 fi fi - if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then + cleanup_owned_residue + cleanup_rc=$? + if [ "${cleanup_rc}" -ne 0 ]; then + receipt cleanup failed "exact_stage_candidate_or_rollback_residue_cleanup_failed" + rc=92 + else + receipt cleanup pass "exact_stage_candidate_and_rollback_residue_absence_verified" + fi + if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ] && [ "${cleanup_rc}" -eq 0 ]; then terminal=pass fi receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}" @@ -455,10 +617,22 @@ trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM +for lock_path in "${LOCK_FILE}" "${BACKUP_OPERATION_LOCK_FILE}" "${CANARY_LOCK_FILE}"; do + [ ! -L "${lock_path}" ] \ + || { receipt check failed operation_lock_symlink_unsafe; exit 75; } + if [ -e "${lock_path}" ]; then + [ -f "${lock_path}" ] \ + || { receipt check failed operation_lock_not_regular; exit 75; } + fi +done exec 9>>"${LOCK_FILE}" flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; } +exec 7>>"${CANARY_LOCK_FILE}" +flock -n 7 || { receipt check failed backup_canary_lock_held; exit 75; } +exec 8>>"${BACKUP_OPERATION_LOCK_FILE}" +flock -n 8 || { receipt check failed backup_operation_lock_held; exit 75; } -for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss sudo timeout; do +for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss stat sudo timeout; do command -v "${command_name}" >/dev/null 2>&1 \ || { receipt check failed "required_command_missing_${command_name}"; exit 69; } done @@ -474,21 +648,39 @@ done compose_base_config compose_stage_config -env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \ +compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \ | grep -qx "${REMOTE_SERVICE}" -if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then - receipt check failed active_signoz_backup_detected - exit 75 -fi -active_native_backups="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ - "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" -[ "${active_native_backups}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; } +require_backup_idle() { + local phase="$1" active_native_backups process_rc + set +e + pgrep_cmd -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' \ + >/dev/null 2>&1 + process_rc=$? + set -e + case "${process_rc}" in + 0) + receipt check failed "active_signoz_backup_or_restore_process_detected_${phase}" + return 75 + ;; + 1) ;; + *) + receipt check failed "signoz_backup_process_probe_failed_${phase}_exit_${process_rc}" + return 69 + ;; + esac + active_native_backups="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" \ + || return 1 + [ "${active_native_backups}" = 0 ] \ + || { receipt check failed "active_native_backup_or_restore_${phase}"; return 75; } + receipt check pass "backup_idle_${phase}_shared_operation_and_canary_locks_held" +} + +require_backup_idle preflight for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do - [ "$(docker inspect --format '{{.State.Running}}' "${container}")" = true ] + [ "$(docker_cmd inspect --format '{{.State.Running}}' "${container}")" = true ] done wait_clickhouse wait_api_and_listeners @@ -505,42 +697,56 @@ fi || { receipt check failed partial_managed_file_drift; exit 78; } if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then compose_managed_config - cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml" - cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml" + PRIOR_OVERRIDE_HASH="$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" + PRIOR_OVERRIDE_MODE="$(stat -c '%a' "${REMOTE_OVERRIDE}")" + PRIOR_OVERRIDE_UID="$(stat -c '%u' "${REMOTE_OVERRIDE}")" + PRIOR_OVERRIDE_GID="$(stat -c '%g' "${REMOTE_OVERRIDE}")" + PRIOR_CONFIG_HASH="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" + PRIOR_CONFIG_MODE="$(stat -c '%a' "${REMOTE_CONFIG}")" + PRIOR_CONFIG_UID="$(stat -c '%u' "${REMOTE_CONFIG}")" + PRIOR_CONFIG_GID="$(stat -c '%g' "${REMOTE_CONFIG}")" + bounded_cmd cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml" + bounded_cmd cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml" + [ "$(sha256sum "${RECEIPT_DIR}/prior-override.yaml" | awk '{print $1}')" = "${PRIOR_OVERRIDE_HASH}" ] + [ "$(sha256sum "${RECEIPT_DIR}/prior-backup-disk.xml" | awk '{print $1}')" = "${PRIOR_CONFIG_HASH}" ] fi -PRIOR_DISK_COUNT="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ +PRIOR_DISK_COUNT="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" -BEFORE_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" -BEFORE_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +BEFORE_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" +BEFORE_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" [ -n "${BEFORE_DATA_VOLUME}" ] -BEFORE_COLLECTOR_ID="$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" -BEFORE_COLLECTOR_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" -BEFORE_SIGNOZ_ID="$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" -BEFORE_SIGNOZ_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" -BEFORE_ZOOKEEPER_ID="$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" -BEFORE_ZOOKEEPER_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" +BEFORE_COLLECTOR_ID="$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" +BEFORE_COLLECTOR_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" +BEFORE_SIGNOZ_ID="$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" +BEFORE_SIGNOZ_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" +BEFORE_ZOOKEEPER_ID="$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" +BEFORE_ZOOKEEPER_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid" receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}" -receipt source_of_truth_diff pass "prior_override_${PRIOR_OVERRIDE_PRESENT}_prior_config_${PRIOR_CONFIG_PRESENT}_prior_disk_count_${PRIOR_DISK_COUNT}" +receipt source_of_truth_diff pass \ + "prior_override_${PRIOR_OVERRIDE_PRESENT}_hash_${PRIOR_OVERRIDE_HASH}_mode_${PRIOR_OVERRIDE_MODE}_uid_${PRIOR_OVERRIDE_UID}_gid_${PRIOR_OVERRIDE_GID}_prior_config_${PRIOR_CONFIG_PRESENT}_hash_${PRIOR_CONFIG_HASH}_mode_${PRIOR_CONFIG_MODE}_uid_${PRIOR_CONFIG_UID}_gid_${PRIOR_CONFIG_GID}_prior_disk_count_${PRIOR_DISK_COUNT}" receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only" receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback" -receipt check pass "compose_config_q_stage_hashes_runtime_preflight_backup_idle" +receipt check pass \ + "compose_config_q_stage_hashes_runtime_preflight_dual_locks_backup_idle" + +require_backup_idle pre_mutation APPLY_STARTED=1 if [ -e "${HOST_BACKUP_DIR}" ]; then [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ] [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ] else - sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}" + bounded_cmd sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}" HOST_DIR_CREATED=1 fi -install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" -install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}" -mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}" -mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}" +bounded_cmd install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" +bounded_cmd install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}" +bounded_cmd mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}" +bounded_cmd mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}" [ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ] [ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ] compose_managed_config @@ -548,39 +754,39 @@ compose_managed_config recreate_managed wait_clickhouse -AFTER_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" -AFTER_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +AFTER_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" +AFTER_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" [ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ] [ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ] -backup_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" -config_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +backup_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +config_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" [ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ] [ "${config_mount}" = false ] [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ] -server_uid="$(docker exec "${CLICKHOUSE_CONTAINER}" \ +server_uid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \ awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)" -server_gid="$(docker exec "${CLICKHOUSE_CONTAINER}" \ +server_gid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \ awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)" [ "${server_uid}" = 101 ] [ "${server_gid}" = 101 ] -docker exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \ +docker_cmd exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \ 'test -d /backups && test -w /backups' -disk_state="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ +disk_state="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ "SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" [ "${disk_state}" = "Local|0|0|0|1" ] -[ "$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ +[ "$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ "EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ] -docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \ +docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \ | grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)' -[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] -[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] -[ "$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ] -[ "$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ] -[ "$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ] -[ "$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ] +[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] +[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] +[ "$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ] +[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ] +[ "$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ] +[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ] wait_api_and_listeners receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged" @@ -589,6 +795,4 @@ receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_execute DEPLOY_VERIFIED=1 REMOTE_APPLY -trap - EXIT -cleanup_stage -local_receipt terminal pass "remote_controlled_apply_and_postverify_complete" +LOCAL_APPLY_VERIFIED=1 diff --git a/scripts/ops/deploy-signoz-native-backup-toolchain.sh b/scripts/ops/deploy-signoz-native-backup-toolchain.sh index 02ae66a68..6d4efb305 100755 --- a/scripts/ops/deploy-signoz-native-backup-toolchain.sh +++ b/scripts/ops/deploy-signoz-native-backup-toolchain.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# P0-OBS-002 - atomically deploy the host110 SigNoz native backup toolchain. +# P0-OBS-002 - bounded controlled deploy of the host110 SigNoz native backup toolchain. # # This deployer owns seven repo-sourced files only. It does not run a backup or # restore, restart a container, pull an image, or inspect a secret-bearing @@ -12,6 +12,13 @@ TARGET_HOST="wooo@192.168.0.110" SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" SAFE_HOME="/home/wooo" API_URL="http://127.0.0.1:8080/api/v1/health" +CANARY_LOCK="/tmp/awoooi-signoz-backup-canary.lock" +OPERATION_LOCK="/tmp/awoooi-signoz-backup-operation.lock" + +SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}" +SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}" +DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}" +TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}" REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}" REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}" @@ -61,10 +68,18 @@ Optional safe absolute path environment: SIGNOZ_TOOLCHAIN_RECEIPT_ROOT SIGNOZ_TOOLCHAIN_STAGE_ROOT +Optional bounded-command timeout environment (positive integer seconds): + SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS + SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS + SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS + SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS + --check validates local sources and performs a remote read-only target diff and runtime readiness check. --apply stages all seven sources, validates hashes and -syntax, atomically replaces exact targets, and rolls every target back to its -prior hash/mode/owner if any bounded verifier fails. +syntax, uses a same-filesystem atomic replacement for each exact target, and +rolls every target back to its prior hash/mode/owner if any bounded verifier +fails. SSH, SCP, and Docker calls are bounded, and apply holds both the SigNoz +canary lock and backup operation lock through post-verification or rollback. Flat restore-driver invocation must set: CLICKHOUSE_RESTORE_INVENTORY_HELPER=/clickhouse-restore-inventory.py @@ -89,6 +104,10 @@ valid_identity() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] } +valid_positive_integer() { + [[ "$1" =~ ^[1-9][0-9]*$ ]] +} + safe_absolute_path() { local path="$1" [[ "${path}" == /* ]] || return 1 @@ -111,6 +130,24 @@ for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do } done +for value in \ + "${SSH_COMMAND_TIMEOUT_SECONDS}" \ + "${SCP_COMMAND_TIMEOUT_SECONDS}" \ + "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \ + "${TIMEOUT_KILL_AFTER_SECONDS}"; do + valid_positive_integer "${value}" || { + printf 'toolchain command timeouts must be positive integer seconds\n' >&2 + exit 64 + } +done +[ "${SSH_COMMAND_TIMEOUT_SECONDS}" -le 1200 ] \ + && [ "${SCP_COMMAND_TIMEOUT_SECONDS}" -le 600 ] \ + && [ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" -le 300 ] \ + && [ "${TIMEOUT_KILL_AFTER_SECONDS}" -le 60 ] || { + printf 'toolchain command timeout exceeds bounded maximum\n' >&2 + exit 64 +} + for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do safe_absolute_path "${path}" || { printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2 @@ -133,13 +170,25 @@ if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \ exit 64 fi -for command_name in bash python3 scp ssh; do +for command_name in bash python3 scp ssh timeout; do command -v "${command_name}" >/dev/null 2>&1 || { printf 'required local command missing: %s\n' "${command_name}" >&2 exit 69 } done + +bounded_ssh() { + timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${SSH_COMMAND_TIMEOUT_SECONDS}" ssh "$@" +} + + +bounded_scp() { + timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${SCP_COMMAND_TIMEOUT_SECONDS}" scp "$@" +} + hash_file() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}' @@ -241,9 +290,10 @@ local_receipt() { } remote_read_only_check() { - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ + bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \ + "${DOCKER_COMMAND_TIMEOUT_SECONDS}" "${TIMEOUT_KILL_AFTER_SECONDS}" \ "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK' set -euo pipefail @@ -254,7 +304,9 @@ REMOTE_SCRIPT_DIR="$4" REMOTE_CONFIG_ROOT="$5" RECEIPT_ROOT="$6" STAGE_ROOT="$7" -shift 7 +DOCKER_COMMAND_TIMEOUT_SECONDS="$8" +TIMEOUT_KILL_AFTER_SECONDS="$9" +shift 9 EXPECTED_HASHES=("$@") LABELS=( backup-signoz.sh @@ -279,10 +331,18 @@ API_URL="http://127.0.0.1:8080/api/v1/health" [ "${#EXPECTED_HASHES[@]}" -eq 7 ] [ -d /backup ] && [ ! -L /backup ] -for command_name in awk curl docker pgrep sha256sum ss stat; do +for command_name in awk curl docker pgrep sha256sum ss stat timeout; do command -v "${command_name}" >/dev/null 2>&1 done +[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]] +[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]] + +bounded_docker() { + timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@" +} + listener_up() { local port="$1" ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' @@ -305,7 +365,7 @@ validate_existing_ancestors() { runtime_line() { local container="$1" value running health - value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" + value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" IFS='|' read -r _ _ _ running health <<<"${value}" [ "${running}" = true ] case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac @@ -322,7 +382,7 @@ validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d" validate_existing_ancestors "${RECEIPT_ROOT}" validate_existing_ancestors "${STAGE_ROOT}" [ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ] -active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \ +active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \ "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" [ "${active_operations}" = 0 ] @@ -373,7 +433,7 @@ fi STAGE_CREATED=0 cleanup_stage() { if [ "${STAGE_CREATED}" -eq 1 ]; then - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true + bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true set -u stage_dir="$1" if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then @@ -393,7 +453,7 @@ REMOTE_CLEANUP } trap cleanup_stage EXIT -ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' +bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' set -euo pipefail stage_dir="$1" [ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ] @@ -405,13 +465,15 @@ REMOTE_STAGE STAGE_CREATED=1 for index in "${!LOCAL_SOURCES[@]}"; do - scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \ + bounded_scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \ "${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}" done -ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ +bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \ + "${CANARY_LOCK}" "${OPERATION_LOCK}" "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \ + "${TIMEOUT_KILL_AFTER_SECONDS}" \ "${SOURCE_HASHES[@]}" <<'REMOTE_APPLY' set -euo pipefail @@ -422,7 +484,11 @@ REMOTE_SCRIPT_DIR="$4" REMOTE_CONFIG_ROOT="$5" RECEIPT_ROOT="$6" STAGE_DIR="$7" -shift 7 +CANARY_LOCK="$8" +OPERATION_LOCK="$9" +DOCKER_COMMAND_TIMEOUT_SECONDS="${10}" +TIMEOUT_KILL_AFTER_SECONDS="${11}" +shift 11 EXPECTED_HASHES=("$@") LABELS=( backup-signoz.sh @@ -472,6 +538,8 @@ APPLY_STARTED=0 DEPLOY_VERIFIED=0 RUNTIME_BASELINE_CAPTURED=0 LOCK_HELD=0 +CANARY_LOCK_HELD=0 +OPERATION_LOCK_HELD=0 REMOTE_UID="$(id -u)" REMOTE_GID="$(id -g)" CREATED_DIRS=() @@ -481,7 +549,7 @@ PRIOR_UIDS=() PRIOR_GIDS=() PRIOR_HASHES=() -for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat; do +for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat timeout; do command -v "${command_name}" >/dev/null 2>&1 || { printf 'required remote command missing: %s\n' "${command_name}" >&2 exit 69 @@ -493,6 +561,10 @@ if [ "${REMOTE_UID}" -ne 0 ]; then fi [ "${#EXPECTED_HASHES[@]}" -eq 7 ] [ -d /backup ] && [ ! -L /backup ] +[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]] +[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]] +[ "${CANARY_LOCK}" = "/tmp/awoooi-signoz-backup-canary.lock" ] +[ "${OPERATION_LOCK}" = "/tmp/awoooi-signoz-backup-operation.lock" ] run_root() { if [ "${REMOTE_UID}" -eq 0 ]; then @@ -502,6 +574,11 @@ run_root() { fi } +bounded_docker() { + timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@" +} + root_hash() { if [ "${REMOTE_UID}" -eq 0 ]; then sha256sum "$1" | awk '{print $1}' @@ -550,7 +627,7 @@ listener_up() { runtime_snapshot() { local container value running health port api_code for container in "${CONTAINERS[@]}"; do - value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" + value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" IFS='|' read -r _ _ _ running health <<<"${value}" [ "${running}" = true ] case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac @@ -673,6 +750,14 @@ on_exit() { if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then rc=92 fi + if [ "${OPERATION_LOCK_HELD}" -eq 1 ]; then + flock -u 7 >/dev/null 2>&1 + exec 7>&- + fi + if [ "${CANARY_LOCK_HELD}" -eq 1 ]; then + flock -u 8 >/dev/null 2>&1 + exec 8>&- + fi if [ "${LOCK_HELD}" -eq 1 ]; then flock -u 9 >/dev/null 2>&1 exec 9>&- @@ -694,11 +779,33 @@ exec 9>>"${LOCK_FILE}" flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; } LOCK_HELD=1 +[ ! -L "${CANARY_LOCK}" ] \ + || { receipt check failed canary_lock_symlink_unsafe; exit 75; } +if [ -e "${CANARY_LOCK}" ]; then + [ -f "${CANARY_LOCK}" ] \ + || { receipt check failed canary_lock_not_regular_file; exit 75; } +fi +exec 8>>"${CANARY_LOCK}" +flock -n 8 \ + || { receipt check failed another_signoz_canary_holds_canary_lock; exit 75; } +CANARY_LOCK_HELD=1 + +[ ! -L "${OPERATION_LOCK}" ] \ + || { receipt check failed backup_operation_lock_symlink_unsafe; exit 75; } +if [ -e "${OPERATION_LOCK}" ]; then + [ -f "${OPERATION_LOCK}" ] \ + || { receipt check failed backup_operation_lock_not_regular_file; exit 75; } +fi +exec 7>>"${OPERATION_LOCK}" +flock -n 7 \ + || { receipt check failed another_signoz_backup_holds_operation_lock; exit 75; } +OPERATION_LOCK_HELD=1 + if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then receipt check failed active_signoz_backup_toolchain_process exit 75 fi -active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \ +active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \ "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" [ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; } @@ -744,9 +851,9 @@ done receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources" receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2" receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}" -receipt ai_decision candidate "atomic_install_native_backup_toolchain_only" -receipt risk_policy pass "risk_medium_bounded_files_only_no_runtime_execution_no_container_change_full_rollback" -receipt check pass "runtime_baseline_active_operations_0_hash_bash_xml_py_compile" +receipt ai_decision candidate "same_filesystem_candidate_replace_native_backup_toolchain_only" +receipt risk_policy pass "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_backup_operation_locks_no_runtime_execution_no_container_change_full_rollback" +receipt check pass "dual_signoz_canary_and_backup_operation_locks_held_runtime_baseline_active_operations_0_hash_bash_xml_py_compile" APPLY_STARTED=1 for directory in "${TARGET_DIRS[@]}"; do @@ -790,7 +897,7 @@ done runtime_snapshot > "${RUNTIME_AFTER}" cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}" -receipt execution pass "seven_candidates_atomically_replaced_expected_hashes_and_modes" +receipt execution pass "seven_same_filesystem_candidates_replaced_with_full_rollback_expected_hashes_and_modes" receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged" receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull" DEPLOY_VERIFIED=1 diff --git a/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py index 504e2ca11..23d7597d2 100644 --- a/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py +++ b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py @@ -2,6 +2,7 @@ from __future__ import annotations import os from pathlib import Path +import re import subprocess import xml.etree.ElementTree as ET @@ -10,6 +11,12 @@ ROOT = Path(__file__).resolve().parents[3] CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml" OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml" DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh" +BACKUP_SCRIPT = ROOT / "scripts/backup/backup-signoz.sh" + + +def write_executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(0o755) def executable_text() -> str: @@ -105,9 +112,9 @@ def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> No text = DEPLOYER.read_text(encoding="utf-8") check_terminal = text.index('if [ "${MODE}" = "--check" ]') stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.') - first_scp = text.index("scp -q") + first_transfer = text.index('bounded_scp "${LOCAL_OVERRIDE}"') - assert check_terminal < stage_assignment < first_scp + assert check_terminal < stage_assignment < first_transfer assert "-f '${REMOTE_BASE}' -f - config -q" in text assert '< "${LOCAL_OVERRIDE}"' in text assert "check_pass_no_write" in text @@ -120,7 +127,12 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None: assert 'REMOTE_SERVICE="clickhouse"' in text assert "--env-file /dev/null" in text assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text - assert "timeout --signal=TERM --kill-after=30 300" in text + assert "bounded_ssh" in text + assert "bounded_scp" in text + assert "docker_cmd" in text + assert "compose_config_cmd" in text + assert "compose_apply_cmd" in text + assert "REMOTE_APPLY_TIMEOUT_SECONDS=3600" in text assert "up -d --no-deps --pull never --force-recreate" in text assert '"${REMOTE_SERVICE}"' in text assert "docker compose down" not in executable @@ -133,15 +145,135 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None: def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None: text = DEPLOYER.read_text(encoding="utf-8") - assert "active_signoz_backup_detected" in text + assert "active_signoz_backup_or_restore_process_detected" in text assert "active_native_backup_or_restore" in text + assert "signoz_backup_process_probe_failed" in text assert "partial_managed_file_drift" in text assert "another_deploy_holds_lock" in text - assert "backup-signoz[.]sh" in text + for process in ( + "backup-signoz", + "clickhouse-native-backup", + "clickhouse-native-restore-drill", + "run-signoz-backup-canary", + ): + assert process in text + probe_block = text[ + text.index("require_backup_idle()") : text.index( + "require_backup_idle preflight" + ) + ] + assert "process_rc=$?" in probe_block + assert 'case "${process_rc}" in' in probe_block + assert "1) ;;" in probe_block + assert "return 69" in probe_block assert "CREATING_BACKUP" in text assert "RESTORING" in text +def test_target_host_is_fixed_and_ignores_ambient_override(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + ssh_log = tmp_path / "ssh.log" + write_executable( + fake_bin / "ssh", + """#!/bin/bash +printf '%s\\n' "$*" >> "${FAKE_SSH_LOG:?}" +cat >/dev/null +""", + ) + write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n") + env = os.environ.copy() + env.update( + { + "PATH": f"{fake_bin}:{env['PATH']}", + "TRACE_ID": "P0-OBS-002-test", + "RUN_ID": "fixed-host-test", + "WORK_ITEM_ID": "P0-OBS-002", + "TARGET_HOST": "attacker@example.invalid", + "FAKE_SSH_LOG": str(ssh_log), + } + ) + + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 0, result.stderr + invocations = ssh_log.read_text(encoding="utf-8").splitlines() + assert len(invocations) == 2 + assert all("wooo@192.168.0.110" in line for line in invocations) + assert all("attacker@example.invalid" not in line for line in invocations) + + +def test_all_transport_and_remote_docker_operations_are_bounded() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + executable = executable_text() + + assert executable.count(' ssh "${SSH_OPTS[@]}"') == 1 + assert executable.count(' scp -q "${SSH_OPTS[@]}"') == 1 + assert not re.search(r"\bdocker (inspect|exec)\b", executable) + assert 'timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}"' in text + assert 'docker_cmd exec "${CLICKHOUSE_CONTAINER}"' in text + assert "ss_cmd -H -lnt" in text + for mutation in ( + "bounded_cmd cp -p", + "bounded_cmd install -m", + "bounded_cmd mv -f", + "bounded_cmd sudo -n install", + "bounded_cmd rm -f", + ): + assert mutation in text + + +def test_stage_cleanup_failure_changes_apply_terminal_to_failed(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + ssh_count = tmp_path / "ssh-count" + write_executable( + fake_bin / "ssh", + """#!/bin/bash +count=0 +if [ -f "${FAKE_SSH_COUNT:?}" ]; then + count="$(cat "${FAKE_SSH_COUNT}")" +fi +count=$((count + 1)) +printf '%s\\n' "${count}" > "${FAKE_SSH_COUNT}" +cat >/dev/null +if [ "${count}" -eq 5 ]; then + exit 1 +fi +""", + ) + write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n") + env = os.environ.copy() + env.update( + { + "PATH": f"{fake_bin}:{env['PATH']}", + "TRACE_ID": "P0-OBS-002-test", + "RUN_ID": "cleanup-failure-test", + "WORK_ITEM_ID": "P0-OBS-002", + "FAKE_SSH_COUNT": str(ssh_count), + } + ) + + result = subprocess.run( + ["bash", str(DEPLOYER), "--apply"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 92 + assert '"phase":"cleanup","result":"failed"' in result.stdout + assert '"phase":"terminal","result":"failed"' in result.stdout + assert "failed_or_residue_present" in result.stdout + + def test_apply_preserves_image_data_volume_and_non_target_containers() -> None: text = DEPLOYER.read_text(encoding="utf-8") assert "BEFORE_IMAGE_ID" in text @@ -166,13 +298,13 @@ def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None: assert "101:101:750" in text assert "server_uid" in text assert "server_gid" in text - assert "docker exec -u 101:101" in text + assert "docker_cmd exec -u 101:101" in text assert "test -w /backups" in text assert "awk '/^Uid:/{print $2}' /proc/1/status" in text assert "awk '/^Gid:/{print $2}' /proc/1/status" in text identity_start = text.index('server_uid="') identity_block = text[ - identity_start : text.index("docker exec -u 101:101", identity_start) + identity_start : text.index("docker_cmd exec -u 101:101", identity_start) ] assert "sh -c" not in identity_block assert "concurrency_disabled" in text @@ -194,6 +326,48 @@ def test_failed_apply_restores_prior_managed_compose_state() -> None: assert "retained_nonempty_no_delete" in text assert "rc=91" in text assert "trap on_exit EXIT" in text + for token in ( + "PRIOR_OVERRIDE_HASH", + "PRIOR_OVERRIDE_MODE", + "PRIOR_OVERRIDE_UID", + "PRIOR_OVERRIDE_GID", + "PRIOR_CONFIG_HASH", + "PRIOR_CONFIG_MODE", + "PRIOR_CONFIG_UID", + "PRIOR_CONFIG_GID", + "file_matches_metadata", + "restore_exact_file", + "prior_compose_state_restored_exact_hash_mode_uid_gid", + ): + assert token in text + assert "exact_stage_candidate_and_rollback_residue_absence_verified" in text + + +def test_deployer_and_backup_execution_share_operation_lock_contract() -> None: + deployer = DEPLOYER.read_text(encoding="utf-8") + backup = BACKUP_SCRIPT.read_text(encoding="utf-8") + operation_lock = "/tmp/awoooi-signoz-backup-operation.lock" + canary_lock = "/tmp/awoooi-signoz-backup-canary.lock" + + assert operation_lock in deployer + assert operation_lock in backup + assert canary_lock in deployer + assert 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' in deployer + assert 'exec 7>>"${CANARY_LOCK_FILE}"' in deployer + assert "flock -n 8 || { receipt check failed backup_operation_lock_held" in deployer + assert "flock -n 7 || { receipt check failed backup_canary_lock_held" in deployer + assert 'exec 8>>"${OPERATION_LOCK_FILE}"' in backup + assert "require_backup_idle pre_mutation" in deployer + assert deployer.index('exec 7>>"${CANARY_LOCK_FILE}"') < deployer.index( + 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' + ) + assert deployer.index('exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"') < deployer.index( + "require_backup_idle pre_mutation" + ) + remote_cleanup_call = deployer.rindex(" cleanup_owned_residue") + assert remote_cleanup_call < deployer.index( + 'receipt terminal "${terminal}"', remote_cleanup_call + ) def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None: diff --git a/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py index 9d105ae82..0abf7e6a1 100644 --- a/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py +++ b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py @@ -3,10 +3,13 @@ from __future__ import annotations import os from pathlib import Path import subprocess +import time ROOT = Path(__file__).resolve().parents[3] DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh" +BACKUP = ROOT / "scripts/backup/backup-signoz.sh" +CANARY = ROOT / "scripts/backup/run-signoz-backup-canary.sh" def deployer_text() -> str: @@ -119,6 +122,33 @@ def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> Non assert "must not overlap" in result.stderr +def test_timeout_overrides_fail_closed_before_network_access() -> None: + for key, value, expected in ( + ( + "SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS", + "0", + "positive integer seconds", + ), + ( + "SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS", + "301", + "exceeds bounded maximum", + ), + ): + env = base_env() + env[key] = value + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 64 + assert expected in result.stderr + assert "ssh:" not in result.stderr + + def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None: text = deployer_text() assert 'TARGET_HOST="wooo@192.168.0.110"' in text @@ -229,6 +259,70 @@ def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None: assert "check_pass_no_write" in result.stdout +def test_hung_ssh_is_bounded_and_cannot_claim_check_pass(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + ssh = fake_bin / "ssh" + ssh.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8") + scp = fake_bin / "scp" + scp.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8") + ssh.chmod(0o755) + scp.chmod(0o755) + + env = base_env() + env["PATH"] = f"{fake_bin}:{env['PATH']}" + env["SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS"] = "1" + env["SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS"] = "1" + started = time.monotonic() + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + timeout=5, + ) + elapsed = time.monotonic() - started + + assert result.returncode == 124 + assert elapsed < 4 + assert "check_pass_no_write" not in result.stdout + + +def test_all_ssh_scp_and_docker_calls_use_bounded_wrappers() -> None: + text = deployer_text() + executable = executable_text() + + assert ( + 'SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"' + in text + ) + assert ( + 'SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"' + in text + ) + assert ( + 'DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"' + in text + ) + assert ( + 'TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"' + in text + ) + assert "bounded_ssh()" in text + assert "bounded_scp()" in text + assert text.count('ssh "$@"') == 1 + assert text.count('scp "$@"') == 1 + assert executable.count("bounded_docker()") == 2 + assert "$(docker inspect" not in executable + assert "$(docker exec" not in executable + assert "\n docker inspect" not in executable + assert "\n docker exec" not in executable + assert executable.count("bounded_docker inspect") == 2 + assert executable.count("bounded_docker exec") == 2 + assert "timeout --signal=TERM" in executable + + def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None: text = deployer_text() for token in ( @@ -250,18 +344,60 @@ def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None: assert "bounded_execution_not_started_targets_unchanged" in text -def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None: +def test_apply_uses_same_filesystem_candidates_and_accurate_replace_receipt() -> None: text = deployer_text() assert '"${target}.candidate.${RUN_ID}"' in text assert ( 'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"' ) in text assert "deployed-manifest.tsv" in text - assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text + assert ( + "seven_same_filesystem_candidates_replaced_with_full_rollback_" + "expected_hashes_and_modes" + ) in text + assert "seven_candidates_atomically_replaced" not in text + assert "atomically replaces exact targets" not in text assert "stage_candidate_residue_0" in text assert "verify_residue_zero" in text +def test_apply_holds_dual_backup_locks_through_postverify_or_rollback() -> None: + text = deployer_text() + backup_text = BACKUP.read_text(encoding="utf-8") + canary_text = CANARY.read_text(encoding="utf-8") + canary_lock = "/tmp/awoooi-signoz-backup-canary.lock" + operation_lock = "/tmp/awoooi-signoz-backup-operation.lock" + + assert f'CANARY_LOCK="{canary_lock}"' in text + assert f'OPERATION_LOCK="{operation_lock}"' in text + assert canary_lock in canary_text + assert operation_lock in backup_text + canary_acquire = text.index('exec 8>>"${CANARY_LOCK}"') + operation_acquire = text.index('exec 7>>"${OPERATION_LOCK}"') + active_check = text.index("if pgrep -af", operation_acquire) + apply_started = text.index("APPLY_STARTED=1", operation_acquire) + target_replace = text.index( + 'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}"', apply_started + ) + assert canary_acquire < operation_acquire < active_check < apply_started + assert apply_started < target_replace + + on_exit = text[text.index("on_exit() {") : text.index("trap on_exit EXIT")] + rollback = on_exit.index("rollback_targets") + release_operation = on_exit.index("flock -u 7") + release_canary = on_exit.index("flock -u 8") + assert rollback < release_operation < release_canary + assert 'rm -f "${OPERATION_LOCK}"' not in text + assert 'rm -f "${CANARY_LOCK}"' not in text + assert "another_signoz_backup_holds_operation_lock" in text + assert "another_signoz_canary_holds_canary_lock" in text + assert ( + "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_" + "backup_operation_locks_" + "no_runtime_execution_no_container_change_full_rollback" + ) in text + + def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None: text = deployer_text() for container in ( diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index a24b7e37f..e3d857987 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -9,6 +9,7 @@ only from the committed post-closure receipt. from __future__ import annotations import argparse +import hashlib import json from pathlib import Path from typing import Any @@ -30,10 +31,45 @@ EXPECTED_POST_CLOSURE_BLOCKERS = [ "signoz_backup_offsite_dr_pending", "clickhouse_native_failed_artifact_retention_pending", "clickhouse_native_resume_submitted_inventory_pending", + "clickhouse_native_current_source_toolchain_pending", "service_registry_runtime_mirror_reconciliation_pending", "github_freeze_legacy_asset_retirement_pending", "monitoring_generator_duplicate_awoooi_api_identity_pending", ] +TOOLCHAIN_SOURCE_PATHS = { + "backup_orchestrator_sha256": "scripts/backup/backup-signoz.sh", + "native_backup_sha256": "scripts/backup/clickhouse-native-backup.sh", + "isolated_restore_sha256": "scripts/backup/clickhouse-native-restore-drill.sh", + "inventory_verifier_sha256": "scripts/backup/clickhouse-restore-inventory.py", + "canary_wrapper_sha256": "scripts/backup/run-signoz-backup-canary.sh", + "backup_disk_config_sha256": "ops/signoz/clickhouse/config.d/backup_disk.xml", + "isolated_cluster_config_sha256": ( + "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + ), +} +HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES = { + "backup_orchestrator_sha256": ( + "2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520" + ), + "native_backup_sha256": ( + "754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a" + ), + "isolated_restore_sha256": ( + "57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7" + ), + "inventory_verifier_sha256": ( + "79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c" + ), + "canary_wrapper_sha256": ( + "64d23d9f7a64d1e4e1347b98d45e50e26eadea1741d0e28a5e2e5d74b22322d8" + ), + "backup_disk_config_sha256": ( + "c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9" + ), + "isolated_cluster_config_sha256": ( + "e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b" + ), +} def _load_yaml(path: Path) -> Any: @@ -45,6 +81,27 @@ def _read(root: Path, relative: str) -> str: return (root / relative).read_text(encoding="utf-8") +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _current_toolchain_source_hashes(root: Path) -> dict[str, str]: + hashes: dict[str, str] = {} + for field, relative in TOOLCHAIN_SOURCE_PATHS.items(): + source = root / relative + if not source.is_file() or source.is_symlink(): + return {} + try: + hashes[field] = _sha256_file(source) + except OSError: + return {} + return hashes + + def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: contract = _load_yaml(contract_path) post_closure = contract.get("runtime_observations", {}).get( @@ -425,17 +482,61 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: native_restic = native_apply4.get("restic", {}) native_post = native_apply4.get("independent_post_verifier", {}) toolchain_apply = native_canary.get("toolchain_controlled_apply", {}) + historical_native_source = native_canary.get("source", {}) + current_native_source = native_canary.get("current_source", {}) + historical_source_hashes = { + field: historical_native_source.get(field) for field in TOOLCHAIN_SOURCE_PATHS + } + declared_current_hashes = current_native_source.get("hashes", {}) + actual_current_hashes = _current_toolchain_source_hashes(root) + actual_historical_drift = sorted( + field + for field in TOOLCHAIN_SOURCE_PATHS + if actual_current_hashes.get(field) != historical_source_hashes.get(field) + ) + declared_historical_drift = current_native_source.get( + "drift_from_historical_production_receipt", [] + ) + checks["native_backup_historical_source_receipt_preserved"] = ( + historical_native_source.get("evidence_scope") + == "historical_production_receipt" + and historical_native_source.get("deployment_status") == "deployed_verified" + and historical_source_hashes == HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES + ) + checks["native_backup_current_source_hashes_bound"] = ( + current_native_source.get("evidence_scope") == "committed_current_source" + and current_native_source.get("expected_file_count") + == len(TOOLCHAIN_SOURCE_PATHS) + and set(declared_current_hashes) == set(TOOLCHAIN_SOURCE_PATHS) + and declared_current_hashes == actual_current_hashes + and declared_historical_drift == actual_historical_drift + ) + pending_source_state = ( + current_native_source.get("status") + == "pending_toolchain_deploy_and_runtime_canary" + and current_native_source.get("deployment_status") == "pending" + and current_native_source.get("current_source_deployed") is False + and current_native_source.get("runtime_verifier_terminal") == "pending" + ) + deployed_source_state = ( + current_native_source.get("status") == "deployed_verified" + and current_native_source.get("deployment_status") == "deployed" + and current_native_source.get("current_source_deployed") is True + and current_native_source.get("runtime_verifier_terminal") == "pass" + and current_native_source.get("deployed_source_hashes") == actual_current_hashes + and bool(current_native_source.get("toolchain_apply_run_id")) + and bool(current_native_source.get("runtime_canary_run_id")) + ) + checks["native_backup_current_source_deployment_state_explicit"] = ( + current_native_source.get("historical_production_receipt_preserved") is True + and (pending_source_state or deployed_source_state) + ) checks["native_backup_restore_canary_closed"] = ( native_canary.get("completion_scope") == "clickhouse_native_only" - and native_canary.get("source", {}).get("native_backup_sha256") - == "754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a" - and native_canary.get("source", {}).get("isolated_restore_sha256") - == "57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7" - and native_canary.get("source", {}).get("post_canary_focused_tests_passed") - == 94 - and native_canary.get("source", {}).get("post_format_focused_tests_passed") - == 94 - and native_canary.get("source", {}).get("ruff_format") == "pass" + and checks["native_backup_historical_source_receipt_preserved"] + and historical_native_source.get("post_canary_focused_tests_passed") == 94 + and historical_native_source.get("post_format_focused_tests_passed") == 94 + and historical_native_source.get("ruff_format") == "pass" and toolchain_apply.get("apply_run_id") == "P0-OBS-002-native-toolchain-20260715T001252Z-apply4" and toolchain_apply.get("apply_terminal") == "pass" @@ -568,6 +669,9 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: resume_inventory = pending_verifiers.get( "clickhouse_native_resume_submitted_inventory", {} ) + current_source_toolchain = pending_verifiers.get( + "clickhouse_native_current_source_toolchain", {} + ) checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = ( native_migration.get("prior_run_id") == retry2.get("run_id") and native_migration.get("run_id") == native_apply4.get("run_id") @@ -595,6 +699,14 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == "pending_durable_submitted_inventory_contract" and resume_inventory.get("same_run_resume_safe") is False and len(resume_inventory.get("required_preconditions", [])) == 4 + and current_source_toolchain.get("status") + == current_native_source.get("status") + == "pending_toolchain_deploy_and_runtime_canary" + and current_source_toolchain.get("current_source_deployed") is False + and current_source_toolchain.get("runtime_verifier_terminal") == "pending" + and current_source_toolchain.get("historical_production_receipt_preserved") + is True + and len(current_source_toolchain.get("required_preconditions", [])) == 3 ) checks["post_closure_contract_mirror_consistent"] = ( @@ -621,6 +733,10 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == toolchain_apply.get("apply_run_id") and post_closure.get("clickhouse_native_backup_toolchain_postcheck_run_id") == toolchain_apply.get("postcheck_run_id") + and post_closure.get("clickhouse_native_current_source_status") + == current_native_source.get("status") + and post_closure.get("clickhouse_native_current_source_deployed") + == current_native_source.get("current_source_deployed") and post_closure.get("clickhouse_native_restore_terminal") == "verified" and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a" and post_closure.get("clickhouse_native_offsite_status") == "pending" @@ -684,10 +800,21 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: "phase": "wave_a_dual_allow", "source_ready": source_ready, "runtime_status": terminal.get("status", "unknown"), + "backup_toolchain_current_source_status": current_native_source.get( + "status", "unknown" + ), + "backup_toolchain_current_source_deployed": current_native_source.get( + "current_source_deployed", False + ), + "backup_toolchain_runtime_verifier_terminal": current_native_source.get( + "runtime_verifier_terminal", "unknown" + ), + "backup_toolchain_source_drift_fields": actual_historical_drift, "active_route": "incumbent", "writes_performed": 0, "checks": checks, "blockers": blockers, + "post_closure_blockers": list(regression_terminal.get("blockers", [])), "terminal": "pass" if source_ready else "failed", } diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index 831e82f3d..463fc9496 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import importlib.util import json import os import subprocess @@ -25,6 +27,16 @@ GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml" POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml" +def _load_preflight_module(): + spec = importlib.util.spec_from_file_location( + "signoz_canonical_route_preflight_under_test", PREFLIGHT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None: payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8")) assert ( @@ -86,11 +98,28 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: assert payload["writes_performed"] == 0 assert payload["terminal"] == "pass" assert payload["runtime_status"] == "partial_degraded" + assert ( + payload["backup_toolchain_current_source_status"] + == "pending_toolchain_deploy_and_runtime_canary" + ) + assert payload["backup_toolchain_current_source_deployed"] is False + assert payload["backup_toolchain_runtime_verifier_terminal"] == "pending" + assert payload["backup_toolchain_source_drift_fields"] == [ + "backup_orchestrator_sha256", + "canary_wrapper_sha256", + "inventory_verifier_sha256", + ] assert payload["active_route"] == "incumbent" assert payload["checks"] and all(payload["checks"].values()) assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True assert payload["checks"]["post_closure_schema_v2"] is True + assert payload["checks"]["native_backup_current_source_hashes_bound"] is True + assert ( + payload["checks"]["native_backup_current_source_deployment_state_explicit"] + is True + ) + assert payload["checks"]["native_backup_historical_source_receipt_preserved"] assert payload["checks"]["post_release_600s_verifier_consistent"] is True assert ( payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"] @@ -621,6 +650,69 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No assert canary["terminal"] == "production_closed_clickhouse_native_scope" +def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt() -> ( + None +): + module = _load_preflight_module() + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"] + historical = canary["source"] + current = canary["current_source"] + + historical_hashes = { + field: historical[field] for field in module.TOOLCHAIN_SOURCE_PATHS + } + assert historical["evidence_scope"] == "historical_production_receipt" + assert historical["deployment_status"] == "deployed_verified" + assert historical_hashes == module.HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES + assert historical["inventory_verifier_sha256"] == ( + "79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c" + ) + + current_hashes = { + field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() + for field, relative in module.TOOLCHAIN_SOURCE_PATHS.items() + } + assert current["hashes"] == current_hashes + assert current["hashes"]["inventory_verifier_sha256"] == ( + "c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863" + ) + assert current["status"] == "pending_toolchain_deploy_and_runtime_canary" + assert current["deployment_status"] == "pending" + assert current["current_source_deployed"] is False + assert current["runtime_verifier_terminal"] == "pending" + assert current["historical_production_receipt_preserved"] is True + assert current["drift_from_historical_production_receipt"] == [ + "backup_orchestrator_sha256", + "canary_wrapper_sha256", + "inventory_verifier_sha256", + ] + + +def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale( + monkeypatch, +) -> None: + module = _load_preflight_module() + real_sha256_file = module._sha256_file + + def stale_inventory_hash(path: Path) -> str: + if path.name == "clickhouse-restore-inventory.py": + return "0" * 64 + return real_sha256_file(path) + + monkeypatch.setattr(module, "_sha256_file", stale_inventory_hash) + payload = module.evaluate(ROOT, CONTRACT) + + assert payload["source_ready"] is False + assert payload["terminal"] == "failed" + assert payload["checks"]["native_backup_current_source_hashes_bound"] is False + assert payload["checks"]["native_backup_historical_source_receipt_preserved"] + assert ( + payload["backup_toolchain_current_source_status"] + == "pending_toolchain_deploy_and_runtime_canary" + ) + + def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None: receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) verifier = receipt["post_release_runtime_verifier"] @@ -667,6 +759,14 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> ] assert resume["same_run_resume_safe"] is False assert len(resume["required_preconditions"]) == 4 + current_source = receipt["pending_verifiers"][ + "clickhouse_native_current_source_toolchain" + ] + assert current_source["status"] == "pending_toolchain_deploy_and_runtime_canary" + assert current_source["current_source_deployed"] is False + assert current_source["runtime_verifier_terminal"] == "pending" + assert current_source["historical_production_receipt_preserved"] is True + assert len(current_source["required_preconditions"]) == 3 assets = receipt["asset_reconciliation"] assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001" assert assets["source_service_count"] == 27 @@ -697,6 +797,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> "signoz_backup_offsite_dr_pending", "clickhouse_native_failed_artifact_retention_pending", "clickhouse_native_resume_submitted_inventory_pending", + "clickhouse_native_current_source_toolchain_pending", "service_registry_runtime_mirror_reconciliation_pending", "github_freeze_legacy_asset_retirement_pending", "monitoring_generator_duplicate_awoooi_api_identity_pending", @@ -737,6 +838,10 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == ( "P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4" ) + assert post_closure["clickhouse_native_current_source_status"] == ( + "pending_toolchain_deploy_and_runtime_canary" + ) + assert post_closure["clickhouse_native_current_source_deployed"] is False assert post_closure["clickhouse_native_restore_terminal"] == "verified" assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a" assert post_closure["clickhouse_native_offsite_status"] == "pending"