wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(observability): harden native backup controlled apply

Browse Source
This commit is contained in:
ogt
2026-07-15 09:43:40 +08:00
parent 322e812d93
commit 032be5f400
12 changed files with 1849 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
scripts/backup/tests/test_backup_signoz_collector_restore.py
View File

@@ -25,7 +25,9 @@ def _run_backup(
native_apply_status: int = 0,
continuity_mode: str = "stable",
restic_backup_status: int = 0,
skip_retention: bool = False,
skip_retention: bool = True,
operation_lock_status: int = 0,
workspace_root_symlink: bool = False,
) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
harness = tmp_path / "backup"
fake_bin = tmp_path / "bin"
@@ -33,6 +35,7 @@ def _run_backup(
event_log = tmp_path / "events.log"
collector_id = tmp_path / "collector.id"
collector_state = tmp_path / "collector.state"
workspace_root = tmp_path / "workspace-root"
harness.mkdir()
fake_bin.mkdir()
(backup_base / "signoz" / "data").mkdir(parents=True)
@@ -43,6 +46,12 @@ def _run_backup(
"true\n" if initial_running else "false\n",
encoding="utf-8",
)
if workspace_root_symlink:
workspace_target = tmp_path / "workspace-target"
workspace_target.mkdir()
workspace_root.symlink_to(workspace_target, target_is_directory=True)
else:
workspace_root.mkdir()
(harness / "common.sh").write_text(
"""\
@@ -139,6 +148,15 @@ case " $* " in
printf '[{"short_id":"native123"}]\\n'
;;
esac
""",
)
_write_executable(
fake_bin / "flock",
"""\
#!/bin/bash
set -eu
printf 'flock %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
exit "${FAKE_OPERATION_LOCK_STATUS:-0}"
""",
)
@@ -161,6 +179,9 @@ esac
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
"TIMEOUT_KILL_AFTER_SECONDS": "1",
"BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0",
"SIGNOZ_BACKUP_OPERATION_LOCK_FILE": str(tmp_path / "operation.lock"),
"FAKE_OPERATION_LOCK_STATUS": str(operation_lock_status),
"SIGNOZ_BACKUP_TMP_ROOT": str(workspace_root),
}
process = subprocess.Popen(
["bash", str(harness / SCRIPT.name)],
@@ -177,7 +198,13 @@ esac
stderr,
)
events = event_log.read_text(encoding="utf-8").splitlines()
return result, events, Path(f"/tmp/signoz-backup-{process.pid}")
workspace_events = _events(events, "info 建立 SigNoz private workspace: ")
dump_dir = (
Path(workspace_events[-1].split(": ", 1)[1])
if workspace_events
else tmp_path / "workspace-not-created"
)
return result, events, dump_dir
def _events(events: list[str], prefix: str) -> list[str]:
@@ -201,6 +228,12 @@ def test_source_sunsets_raw_volumes_and_collector_mutation() -> None:
assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
assert 'notify_clawbot "success"' not in source
assert "no-downtime continuity verifier" in source
assert "awoooi-signoz-backup-operation.lock" in source
assert "flock -n 8" in source
assert "cleanup_old_backups" not in source
assert 'mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX"' in source
assert "/tmp/signoz-backup-$$" not in source
assert "WORKSPACE_CREATED=1" in source
def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
@@ -233,6 +266,32 @@ def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
assert not dump_dir.exists()
def test_operation_lock_contention_fails_before_native_or_restic(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, operation_lock_status=1)
assert result.returncode != 0
assert _events(events, "flock -n 8")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert len(_events(events, "notify failed signoz ")) == 1
assert not dump_dir.exists()
def test_symlink_workspace_root_fails_before_native_or_restic(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, workspace_root_symlink=True)
assert result.returncode != 0
assert _events(events, "flock -n 8")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert any("workspace root" in event for event in events)
assert not dump_dir.exists()
def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None:
result, events, dump_dir = _run_backup(tmp_path, native_check_status=19)
@@ -307,6 +366,20 @@ def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None:
assert not _events(events, "retention ")
def test_inline_destructive_retention_request_fails_before_backup(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, skip_retention=False)
assert result.returncode != 0
assert not _events(events, "flock ")
assert not _events(events, "native ")
assert not _events(events, "restic ")
assert not _events(events, "retention ")
assert any("critical break-glass workflow" in event for event in events)
assert not dump_dir.exists()
def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None:
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")

303
scripts/backup/tests/test_signoz_backup_canary_wrapper.py
View File

@@ -7,6 +7,8 @@ import subprocess
import time
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[3]
WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh"
@@ -115,6 +117,9 @@ while [[ "${1:-}" == --kill-after=* ]]; do
done
[ "$#" -gt 1 ]
shift
if [ "${FAKE_DOCKER_TIMEOUT:-0}" = "1" ] && [ "${1:-}" = "docker" ]; then
exit 124
fi
exec "$@"
""",
)
@@ -122,6 +127,9 @@ exec "$@"
fake_bin / "flock",
"""\
#!/bin/bash
if [ -n "${TEST_FLOCK_MARKER:-}" ]; then
printf 'held\n' > "${TEST_FLOCK_MARKER}"
fi
exit 0
""",
)
@@ -171,14 +179,103 @@ print(f"{digest} {sys.argv[1]}")
)
def _restore_hook_source() -> str:
return """\
#!/bin/bash
set -eu
[ "$#" -eq 1 ]
case "$1" in
--check) mode="check" ;;
--apply) mode="apply" ;;
*) exit 64 ;;
esac
receipt_dir="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:?}/${RUN_ID:?}"
mkdir -p "$receipt_dir"
receipt_log="$receipt_dir/receipts.jsonl"
identity_run="$RUN_ID"
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "forged_identity" ]; then
identity_run="stale-$RUN_ID"
fi
invocation_id="${mode}-fake-restore-$$"
emit() {
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","stage":"%s","terminal":"%s","detail":"fake"}\n' \
"$TRACE_ID" "$identity_run" "$WORK_ITEM_ID" "$invocation_id" "$1" "$2" \
>> "$receipt_log"
}
if [ "$mode" = "check" ]; then
emit check check_pass_no_runtime_write
emit cleanup pass
emit terminal check_pass_no_runtime_write
exit 0
fi
emit check pass
emit independent_post_verifier pass
emit cleanup pass
emit terminal verified
if [ "${FAKE_RECEIPT_FAULT:-valid}" != "missing_status" ]; then
cat > "$receipt_dir/status.json" <<EOF
{
"schema_version":"clickhouse_native_restore_drill_status_v1",
"trace_id":"$TRACE_ID",
"run_id":"$identity_run",
"work_item_id":"$WORK_ITEM_ID",
"mode":"apply",
"terminal":"verified",
"exit_code":0,
"network_mode":"docker_internal_only",
"staging_network_mode":"none",
"production_network_attached":false,
"production_restore_performed":false,
"allow_non_empty_tables":false,
"isolated_keeper":true,
"artifact_staging_verified":true,
"clickhouse_artifact_readback_verified":true,
"ephemeral_runtime_created":true,
"check_pass":true,
"apply_pass":true,
"restore_terminal":"RESTORED",
"manifest_parity":1,
"source_database_count":6,
"restored_database_count":6,
"source_table_count":100,
"restored_table_count":100,
"check_table_count":44,
"check_table_pass_count":44,
"critical_nonzero_count":4,
"cleanup_verified":true
}
EOF
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "symlink_status" ]; then
mv "$receipt_dir/status.json" "$receipt_dir/status-target.json"
ln -s "status-target.json" "$receipt_dir/status.json"
fi
fi
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "malformed_restore_receipt" ]; then
printf '{malformed-json\n' >> "$receipt_log"
fi
"""
def _backup_source() -> str:
return """\
#!/bin/bash
set -eu
[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80
[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ]
[ "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}" = "${TEST_RESTORE_HOOK:?}" ] || exit 82
[ "${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-}" = "${TEST_INVENTORY_HELPER:?}" ] || exit 83
[ "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-}" = "${TEST_BACKUP_DISK_CONFIG:?}" ] || exit 84
[ "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-}" = "${TEST_CLUSTER_CONFIG:?}" ] || exit 85
[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-}" = "${TEST_NATIVE_RECEIPT_ROOT:?}" ] || exit 86
[ "${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-}" = "${TEST_RESTORE_RECEIPT_ROOT:?}" ] || exit 87
printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
> "${TEST_BACKUP_IDENTITIES:?}"
printf '%s\t%s\t%s\t%s\n' \
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" \
"$CLICKHOUSE_RESTORE_INVENTORY_HELPER" \
"$CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" \
"$CLICKHOUSE_RESTORE_CLUSTER_CONFIG" \
> "${TEST_RESTORE_CONTRACT:?}"
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}"
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
@@ -204,6 +301,23 @@ if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
else
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
native_dir="$CLICKHOUSE_NATIVE_RECEIPT_ROOT/$RUN_ID"
mkdir -p "$native_dir"
native_receipts="$native_dir/receipts.jsonl"
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"check-fake-native","stage":"terminal","terminal":"check_pass_no_write","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
if [ "${FAKE_RECEIPT_FAULT:-valid}" != "no_hook" ]; then
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --check
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_check_exit_0_stdout_fake_stderr_fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --apply
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_exit_0_stdout_fake_stderr_fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
fi
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"independent_verifier","terminal":"pass","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"terminal","terminal":"pass","detail":"fake"}\n' \
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
fi
if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then
printf '4317\n' > "${TEST_LISTENERS:?}"
@@ -229,13 +343,30 @@ def _run_canary(
duplicate_cron: bool = False,
drop_listener: bool = False,
expect_collector_stop: bool = True,
native_restore_contract: str = "valid",
native_receipt_state: str = "valid",
docker_timeout: bool = False,
) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]:
if native_receipt_state not in {
"valid",
"no_hook",
"forged_identity",
"missing_status",
"symlink_status",
"malformed_restore_receipt",
}:
raise ValueError(f"unsupported native receipt state: {native_receipt_state}")
fake_bin = tmp_path / "bin"
cooldown_dir = tmp_path / "cooldown"
receipt_root = tmp_path / "receipts"
native_receipt_root = tmp_path / "native-receipts"
restore_receipt_root = tmp_path / "restore-receipts"
fake_bin.mkdir()
cooldown_dir.mkdir()
receipt_root.mkdir()
native_receipt_root.mkdir()
restore_receipt_root.mkdir()
monitor_script = tmp_path / "docker-health-monitor.sh"
monitor_log = tmp_path / "monitor.log"
@@ -244,6 +375,12 @@ def _run_canary(
listeners = tmp_path / "listeners"
identities = tmp_path / "backup-identities.tsv"
lease_observed = tmp_path / "lease-observed.txt"
restore_contract_observed = tmp_path / "restore-contract.tsv"
flock_marker = tmp_path / "canary-lock-held.txt"
restore_hook = tmp_path / "clickhouse-native-restore-drill.sh"
inventory_helper = tmp_path / "clickhouse-restore-inventory.py"
backup_disk_config = tmp_path / "backup_disk.xml"
cluster_config = tmp_path / "isolated-cluster.xml"
cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown"
run_id = f"canary-{tmp_path.name}"
@@ -252,6 +389,19 @@ def _run_canary(
_monitor_source(cooldown_dir, omit_contract=omit_monitor_contract),
)
_write_executable(backup_script, _backup_source())
if native_restore_contract == "symlink":
restore_hook_target = tmp_path / "restore-hook-target.sh"
_write_executable(restore_hook_target, _restore_hook_source())
restore_hook.symlink_to(restore_hook_target)
elif native_restore_contract == "valid":
_write_executable(restore_hook, _restore_hook_source())
elif native_restore_contract != "missing":
raise ValueError(
f"unsupported native restore contract: {native_restore_contract}"
)
_write_executable(inventory_helper, "#!/bin/sh\nexit 0\n")
backup_disk_config.write_text("<clickhouse/>\n", encoding="utf-8")
cluster_config.write_text("<clickhouse/>\n", encoding="utf-8")
_install_fake_commands(fake_bin)
monitor_log.write_text("[test] monitor seed\n", encoding="utf-8")
collector_state.write_text("true\n", encoding="utf-8")
@@ -274,10 +424,17 @@ def _run_canary(
"SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"),
"SIGNOZ_CANARY_TIMEOUT_SECONDS": "5",
"SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2",
"SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS": "2",
"SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2",
"SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds),
"SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05",
"SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"),
"CLICKHOUSE_NATIVE_POST_VERIFY_HOOK": str(restore_hook),
"CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(inventory_helper),
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_disk_config),
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config),
"CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(native_receipt_root),
"CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(restore_receipt_root),
"TEST_MONITOR_SCRIPT": str(monitor_script),
"TEST_MONITOR_LOG": str(monitor_log),
"TEST_COLLECTOR_STATE": str(collector_state),
@@ -285,11 +442,21 @@ def _run_canary(
"TEST_BACKUP_IDENTITIES": str(identities),
"TEST_COOLDOWN_FILE": str(cooldown_file),
"TEST_LEASE_OBSERVED": str(lease_observed),
"TEST_RESTORE_CONTRACT": str(restore_contract_observed),
"TEST_RESTORE_HOOK": str(restore_hook),
"TEST_INVENTORY_HELPER": str(inventory_helper),
"TEST_BACKUP_DISK_CONFIG": str(backup_disk_config),
"TEST_CLUSTER_CONFIG": str(cluster_config),
"TEST_NATIVE_RECEIPT_ROOT": str(native_receipt_root),
"TEST_RESTORE_RECEIPT_ROOT": str(restore_receipt_root),
"TEST_FLOCK_MARKER": str(flock_marker),
"FAKE_MONITOR_MODE": monitor_mode,
"FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds),
"FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0",
"FAKE_DROP_LISTENER": "1" if drop_listener else "0",
"FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0",
"FAKE_RECEIPT_FAULT": native_receipt_state,
"FAKE_DOCKER_TIMEOUT": "1" if docker_timeout else "0",
}
started_at = int(time.time())
result = subprocess.run(
@@ -308,6 +475,13 @@ def _run_canary(
"lease_observed": lease_observed,
"collector_state": collector_state,
"listeners": listeners,
"restore_contract": restore_contract_observed,
"restore_hook": restore_hook,
"flock_marker": flock_marker,
"native_receipts": native_receipt_root / run_id / "receipts.jsonl",
"restore_receipts": restore_receipt_root / run_id / "receipts.jsonl",
"restore_status": restore_receipt_root / run_id / "status.json",
"backup_output": receipt_root / run_id / "backup-output.log",
}
return result, paths, str(started_at)
@@ -337,7 +511,9 @@ def test_existing_lease_is_atomically_replaced_and_exactly_restored(
assert identities[2] == "P0-OBS-002\n"
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "runtime_operation") == "pass"
assert _terminal(receipts, "closure_writeback") == "partial_degraded"
assert _terminal(receipts, "terminal") == "partial_degraded"
metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8"))
assert metadata["trace_id"] == "trace-P0-OBS-002"
assert metadata["prior_present"] == 1
@@ -368,7 +544,7 @@ def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None:
assert not paths["cooldown"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
@@ -380,13 +556,30 @@ def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
assert not paths["lease_observed"].exists()
assert paths["collector_state"].read_text(encoding="utf-8") == "true\n"
assert paths["restore_contract"].read_text(encoding="utf-8").split("\t") == [
str(paths["restore_hook"]),
str(tmp_path / "clickhouse-restore-inventory.py"),
str(tmp_path / "backup_disk.xml"),
f"{tmp_path / 'isolated-cluster.xml'}\n",
]
receipts = _receipts(paths["receipts"])
assert paths["native_receipts"].is_file()
assert paths["restore_receipts"].is_file()
restore_status = json.loads(paths["restore_status"].read_text(encoding="utf-8"))
assert restore_status["terminal"] == "verified"
assert restore_status["check_table_count"] == 44
assert restore_status["check_table_pass_count"] == 44
assert any(
item["stage"] == "native_restore_receipt_verifier"
and item["terminal"] == "pass"
for item in receipts
)
assert any(
item["stage"] == "lease" and item["terminal"] == "not_required"
for item in receipts
)
assert _terminal(receipts, "rollback") == "no_write"
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
@@ -406,7 +599,7 @@ def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
item for item in receipts if item["stage"] == "source_of_truth_diff"
][-1]
assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"]
assert _terminal(receipts, "terminal") == "pass"
assert _terminal(receipts, "terminal") == "partial_degraded"
def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None:
@@ -496,6 +689,34 @@ def test_wrapper_deployment_and_static_contracts_are_owned() -> None:
assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper
assert "monitor_auto_repair_contaminated_canary" in wrapper
assert "collector_post_backup_runtime_failed" in wrapper
assert "/backup/scripts/clickhouse-native-restore-drill.sh" in wrapper
assert "/backup/scripts/clickhouse-restore-inventory.py" in wrapper
assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in wrapper
assert (
"/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
in wrapper
)
assert (
'CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}"'
in wrapper
)
assert (
'CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}"' in wrapper
)
assert (
'CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}"'
in wrapper
)
main_source = wrapper[wrapper.index("main() {") :]
assert main_source.index("acquire_canary_lock") < main_source.index(
"require_native_restore_contract"
)
assert "same_run_native_restore_receipt_validation_failed" in wrapper
assert '"check_table_count"' in wrapper
assert "mandatory_isolated_restore_verifier" in wrapper
assert "SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS" in wrapper
assert 'emit_receipt "runtime_operation" "pass"' in wrapper
assert 'emit_receipt "closure_writeback" "partial_degraded"' in wrapper
assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8")
assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text(
encoding="utf-8"
@@ -518,6 +739,80 @@ def test_check_mode_proves_contract_without_lease_or_backup_write(
assert _terminal(check_receipts, "terminal") == "check_pass_no_write"
@pytest.mark.parametrize("contract_state", ["missing", "symlink"])
def test_native_check_fails_closed_for_unsafe_restore_contract(
tmp_path: Path,
contract_state: str,
) -> None:
result, paths, _ = _run_canary(
tmp_path,
mode="check",
expect_collector_stop=False,
native_restore_contract=contract_state,
)
assert result.returncode != 0
assert "native_restore_contract_CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" in result.stderr
assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n"
assert not paths["identities"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "terminal") == "failed"
@pytest.mark.parametrize(
"receipt_state",
[
"no_hook",
"forged_identity",
"missing_status",
"symlink_status",
"malformed_restore_receipt",
],
)
def test_native_apply_rejects_missing_or_forged_machine_receipts(
tmp_path: Path,
receipt_state: str,
) -> None:
result, paths, _ = _run_canary(
tmp_path,
expect_collector_stop=False,
native_receipt_state=receipt_state,
)
assert result.returncode != 0
assert "========== SigNoz ClickHouse native 備份完成 (1s) ==========" in paths[
"backup_output"
].read_text(encoding="utf-8")
assert (
"same_run_native_restore_receipt_validation_failed" in result.stderr
or "same_run_native_restore_receipt_missing_unreadable_or_symlink"
in result.stderr
)
assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n"
receipts = _receipts(paths["receipts"])
assert not any(
item["stage"] == "native_restore_receipt_verifier"
and item["terminal"] == "pass"
for item in receipts
)
assert _terminal(receipts, "terminal") == "failed"
def test_docker_metadata_timeout_fails_before_backup(tmp_path: Path) -> None:
result, paths, _ = _run_canary(
tmp_path,
mode="check",
expect_collector_stop=False,
docker_timeout=True,
)
assert result.returncode != 0
assert "collector_runtime_metadata_unreadable" in result.stderr
assert not paths["identities"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "terminal") == "failed"
def test_wrapper_is_valid_bash() -> None:
result = subprocess.run(
["bash", "-n", str(WRAPPER)],