Files
awoooi/scripts/backup/tests/test_signoz_backup_canary_wrapper.py

529 lines
17 KiB
Python

from __future__ import annotations
import json
import os
import stat
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh"
PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
ANSIBLE_VALIDATE = ROOT / "scripts" / "ops" / "ansible-validate.sh"
READINESS_AUDIT = (
ROOT / "scripts" / "reboot-recovery" / "reboot-recovery-readiness-audit.sh"
)
def _write_executable(path: Path, text: str) -> None:
path.write_text(text, encoding="utf-8")
path.chmod(0o755)
def _monitor_source(cooldown_dir: Path, *, omit_contract: bool = False) -> str:
cooldown_log = (
'log "COOLDOWN: ${container} skipped"'
if not omit_contract
else 'log "maintenance skip for ${container}"'
)
return f"""\
#!/bin/bash
: "${{ACTION_COOLDOWN_SECONDS:=${{SEND_COOLDOWN_SECONDS}}}}"
: "${{COOLDOWN_DIR:={cooldown_dir}}}"
: "${{EXCLUDE_CONTAINERS:=signoz-clickhouse}}"
is_in_cooldown() {{
local container="$1"
local cooldown_file="${{COOLDOWN_DIR}}/${{container}}.cooldown"
local last_sent now elapsed
last_sent=$(cat "$cooldown_file")
now=$(date +%s)
elapsed=$(( now - last_sent ))
if (( elapsed < ACTION_COOLDOWN_SECONDS )); then
{cooldown_log}
return 0
fi
return 1
}}
set_cooldown() {{ :; }}
# A second monitor path may use the same elapsed expression. The wrapper must
# validate the expression inside is_in_cooldown, not require global uniqueness.
# elapsed=$(( now - last_sent ))
while read -r container_name; do
is_in_cooldown "$container_name" && continue
set_cooldown "$container_name"
container="$container_name"
log "AUTO_REPAIR: docker restart ${{container}}"
if docker restart "$container"; then
:
fi
done
"""
def _install_fake_commands(fake_bin: Path) -> None:
_write_executable(
fake_bin / "crontab",
"""\
#!/bin/bash
set -eu
[ "${1:-}" = "-l" ]
printf '*/5 * * * * %s >> %s 2>&1\n' \
"${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
if [ "${FAKE_DUPLICATE_CRON:-0}" = "1" ]; then
printf '*/5 * * * * %s >> %s 2>&1\n' \
"${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
fi
""",
)
_write_executable(
fake_bin / "docker",
"""\
#!/bin/bash
set -eu
[ "${1:-}" = "inspect" ] || exit 70
case " $* " in
*"{{.State.StartedAt}}"*)
printf 'sha256:collector-test-id\\t%s\\t2026-07-15T00:00:00Z\\t0\\thealthy\\n' \
"$(cat "${TEST_COLLECTOR_STATE:?}")"
;;
*"{{.Id}}"*) printf 'sha256:collector-test-id\n' ;;
*"{{.State.Running}}"*) cat "${TEST_COLLECTOR_STATE:?}" ;;
*) exit 71 ;;
esac
""",
)
_write_executable(
fake_bin / "ss",
"""\
#!/bin/bash
set -eu
while IFS= read -r port; do
[ -n "$port" ] || continue
printf 'LISTEN 0 4096 *:%s *:*\n' "$port"
done < "${TEST_LISTENERS:?}"
""",
)
_write_executable(
fake_bin / "timeout",
"""\
#!/bin/bash
set -eu
while [[ "${1:-}" == --kill-after=* ]]; do
shift
done
[ "$#" -gt 1 ]
shift
exec "$@"
""",
)
_write_executable(
fake_bin / "flock",
"""\
#!/bin/bash
exit 0
""",
)
_write_executable(
fake_bin / "pgrep",
"""\
#!/bin/bash
exit 1
""",
)
_write_executable(
fake_bin / "stat",
"""\
#!/usr/bin/env python3
import os
import stat
import sys
if len(sys.argv) != 4 or sys.argv[1] != "-c":
raise SystemExit(64)
value = os.stat(sys.argv[3])
formats = {
"%a": format(stat.S_IMODE(value.st_mode), "o"),
"%u": str(value.st_uid),
"%g": str(value.st_gid),
"%s": str(value.st_size),
"%d:%i": f"{value.st_dev}:{value.st_ino}",
}
if sys.argv[2] not in formats:
raise SystemExit(65)
print(formats[sys.argv[2]])
""",
)
_write_executable(
fake_bin / "sha256sum",
"""\
#!/usr/bin/env python3
import hashlib
import sys
if len(sys.argv) != 2:
raise SystemExit(64)
with open(sys.argv[1], "rb") as source:
digest = hashlib.sha256(source.read()).hexdigest()
print(f"{digest} {sys.argv[1]}")
""",
)
def _backup_source() -> str:
return """\
#!/bin/bash
set -eu
[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80
[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ]
printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
> "${TEST_BACKUP_IDENTITIES:?}"
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}"
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
case "${FAKE_MONITOR_MODE:-cooldown}" in
cooldown)
printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
>> "${TEST_MONITOR_LOG:?}"
printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
>> "${TEST_MONITOR_LOG:?}"
;;
auto_repair)
printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
>> "${TEST_MONITOR_LOG:?}"
printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
>> "${TEST_MONITOR_LOG:?}"
printf '[test] AUTO_REPAIR: docker restart signoz-otel-collector\n' \
>> "${TEST_MONITOR_LOG:?}"
;;
none) : ;;
*) exit 81 ;;
esac
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
else
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
fi
if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then
printf '4317\n' > "${TEST_LISTENERS:?}"
fi
printf '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)\n'
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
printf '[SUCCESS] ========== SignOz 備份完成 (1s) ==========\n'
else
printf '[SUCCESS] ========== SigNoz ClickHouse native 備份完成 (1s) ==========\n'
fi
"""
def _run_canary(
tmp_path: Path,
*,
mode: str = "apply",
prior_lease: bool = True,
monitor_mode: str = "cooldown",
backup_sleep_seconds: float = 0.2,
monitor_interval_seconds: int = 300,
omit_monitor_contract: bool = False,
duplicate_cron: bool = False,
drop_listener: bool = False,
expect_collector_stop: bool = True,
) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]:
fake_bin = tmp_path / "bin"
cooldown_dir = tmp_path / "cooldown"
receipt_root = tmp_path / "receipts"
fake_bin.mkdir()
cooldown_dir.mkdir()
receipt_root.mkdir()
monitor_script = tmp_path / "docker-health-monitor.sh"
monitor_log = tmp_path / "monitor.log"
backup_script = tmp_path / "backup-signoz.sh"
collector_state = tmp_path / "collector.state"
listeners = tmp_path / "listeners"
identities = tmp_path / "backup-identities.tsv"
lease_observed = tmp_path / "lease-observed.txt"
cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown"
run_id = f"canary-{tmp_path.name}"
_write_executable(
monitor_script,
_monitor_source(cooldown_dir, omit_contract=omit_monitor_contract),
)
_write_executable(backup_script, _backup_source())
_install_fake_commands(fake_bin)
monitor_log.write_text("[test] monitor seed\n", encoding="utf-8")
collector_state.write_text("true\n", encoding="utf-8")
listeners.write_text("4317\n4318\n", encoding="utf-8")
if prior_lease:
cooldown_file.write_text("123\n", encoding="utf-8")
cooldown_file.chmod(0o640)
env = {
**os.environ,
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"TRACE_ID": "trace-P0-OBS-002",
"RUN_ID": run_id,
"WORK_ITEM_ID": "P0-OBS-002",
"SIGNOZ_CANARY_BACKUP_SCRIPT": str(backup_script),
"SIGNOZ_CANARY_MONITOR_SCRIPT": str(monitor_script),
"SIGNOZ_CANARY_MONITOR_LOG": str(monitor_log),
"SIGNOZ_CANARY_COOLDOWN_DIR": str(cooldown_dir),
"SIGNOZ_CANARY_RECEIPT_ROOT": str(receipt_root),
"SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"),
"SIGNOZ_CANARY_TIMEOUT_SECONDS": "5",
"SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2",
"SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2",
"SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds),
"SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05",
"SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"),
"TEST_MONITOR_SCRIPT": str(monitor_script),
"TEST_MONITOR_LOG": str(monitor_log),
"TEST_COLLECTOR_STATE": str(collector_state),
"TEST_LISTENERS": str(listeners),
"TEST_BACKUP_IDENTITIES": str(identities),
"TEST_COOLDOWN_FILE": str(cooldown_file),
"TEST_LEASE_OBSERVED": str(lease_observed),
"FAKE_MONITOR_MODE": monitor_mode,
"FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds),
"FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0",
"FAKE_DROP_LISTENER": "1" if drop_listener else "0",
"FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0",
}
started_at = int(time.time())
result = subprocess.run(
["bash", str(WRAPPER), f"--{mode}"],
env=env,
text=True,
capture_output=True,
timeout=15,
check=False,
)
paths = {
"cooldown": cooldown_file,
"receipts": receipt_root / run_id / "receipts.jsonl",
"rollback_metadata": receipt_root / run_id / "cooldown.rollback.json",
"identities": identities,
"lease_observed": lease_observed,
"collector_state": collector_state,
"listeners": listeners,
}
return result, paths, str(started_at)
def _receipts(path: Path) -> list[dict[str, object]]:
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
def _terminal(receipts: list[dict[str, object]], stage: str) -> str:
return str([item for item in receipts if item["stage"] == stage][-1]["terminal"])
def test_existing_lease_is_atomically_replaced_and_exactly_restored(
tmp_path: Path,
) -> None:
result, paths, started_at = _run_canary(tmp_path)
assert result.returncode == 0, result.stdout + result.stderr
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
assert int(paths["lease_observed"].read_text(encoding="utf-8")) >= (
int(started_at) + 7
)
identities = paths["identities"].read_text(encoding="utf-8").split("\t")
assert identities[0] == "trace-P0-OBS-002"
assert identities[1].startswith("canary-")
assert identities[2] == "P0-OBS-002\n"
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "pass"
metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8"))
assert metadata["trace_id"] == "trace-P0-OBS-002"
assert metadata["prior_present"] == 1
stages = [str(item["stage"]) for item in receipts]
ordered = [
"sensor_source",
"normalized_asset_identity",
"source_of_truth_diff",
"ai_decision",
"risk_policy",
"check",
"execution",
"post_verifier",
"rollback",
"closure_writeback",
"terminal",
]
assert [stages.index(stage) for stage in ordered] == sorted(
stages.index(stage) for stage in ordered
)
def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None:
result, paths, _ = _run_canary(tmp_path, prior_lease=False)
assert result.returncode == 0, result.stdout + result.stderr
assert not paths["cooldown"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "pass"
def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
tmp_path: Path,
) -> None:
result, paths, _ = _run_canary(tmp_path, expect_collector_stop=False)
assert result.returncode == 0, result.stdout + result.stderr
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
assert not paths["lease_observed"].exists()
assert paths["collector_state"].read_text(encoding="utf-8") == "true\n"
receipts = _receipts(paths["receipts"])
assert any(
item["stage"] == "lease" and item["terminal"] == "not_required"
for item in receipts
)
assert _terminal(receipts, "rollback") == "no_write"
assert _terminal(receipts, "terminal") == "pass"
def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
tmp_path: Path,
) -> None:
result, paths, _ = _run_canary(
tmp_path,
expect_collector_stop=False,
omit_monitor_contract=True,
duplicate_cron=True,
)
assert result.returncode == 0, result.stdout + result.stderr
assert not paths["lease_observed"].exists()
receipts = _receipts(paths["receipts"])
source_diff = [
item for item in receipts if item["stage"] == "source_of_truth_diff"
][-1]
assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"]
assert _terminal(receipts, "terminal") == "pass"
def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None:
result, paths, _ = _run_canary(tmp_path, monitor_mode="auto_repair")
assert result.returncode != 0
assert "monitor_auto_repair_contaminated_canary" in result.stderr
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "failed"
def test_cron_crossing_requires_monitor_cooldown_receipt(tmp_path: Path) -> None:
result, paths, _ = _run_canary(
tmp_path,
monitor_mode="none",
backup_sleep_seconds=1.2,
monitor_interval_seconds=1,
)
assert result.returncode != 0
assert "cron_crossed_without_collector_cooldown_receipt" in result.stderr
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
def test_cron_crossing_with_detected_and_cooldown_receipts_passes(
tmp_path: Path,
) -> None:
result, paths, _ = _run_canary(
tmp_path,
monitor_mode="cooldown",
backup_sleep_seconds=1.2,
monitor_interval_seconds=1,
)
assert result.returncode == 0, result.stdout + result.stderr
receipts = _receipts(paths["receipts"])
monitor_receipt = [
item for item in receipts if item["stage"] == "monitor_verifier"
][-1]
assert monitor_receipt["terminal"] == "pass"
assert "cron_crossed_1" in str(monitor_receipt["detail"])
def test_missing_monitor_contract_fails_before_lease_or_backup(tmp_path: Path) -> None:
result, paths, _ = _run_canary(tmp_path, omit_monitor_contract=True)
assert result.returncode != 0
assert "monitor_contract_ambiguous_fragment_count_0" in result.stderr
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
assert not paths["lease_observed"].exists()
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "not_armed"
assert _terminal(receipts, "terminal") == "failed"
def test_ambiguous_cron_contract_fails_before_lease_or_backup(
tmp_path: Path,
) -> None:
result, paths, _ = _run_canary(tmp_path, duplicate_cron=True)
assert result.returncode != 0
assert "monitor_cron_contract_ambiguous_count_2" in result.stderr
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
assert not paths["lease_observed"].exists()
def test_post_backup_listener_failure_fails_and_restores_lease(
tmp_path: Path,
) -> None:
result, paths, _ = _run_canary(tmp_path, drop_listener=True)
assert result.returncode != 0
assert "collector_post_backup_runtime_failed" in result.stderr
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
receipts = _receipts(paths["receipts"])
assert _terminal(receipts, "rollback") == "restored"
assert _terminal(receipts, "terminal") == "failed"
def test_wrapper_deployment_and_static_contracts_are_owned() -> None:
wrapper = WRAPPER.read_text(encoding="utf-8")
assert "BACKUP_SKIP_RETENTION_CLEANUP=1" in wrapper
assert 'TRACE_ID="${TRACE_ID:-}"' in wrapper
assert 'RUN_ID="${RUN_ID:-}"' in wrapper
assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper
assert "monitor_auto_repair_contaminated_canary" in wrapper
assert "collector_post_backup_runtime_failed" in wrapper
assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8")
assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text(
encoding="utf-8"
)
assert str(WRAPPER.relative_to(ROOT)) in READINESS_AUDIT.read_text(encoding="utf-8")
def test_check_mode_proves_contract_without_lease_or_backup_write(
tmp_path: Path,
) -> None:
result, paths, _ = _run_canary(tmp_path, mode="check")
assert result.returncode == 0, result.stdout + result.stderr
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
assert not paths["identities"].exists()
assert not paths["lease_observed"].exists()
check_receipts = _receipts(paths["receipts"])
assert _terminal(check_receipts, "rollback") == "no_write"
assert _terminal(check_receipts, "terminal") == "check_pass_no_write"
def test_wrapper_is_valid_bash() -> None:
result = subprocess.run(
["bash", "-n", str(WRAPPER)],
text=True,
capture_output=True,
check=False,
)
assert result.returncode == 0, result.stderr