fix(ops): bound CD DB probes and restore SigNoz collector

scripts/backup/tests/test_backup_signoz_collector_restore.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
+
+
+def _write_executable(path: Path, text: str) -> None:
+    path.write_text(text, encoding="utf-8")
+    path.chmod(0o755)
+
+
+def _run_backup(
+    tmp_path: Path,
+    *,
+    docker_run_mode: str = "success",
+    docker_start_fail_once: bool = False,
+    restic_backup_status: int = 0,
+) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
+    harness = tmp_path / "backup"
+    fake_bin = tmp_path / "bin"
+    backup_base = tmp_path / "repository"
+    docker_log = tmp_path / "docker.log"
+    harness.mkdir()
+    fake_bin.mkdir()
+    (backup_base / "signoz" / "data").mkdir(parents=True)
+    shutil.copy2(SCRIPT, harness / SCRIPT.name)
+
+    (harness / "common.sh").write_text(
+        """\
+BACKUP_BASE="${TEST_BACKUP_BASE:?}"
+RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password"
+log_info() { :; }
+log_warn() { :; }
+log_error() { :; }
+log_success() { :; }
+notify_clawbot() { :; }
+build_tags() { :; }
+cleanup_old_backups() { :; }
+""",
+        encoding="utf-8",
+    )
+
+    _write_executable(
+        fake_bin / "docker",
+        """\
+#!/bin/bash
+set -eu
+printf '%s\n' "$*" >> "${TEST_DOCKER_LOG:?}"
+case "${1:-}" in
+    stop)
+        exit 0
+        ;;
+    start)
+        if [ "${FAKE_DOCKER_START_FAIL_ONCE:-0}" = "1" ] && \
+           [ ! -e "${TEST_DOCKER_START_MARKER:?}" ]; then
+            : > "${TEST_DOCKER_START_MARKER}"
+            exit 1
+        fi
+        exit 0
+        ;;
+    run)
+        if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then
+            kill -TERM "${PPID}"
+            sleep 0.1
+            exit 0
+        fi
+        if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "empty" ]; then
+            exit 0
+        fi
+        printf 'archive-data\n'
+        ;;
+esac
+""",
+    )
+    _write_executable(
+        fake_bin / "restic",
+        """\
+#!/bin/bash
+set -eu
+case " $* " in
+    *" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
+    *" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
+esac
+""",
+    )
+    _write_executable(
+        fake_bin / "du",
+        """\
+#!/bin/bash
+printf '4K\t%s\n' "${2:-unknown}"
+""",
+    )
+
+    env = {
+        **os.environ,
+        "PATH": f"{fake_bin}:{os.environ['PATH']}",
+        "TEST_BACKUP_BASE": str(backup_base),
+        "TEST_DOCKER_LOG": str(docker_log),
+        "TEST_DOCKER_START_MARKER": str(tmp_path / "docker-start-failed-once"),
+        "FAKE_DOCKER_RUN_MODE": docker_run_mode,
+        "FAKE_DOCKER_START_FAIL_ONCE": "1" if docker_start_fail_once else "0",
+        "FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
+    }
+    process = subprocess.Popen(
+        ["bash", str(harness / SCRIPT.name)],
+        env=env,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = process.communicate(timeout=10)
+    result = subprocess.CompletedProcess(
+        process.args,
+        process.returncode,
+        stdout,
+        stderr,
+    )
+    docker_calls = docker_log.read_text(encoding="utf-8").splitlines()
+    return result, docker_calls, Path(f"/tmp/signoz-backup-{process.pid}")
+
+
+def _collector_start_count(docker_calls: list[str]) -> int:
+    return docker_calls.count("start signoz-otel-collector")
+
+
+def test_collector_is_restored_once_when_backup_receives_term(tmp_path: Path) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_run_mode="signal",
+    )
+
+    assert result.returncode != 0
+    assert "stop signoz-otel-collector" in docker_calls
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_collector_is_restored_once_when_clickhouse_backup_fails(tmp_path: Path) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_run_mode="empty",
+    )
+
+    assert result.returncode == 1
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_exit_cleanup_does_not_restart_an_already_restored_collector(
+    tmp_path: Path,
+) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        restic_backup_status=17,
+    )
+
+    assert result.returncode == 17
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_exit_cleanup_retries_only_after_explicit_restore_fails(
+    tmp_path: Path,
+) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_start_fail_once=True,
+        restic_backup_status=17,
+    )
+
+    assert result.returncode == 17
+    assert _collector_start_count(docker_calls) == 2
+    assert not dump_dir.exists()