fix(ops): bound CD DB probes and restore SigNoz collector

scripts/backup/backup-signoz.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 # =============================================================================
 # WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
-# 版本: 1.1.0
+# 版本: 1.2.0
 # 建立日期: 2026-04-05
 # 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
 # 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
+# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
 # =============================================================================
 
 set -euo pipefail
@@ -14,11 +15,42 @@ source "$(dirname "$0")/common.sh"
 SERVICE="signoz"
 LOCAL_REPO="${BACKUP_BASE}/signoz"
 DUMP_DIR="/tmp/signoz-backup-$$"
+COLLECTOR_RESTORE_PENDING=0
+CLEANUP_COMPLETE=0
+
+restore_collector() {
+    if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
+        return 0
+    fi
+
+    log_info "重啟 signoz-otel-collector..."
+    if docker start signoz-otel-collector 2>/dev/null; then
+        COLLECTOR_RESTORE_PENDING=0
+        return 0
+    fi
+
+    log_warn "signoz-otel-collector 重啟失敗"
+    return 1
+}
 
 cleanup() {
-    # 確保 collector 已重啟
-    docker start signoz-otel-collector 2>/dev/null || true
+    local exit_code=$?
+
+    if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
+        return "${exit_code}"
+    fi
+    CLEANUP_COMPLETE=1
+
+    # cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
+    trap '' HUP INT TERM
+    restore_collector || true
     rm -rf "${DUMP_DIR}"
+    return "${exit_code}"
+}
+
+on_signal() {
+    local signal_number="$1"
+    exit "$((128 + signal_number))"
 }
 
 backup_volume() {
@@ -44,6 +76,12 @@ backup_volume() {
 
 main() {
     local start_time=$(date +%s)
+
+    trap cleanup EXIT
+    trap 'on_signal 1' HUP
+    trap 'on_signal 2' INT
+    trap 'on_signal 15' TERM
+
     log_info "========== 開始 SignOz 備份 =========="
     mkdir -p "${DUMP_DIR}"
 
@@ -51,13 +89,14 @@ main() {
 
     # Step 1: 停止 OTEL Collector 確保數據一致性
     log_info "暫停 signoz-otel-collector 以確保數據一致性..."
+    # 先 arm 還原再停止，消除 signal 發生在 stop 與狀態記錄之間的空窗。
+    COLLECTOR_RESTORE_PENDING=1
     docker stop signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 未在運行，繼續"
     docker stop signoz-telemetrystore-migrator 2>/dev/null || true
 
     # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
     backup_volume "signoz-clickhouse" "${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" "--exclude=/data/tmp" || {
         log_error "ClickHouse volume 備份失敗"
-        cleanup
         notify_clawbot "failed" "${SERVICE}" "SignOz ClickHouse 備份失敗"
         exit 1
     }
@@ -68,15 +107,14 @@ main() {
     }
 
     # Step 4: 重啟 Collector
-    log_info "重啟 signoz-otel-collector..."
-    docker start signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 重啟失敗"
+    # 若此處失敗，保留 pending 讓 EXIT cleanup 再做一次有界還原。
+    restore_collector || true
 
     # Step 5: 初始化 Restic 倉庫
     if [ ! -d "${LOCAL_REPO}/data" ]; then
         log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
         restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
             log_error "Restic 倉庫初始化失敗"
-            rm -rf "${DUMP_DIR}"
             exit 1
         }
     fi
@@ -92,8 +130,6 @@ main() {
     # Step 7: GFS 清理
     cleanup_old_backups "${LOCAL_REPO}"
 
-    rm -rf "${DUMP_DIR}"
-
     local end_time=$(date +%s)
     local duration=$((end_time - start_time))
     log_success "========== SignOz 備份完成 (${duration}s) =========="

scripts/backup/tests/test_backup_signoz_collector_restore.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
+
+
+def _write_executable(path: Path, text: str) -> None:
+    path.write_text(text, encoding="utf-8")
+    path.chmod(0o755)
+
+
+def _run_backup(
+    tmp_path: Path,
+    *,
+    docker_run_mode: str = "success",
+    docker_start_fail_once: bool = False,
+    restic_backup_status: int = 0,
+) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
+    harness = tmp_path / "backup"
+    fake_bin = tmp_path / "bin"
+    backup_base = tmp_path / "repository"
+    docker_log = tmp_path / "docker.log"
+    harness.mkdir()
+    fake_bin.mkdir()
+    (backup_base / "signoz" / "data").mkdir(parents=True)
+    shutil.copy2(SCRIPT, harness / SCRIPT.name)
+
+    (harness / "common.sh").write_text(
+        """\
+BACKUP_BASE="${TEST_BACKUP_BASE:?}"
+RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password"
+log_info() { :; }
+log_warn() { :; }
+log_error() { :; }
+log_success() { :; }
+notify_clawbot() { :; }
+build_tags() { :; }
+cleanup_old_backups() { :; }
+""",
+        encoding="utf-8",
+    )
+
+    _write_executable(
+        fake_bin / "docker",
+        """\
+#!/bin/bash
+set -eu
+printf '%s\n' "$*" >> "${TEST_DOCKER_LOG:?}"
+case "${1:-}" in
+    stop)
+        exit 0
+        ;;
+    start)
+        if [ "${FAKE_DOCKER_START_FAIL_ONCE:-0}" = "1" ] && \
+           [ ! -e "${TEST_DOCKER_START_MARKER:?}" ]; then
+            : > "${TEST_DOCKER_START_MARKER}"
+            exit 1
+        fi
+        exit 0
+        ;;
+    run)
+        if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then
+            kill -TERM "${PPID}"
+            sleep 0.1
+            exit 0
+        fi
+        if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "empty" ]; then
+            exit 0
+        fi
+        printf 'archive-data\n'
+        ;;
+esac
+""",
+    )
+    _write_executable(
+        fake_bin / "restic",
+        """\
+#!/bin/bash
+set -eu
+case " $* " in
+    *" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
+    *" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
+esac
+""",
+    )
+    _write_executable(
+        fake_bin / "du",
+        """\
+#!/bin/bash
+printf '4K\t%s\n' "${2:-unknown}"
+""",
+    )
+
+    env = {
+        **os.environ,
+        "PATH": f"{fake_bin}:{os.environ['PATH']}",
+        "TEST_BACKUP_BASE": str(backup_base),
+        "TEST_DOCKER_LOG": str(docker_log),
+        "TEST_DOCKER_START_MARKER": str(tmp_path / "docker-start-failed-once"),
+        "FAKE_DOCKER_RUN_MODE": docker_run_mode,
+        "FAKE_DOCKER_START_FAIL_ONCE": "1" if docker_start_fail_once else "0",
+        "FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
+    }
+    process = subprocess.Popen(
+        ["bash", str(harness / SCRIPT.name)],
+        env=env,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    stdout, stderr = process.communicate(timeout=10)
+    result = subprocess.CompletedProcess(
+        process.args,
+        process.returncode,
+        stdout,
+        stderr,
+    )
+    docker_calls = docker_log.read_text(encoding="utf-8").splitlines()
+    return result, docker_calls, Path(f"/tmp/signoz-backup-{process.pid}")
+
+
+def _collector_start_count(docker_calls: list[str]) -> int:
+    return docker_calls.count("start signoz-otel-collector")
+
+
+def test_collector_is_restored_once_when_backup_receives_term(tmp_path: Path) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_run_mode="signal",
+    )
+
+    assert result.returncode != 0
+    assert "stop signoz-otel-collector" in docker_calls
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_collector_is_restored_once_when_clickhouse_backup_fails(tmp_path: Path) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_run_mode="empty",
+    )
+
+    assert result.returncode == 1
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_exit_cleanup_does_not_restart_an_already_restored_collector(
+    tmp_path: Path,
+) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        restic_backup_status=17,
+    )
+
+    assert result.returncode == 17
+    assert _collector_start_count(docker_calls) == 1
+    assert not dump_dir.exists()
+
+
+def test_exit_cleanup_retries_only_after_explicit_restore_fails(
+    tmp_path: Path,
+) -> None:
+    result, docker_calls, dump_dir = _run_backup(
+        tmp_path,
+        docker_start_fail_once=True,
+        restic_backup_status=17,
+    )
+
+    assert result.returncode == 17
+    assert _collector_start_count(docker_calls) == 2
+    assert not dump_dir.exists()

scripts/reboot-recovery/signoz-canonical-route-preflight.py

@@ -22,6 +22,9 @@ EXPECTED_BLOCKERS = [
     "direct_ingress_policy_not_closed",
     "http_bridge_not_supervised",
     "filelog_normalization_degraded",
+    "api_runtime_cooldown_pending",
+    "cd_probe_safety_fix_not_deployed",
+    "backup_collector_restore_guard_not_deployed",
     "wave_b_not_authorized",
 ]
 

scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py

@@ -23,6 +23,7 @@ GRPC_BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
 GRPC_UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
 GRPC_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
 GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml"
+POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml"
 
 
 def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
@@ -266,6 +267,14 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
     assert receipt["completion"]["grpc_runtime_asset"] == "closed"
     assert receipt["completion"]["program"] == "partial_degraded"
 
+    regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+    assert regression["baseline"]["verifier_terminal"] == "pass"
+    assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False
+    assert regression["controlled_recovery"]["stateful_services_touched"] == 0
+    assert regression["controlled_recovery"]["grpc_4317_listening"] is True
+    assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False
+    assert regression["terminal"]["status"] == "partial_degraded"
+
 
 def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
     env = os.environ.copy()