From bfb2d2d040faba6d07a16067f2bfc09435fe9aee Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 02:29:51 +0800 Subject: [PATCH] fix(ops): bound CD DB probes and restore SigNoz collector --- .gitea/workflows/cd.yaml | 37 +++- .../test_executor_network_policy_boundary.py | 8 +- ..._signal_worker_ansible_executor_binding.py | 4 +- ...t_workload_database_identity_projection.py | 48 ++++- docs/LOGBOOK.md | 16 ++ ...rganization-canonical-route-migration.yaml | 10 + .../p0-obs-002-post-closure-regression.yaml | 93 +++++++++ scripts/backup/backup-signoz.sh | 54 +++++- .../test_backup_signoz_collector_restore.py | 179 ++++++++++++++++++ .../signoz-canonical-route-preflight.py | 3 + .../test_signoz_canonical_route_migration.py | 9 + 11 files changed, 442 insertions(+), 19 deletions(-) create mode 100644 ops/signoz/p0-obs-002-post-closure-regression.yaml create mode 100644 scripts/backup/tests/test_backup_signoz_collector_restore.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 6205e9158..35d26af2d 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -3868,6 +3868,25 @@ jobs: API_DESIRED_REPLICA_COUNT=$($KUBECTL get deployment awoooi-api \ -n awoooi-prod -o jsonpath='{.spec.replicas}') + API_HTTP_PROBE_SAFETY_RESERVE=4 + API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE=$(( + API_DB_AVAILABLE_CONNECTION_HEADROOM - API_HTTP_PROBE_SAFETY_RESERVE + )) + if [ "$API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE" -lt 0 ]; then + API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE=0 + fi + API_HTTP_CONCURRENCY_WORKERS=$API_HTTP_PROBE_AVAILABLE_AFTER_RESERVE + if [ "$API_HTTP_CONCURRENCY_WORKERS" -gt 4 ]; then + API_HTTP_CONCURRENCY_WORKERS=4 + fi + API_HTTP_CONCURRENCY_VERIFIED=false + API_HTTP_PROBE_SKIP_REASON="" + if [ "$CONNECTION_BUDGET_VERIFIED" != "true" ]; then + API_HTTP_PROBE_SKIP_REASON=connection_budget_not_verified + elif [ "$API_HTTP_CONCURRENCY_WORKERS" -lt 1 ]; then + API_HTTP_PROBE_SKIP_REASON=connection_headroom_reserved_for_runtime + else + export API_HTTP_CONCURRENCY_WORKERS API_HTTP_CONCURRENCY_VERIFIED=$(python3 - <<'PY' import concurrent.futures import os @@ -3904,11 +3923,14 @@ jobs: "API_PUBLIC_READINESS_DELAY_SECONDS", 5, 0, 30 ) concurrency_attempts = bounded_int( - "API_HTTP_CONCURRENCY_ATTEMPTS", 4, 1, 8 + "API_HTTP_CONCURRENCY_ATTEMPTS", 1, 1, 1 ) concurrency_delay_seconds = bounded_int( "API_HTTP_CONCURRENCY_DELAY_SECONDS", 5, 0, 30 ) + probe_worker_count = bounded_int( + "API_HTTP_CONCURRENCY_WORKERS", 4, 1, 4 + ) def fetch_status(url: str, timeout: int) -> tuple[int | None, str]: @@ -3955,18 +3977,18 @@ jobs: def fetch(url: str) -> int: - status, error_type = fetch_status(url, timeout=20) + status, error_type = fetch_status(url, timeout=5) if status is None: raise RuntimeError(error_type) return status last_result = "unavailable" - concurrency_urls = (db_probe_url,) * 8 + concurrency_urls = (db_probe_url,) * probe_worker_count for attempt in range(1, concurrency_attempts + 1): try: with concurrent.futures.ThreadPoolExecutor( - max_workers=8 + max_workers=probe_worker_count ) as executor: statuses = list(executor.map(fetch, concurrency_urls)) except Exception as exc: @@ -4000,6 +4022,10 @@ jobs: print("false") PY ) + fi + if [ -n "$API_HTTP_PROBE_SKIP_REASON" ]; then + echo "API database concurrency probe skipped: ${API_HTTP_PROBE_SKIP_REASON} headroom=${API_DB_AVAILABLE_CONNECTION_HEADROOM} reserve=${API_HTTP_PROBE_SAFETY_RESERVE}" >&2 + fi if [ "$API_HTTP_CONCURRENCY_VERIFIED" != "true" ]; then CONNECTION_BUDGET_VERIFIED=false fi @@ -4100,6 +4126,9 @@ jobs: --from-literal=broker_sequence_privilege_gap_count="$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \ --from-literal=representative_db_preflights_verified="$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" \ --from-literal=api_http_concurrency_probe_passed="$API_HTTP_CONCURRENCY_VERIFIED" \ + --from-literal=api_http_concurrency_probe_workers="$API_HTTP_CONCURRENCY_WORKERS" \ + --from-literal=api_http_concurrency_probe_safety_reserve="$API_HTTP_PROBE_SAFETY_RESERVE" \ + --from-literal=api_http_concurrency_probe_skip_reason="$API_HTTP_PROBE_SKIP_REASON" \ --from-literal=api_http_probe_restart_free="$API_PROBE_RESTART_FREE" \ --from-literal=api_restart_count_before="$API_RESTART_COUNT_BEFORE" \ --from-literal=api_restart_count_after="$API_RESTART_COUNT_AFTER" \ diff --git a/apps/api/tests/test_executor_network_policy_boundary.py b/apps/api/tests/test_executor_network_policy_boundary.py index c783c9606..027581780 100644 --- a/apps/api/tests/test_executor_network_policy_boundary.py +++ b/apps/api/tests/test_executor_network_policy_boundary.py @@ -208,7 +208,10 @@ def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None: assert "import sys" in probe assert "concurrency_urls =" in probe assert "ThreadPoolExecutor(" in probe - assert "max_workers=8" in probe + assert "probe_worker_count" in probe + assert "max_workers=probe_worker_count" in probe + assert "max_workers=8" not in probe + assert "API_HTTP_CONCURRENCY_WORKERS" in probe assert "for attempt in range(1, concurrency_attempts + 1):" in probe assert "if attempt < concurrency_attempts:" in probe assert "transient_http_statuses" in probe @@ -220,6 +223,9 @@ def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None: 'api_http_concurrency_probe_passed="$API_HTTP_CONCURRENCY_VERIFIED"' in workflow ) + assert "API_HTTP_PROBE_SAFETY_RESERVE=4" in workflow + assert "connection_budget_not_verified" in workflow + assert 'api_http_concurrency_probe_safety_reserve="$API_HTTP_PROBE_SAFETY_RESERVE"' in workflow assert 'connection_budget_verified="$CONNECTION_BUDGET_VERIFIED"' in workflow assert "workload DB identity/budget remains in progress" in workflow diff --git a/apps/api/tests/test_signal_worker_ansible_executor_binding.py b/apps/api/tests/test_signal_worker_ansible_executor_binding.py index cd7df1b99..013c7d594 100644 --- a/apps/api/tests/test_signal_worker_ansible_executor_binding.py +++ b/apps/api/tests/test_signal_worker_ansible_executor_binding.py @@ -272,7 +272,9 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None: assert "AWOOOI_DB_REQUIRED_CONNECTION_BUDGET" in workflow assert "api_http_concurrency_probe_passed" in workflow assert "ThreadPoolExecutor" in workflow - assert "max_workers=8" in workflow + assert "max_workers=probe_worker_count" in workflow + assert "max_workers=8" not in workflow + assert "api_http_concurrency_probe_safety_reserve" in workflow assert "WORKLOAD_DB_ROLLBACK_FILE" in workflow assert "restore_workload_database_projection" in workflow assert "awoooi-workload-db-identity-bootstrap.sh" in workflow diff --git a/apps/api/tests/test_workload_database_identity_projection.py b/apps/api/tests/test_workload_database_identity_projection.py index d41ac01bb..3e3d7e45c 100644 --- a/apps/api/tests/test_workload_database_identity_projection.py +++ b/apps/api/tests/test_workload_database_identity_projection.py @@ -280,13 +280,22 @@ def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None: assert "workload DB identity attempts exhausted before receipt write" in workflow assert "API_PUBLIC_READINESS_ATTEMPTS" in workflow assert "API_HTTP_CONCURRENCY_ATTEMPTS" in workflow + assert "API_HTTP_PROBE_SAFETY_RESERVE=4" in workflow + assert "API_HTTP_CONCURRENCY_WORKERS" in workflow assert "API sequential readiness permanent HTTP status" in workflow assert "api/v1/health/ready" in workflow assert "events/dossier/coverage?project_id=awoooi&limit=1" not in workflow assert "api/v1/platform/cicd/events" in workflow assert "?project_id=awoooi&limit=1" in workflow - assert "concurrency_urls = (db_probe_url,) * 8" in workflow + assert "concurrency_urls = (db_probe_url,) * probe_worker_count" in workflow + assert "max_workers=probe_worker_count" in workflow + assert "concurrency_urls = (db_probe_url,) * 8" not in workflow + assert "max_workers=8" not in workflow assert "concurrency_urls = urls * 6" not in workflow + assert "connection_budget_not_verified" in workflow + assert "connection_headroom_reserved_for_runtime" in workflow + assert "api_http_concurrency_probe_safety_reserve" in workflow + assert "api_http_concurrency_probe_skip_reason" in workflow assert "API_RESTART_COUNT_BEFORE=$(api_restart_total)" in workflow assert "API_RESTART_COUNT_AFTER=$(api_restart_total)" in workflow assert '"$API_PROBE_RESTART_FREE" != "true"' in workflow @@ -353,7 +362,7 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: - responses = [503, 200, *([200] * 8)] + responses = [503, 200, *([200] * 4)] calls: list[str] = [] def fake_urlopen(url: str, *, timeout: int) -> _FakeHTTPResponse: @@ -367,8 +376,9 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency( monkeypatch.setattr(time, "sleep", lambda _seconds: None) monkeypatch.setenv("API_PUBLIC_READINESS_ATTEMPTS", "3") monkeypatch.setenv("API_PUBLIC_READINESS_DELAY_SECONDS", "0") - monkeypatch.setenv("API_HTTP_CONCURRENCY_ATTEMPTS", "2") + monkeypatch.setenv("API_HTTP_CONCURRENCY_ATTEMPTS", "1") monkeypatch.setenv("API_HTTP_CONCURRENCY_DELAY_SECONDS", "0") + monkeypatch.setenv("API_HTTP_CONCURRENCY_WORKERS", "4") exec( compile( @@ -382,13 +392,43 @@ def test_cd_api_readiness_retries_http_503_then_runs_all_200_concurrency( output = capsys.readouterr() assert output.out.strip() == "true" assert "api_sequential_readiness=ready;attempt=2" in output.err - assert len(calls) == 10 + assert len(calls) == 6 assert calls[0].startswith("https://awoooi.wooo.work/api/v1/health/ready|") assert calls[1].startswith("https://awoooi.wooo.work/api/v1/health/ready|") assert all("platform/cicd/events" in call for call in calls[2:]) assert responses == [] +def test_cd_api_concurrency_workers_are_capped_at_four( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + responses = [200, *([200] * 4)] + calls: list[str] = [] + + def fake_urlopen(url: str, *, timeout: int) -> _FakeHTTPResponse: + calls.append(f"{url}|{timeout}") + return _FakeHTTPResponse(responses.pop(0)) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + monkeypatch.setenv("API_HTTP_CONCURRENCY_WORKERS", "8") + + exec( + compile( + _api_http_readiness_and_concurrency_python(), + "", + "exec", + ), + {}, + ) + + output = capsys.readouterr() + assert output.out.strip() == "true" + assert len(calls) == 5 + assert all("platform/cicd/events" in call for call in calls[1:]) + assert responses == [] + + def test_cd_api_readiness_exhaustion_stops_before_green_receipt_write( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 66e446276..44dde0a64 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,19 @@ +## 2026-07-15 — P0-OBS-002 post-closure runtime regression 與 guard 修復 + +**runtime evidence**： +- gRPC bridge run `P0-OBS-002-20260715T013800+0800` 先通過完整 600 秒 verifier；後續 revision `2004` 於 02:11:22 起發生 liveness/readiness timeout，最低 `0/2 Ready`，兩 pod 各累積 restart `4`。bridge 維持同 PID、NRestarts `0`，因此不執行 bridge rollback。 +- API log 捕獲 `too many connections for role awoooi_api_runtime`；既有 CD verifier 在 role limit `12`、最低 headroom `8` 的條件下仍固定發出 8 個 DB-backed requests，存在把 `4 + 8` 打滿 `12/12` 的結構性風險。已完成的 #5119 receipt 本身顯示 probe 當時 active=`0`、restart `0->0`，故只把此問題列為高信心 structural amplifier，不誤報為唯一直接觸發。 +- 每日 `backup-all` 02:12 執行 SignOz 備份時停止 collector；volume backup 立即失敗，總備份僅 `3/9` 成功且 collector 未恢復。`P0-OBS-002-static-recovery-20260715T021600+0800` 只恢復 stateless collector，4317/4318、OTLP HTTP `200`、`.120/.121` gRPC TCP 全數通過，stateful services touched=`0`。 + +**source/test evidence**： +- CD probe 改為先檢查 connection budget、保留 4 條安全餘量、最多 4 workers、單輪、每 request 5 秒；新增 workers/reserve/skip reason machine-readable receipt fields。 +- `backup-signoz.sh` 在 stop 前先 arm restore，加入 `EXIT/HUP/INT/TERM` trap、防重入 cleanup 與首次 restart 失敗後的有界 EXIT retry。 +- 新增 `ops/signoz/p0-obs-002-post-closure-regression.yaml`，分離 API probe starvation、collector backup interruption、controlled recovery、source candidate 與 scope boundary。 + +**仍維持**： +- API 最近一次讀回雖恢復 `2/2`，但 restart 已是 `4/4` 且較新 Gitea CD 仍連續進行，尚未完成 10 分鐘 cooldown；program 仍是 `partial_degraded`。 +- 兩個 source guard 尚未由 main/CD/runtime readback 證明；filelog normalization、organization、direct ingress policy、HTTP bridge supervision 與 producer promotion 仍未完成。 + ## 2026-07-15 — P0-OBS-002 production OTLP/gRPC runtime closure **完成內容**： diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index e7e61f648..7aeebbc3c 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -152,6 +152,13 @@ runtime_observations: real_application_trace_freshness_inferred: false producer_queue_failure_gate_inferred: false promotion_authorized: false + post_closure_regression: + snapshot: ops/signoz/p0-obs-002-post-closure-regression.yaml + grpc_bridge_rollback_indicated: false + canonical_collector_recovered: true + api_runtime_cooldown_closed: false + cd_probe_safety_fix_deployed: false + backup_collector_restore_guard_deployed: false terminal: status: partial_degraded @@ -160,4 +167,7 @@ terminal: - direct_ingress_policy_not_closed - http_bridge_not_supervised - filelog_normalization_degraded + - api_runtime_cooldown_pending + - cd_probe_safety_fix_not_deployed + - backup_collector_restore_guard_not_deployed - wave_b_not_authorized diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml new file mode 100644 index 000000000..6ff20acab --- /dev/null +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -0,0 +1,93 @@ +schema: awoooi_signoz_post_closure_regression_v1 +work_item_id: P0-OBS-002 +trace_id: P0-OBS-002 +observed_at: "2026-07-15T02:20:00+08:00" + +baseline: + grpc_runtime_run_id: P0-OBS-002-20260715T013800+0800 + verifier_window_seconds: 600 + verifier_terminal: pass + api_restart_delta: 0 + trace_count: 1098 + metric_sample_count: 60180 + +regressions: + api_runtime: + first_probe_timeout_at: "2026-07-15T02:11:22+08:00" + deployed_source_sha: 31de3a709815f33a11ba6de6e78b771b7160a245 + deploy_marker_sha: f331ae87c27f42ecc96d3cd4e86f30499b5fbfba + image_digest: sha256:a99b3b276def966c03be56f7e8e15feb69d5e2029c8d4ec11dd4b839709351b8 + minimum_ready_replicas_observed: 0 + expected_replicas: 2 + restart_count_per_pod_at_recovery: 4 + last_observed_ready_replicas: 2 + last_observed_cooldown_closed: false + primary_classification: db_connection_budget_and_request_pressure + structural_risk_confidence: 0.98 + direct_application_commit_regression_confidence: 0.12 + evidence: + - too_many_connections_for_awoooi_api_runtime + - constant_response_liveness_and_readiness_timed_out + - no_oom_or_cgroup_cpu_throttle + - fixed_eight_worker_cd_db_probe_can_exhaust_role_limit_twelve + canonical_collector: + stopped_at: "2026-07-15T02:12:15+08:00" + exit_code: 0 + oom_killed: false + trigger: daily_backup_signoz_collector_stop + backup_terminal: failed + backup_services_succeeded: 3 + backup_services_total: 9 + automatic_restore_observed: false + +controlled_recovery: + run_id: P0-OBS-002-static-recovery-20260715T021600+0800 + collector_only: true + stateful_services_touched: 0 + collector_running: true + grpc_4317_listening: true + http_4318_listening: true + empty_otlp_http_status: 200 + grpc_bridge_enabled_active: true + grpc_bridge_restart_count: 0 + source_nodes_tcp_verified: + - 192.168.0.120 + - 192.168.0.121 + +source_candidates: + cd_probe: + path: .gitea/workflows/cd.yaml + connection_budget_precheck_required: true + max_workers: 4 + max_attempts: 1 + request_timeout_seconds: 5 + safety_reserve_connections: 4 + receipt_fields_added: true + deployment_status: pending + backup_restore_guard: + path: scripts/backup/backup-signoz.sh + trap_exit_hup_int_term: true + restore_armed_before_stop: true + bounded_exit_retry: true + deployment_status: pending + +scope_boundaries: + grpc_bridge_rollback_indicated: false + api_manual_rollback_performed: false + firewall_changed: false + database_changed: false + stateful_volume_read: false + secret_read: false + github_used: false + +terminal: + status: partial_degraded + blockers: + - api_runtime_cooldown_pending + - cd_probe_safety_fix_not_deployed + - backup_collector_restore_guard_not_deployed + - newer_main_cd_nonterminal + safe_next_action: >- + Merge current Gitea main, validate and deploy the bounded CD probe and backup + restore guard, then require a fresh CD terminal plus ten-minute API and OTLP + runtime verifier with zero restart and exporter-error deltas. diff --git a/scripts/backup/backup-signoz.sh b/scripts/backup/backup-signoz.sh index 9fb6464d6..b37c6c246 100755 --- a/scripts/backup/backup-signoz.sh +++ b/scripts/backup/backup-signoz.sh @@ -1,10 +1,11 @@ #!/bin/bash # ============================================================================= # WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite) -# 版本: 1.1.0 +# 版本: 1.2.0 # 建立日期: 2026-04-05 # 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計 # 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true +# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector # ============================================================================= set -euo pipefail @@ -14,11 +15,42 @@ source "$(dirname "$0")/common.sh" SERVICE="signoz" LOCAL_REPO="${BACKUP_BASE}/signoz" DUMP_DIR="/tmp/signoz-backup-$$" +COLLECTOR_RESTORE_PENDING=0 +CLEANUP_COMPLETE=0 + +restore_collector() { + if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then + return 0 + fi + + log_info "重啟 signoz-otel-collector..." + if docker start signoz-otel-collector 2>/dev/null; then + COLLECTOR_RESTORE_PENDING=0 + return 0 + fi + + log_warn "signoz-otel-collector 重啟失敗" + return 1 +} cleanup() { - # 確保 collector 已重啟 - docker start signoz-otel-collector 2>/dev/null || true + local exit_code=$? + + if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then + return "${exit_code}" + fi + CLEANUP_COMPLETE=1 + + # cleanup 期間忽略重複 signal，避免還原與刪除流程重入。 + trap '' HUP INT TERM + restore_collector || true rm -rf "${DUMP_DIR}" + return "${exit_code}" +} + +on_signal() { + local signal_number="$1" + exit "$((128 + signal_number))" } backup_volume() { @@ -44,6 +76,12 @@ backup_volume() { main() { local start_time=$(date +%s) + + trap cleanup EXIT + trap 'on_signal 1' HUP + trap 'on_signal 2' INT + trap 'on_signal 15' TERM + log_info "========== 開始 SignOz 備份 ==========" mkdir -p "${DUMP_DIR}" @@ -51,13 +89,14 @@ main() { # Step 1: 停止 OTEL Collector 確保數據一致性 log_info "暫停 signoz-otel-collector 以確保數據一致性..." + # 先 arm 還原再停止，消除 signal 發生在 stop 與狀態記錄之間的空窗。 + COLLECTOR_RESTORE_PENDING=1 docker stop signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 未在運行，繼續" docker stop signoz-telemetrystore-migrator 2>/dev/null || true # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積) backup_volume "signoz-clickhouse" "${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" "--exclude=/data/tmp" || { log_error "ClickHouse volume 備份失敗" - cleanup notify_clawbot "failed" "${SERVICE}" "SignOz ClickHouse 備份失敗" exit 1 } @@ -68,15 +107,14 @@ main() { } # Step 4: 重啟 Collector - log_info "重啟 signoz-otel-collector..." - docker start signoz-otel-collector 2>/dev/null || log_warn "signoz-otel-collector 重啟失敗" + # 若此處失敗，保留 pending 讓 EXIT cleanup 再做一次有界還原。 + restore_collector || true # Step 5: 初始化 Restic 倉庫 if [ ! -d "${LOCAL_REPO}/data" ]; then log_info "初始化 Restic 倉庫: ${LOCAL_REPO}" restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || { log_error "Restic 倉庫初始化失敗" - rm -rf "${DUMP_DIR}" exit 1 } fi @@ -92,8 +130,6 @@ main() { # Step 7: GFS 清理 cleanup_old_backups "${LOCAL_REPO}" - rm -rf "${DUMP_DIR}" - local end_time=$(date +%s) local duration=$((end_time - start_time)) log_success "========== SignOz 備份完成 (${duration}s) ==========" diff --git a/scripts/backup/tests/test_backup_signoz_collector_restore.py b/scripts/backup/tests/test_backup_signoz_collector_restore.py new file mode 100644 index 000000000..fcab0e174 --- /dev/null +++ b/scripts/backup/tests/test_backup_signoz_collector_restore.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh" + + +def _write_executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + + +def _run_backup( + tmp_path: Path, + *, + docker_run_mode: str = "success", + docker_start_fail_once: bool = False, + restic_backup_status: int = 0, +) -> tuple[subprocess.CompletedProcess[str], list[str], Path]: + harness = tmp_path / "backup" + fake_bin = tmp_path / "bin" + backup_base = tmp_path / "repository" + docker_log = tmp_path / "docker.log" + harness.mkdir() + fake_bin.mkdir() + (backup_base / "signoz" / "data").mkdir(parents=True) + shutil.copy2(SCRIPT, harness / SCRIPT.name) + + (harness / "common.sh").write_text( + """\ +BACKUP_BASE="${TEST_BACKUP_BASE:?}" +RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password" +log_info() { :; } +log_warn() { :; } +log_error() { :; } +log_success() { :; } +notify_clawbot() { :; } +build_tags() { :; } +cleanup_old_backups() { :; } +""", + encoding="utf-8", + ) + + _write_executable( + fake_bin / "docker", + """\ +#!/bin/bash +set -eu +printf '%s\n' "$*" >> "${TEST_DOCKER_LOG:?}" +case "${1:-}" in + stop) + exit 0 + ;; + start) + if [ "${FAKE_DOCKER_START_FAIL_ONCE:-0}" = "1" ] && \ + [ ! -e "${TEST_DOCKER_START_MARKER:?}" ]; then + : > "${TEST_DOCKER_START_MARKER}" + exit 1 + fi + exit 0 + ;; + run) + if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "signal" ]; then + kill -TERM "${PPID}" + sleep 0.1 + exit 0 + fi + if [ "${FAKE_DOCKER_RUN_MODE:-success}" = "empty" ]; then + exit 0 + fi + printf 'archive-data\n' + ;; +esac +""", + ) + _write_executable( + fake_bin / "restic", + """\ +#!/bin/bash +set -eu +case " $* " in + *" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;; + *" snapshots "*) printf '[{"short_id":"test123"}]\n' ;; +esac +""", + ) + _write_executable( + fake_bin / "du", + """\ +#!/bin/bash +printf '4K\t%s\n' "${2:-unknown}" +""", + ) + + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "TEST_BACKUP_BASE": str(backup_base), + "TEST_DOCKER_LOG": str(docker_log), + "TEST_DOCKER_START_MARKER": str(tmp_path / "docker-start-failed-once"), + "FAKE_DOCKER_RUN_MODE": docker_run_mode, + "FAKE_DOCKER_START_FAIL_ONCE": "1" if docker_start_fail_once else "0", + "FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status), + } + process = subprocess.Popen( + ["bash", str(harness / SCRIPT.name)], + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = process.communicate(timeout=10) + result = subprocess.CompletedProcess( + process.args, + process.returncode, + stdout, + stderr, + ) + docker_calls = docker_log.read_text(encoding="utf-8").splitlines() + return result, docker_calls, Path(f"/tmp/signoz-backup-{process.pid}") + + +def _collector_start_count(docker_calls: list[str]) -> int: + return docker_calls.count("start signoz-otel-collector") + + +def test_collector_is_restored_once_when_backup_receives_term(tmp_path: Path) -> None: + result, docker_calls, dump_dir = _run_backup( + tmp_path, + docker_run_mode="signal", + ) + + assert result.returncode != 0 + assert "stop signoz-otel-collector" in docker_calls + assert _collector_start_count(docker_calls) == 1 + assert not dump_dir.exists() + + +def test_collector_is_restored_once_when_clickhouse_backup_fails(tmp_path: Path) -> None: + result, docker_calls, dump_dir = _run_backup( + tmp_path, + docker_run_mode="empty", + ) + + assert result.returncode == 1 + assert _collector_start_count(docker_calls) == 1 + assert not dump_dir.exists() + + +def test_exit_cleanup_does_not_restart_an_already_restored_collector( + tmp_path: Path, +) -> None: + result, docker_calls, dump_dir = _run_backup( + tmp_path, + restic_backup_status=17, + ) + + assert result.returncode == 17 + assert _collector_start_count(docker_calls) == 1 + assert not dump_dir.exists() + + +def test_exit_cleanup_retries_only_after_explicit_restore_fails( + tmp_path: Path, +) -> None: + result, docker_calls, dump_dir = _run_backup( + tmp_path, + docker_start_fail_once=True, + restic_backup_status=17, + ) + + assert result.returncode == 17 + assert _collector_start_count(docker_calls) == 2 + assert not dump_dir.exists() diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index 223786dc6..5661913e7 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -22,6 +22,9 @@ EXPECTED_BLOCKERS = [ "direct_ingress_policy_not_closed", "http_bridge_not_supervised", "filelog_normalization_degraded", + "api_runtime_cooldown_pending", + "cd_probe_safety_fix_not_deployed", + "backup_collector_restore_guard_not_deployed", "wave_b_not_authorized", ] diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index e082704c9..edb71cfbe 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -23,6 +23,7 @@ GRPC_BRIDGE = ROOT / "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh" GRPC_UNIT = ROOT / "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service" GRPC_VERIFIER = ROOT / "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh" GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml" +POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml" def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None: @@ -266,6 +267,14 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate( assert receipt["completion"]["grpc_runtime_asset"] == "closed" assert receipt["completion"]["program"] == "partial_degraded" + regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + assert regression["baseline"]["verifier_terminal"] == "pass" + assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False + assert regression["controlled_recovery"]["stateful_services_touched"] == 0 + assert regression["controlled_recovery"]["grpc_4317_listening"] is True + assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False + assert regression["terminal"]["status"] == "partial_degraded" + def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None: env = os.environ.copy()