diff --git a/.agents/skills/04-awoooi-devops-commander.md b/.agents/skills/04-awoooi-devops-commander.md
index 1676868a0..5dbd79d3a 100644
--- a/.agents/skills/04-awoooi-devops-commander.md
+++ b/.agents/skills/04-awoooi-devops-commander.md
@@ -3,6 +3,7 @@
> **管轄範圍**: Docker, K3s, Nginx, NetworkPolicy, CI/CD
> **觸發條件**: 修改 `k8s/`, `Dockerfile`, `.github/workflows/`, `nginx`
+> **治理覆寫**: `superseded_by=global_product_governance_v2`；本文 GitHub/GitHub Actions 段落只保留歷史背景，不得執行。現行 source truth 與 workflow 一律使用 Gitea；舊 SigNoz `192.168.0.188:3301` UI/query 路由同樣失效。
---
@@ -190,12 +191,12 @@ echo "${{ secrets.SUDO_PASSWORD }}" | sudo -S kubectl ...
| 主機 | IP | 角色 | 部署內容 |
|------|-----|------|----------|
-| DevOps | 192.168.0.110 | 金庫 | Harbor:5000, GitHub Runner, Sentry:9000, Langfuse:3100 |
+| DevOps | 192.168.0.110 | 金庫 | Harbor:5000, Gitea Actions Runner, Sentry:9000, Langfuse:3100, SigNoz health:8080 |
| Security | 192.168.0.112 | 安全 | Kali Scanner:8080 |
| K3s Server #1 | 192.168.0.120 | 叢集 | Control-Plane MASTER (keepalived priority=101) |
| K3s Server #2 | 192.168.0.121 | 叢集 | Control-Plane BACKUP (keepalived priority=100) |
| **VIP** | **192.168.0.125** | **HA 入口** | **K3s API:6443 + NodePort (32334/32335)** |
-| AI/Web | 192.168.0.188 | 大腦 | Nginx, **PostgreSQL:5432 (K3s Datastore)**, Redis:6380, Ollama:11434, OpenClaw:8089, SigNoz:3301 |
+| AI/Web | 192.168.0.188 | 大腦 | Nginx, **PostgreSQL:5432 (K3s Datastore)**, Redis:6380, Ollama:11434, OpenClaw:8089, OTLP bridge:24317/24318 |
### 絕對禁令
@@ -421,7 +422,8 @@ kubectl --server=https://192.168.0.125:6443 get nodes
| 服務 | 端點 | 用途 |
|------|------|------|
-| **SignOz Web UI** | `http://192.168.0.188:3301` | APM Dashboard |
+| **SigNoz machine health** | `http://192.168.0.110:8080/api/v1/health` | service-to-service health/readback |
+| **SigNoz Web UI** | `https://signoz.wooo.work` | human APM dashboard / deep links |
| **OTEL gRPC** | `192.168.0.188:24317` | K8s/應用程式 Traces |
| **OTEL HTTP** | `http://192.168.0.188:24318` | CI/CD workflows |
| **Sentry** | `http://192.168.0.110:9000` | Error Tracking |
@@ -451,8 +453,8 @@ OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318
### 驗證指令
```bash
-# 檢查 SignOz 容器運行狀態
-ssh ollama@192.168.0.188 "docker ps | grep signoz"
+# 檢查 SigNoz machine health
+curl -fsS http://192.168.0.110:8080/api/v1/health | jq .
# 測試 OTEL 端點
curl -s http://192.168.0.188:24318/v1/traces -X POST | head -c 100
@@ -1165,7 +1167,7 @@ def verify_github_signature(payload: bytes, signature: str, secret: str) -> bool
┌─────────────┐ trace_id ┌─────────────┐ trace_id ┌─────────────┐
│ Sentry │ ◄────────────────► │ SignOz │ ◄────────────────► │ Langfuse │
│ Errors │ │ Traces │ │ LLMOps │
-│ :9000 (.110)│ │ :3301 (.188)│ │ :3100 (.110)│
+│ :9000 (.110)│ │ signoz.wooo │ │ :3100 (.110)│
└─────────────┘ └─────────────┘ └─────────────┘
```
@@ -1173,7 +1175,7 @@ def verify_github_signature(payload: bytes, signature: str, secret: str) -> bool
| 系統 | URL 格式 |
|------|----------|
-| SignOz Trace | `http://192.168.0.188:3301/trace/{trace_id}` |
+| SigNoz Trace | `https://signoz.wooo.work/trace/{trace_id}` |
| Langfuse Trace | `http://192.168.0.110:3100/project/awoooi-openclaw/traces/{id}` |
| Sentry Issue | `http://192.168.0.110:9000/organizations/sentry/issues/{id}/` |
diff --git a/.agents/skills/05-awoooi-sre-qa.md b/.agents/skills/05-awoooi-sre-qa.md
index b8e0cac10..732afc551 100644
--- a/.agents/skills/05-awoooi-sre-qa.md
+++ b/.agents/skills/05-awoooi-sre-qa.md
@@ -3,6 +3,7 @@
> **管轄範圍**: Playwright, API Testing, Health Monitoring
> **觸發條件**: 懷疑 UI 異常、API 錯誤、部署驗證
+> **治理覆寫**: `superseded_by=global_product_governance_v2`；本文 `.github/workflows` 與舊 SigNoz `192.168.0.188:3301` 測試僅屬歷史背景。現行 CI/CD 使用 Gitea，SigNoz health/UI/OTLP 依 canonical route 分流。
---
@@ -494,8 +495,8 @@ kubectl rollout undo deployment/awoooi-api -n awoooi-prod
### 觀測系統健康檢查
```bash
-# SignOz (Traces)
-curl -f http://192.168.0.188:3301/api/v1/services | jq '.data[] | .serviceName'
+# SigNoz control-plane health (machine readback)
+curl -fsS http://192.168.0.110:8080/api/v1/health | jq .
# Langfuse (LLMOps)
curl -f http://192.168.0.110:3100/api/health
@@ -511,7 +512,7 @@ curl -f http://192.168.0.110:9000/api/0/internal/health/
from src.core.deep_linking import DeepLinking
# 應返回有效 URL
-assert DeepLinking.signoz_trace_url("abc123") == "http://192.168.0.188:3301/trace/abc123"
+assert DeepLinking.signoz_trace_url("abc123") == "https://signoz.wooo.work/trace/abc123"
assert DeepLinking.langfuse_trace_url("lf-123") != ""
assert DeepLinking.sentry_issue_url("456") != ""
```
@@ -539,7 +540,7 @@ with restore_trace_context({"trace_id": "", "span_id": ""}) as span:
| 系統 | URL | 狀態 |
|------|-----|------|
-| SignOz | http://192.168.0.188:3301 | ✅ |
+| SigNoz | https://signoz.wooo.work (machine health: `192.168.0.110:8080/api/v1/health`) | ✅ |
| Langfuse | http://192.168.0.110:3100 | ✅ |
| Sentry | http://192.168.0.110:9000 | ✅ |
| Deep Linking | N/A (內建) | ✅ |
diff --git a/.agents/skills/07-tool-integration-expert.md b/.agents/skills/07-tool-integration-expert.md
index af8df8103..87a427466 100644
--- a/.agents/skills/07-tool-integration-expert.md
+++ b/.agents/skills/07-tool-integration-expert.md
@@ -3,6 +3,7 @@
> **管轄範圍**: MCP Bridge, 外部系統連接, RAG 向量化
> **觸發條件**: 修改 `plugins/mcp/`, `services/*_tool.py`, 向量資料庫
+> **治理覆寫**: `superseded_by=global_product_governance_v2`；舊 SigNoz `192.168.0.188:3301` query/UI route 已失效，machine health、public query/UI 與 OTLP ingestion 必須使用下列 canonical endpoints。
---
@@ -100,7 +101,11 @@ Phase 13.2 Tool 實作 (P0 最優先):
```
ClickHouse HTTP API: http://192.168.0.188:8123
-SignOz Query API: http://192.168.0.188:3301/api/v3
+SigNoz machine health: http://192.168.0.110:8080/api/v1/health
+SigNoz Query API (machine/MCP): http://192.168.0.110:8080/api/v3
+SigNoz UI / deep links: https://signoz.wooo.work
+OTLP gRPC: http://192.168.0.188:24317
+OTLP HTTP: http://192.168.0.188:24318
```
### 查詢範例
diff --git a/.awoooi-agent-rules.md b/.awoooi-agent-rules.md
index 6269c6ed3..23b88dc54 100644
--- a/.awoooi-agent-rules.md
+++ b/.awoooi-agent-rules.md
@@ -4,6 +4,7 @@
> **生效日期**: 2026-03-21
> **適用對象**: Claude Code / Cursor / GitHub Copilot
> **強制等級**: 絕對遵守 (違反等同事故)
+> **治理覆寫**: `superseded_by=global_product_governance_v2`；本文內以 GitHub 為 source truth、以 `192.168.0.188:3301` 存取 SigNoz UI/query，或以未受控人工 gate 作為終局的舊指令均已失效。現行 source truth/CI/CD 僅使用 Gitea；SigNoz machine health、human deep link 與 OTLP ingestion 必須分流。
---
@@ -310,9 +311,9 @@ import { Eye, Sparkles } from 'lucide-react'
| IP | 名稱 | 職責 |
|-----|------|------|
-| **192.168.0.110** | DevOps 金庫 | Harbor:5000 (`awoooi/` Project), GH Runner (`awoooi-runner`) |
+| **192.168.0.110** | DevOps 金庫 | Harbor:5000 (`awoooi/` Project), Gitea:3001, **SigNoz machine health:8080 (`/api/v1/health`)** |
| **192.168.0.112** | Kali Security | Scanner API:8080 (Header 區分: `X-Source: awoooi`) |
-| **192.168.0.188** | AI+Web 中心 | Nginx SSL, Ollama:11434, **OpenClaw:8089**, **SigNoz:3301 (OTEL 收集器)** |
+| **192.168.0.188** | AI+Web 中心 | Nginx SSL, Ollama:11434, **OpenClaw:8089**, **OTLP bridge:24317/24318** |
| **192.168.0.120** | K3s Master | **awoooi-prod (API:32334, Frontend:32335)** ⚠️ 唯一部署目標 |
| **192.168.0.121** | K3s Worker | HA 備援 |
@@ -344,10 +345,10 @@ import { Eye, Sparkles } from 'lucide-react'
| 1 | **kustomization 禁寫 newTag** | 導致 Ghost Rollback，Tag 由 CI 動態注入 |
| 2 | **CI/CD 禁止 git push 回 main** | GitOps Lite 已永久停用 |
| 3 | **映像標籤必須唯一不可變** | 格式: `{sha}-{run_id}`，禁用 latest/main/dev |
-| 4 | **禁止 Gitea ↔ GitHub 雙向同步** | 唯一來源: GitHub |
+| 4 | **禁止 Gitea ↔ GitHub 雙向同步** | 唯一來源: Gitea；GitHub 全面停用 |
| 5 | **LLM 調用必須走快取** | 使用 `_call_with_cache`，禁止直接調用 |
| 6 | **Markdown 產出必須遵守格式** | 頭部元數據 + 更新 LOGBOOK |
-| 7 | **可觀測性必須打向 SigNoz** | OTEL Traces/Logs → 192.168.0.188:3301 |
+| 7 | **SigNoz 流量必須依用途分流** | machine health → `192.168.0.110:8080/api/v1/health`；human UI → `https://signoz.wooo.work`；OTLP → `192.168.0.188:24317/24318` |
| 8 | **UI 文字禁止硬編碼** | 100% next-intl 字典檔 (zh-TW/en) |
---
@@ -366,12 +367,14 @@ from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
-# SigNoz 端點 (絕對禁止寫死其他地址)
-SIGNOZ_ENDPOINT = "http://192.168.0.188:4317"
+# SigNoz canonical endpoints：健康、UI、遙測攝取不可混用
+SIGNOZ_HEALTH_URL = "http://192.168.0.110:8080/api/v1/health"
+SIGNOZ_PUBLIC_URL = "https://signoz.wooo.work"
+OTEL_GRPC_ENDPOINT = "http://192.168.0.188:24317"
def setup_telemetry(app):
provider = TracerProvider()
- processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=SIGNOZ_ENDPOINT))
+ processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=OTEL_GRPC_ENDPOINT))
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)
FastAPIInstrumentor.instrument_app(app)
@@ -440,9 +443,9 @@ AI 可跨資料夾讀取以下舊專案文檔 (唯讀)：
| IP | 職責 | 允許操作 |
|-----|------|----------|
-| **192.168.0.110** | DevOps 金庫 (Harbor) | Docker Registry 操作 |
+| **192.168.0.110** | DevOps 金庫 (Harbor + SigNoz control plane) | Docker Registry、SigNoz machine health (`:8080/api/v1/health`) |
| **192.168.0.112** | Kali Security | 安全掃描 API 呼叫 |
-| **192.168.0.188** | AI+Web 中心 | **Nginx 部署**、Ollama、SigNoz |
+| **192.168.0.188** | AI+Web 中心 | **Nginx 部署**、Ollama、OTLP bridge (`:24317/:24318`) |
| **192.168.0.120** | K3s Master | kubectl 操作、NodePort 存取 |
| **192.168.0.121** | K3s Worker | HA 備援 (禁止直接操作) |
diff --git a/apps/api/tests/test_signoz_canonical_health_route.py b/apps/api/tests/test_signoz_canonical_health_route.py
index 80946b9fa..cc83758b2 100644
--- a/apps/api/tests/test_signoz_canonical_health_route.py
+++ b/apps/api/tests/test_signoz_canonical_health_route.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+import subprocess
+import sys
from datetime import UTC, datetime
from pathlib import Path
from types import SimpleNamespace
@@ -15,9 +17,15 @@ from src.services import signoz_client as signoz_client_module
from src.services.host_aggregator import HOST_CONFIGS
REPO_ROOT = Path(__file__).resolve().parents[3]
+ALERT_CHAIN_SMOKE = REPO_ROOT / "scripts/alert_chain_smoke_test.py"
+PROMETHEUS_ROUTE_DEPLOY = (
+ REPO_ROOT / "scripts/ops/deploy-signoz-prometheus-route.sh"
+)
+LEGACY_PROMETHEUS_DEPLOY = REPO_ROOT / "k8s/monitoring/deploy-prometheus-config.sh"
INTERNAL_URL = "http://192.168.0.110:8080"
PUBLIC_URL = "https://signoz.wooo.work"
LEGACY_URL = "192.168.0.188:3301"
+INTERNAL_HEALTH_URL = f"{INTERNAL_URL}/api/v1/health"
def _configmap(relative_path: str) -> dict:
@@ -81,7 +89,9 @@ def test_user_facing_deep_links_use_public_route(
) -> None:
monkeypatch.setattr(deep_linking_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL)
monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL)
- monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL)
+ monkeypatch.setattr(
+ signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL
+ )
trace_id = "0af7651916cd43dd8448eb211c80319c"
assert deep_linking_module.DeepLinking.signoz_trace_url(trace_id) == (
@@ -165,9 +175,176 @@ def test_active_api_source_has_no_legacy_signoz_ui_route() -> None:
assert offenders == []
+def test_cd_alert_chain_smoke_uses_canonical_public_signoz_route() -> None:
+ text = ALERT_CHAIN_SMOKE.read_text(encoding="utf-8")
+
+ assert '"SIGNOZ_PUBLIC_URL"' in text
+ assert PUBLIC_URL in text
+ assert LEGACY_URL not in text
+
+
def test_host_asset_map_places_signoz_on_the_canonical_query_host() -> None:
canonical_services = HOST_CONFIGS["192.168.0.110"]["services"]
legacy_services = HOST_CONFIGS["192.168.0.188"]["services"]
assert ("SigNoz", 8080, "http", "/api/v1/health") in canonical_services
assert not any(service[0] == "SigNoz" for service in legacy_services)
+
+
+def test_monitoring_registry_splits_query_health_from_otlp_transport() -> None:
+ registry = _configmap("ops/monitoring/service-registry.yaml")
+ services = {service["name"]: service for service in registry["services"]}
+
+ query = services["signoz-ui"]
+ assert query["host"] == "192.168.0.110"
+ assert query["port"] == 8080
+ assert query["health_endpoint"] == "/api/v1/health"
+ assert query["health_type"] == "http"
+ assert query["monitoring"]["prometheus"] is True
+ assert query["monitoring"]["prometheus_scrape"] is False
+
+ collector = services["signoz-collector"]
+ assert collector["host"] == "192.168.0.188"
+ assert collector["port"] == 24317
+ assert collector["health_type"] == "grpc"
+ assert collector["monitoring"]["prometheus_scrape"] is False
+ assert collector["monitoring"]["external_verifier"] == (
+ "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
+ )
+
+
+def test_stateful_registry_places_clickhouse_on_canonical_host() -> None:
+ source_registry = _configmap("ops/config/service-registry.yaml")
+ configmap = _configmap("k8s/awoooi-prod/15-service-registry-configmap.yaml")
+ runtime_registry = yaml.safe_load(configmap["data"]["service-registry.yaml"])
+
+ for registry in (source_registry, runtime_registry):
+ services = {service["name"]: service for service in registry["services"]}
+ clickhouse = services["signoz-clickhouse"]
+ assert clickhouse["host"] == "192.168.0.110"
+ assert clickhouse["stateful_level"] == "BLOCK"
+ assert clickhouse["alert_only"] is True
+
+
+def test_active_prometheus_targets_canonical_http_health_not_legacy_tcp() -> None:
+ config = _configmap("k8s/monitoring/prometheus.yml")
+ jobs = {job["job_name"]: job for job in config["scrape_configs"]}
+ http_job = jobs["blackbox-http"]
+ http_targets = http_job["static_configs"][0]["targets"]
+ tcp_targets = jobs["blackbox-tcp"]["static_configs"][0]["targets"]
+
+ assert INTERNAL_HEALTH_URL in http_targets
+ assert LEGACY_URL not in tcp_targets
+ assert http_job["relabel_configs"] == [
+ {
+ "source_labels": ["__address__"],
+ "target_label": "__param_target",
+ },
+ {
+ "source_labels": ["__param_target"],
+ "target_label": "instance",
+ },
+ {
+ "target_label": "__address__",
+ "replacement": "192.168.0.110:9115",
+ },
+ ]
+
+
+def test_signoz_prometheus_route_deploy_is_target_first_and_rollback_safe() -> None:
+ text = PROMETHEUS_ROUTE_DEPLOY.read_text(encoding="utf-8")
+
+ assert 'MODE="${1:-apply}"' in text
+ assert "promtool check config" in text
+ assert "cp -p \"${REMOTE_CONFIG}\" \"${BACKUP_CONFIG}\"" in text
+ assert 'action=target_first_config_apply' in text
+ assert text.index("promtool check config") < text.index(
+ 'action=target_first_config_apply'
+ )
+ assert text.index('action=target_first_config_apply') < text.index(
+ "terminal=target_up_two_scrapes"
+ )
+ assert "rollback_deploy" in text
+ assert INTERNAL_HEALTH_URL in text
+ assert LEGACY_URL in text
+
+
+def test_legacy_prometheus_deployer_routes_to_bounded_host110_replacement() -> None:
+ text = LEGACY_PROMETHEUS_DEPLOY.read_text(encoding="utf-8")
+
+ assert "superseded_by=global_product_governance_v2" in text
+ assert "expiry=2026-07-15" in text
+ assert "scripts/ops/deploy-signoz-prometheus-route.sh" in text
+ assert "192.168.0.188" not in text
+
+
+@pytest.mark.parametrize(
+ "relative_path",
+ [
+ "ops/monitoring/alerts-unified.yml",
+ "ops/monitoring/alerts.yml",
+ ],
+)
+def test_signoz_down_alert_is_exact_and_fails_closed_on_missing_series(
+ relative_path: str,
+) -> None:
+ payload = _configmap(relative_path)
+ rules = [
+ rule
+ for group in payload["groups"]
+ for rule in group.get("rules", [])
+ if rule.get("alert") == "SignOzDown"
+ ]
+
+ assert len(rules) == 1
+ rule = rules[0]
+ assert 'job="blackbox-http"' in rule["expr"]
+ assert INTERNAL_HEALTH_URL in rule["expr"]
+ assert "absent(" in rule["expr"]
+ assert LEGACY_URL not in rule["expr"]
+ assert rule["labels"]["host"] == "110"
+ assert rule["labels"]["target_host"] == "192.168.0.110"
+
+
+def test_legacy_188_rule_plane_does_not_duplicate_canonical_signoz_alert() -> None:
+ payload = _configmap("k8s/monitoring/k3s-alerts.yaml")
+ signoz_rules = [
+ rule
+ for group in payload["groups"]
+ for rule in group.get("rules", [])
+ if rule.get("alert") == "SignOzDown"
+ ]
+ text = (REPO_ROOT / "k8s/monitoring/k3s-alerts.yaml").read_text(encoding="utf-8")
+
+ assert signoz_rules == []
+ assert LEGACY_URL not in text
+
+
+def test_monitoring_generator_uses_blackbox_without_scraping_signoz_spa(
+ tmp_path: Path,
+) -> None:
+ subprocess.run(
+ [
+ sys.executable,
+ str(REPO_ROOT / "ops/monitoring/generate_monitoring.py"),
+ "--output-dir",
+ str(tmp_path),
+ ],
+ cwd=REPO_ROOT,
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+
+ scrape = _configmap(str(tmp_path / "prometheus-scrape-generated.yaml"))
+ blackbox = yaml.safe_load(
+ (tmp_path / "blackbox-targets-generated.yaml").read_text(encoding="utf-8")
+ )
+ generated_jobs = {job["job_name"] for job in scrape["scrape_configs"]}
+ assert "signoz-ui" not in generated_jobs
+ assert "signoz-collector" not in generated_jobs
+ assert any(
+ target["labels"].get("service") == "signoz-ui"
+ and target["url"] == INTERNAL_HEALTH_URL
+ for target in blackbox
+ )
diff --git a/apps/sensor/agent.py b/apps/sensor/agent.py
index a873d31dc..7473bbcae 100644
--- a/apps/sensor/agent.py
+++ b/apps/sensor/agent.py
@@ -66,7 +66,6 @@ HOST_CONFIGS: dict[str, dict] = {
{"name": "Redis", "host": "127.0.0.1", "port": 6380},
{"name": "Ollama", "host": "127.0.0.1", "port": 11434},
{"name": "Nginx", "host": "127.0.0.1", "port": 80},
- {"name": "SigNoz", "host": "127.0.0.1", "port": 3301},
# OpenClaw 跑在 K3s (awoooi-prod)，非 188 本機服務，不在此探測
],
},
@@ -77,7 +76,9 @@ HOST_CONFIGS: dict[str, dict] = {
"services": [
{"name": "Harbor", "host": "127.0.0.1", "port": 5000},
{"name": "Gitea", "host": "127.0.0.1", "port": 3001},
- {"name": "GH-Runner", "host": "127.0.0.1", "port": 8080},
+ # TCP availability for canonical machine health endpoint:
+ # http://192.168.0.110:8080/api/v1/health
+ {"name": "SigNoz", "host": "127.0.0.1", "port": 8080},
],
},
}
diff --git a/apps/web/src/app/[locale]/classic/page.tsx b/apps/web/src/app/[locale]/classic/page.tsx
index 003feaa3e..091de2970 100644
--- a/apps/web/src/app/[locale]/classic/page.tsx
+++ b/apps/web/src/app/[locale]/classic/page.tsx
@@ -31,6 +31,7 @@ import { HostGrid, type HostInfo, type HostService } from '@/components/infra/ho
import { AppLayout } from '@/components/layout'
const API_BASE = process.env.NEXT_PUBLIC_API_URL ?? ''
+const SIGNOZ_PUBLIC_URL = 'https://signoz.wooo.work'
// =============================================================================
// Types
@@ -152,7 +153,7 @@ function MonitoringTools() {
const statusText = isUp ? (hasFiring ? `${tool.firing_count} ${tDash('monitoringStatus.firing')}` : tDash('monitoringStatus.up')) : tDash('monitoringStatus.down')
const accentColor = TOOL_ACCENT_COLOR[tool.name] ?? '#b0ad9f'
const icon = TOOL_ICON[tool.name] ??
- const link = safeExternalMonitoringUrl(tool.url)
+ const link = tool.name === 'SigNoz' ? SIGNOZ_PUBLIC_URL : safeExternalMonitoringUrl(tool.url)
const timeStr = (() => {
try { return new Date(tool.checked_at).toLocaleTimeString('zh-TW', { timeZone: 'Asia/Taipei', hour: '2-digit', minute: '2-digit' }) }
catch { return '--' }
@@ -294,6 +295,7 @@ const HOST_CATALOG: Record = {
'110:3002': 'host:public-gateway/grafana',
'110:3100': 'host:public-gateway/langfuse',
'110:5000': 'host:public-gateway/registry',
+ '110:8080': 'host:public-gateway/signoz-health',
'110:9000': 'host:public-gateway/sentry',
'112:8080': 'host:kali-readonly/scanner',
'188:11434': 'host:observability-a/ollama',
'188:8088': 'host:observability-a/openclaw-legacy',
'188:8089': 'host:observability-a/openclaw',
- '188:3301': 'host:observability-a/signoz',
+ // Historical scrub-only alias; never use this legacy route for live access.
+ '188:3301': 'host:observability-a/signoz-legacy',
'188:5432': 'host:observability-a/postgres',
'188:6380': 'host:observability-a/redis',
}
diff --git a/apps/web/src/components/dashboard/live-dashboard.tsx b/apps/web/src/components/dashboard/live-dashboard.tsx
index b4c096bef..5122e979d 100644
--- a/apps/web/src/components/dashboard/live-dashboard.tsx
+++ b/apps/web/src/components/dashboard/live-dashboard.tsx
@@ -81,6 +81,7 @@ export function LiveDashboard({ locale: _locale }: LiveDashboardProps) {
services: [
{ name: 'Harbor', status: 'idle', port: 5000 },
{ name: 'GH Runner', status: 'idle' },
+ { name: 'SigNoz', status: 'idle', port: 8080 },
{ name: 'Docker', status: 'idle', port: 2375 },
],
},
@@ -111,7 +112,6 @@ export function LiveDashboard({ locale: _locale }: LiveDashboardProps) {
{ name: 'Redis', status: 'idle', port: 6380 },
{ name: 'Ollama', status: 'idle', port: 11434 },
{ name: 'OpenClaw', status: 'idle', port: 8089 },
- { name: 'SigNoz', status: 'idle', port: 3301 },
],
},
}
diff --git a/infra/ansible/playbooks/110-devops.yml b/infra/ansible/playbooks/110-devops.yml
index a0db8eb98..9130aefc1 100644
--- a/infra/ansible/playbooks/110-devops.yml
+++ b/infra/ansible/playbooks/110-devops.yml
@@ -242,6 +242,10 @@
- backup-awoooi.sh
- backup-awoooi-frequent.sh
- backup-signoz.sh
+ - clickhouse-native-backup.sh
+ - clickhouse-native-restore-drill.sh
+ - clickhouse-restore-inventory.py
+ - run-signoz-backup-canary.sh
- backup-configs.sh
- check-backup-integrity.sh
- verify-gitea-backup.sh
@@ -250,6 +254,36 @@
- verify-offsite-full-sync.sh
tags: backup_jobs
+ - name: "Backup | 建立 SigNoz native backup/restore config 目錄"
+ ansible.builtin.file:
+ path: "{{ item }}"
+ state: directory
+ owner: wooo
+ group: wooo
+ mode: "0755"
+ loop:
+ - /backup/config/signoz/clickhouse/config.d
+ - /backup/config/signoz/clickhouse/restore-drill/config.d
+ tags: backup_jobs
+
+ - name: "Backup | 部署 SigNoz native backup disk config"
+ ansible.builtin.copy:
+ src: "{{ playbook_dir }}/../../../ops/signoz/clickhouse/config.d/backup_disk.xml"
+ dest: /backup/config/signoz/clickhouse/config.d/backup_disk.xml
+ owner: wooo
+ group: wooo
+ mode: "0644"
+ tags: backup_jobs
+
+ - name: "Backup | 部署 SigNoz isolated restore cluster config"
+ ansible.builtin.copy:
+ src: "{{ playbook_dir }}/../../../ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
+ dest: /backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml
+ owner: wooo
+ group: wooo
+ mode: "0644"
+ tags: backup_jobs
+
- name: "Backup | 安裝 AWOOOI daily backup Telegram heartbeat"
ansible.builtin.cron:
name: "AWOOOI daily backup Telegram heartbeat"
diff --git a/k8s/awoooi-prod/15-service-registry-configmap.yaml b/k8s/awoooi-prod/15-service-registry-configmap.yaml
index fefaaaf2d..7a040908d 100644
--- a/k8s/awoooi-prod/15-service-registry-configmap.yaml
+++ b/k8s/awoooi-prod/15-service-registry-configmap.yaml
@@ -70,9 +70,9 @@ data:
- name: signoz-clickhouse
display_name: "ClickHouse (SignOz)"
- host: "192.168.0.188"
+ host: "192.168.0.110"
stateful_level: BLOCK
- reason: "列欄式 OLAP 資料庫，寫入中重啟可能損壞列欄檔案"
+ reason: "列欄式 OLAP 資料庫；只允許具備 native backup、bounded apply、rollback 與 post-verifier 的受控重建"
alert_only: true
containers: ["signoz-clickhouse"]
diff --git a/k8s/monitoring/deploy-prometheus-config.sh b/k8s/monitoring/deploy-prometheus-config.sh
index 18ed07200..ee398e098 100755
--- a/k8s/monitoring/deploy-prometheus-config.sh
+++ b/k8s/monitoring/deploy-prometheus-config.sh
@@ -1,118 +1,18 @@
-#!/bin/bash
-# =============================================================================
-# Prometheus Config GitOps 部署腳本
-# =============================================================================
-# 用途: 將 Git 管理的 Prometheus 配置同步到 192.168.0.188
-# 建立者: Claude Code (首席架構師)
-# 日期: 2026-03-29 (台北時間)
+#!/usr/bin/env bash
+# Legacy compatibility entrypoint.
+# superseded_by=global_product_governance_v2
+# owner=observability-platform
+# expiry=2026-07-15T00:00:00+08:00
+# replacement=scripts/ops/deploy-signoz-prometheus-route.sh
+# verifier=promtool_then_two_successful_blackbox_scrapes
#
-# 使用方式:
-# ./deploy-prometheus-config.sh [--dry-run]
-#
-# 依賴:
-# - SSH 免密碼登入 ollama@192.168.0.188
-# - 遠端主機已安裝 Prometheus
-# =============================================================================
+# The historical implementation appended partial config to the retired host188
+# Prometheus plane. Full-file replacement is also unsafe while independently
+# governed production targets differ from the repository snapshot. Route this
+# entrypoint to the bounded, target-only host110 deployer instead.
set -euo pipefail
-# 配置
-REMOTE_HOST="ollama@192.168.0.188"
-REMOTE_CONFIG="/home/ollama/momo-pro/monitoring/prometheus.yml"
-LOCAL_ADDITIONS="$(dirname "$0")/prometheus-config-additions.yaml"
-DRY_RUN="${1:-}"
-
-# 顏色輸出
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-YELLOW='\033[1;33m'
-NC='\033[0m' # No Color
-
-log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
-log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
-log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
-
-# 檢查本地配置檔案
-if [[ ! -f "$LOCAL_ADDITIONS" ]]; then
- log_error "找不到配置檔案: $LOCAL_ADDITIONS"
- exit 1
-fi
-
-log_info "Prometheus GitOps 部署開始"
-log_info "目標主機: $REMOTE_HOST"
-log_info "遠端配置: $REMOTE_CONFIG"
-
-# 備份遠端配置
-BACKUP_FILE="${REMOTE_CONFIG}.bak.$(date +%Y%m%d_%H%M%S)"
-log_info "備份遠端配置至: $BACKUP_FILE"
-
-if [[ "$DRY_RUN" == "--dry-run" ]]; then
- log_warn "DRY RUN 模式 - 不會實際執行"
- echo ""
- echo "將執行以下操作:"
- echo " 1. ssh $REMOTE_HOST \"cp $REMOTE_CONFIG $BACKUP_FILE\""
- echo " 2. 檢查配置中是否已包含 argocd job"
- echo " 3. 如需更新，附加新配置"
- echo " 4. ssh $REMOTE_HOST \"docker exec prometheus kill -SIGHUP 1\""
- exit 0
-fi
-
-# 執行備份
-ssh "$REMOTE_HOST" "cp $REMOTE_CONFIG $BACKUP_FILE"
-log_info "備份完成"
-
-# 檢查是否已有 argocd job
-ARGOCD_EXISTS=$(ssh "$REMOTE_HOST" "grep -c 'job_name.*argocd' $REMOTE_CONFIG || true")
-
-if [[ "$ARGOCD_EXISTS" -gt 0 ]]; then
- log_info "ArgoCD job 已存在於配置中，跳過新增"
-else
- log_info "新增 ArgoCD 和 Blackbox scrape configs..."
-
- # 提取配置內容 (去除註解標題)
- CONFIG_CONTENT=$(grep -A 100 "^- job_name: argocd" "$LOCAL_ADDITIONS" | head -35)
-
- # 附加到遠端配置
- ssh "$REMOTE_HOST" "cat >> $REMOTE_CONFIG << 'EOF'
-
-# === GitOps 新增 ($(date +%Y-%m-%d)) ===
-$CONFIG_CONTENT
-EOF"
-
- log_info "配置已更新"
-fi
-
-# 驗證配置語法
-log_info "驗證 Prometheus 配置語法..."
-SYNTAX_CHECK=$(ssh "$REMOTE_HOST" "docker exec prometheus promtool check config /etc/prometheus/prometheus.yml 2>&1 || true")
-
-if echo "$SYNTAX_CHECK" | grep -q "SUCCESS"; then
- log_info "配置語法正確"
-else
- log_error "配置語法錯誤!"
- echo "$SYNTAX_CHECK"
- log_warn "正在還原備份..."
- ssh "$REMOTE_HOST" "cp $BACKUP_FILE $REMOTE_CONFIG"
- exit 1
-fi
-
-# 重載 Prometheus
-log_info "重載 Prometheus 配置..."
-ssh "$REMOTE_HOST" "docker exec prometheus kill -SIGHUP 1 || sudo systemctl reload prometheus"
-
-# 等待並驗證
-sleep 3
-log_info "驗證 targets..."
-
-TARGETS=$(curl -s "http://192.168.0.188:9090/api/v1/targets" 2>/dev/null | \
- jq -r '.data.activeTargets[] | "\(.labels.job): \(.health)"' 2>/dev/null || echo "驗證失敗")
-
-echo ""
-echo "═══════════════════════════════════════════════════════"
-echo " Prometheus Targets 狀態"
-echo "═══════════════════════════════════════════════════════"
-echo "$TARGETS" | grep -E "argocd|blackbox" || echo " (無 argocd/blackbox targets)"
-echo "═══════════════════════════════════════════════════════"
-echo ""
-
-log_info "部署完成!"
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+printf 'NOTICE: deploy-prometheus-config.sh is superseded; using the bounded host110 route deployer\n' >&2
+exec "${REPO_ROOT}/scripts/ops/deploy-signoz-prometheus-route.sh" "${1:-apply}"
diff --git a/k8s/monitoring/k3s-alerts.yaml b/k8s/monitoring/k3s-alerts.yaml
index b6bc71302..6682f6bcd 100644
--- a/k8s/monitoring/k3s-alerts.yaml
+++ b/k8s/monitoring/k3s-alerts.yaml
@@ -92,17 +92,9 @@ groups:
# ===== 可觀測性服務告警 =====
- name: observability_alerts
rules:
- # SignOz 離線
- - alert: SignOzDown
- expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:3301"} == 0
- for: 2m
- labels:
- severity: warning
- team: ops
- component: signoz
- annotations:
- summary: "⚠️ SignOz 服務離線"
- description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘"
+ # SigNoz query/UI health belongs to the canonical 110 Prometheus plane in
+ # ops/monitoring/alerts-unified.yml. This legacy 188 rule mirror has no
+ # matching canonical blackbox-http series and must not duplicate it.
# Sentry 離線
- alert: SentryDown
diff --git a/k8s/monitoring/prometheus.yml b/k8s/monitoring/prometheus.yml
index 23db89f82..067b868dd 100644
--- a/k8s/monitoring/prometheus.yml
+++ b/k8s/monitoring/prometheus.yml
@@ -56,6 +56,7 @@ scrape_configs:
- https://aiops.wooo.work
- https://mo.wooo.work
- http://192.168.0.110:3001
+ - http://192.168.0.110:8080/api/v1/health
- http://192.168.0.125:32334/api/v1/health
- http://192.168.0.125:32335
- https://www.tsenyang.com
@@ -87,7 +88,6 @@ scrape_configs:
- 192.168.0.110:5000
- 192.168.0.110:3100
# 188 服務（Ollama 已於 2026-05-06 自 AWOOOI provider pool 退役）
- - 192.168.0.188:3301
- 192.168.0.188:5432
- 192.168.0.188:6380
- 192.168.0.188:8089
diff --git a/ops/config/service-registry.yaml b/ops/config/service-registry.yaml
index 710df45b6..262520d69 100644
--- a/ops/config/service-registry.yaml
+++ b/ops/config/service-registry.yaml
@@ -54,9 +54,9 @@ services:
- name: signoz-clickhouse
display_name: "ClickHouse (SignOz)"
- host: "192.168.0.188"
+ host: "192.168.0.110"
stateful_level: BLOCK
- reason: "列欄式 OLAP 資料庫，寫入中重啟可能損壞列欄檔案"
+ reason: "列欄式 OLAP 資料庫；只允許具備 native backup、bounded apply、rollback 與 post-verifier 的受控重建"
alert_only: true
containers: ["signoz-clickhouse"]
diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml
index f30809f58..c453077fe 100644
--- a/ops/monitoring/alerts-unified.yml
+++ b/ops/monitoring/alerts-unified.yml
@@ -716,21 +716,23 @@ groups:
description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘"
- alert: SignOzDown
- expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0
+ expr: >-
+ (probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"} == 0)
+ or absent(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"})
for: 2m
labels:
severity: warning
- layer: docker-188
+ layer: docker-110
component: signoz
- host: "188"
+ host: "110"
team: ops
auto_repair: "true"
mcp_provider: "ssh_host"
- target_host: "192.168.0.188"
+ target_host: "192.168.0.110"
alert_category: "devops_tool"
annotations:
summary: "SignOz 服務離線"
- description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘"
+ description: "SignOz (http://192.168.0.110:8080/api/v1/health) 已離線或監控序列缺失超過 2 分鐘"
# ---- 110 Docker 層 ----
- alert: SentryDown
diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml
index 3e21b64f2..b437fc24f 100644
--- a/ops/monitoring/alerts.yml
+++ b/ops/monitoring/alerts.yml
@@ -400,21 +400,23 @@ groups:
description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘"
- alert: SignOzDown
- expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0
+ expr: >-
+ (probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"} == 0)
+ or absent(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"})
for: 2m
labels:
severity: warning
- layer: docker-188
+ layer: docker-110
component: signoz
- host: "188"
+ host: "110"
team: ops
auto_repair: "true"
mcp_provider: "ssh_host"
- target_host: "192.168.0.188"
+ target_host: "192.168.0.110"
alert_category: "devops_tool"
annotations:
summary: "SignOz 服務離線"
- description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘"
+ description: "SignOz (http://192.168.0.110:8080/api/v1/health) 已離線或監控序列缺失超過 2 分鐘"
# ---- 110 Docker 層 ----
- alert: SentryDown
diff --git a/ops/monitoring/generate_monitoring.py b/ops/monitoring/generate_monitoring.py
index 1488cc519..6c8c3014a 100755
--- a/ops/monitoring/generate_monitoring.py
+++ b/ops/monitoring/generate_monitoring.py
@@ -54,7 +54,10 @@ def generate_prometheus_scrape_configs(registry: dict) -> list[dict]:
# K8s 服務
for svc in registry.get("services", []):
- if svc.get("monitoring", {}).get("prometheus"):
+ monitoring = svc.get("monitoring", {})
+ if monitoring.get("prometheus") and monitoring.get(
+ "prometheus_scrape", True
+ ):
if svc.get("type") == "k8s-deployment":
# K8s ServiceMonitor 風格
config = {
diff --git a/ops/monitoring/service-registry.yaml b/ops/monitoring/service-registry.yaml
index 10bafca81..ffc41e6fe 100644
--- a/ops/monitoring/service-registry.yaml
+++ b/ops/monitoring/service-registry.yaml
@@ -335,6 +335,8 @@ services:
health_type: grpc
monitoring:
prometheus: true
+ prometheus_scrape: false
+ external_verifier: scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh
sentry: false
alerts:
- service_down
@@ -345,11 +347,13 @@ services:
# --- SignOz UI ---
- name: signoz-ui
type: docker
- host: 192.168.0.188
- port: 3301
- health_endpoint: /
+ host: 192.168.0.110
+ port: 8080
+ health_endpoint: /api/v1/health
+ health_type: http
monitoring:
prometheus: true
+ prometheus_scrape: false
sentry: false
alerts:
- service_down
diff --git a/ops/signoz/alerting/log-rules.md b/ops/signoz/alerting/log-rules.md
index 4c1a418eb..3f9344d4d 100644
--- a/ops/signoz/alerting/log-rules.md
+++ b/ops/signoz/alerting/log-rules.md
@@ -1,11 +1,13 @@
# SigNoz Log-Based Alert Rules
# 2026-04-05 Claude Code: Sprint 2 — 日誌告警
-# 設定位置: http://192.168.0.188:3301/alerts (SigNoz UI)
+# superseded_by=global_product_governance_v2: legacy 192.168.0.188:3301 UI/query route
+# 設定位置: https://signoz.wooo.work/alerts (SigNoz human UI)
+# Machine health: http://192.168.0.110:8080/api/v1/health
# Webhook: http://192.168.0.121:32334/api/v1/webhooks/signoz
## 設定步驟
-1. 開啟 http://192.168.0.188:3301/alerts
+1. 開啟 https://signoz.wooo.work/alerts
2. 點擊 "New Alert Rule" → "Logs Based Alert"
3. 依照下表填入各欄位
4. Notification Channel: 選擇 awoooi-api (指向 /api/v1/webhooks/signoz)
@@ -13,6 +15,8 @@
驗證 Webhook 鏈路：
```bash
+curl -fsS http://192.168.0.110:8080/api/v1/health | jq .
+
curl -X POST http://192.168.0.121:32334/api/v1/webhooks/signoz \
-H 'Content-Type: application/json' \
-d '{"alerts":[{"labels":{"alertname":"SignozWebhookTest","severity":"info","layer":"k8s"},"annotations":{"summary":"Sprint 2 webhook 驗證，請忽略"},"status":"firing"}],"version":"4"}'
diff --git a/ops/signoz/clickhouse/config.d/backup_disk.xml b/ops/signoz/clickhouse/config.d/backup_disk.xml
new file mode 100644
index 000000000..b4d5e4523
--- /dev/null
+++ b/ops/signoz/clickhouse/config.d/backup_disk.xml
@@ -0,0 +1,24 @@
+
+
+
+
+
+
+ local
+ /backups/
+
+
+
+
+ backups
+ /backups/
+ false
+ false
+
+
diff --git a/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml b/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml
new file mode 100644
index 000000000..390587092
--- /dev/null
+++ b/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml
@@ -0,0 +1,35 @@
+
+
+
+
+
+ restore-zookeeper
+ 2181
+
+
+
+
+
+
+ true
+
+ restore-clickhouse
+ 9000
+
+
+
+
+
+
+ 01
+
+
+
diff --git a/ops/signoz/docker-compose.clickhouse-backup.override.yaml b/ops/signoz/docker-compose.clickhouse-backup.override.yaml
new file mode 100644
index 000000000..db51a7bed
--- /dev/null
+++ b/ops/signoz/docker-compose.clickhouse-backup.override.yaml
@@ -0,0 +1,13 @@
+# P0-OBS-002: additive override for the host110 SigNoz ClickHouse service.
+# Always pass this file explicitly after the upstream compose file with -f.
+services:
+ clickhouse:
+ volumes:
+ - type: bind
+ source: /backup/staging/signoz-clickhouse
+ target: /backups
+ read_only: false
+ - type: bind
+ source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml
+ target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml
+ read_only: true
diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml
index e9ee97bc5..3a94d61c3 100644
--- a/ops/signoz/organization-canonical-route-migration.yaml
+++ b/ops/signoz/organization-canonical-route-migration.yaml
@@ -2,7 +2,7 @@ schema: awoooi_signoz_organization_canonical_route_migration_v1
work_item_id: P0-OBS-002
owner_lane: platform-observability
phase: wave_a_dual_allow
-generated_at: "2026-07-15T03:51:00+08:00"
+generated_at: "2026-07-15T06:06:46+08:00"
policy:
superseded_by: global_product_governance_v2
controlled_apply_allowed: true
@@ -154,11 +154,47 @@ runtime_observations:
promotion_authorized: false
post_closure_regression:
snapshot: ops/signoz/p0-obs-002-post-closure-regression.yaml
+ snapshot_schema: awoooi_signoz_post_closure_regression_v2
+ gitea_cd_run_id: 5140
+ release_source_sha: e4da6570a645e504cc93647d396f45f167b3172a
+ release_terminal: success
grpc_bridge_rollback_indicated: false
canonical_collector_recovered: true
- api_runtime_cooldown_closed: false
- cd_probe_safety_fix_deployed: false
- backup_collector_restore_guard_deployed: false
+ api_runtime_cooldown_closed: true
+ cd_probe_safety_fix_deployed: true
+ config_route_patch_run_id: P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1
+ config_route_patch_deployed: true
+ signoz_control_plane_health_route_deployed: true
+ prometheus_target_route_run_id: P0-OBS-002-prometheus-route-20260714T215106Z-apply1
+ prometheus_target_route_terminal: completed_target_first
+ alert_rules_run_id: P0-OBS-002-alert-rules-20260714T215328Z-apply1
+ alert_rules_terminal: completed_verified
+ backup_collector_restore_guard_deployed: true
+ backup_canary_scope: clickhouse_native_only
+ backup_canary_verified: true
+ backup_canary_last_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4
+ backup_canary_last_terminal: pass
+ backup_canary_prior_root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract
+ backup_canary_cooldown_retry_status: superseded_by_online_native_no_stop_contract
+ clickhouse_native_backup_migration_status: production_closed
+ clickhouse_native_backup_disk_run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2
+ clickhouse_native_backup_disk_status: deployed_verified
+ clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4
+ clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4
+ clickhouse_native_restore_terminal: verified
+ clickhouse_native_restic_snapshot_id: 574e2a1a
+ clickhouse_native_offsite_status: pending
+ clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision
+ clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract
+ signoz_sqlite_application_consistent_backup_status: pending
+ service_registry_runtime_mirror_status: partial_degraded
+ service_registry_runtime_mirror_work_item_id: P0-OBS-002-ASSET-DRIFT-001
+ github_freeze_legacy_asset_status: pending_controlled_retirement
+ github_freeze_legacy_asset_work_item_id: P0-OBS-002-ASSET-DRIFT-002
+ monitoring_generator_duplicate_identity_status: pending_source_deduplication
+ monitoring_generator_duplicate_identity_work_item_id: P0-OBS-002-ASSET-DRIFT-003
+ post_release_otlp_grpc_run_id: P0-OBS-002-post-release-20260714T215500Z
+ post_release_otlp_grpc_600s_verifier_status: pass
terminal:
status: partial_degraded
@@ -167,8 +203,4 @@ terminal:
- direct_ingress_policy_not_closed
- http_bridge_not_supervised
- filelog_normalization_degraded
- - api_runtime_cooldown_pending
- - cd_probe_safety_fix_not_deployed
- - backup_collector_restore_guard_not_deployed
- - signoz_control_plane_health_route_not_deployed
- wave_b_not_authorized
diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml
index 7934161f4..0bb28a860 100644
--- a/ops/signoz/p0-obs-002-post-closure-regression.yaml
+++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml
@@ -1,7 +1,7 @@
-schema: awoooi_signoz_post_closure_regression_v1
+schema: awoooi_signoz_post_closure_regression_v2
work_item_id: P0-OBS-002
trace_id: P0-OBS-002
-observed_at: "2026-07-15T02:20:00+08:00"
+observed_at: "2026-07-15T08:16:07+08:00"
baseline:
grpc_runtime_run_id: P0-OBS-002-20260715T013800+0800
@@ -11,6 +11,91 @@ baseline:
trace_count: 1098
metric_sample_count: 60180
+delivery_validation:
+ backup_restore_deployer_governance_focused_tests:
+ passed: 94
+ failed: 0
+ post_format_rerun: pass
+ api_monitoring_alert_adjacent_tests:
+ passed: 115
+ failed: 0
+ environment: explicit_dummy_local_database_and_redis_urls_no_env_file_read
+ web:
+ dependency_install: offline_frozen_lockfile
+ packages_reused: 851
+ packages_downloaded: 0
+ typecheck: pass
+ prettier: legacy_file_style_not_enforced
+ bulk_format_churn_removed: true
+ eslint_errors: 0
+ eslint_existing_warnings: 6
+ production_build: pass
+ build_environment: explicit_local_public_api_and_signoz_urls_no_env_file_read
+ monitoring:
+ generator_validate_only: pass
+ registry_service_count: 26
+ registry_monitored_count: 25
+ registry_coverage_percent: 96.2
+ generated_scrape_config_count: 24
+ generated_blackbox_target_count: 20
+ promtool_rules:
+ alerts: 81
+ alerts_unified: 147
+ k3s_alerts: 14
+ prometheus_config_syntax: pass
+ infrastructure:
+ configmap_client_dry_run: pass
+ ansible_validation_script: pass_with_local_ansible_syntax_check_unavailable
+ ansible_110_playbook_isolated_syntax_check: pass
+ shell_syntax: pass
+ python_compile: pass
+ documentation_secret_sanity_files_scanned: 1090
+ github_used: false
+ secret_read: false
+
+asset_reconciliation:
+ drift_work_item_id: P0-OBS-002-ASSET-DRIFT-001
+ source: ops/config/service-registry.yaml
+ runtime_mirror: k8s/awoooi-prod/15-service-registry-configmap.yaml
+ source_service_count: 27
+ runtime_mirror_service_count: 24
+ exact_shared_service_count: 24
+ shared_service_drift_count: 0
+ source_only_services:
+ - bitan-app
+ - sentry
+ - signoz
+ signoz_clickhouse_exact_match: true
+ live_signoz_query_host: 192.168.0.110
+ source_signoz_host: 192.168.0.188
+ terminal: partial_degraded
+ safe_next_action: >-
+ Reconcile the three source-only services and the stale SignOz query host
+ against production runtime before atomically regenerating the ConfigMap;
+ do not copy the stale 188 SignOz row into the runtime mirror.
+ github_freeze_legacy_asset_drift:
+ drift_work_item_id: P0-OBS-002-ASSET-DRIFT-002
+ superseded_by: global_product_governance_v2
+ status: pending_controlled_retirement
+ active_github_use_in_this_run: false
+ source_references:
+ - ops/monitoring/service-registry.yaml:github-runner
+ - infra/ansible/playbooks/110-devops.yml:github_runner_management
+ - apps/web/src/components/dashboard/live-dashboard.tsx:gh_runner_fallback
+ replacement: gitea_runner_readiness_and_queue_receipts
+ exit_condition: legacy_runner_assets_absent_and_gitea_replacement_verified
+ p0_p1_github_recovery_scheduled: false
+ monitoring_generator_duplicate_identity_drift:
+ drift_work_item_id: P0-OBS-002-ASSET-DRIFT-003
+ status: pending_source_deduplication
+ registry_service_count: 26
+ awoooi_api_registry_entry_count: 2
+ generated_scrape_config_count: 24
+ generated_awoooi_api_job_count: 2
+ generated_blackbox_target_count: 20
+ runtime_apply_performed: false
+ exit_condition: one_canonical_awoooi_api_asset_and_one_generated_scrape_job
+
regressions:
api_runtime:
first_probe_timeout_at: "2026-07-15T02:11:22+08:00"
@@ -21,7 +106,7 @@ regressions:
expected_replicas: 2
restart_count_per_pod_at_recovery: 4
last_observed_ready_replicas: 2
- last_observed_cooldown_closed: false
+ last_observed_cooldown_closed: true
primary_classification: db_connection_budget_and_request_pressure
structural_risk_confidence: 0.98
direct_application_commit_regression_confidence: 0.12
@@ -54,50 +139,508 @@ controlled_recovery:
- 192.168.0.120
- 192.168.0.121
-source_candidates:
- cd_probe:
- path: .gitea/workflows/cd.yaml
- connection_budget_precheck_required: true
- max_workers: 4
- max_attempts: 1
- request_timeout_seconds: 5
- safety_reserve_connections: 4
- receipt_fields_added: true
- deployment_status: pending
- backup_restore_guard:
- path: scripts/backup/backup-signoz.sh
- deployment_playbook: infra/ansible/playbooks/110-devops.yml
- deployment_tag: backup_jobs
- deployment_contract_present: true
- trap_exit_hup_int_term: true
- restore_armed_before_stop: true
- bounded_exit_retry: true
- deployment_status: pending
- canonical_health_route:
- internal_url: http://192.168.0.110:8080
- public_url: https://signoz.wooo.work
- production_networkpolicy_tcp_port: 8080
- legacy_query_route_removed: http://192.168.0.188:3301
- otlp_producer_route_changed: false
- deployment_status: pending
+release_receipts:
+ gitea_cd_5140:
+ gitea_run_id: 5140
+ source_sha: e4da6570a645e504cc93647d396f45f167b3172a
+ workflow_terminal: success
+ focused_tests_passed: 6
+ unit_tests:
+ passed: 4817
+ skipped: 23
+ warnings: 69
+ integration_tests_passed: 5
+ production_readback:
+ build_tag_matched: true
+ desired_tag_matched: true
+ attempt: 1
+ rollout_terminal: success
+ transient_rollout_risk_observed: true
+ postdeploy:
+ alert_chain_terminal: pass
+ alert_chain_checks_passed: 8
+ alert_chain_checks_total: 9
+ playwright_passed: 5
+ notification_mirrored: true
+ cd_probe:
+ receipt_provenance: gitea_run_5140_job_log
+ passed: true
+ workers: 4
+ safety_reserve_connections: 4
+ max_attempts: 1
+ request_timeout_seconds: 5
+ skip_reason: ""
+ restart_free: true
+ restart_count_before: 0
+ restart_count_after: 0
+ ready_replicas: 2
+ desired_replicas: 2
+
+controlled_applies:
+ config_route_patch:
+ trace_id: P0-OBS-002
+ run_id: P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1
+ work_item_id: P0-OBS-002
+ source_commit: 9a9d1464a586ba172d1613db5bc285def43eb3e8
+ included_in_release_sha: e4da6570a645e504cc93647d396f45f167b3172a
+ normalized_asset:
+ canonical_id: signoz-110-query-health-route
+ internal_url: http://192.168.0.110:8080
+ public_url: https://signoz.wooo.work
+ health_path: /api/v1/health
+ production_networkpolicy_tcp_port: 8080
+ source_truth_diff:
+ legacy_query_route_removed: http://192.168.0.188:3301
+ otlp_producer_route_changed: false
+ check_receipt:
+ source_tests_passed: true
+ execution_receipt:
+ gitea_run_id: 5140
+ deployment_status: deployed_verified
+ post_verifier:
+ release_readback_matched: true
+ ready_replicas: 2
+ desired_replicas: 2
+ rollback: not_required
+ terminal: applied_post_verifier_pass
+
+ prometheus_target_route:
+ trace_id: P0-OBS-002
+ run_id: P0-OBS-002-prometheus-route-20260714T215106Z-apply1
+ work_item_id: P0-OBS-002
+ risk: medium
+ check_receipt:
+ promtool: pass
+ source_sha: 462cb5c2397d1a6ec0542f2fde83e06e0baa05d82df299ac9ae9464d08d9947d
+ candidate_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c
+ execution_receipt:
+ action: target_first_config_apply
+ active_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c
+ runtime_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c
+ ready: true
+ backup_path: /home/wooo/monitoring/prometheus.yml.bak.P0-OBS-002-prometheus-route-20260714T215106Z-apply1
+ backup_sha: 462cb5c2397d1a6ec0542f2fde83e06e0baa05d82df299ac9ae9464d08d9947d
+ post_verifier:
+ terminal: target_up_two_scrapes
+ target_count: 1
+ target_health: up
+ first_scrape_at: "2026-07-15T05:52:38.146935853+08:00"
+ second_scrape_at: "2026-07-15T05:52:53.146935853+08:00"
+ probe_success: 1
+ legacy_target_count: 0
+ independent_verifier:
+ mode: independent_readback
+ observed_at: "2026-07-15T05:58:09+08:00"
+ active_sha_matches_runtime: true
+ target_count: 1
+ target_health: up
+ probe_success: 1
+ legacy_target_count: 0
+ terminal: pass
+ rollback:
+ required: false
+ available: true
+ terminal: completed_target_first
+
+ alert_rules:
+ trace_id: P0-OBS-002
+ run_id: P0-OBS-002-alert-rules-20260714T215328Z-apply1
+ work_item_id: P0-OBS-002
+ check_receipt:
+ promtool: pass
+ source_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58
+ execution_receipt:
+ remote_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58
+ canonical_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58
+ runtime_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58
+ hashes_match: true
+ prometheus_ready: true
+ post_verifier:
+ window_seconds: 130
+ terminal: pass
+ signoz_rule_count: 1
+ signoz_rule_health: ok
+ signoz_rule_state: inactive
+ exact_query: '(probe_success{instance="http://192.168.0.110:8080/api/v1/health",job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",job="blackbox-http"})'
+ rollback:
+ required: false
+ available: true
+ terminal: completed_verified
+
+ clickhouse_native_backup_disk:
+ trace_id: P0-OBS-002
+ work_item_id: P0-OBS-002
+ risk: high
+ normalized_asset:
+ canonical_id: host110-signoz-clickhouse-backup-disk
+ compose_project: signoz
+ compose_service: clickhouse
+ clickhouse_disk: backups
+ container_path: /backups/
+ host_path: /backup/staging/signoz-clickhouse
+ source:
+ override_sha256: cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488
+ config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
+ focused_tests_passed: 12
+ bash_syntax: pass
+ check_receipt:
+ run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064800P0800-check2
+ terminal: check_pass_no_write
+ prior_disk_count: 0
+ prior_backup_mount_count: 0
+ collector_running: true
+ collector_restart_count: 0
+ attempts:
+ - run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064300P0800-apply1
+ terminal: failed_rolled_back
+ failed_stage: post_recreate_server_identity_verifier
+ exit_code: 2
+ root_cause: remote_heredoc_expanded_awk_positional_parameter
+ source_fix: direct_awk_arguments_without_remote_shell_expansion
+ rollback:
+ terminal: pass
+ prior_compose_state_restored: true
+ managed_files_absent: true
+ host_backup_directory_absent: true
+ clickhouse_healthy: true
+ clickhouse_restart_count: 0
+ original_data_volume_preserved: true
+ dependent_container_identity_preserved: true
+ active_backup_or_restore_count: 0
+ residue_count: 0
+ - run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2
+ terminal: pass
+ action: clickhouse_only_force_recreate_no_pull
+ execution:
+ image_digest_unchanged: true
+ data_volume_name: signoz-clickhouse
+ data_volume_unchanged: true
+ clickhouse_restart_count: 0
+ post_verifier:
+ independent: true
+ disk_type: Local
+ disk_read_only: false
+ disk_remote: false
+ disk_broken: false
+ disk_free_space_positive: true
+ server_uid: 101
+ server_gid: 101
+ server_write_access: true
+ concurrent_backups_allowed: false
+ concurrent_restores_allowed: false
+ collector_identity_unchanged: true
+ signoz_identity_unchanged: true
+ zookeeper_identity_unchanged: true
+ dependent_restart_count_delta: 0
+ grpc_4317_listening: true
+ http_4318_listening: true
+ health_8080_listening: true
+ api_http_status: 200
+ active_backup_or_restore_count: 0
+ backup_directory_entry_count: 0
+ stage_candidate_process_residue_count: 0
+ terminal: deployed_verified
+
+ clickhouse_native_backup_restore_canary:
+ trace_id: P0-OBS-002
+ work_item_id: P0-OBS-002
+ risk: high
+ completion_scope: clickhouse_native_only
+ source:
+ backup_orchestrator_sha256: 2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520
+ native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a
+ isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7
+ inventory_verifier_sha256: 79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c
+ canary_wrapper_sha256: 64d23d9f7a64d1e4e1347b98d45e50e26eadea1741d0e28a5e2e5d74b22322d8
+ backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
+ isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
+ latest_restore_focused_tests_passed: 12
+ post_canary_focused_tests_passed: 94
+ post_format_focused_tests_passed: 94
+ bash_syntax: pass
+ ruff: pass
+ ruff_format: pass
+ yaml_xml_parse: pass
+ toolchain_controlled_apply:
+ check_run_id: P0-OBS-002-native-toolchain-20260715T001124Z-check5
+ check_terminal: pass
+ check_diff: matching_6_drift_1_absent_0_active_operations_0
+ apply_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4
+ apply_terminal: pass
+ action: seven_candidates_atomically_replaced
+ rollback_available: true
+ runtime_backup_restore_restart_or_pull: false
+ postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4
+ postcheck_terminal: check_pass_no_write
+ postcheck_diff: matching_7_drift_0_absent_0_active_operations_0
+ attempts:
+ - run_id: P0-OBS-002-native-canary-20260714T233524Z-apply1
+ terminal: failed_cleanup_verified
+ failed_stage: backup_submit_parser
+ exit_code: 62
+ root_cause: clickhouse_25_5_async_settings_id_parameter_syntax_invalid
+ backup_submitted: false
+ restore_submitted: false
+ server_artifact_created: false
+ ephemeral_residue_count: 0
+ - run_id: P0-OBS-002-native-canary-20260714T234436Z-apply2
+ terminal: failed_cleanup_verified_artifact_preserved
+ failed_stage: isolated_restore_startup
+ root_cause:
+ - read_only_source_artifact_was_chowned_by_clickhouse_entrypoint
+ - isolated_zookeeper_anonymous_login_contract_missing
+ backup_terminal: BACKUP_CREATED
+ backup_file_count: 3690
+ backup_total_size_bytes: 88261384
+ artifact_size_bytes: 81474109
+ artifact_sha256: 139230bc6aa9f53b7b351b5d8f37eed87a68cfbba31a0b4ff4caa31906dfdaac
+ restore_submitted: false
+ cleanup_verified: true
+ artifact_retention: preserved_failed_run_evidence
+ - run_id: P0-OBS-002-native-canary-20260715T000638Z-apply3
+ terminal: failed_cleanup_verified_artifact_preserved
+ failed_stage: macro_contract
+ exit_code: 47
+ root_cause: clickhouse_25_5_system_macros_uses_macro_and_substitution_columns
+ backup_terminal: BACKUP_CREATED
+ backup_file_count: 4414
+ backup_total_size_bytes: 92901923
+ artifact_size_bytes: 85561114
+ artifact_sha256: e4da648d36462e3362ff6532b8bee8f5c6745c2de4bf188fe95fe68946287993
+ staging_artifact_verified: true
+ restore_submitted: false
+ cleanup_verified: true
+ artifact_retention: preserved_failed_run_evidence
+ - run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4
+ check_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4-check
+ check_terminal: check_pass_no_write
+ terminal: pass
+ exit_code: 0
+ backup:
+ operation_id: awoooi-6d64f408f85ad25cd78a351c0c851db8a29bea958afaad5ba6e07f03ec7a55dc
+ terminal: BACKUP_CREATED
+ error: ""
+ file_count: 4228
+ total_size_bytes: 93405832
+ compressed_size_bytes: 86033128
+ artifact_sha256: 4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc
+ source_inventory_sha256: 1c290facb734d59443eaeeec0d778772f5fd8e06e832a096c0d4f82a4a7210ef
+ isolated_restore:
+ terminal: verified
+ clickhouse_restore_terminal: RESTORED
+ exact_clickhouse_image: true
+ exact_zookeeper_image: true
+ staging_network_mode: none
+ restore_network_mode: docker_internal_only
+ production_network_attached: false
+ production_restore_performed: false
+ artifact_source_mount: read_only_staging_only
+ backup_volume_mount: writable_ephemeral
+ source_staged_clickhouse_artifact_sha_match: true
+ database_count: 6
+ restored_database_count: 6
+ table_count: 100
+ restored_table_count: 100
+ table_engine_schema_manifest_parity: true
+ check_table_count: 44
+ check_table_pass_count: 44
+ critical_nonzero_count: 4
+ source_total_rows: 4772292
+ restored_total_rows: 4752914
+ row_delta: -19378
+ source_total_bytes: 93042212
+ restored_total_bytes: 93243468
+ byte_delta: 201256
+ aggregate_counter_exact_parity_gate: false
+ aggregate_counter_reason: online_ingestion_can_advance_between_source_inventory_and_backup_snapshot
+ restic:
+ snapshot_id: 574e2a1a
+ snapshot_readback: pass
+ repository_check_read_data_subset: 1%
+ repository_scope: local_same_failure_domain
+ offsite_verified: false
+ independent_post_verifier:
+ independent_monitor: pass
+ collector_identity_started_at_restart_health_listeners_unchanged: true
+ production_container_identity_count_unchanged: 4
+ production_restart_count_delta: 0
+ api_http_status: 200
+ new_server_artifact_removed: true
+ apply2_apply3_preserved_artifact_hashes_unchanged: true
+ exact_ephemeral_container_volume_network_process_tmp_residue_count: 0
+ cleanup_verified: true
+ secret_values_collected: false
+ closure_writeback: durable_local_receipts_acknowledged
+ terminal: production_closed_clickhouse_native_scope
+
+ backup_canary:
+ guard_source:
+ path: scripts/backup/run-signoz-backup-canary.sh
+ monitor_cooldown_contract_present: true
+ deployment_status: deployed_verified
+ attempts:
+ - trace_id: P0-OBS-002
+ run_id: P0-OBS-002-backup-canary-20260714T213210Z-retry1
+ work_item_id: P0-OBS-002
+ terminal: failed_rolled_back
+ classification: failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention
+ failed_stage: clickhouse_archive_create
+ archive_container_exit_code: 137
+ root_cause: docker_health_monitor_restarted_collector_during_intentional_backup_stop
+ consistent_archive_valid: false
+ clickhouse_archive_committed: false
+ sqlite_stage_reached: false
+ restic_stage_reached: false
+ retention_stage_reached: false
+ success_stage_reached: false
+ rollback:
+ collector_running: true
+ collector_pid_at_verifier: 1518279
+ collector_started_at: "2026-07-14T21:35:01Z"
+ collector_restart_count: 0
+ grpc_4317_listening: true
+ http_4318_listening: true
+ canary_processes_absent: true
+ canary_container_absent: true
+ current_temp_dir_absent: true
+ partial_artifacts_absent: true
+ residual_cleanup_debt:
+ - /tmp/signoz-backup-3723877
+ - trace_id: P0-OBS-002
+ run_id: P0-OBS-002-backup-canary-20260714T221230Z-retry2
+ work_item_id: P0-OBS-002
+ terminal: failed_rolled_back
+ classification: failed_live_rw_volume_archive_native_backup_required_no_restic_no_retention
+ failed_stage: clickhouse_archive_create
+ backup_script_exit_code: 1
+ archive_container_exit_code: unknown_not_durably_captured
+ archive_stderr_receipt: suppressed_by_legacy_pipeline
+ root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract
+ exact_tar_error_proven: false
+ likely_tar_file_changed_during_read: true
+ consistent_archive_valid: false
+ clickhouse_archive_committed: false
+ sqlite_stage_reached: false
+ restic_stage_reached: false
+ retention_stage_reached: false
+ success_stage_reached: false
+ monitor_window:
+ cron_intervals_crossed: 2
+ collector_detected_during_intentional_stop: true
+ cooldown_receipt_observed: true
+ auto_repair_count: 0
+ rollback:
+ cooldown_lease_restored: true
+ collector_running: true
+ collector_started_at: "2026-07-14T22:20:53Z"
+ collector_restart_count: 0
+ grpc_4317_listening: true
+ http_4318_listening: true
+ health_8080_listening: true
+ canary_processes_absent: true
+ canary_container_absent: true
+ current_temp_dir_absent: true
+ partial_artifacts_absent: true
+ residual_cleanup_debt: []
+
+post_release_runtime_verifier:
+ trace_id: P0-OBS-002
+ run_id: P0-OBS-002-post-release-20260714T215500Z
+ work_item_id: P0-OBS-002
+ window_seconds: 600
+ terminal: pass
+ restart_delta: 0
+ exporter_trace_errors: 0
+ exporter_metric_errors: 0
+ trace_count: 4026
+ metric_sample_count: 76369
+ api_runtime_cooldown_closed: true
+ rollback: not_required
+
+completed_verifiers:
+ clickhouse_native_backup_migration:
+ prior_run_id: P0-OBS-002-backup-canary-20260714T221230Z-retry2
+ run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4
+ status: production_closed_clickhouse_native_scope
+ native_backup_terminal: BACKUP_CREATED
+ isolated_restore_terminal: verified
+ restic_snapshot_id: 574e2a1a
+ independent_verifier: pass
+ exact_cleanup: pass
+
+pending_verifiers:
+ signoz_sqlite_application_consistent_backup:
+ status: pending_supported_non_raw_export_or_coordinated_snapshot
+ raw_sqlite_read_or_sync_allowed: false
+ sqlite_cli_present_in_runtime: false
+ required_preconditions:
+ - supported_metadata_export_or_application_consistent_snapshot_design
+ - independent_restore_verifier_without_raw_session_or_secret_read
+ signoz_backup_offsite_dr:
+ status: pending_offsite_snapshot_and_restore_verifier
+ current_repository_scope: local_same_failure_domain
+ latest_local_snapshot_id: 574e2a1a
+ offsite_verified: false
+ required_preconditions:
+ - immutable_or_versioned_offsite_target
+ - bounded_snapshot_copy_receipt
+ - independent_offsite_restore_drill
+ clickhouse_native_failed_artifact_retention:
+ status: pending_bounded_retention_decision
+ destructive_cleanup_authorized: false
+ active_backup_or_restore_count: 0
+ preserved_artifacts:
+ - run_id: P0-OBS-002-native-canary-20260714T234436Z-apply2
+ path: /backups/signoz-35003fe7ee97b8f74547b5fb2c4b1df01b9f472137e3b8299fd8e95ded6d220e.zip
+ sha256: 139230bc6aa9f53b7b351b5d8f37eed87a68cfbba31a0b4ff4caa31906dfdaac
+ size_bytes: 81474109
+ mode: "0640"
+ - run_id: P0-OBS-002-native-canary-20260715T000638Z-apply3
+ path: /backups/signoz-a011c242647d8a58cf47c38e90605b5325fa0800871256a04e0fbcdeb9b9a289.zip
+ sha256: e4da648d36462e3362ff6532b8bee8f5c6745c2de4bf188fe95fe68946287993
+ size_bytes: 85561114
+ mode: "0640"
+ clickhouse_native_resume_submitted_inventory:
+ status: pending_durable_submitted_inventory_contract
+ same_run_resume_safe: false
+ reason: live_inventory_drift_must_not_rebind_an_existing_backup_artifact
+ required_preconditions:
+ - atomically_persist_inventory_before_first_backup_submit
+ - bind_inventory_hash_operation_id_artifact_hash_and_run_identity
+ - resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered
+ - regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits
scope_boundaries:
grpc_bridge_rollback_indicated: false
api_manual_rollback_performed: false
firewall_changed: false
database_changed: false
- stateful_volume_read: false
+ stateful_volume_read: true
+ stateful_volume_write: true
+ clickhouse_container_recreated: true
+ clickhouse_data_volume_identity_changed: false
secret_read: false
github_used: false
terminal:
status: partial_degraded
blockers:
- - api_runtime_cooldown_pending
- - cd_probe_safety_fix_not_deployed
- - backup_collector_restore_guard_not_deployed
- - newer_main_cd_nonterminal
+ - signoz_sqlite_application_consistent_backup_pending
+ - signoz_backup_offsite_dr_pending
+ - clickhouse_native_failed_artifact_retention_pending
+ - clickhouse_native_resume_submitted_inventory_pending
+ - service_registry_runtime_mirror_reconciliation_pending
+ - github_freeze_legacy_asset_retirement_pending
+ - monitoring_generator_duplicate_awoooi_api_identity_pending
safe_next_action: >-
- Merge current Gitea main, validate and deploy the bounded CD probe and backup
- restore guard, then require a fresh CD terminal plus ten-minute API and OTLP
- runtime verifier with zero restart and exporter-error deltas.
+ Preserve the verified native ClickHouse backup and isolated restore lane.
+ Add a supported application-consistent SigNoz metadata export and restore
+ verifier, copy a bounded snapshot to an immutable or versioned offsite
+ target and restore it independently, then apply an explicit failed-artifact
+ retention decision. Persist the submitted source inventory before BACKUP so
+ same-run resume cannot bind an old artifact to newer live counters. Keep raw
+ SQLite reads and syncs disabled. Reconcile the source-only service registry
+ rows against live runtime before regenerating its K8s ConfigMap mirror, and
+ retire legacy GitHub runner assets only after the Gitea replacement verifier
+ is ready. Deduplicate the monitoring registry's awoooi-api identity before
+ applying regenerated scrape configuration.
diff --git a/scripts/alert_chain_smoke_test.py b/scripts/alert_chain_smoke_test.py
index 3cd6050de..a469c5e2e 100644
--- a/scripts/alert_chain_smoke_test.py
+++ b/scripts/alert_chain_smoke_test.py
@@ -9,7 +9,7 @@ Wave A.6 (ADR-037): 驗證告警鏈路 E2E 完整性
2. Alert Chain Metric — awoooi_alert_chain_last_success_timestamp 不超過 2h
3. Webhook 可達性 — /api/v1/webhooks/alertmanager, /signoz, /sentry health
4. Telegram Secret — K8s Secret 存在且非空
- 5. SigNoz 可達 — 192.168.0.188:3301
+ 5. SigNoz 可達 — canonical public route (https://signoz.wooo.work)
6. Prometheus Alertmanager — 192.168.0.188:9093 (可選)
使用方式:
@@ -56,7 +56,11 @@ def _env_float(name: str, default: float) -> float:
DEFAULT_API_URL = "https://awoooi.wooo.work"
-SIGNOZ_URL = "http://192.168.0.188:3301"
+DEFAULT_SIGNOZ_PUBLIC_URL = "https://signoz.wooo.work"
+SIGNOZ_URL = (
+ os.environ.get("SIGNOZ_PUBLIC_URL", DEFAULT_SIGNOZ_PUBLIC_URL).strip().rstrip("/")
+ or DEFAULT_SIGNOZ_PUBLIC_URL
+)
ALERTMANAGER_URL = "http://192.168.0.188:9093"
PROMETHEUS_URL = "http://192.168.0.110:9090"
diff --git a/scripts/backup/backup-signoz.sh b/scripts/backup/backup-signoz.sh
index a942e4957..406a9ceab 100755
--- a/scripts/backup/backup-signoz.sh
+++ b/scripts/backup/backup-signoz.sh
@@ -1,12 +1,12 @@
#!/bin/bash
# =============================================================================
-# WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
-# 版本: 1.3.0
-# 建立日期: 2026-04-05
-# 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
-# 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
-# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
-# 2026-07-15 Codex: v1.3 加入初始狀態、全 run 有界還原與 success 前獨立 verifier
+# WOOO AIOps - SigNoz ClickHouse native backup orchestrator
+# Version: 2.0.0
+#
+# ClickHouse is backed up online through its native BACKUP engine. This script
+# never reads or archives the live ClickHouse/SQLite Docker volumes. SigNoz
+# metadata/SQLite remains an explicit application-consistent export gap and is
+# recorded in the Restic payload without being misreported as completed.
# =============================================================================
set -euo pipefail
@@ -16,17 +16,31 @@ source "$(dirname "$0")/common.sh"
SERVICE="signoz"
LOCAL_REPO="${BACKUP_BASE}/signoz"
DUMP_DIR="/tmp/signoz-backup-$$"
-COLLECTOR_NAME="signoz-otel-collector"
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}"
+COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}"
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
-COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"
-COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}"
-ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"
-ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"
+COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}"
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"
-COLLECTOR_WAS_RUNNING=0
-COLLECTOR_RESTORE_PENDING=0
-COLLECTOR_RESTORE_ATTEMPTS_USED=0
+RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}"
+RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}"
+RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}"
+RESTIC_READBACK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_READBACK_TIMEOUT_SECONDS:-120}"
+RESTIC_CHECK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_CHECK_TIMEOUT_SECONDS:-1800}"
+
+BACKUP_ID_SEED="$(date -u '+%Y%m%dT%H%M%SZ')-$$"
+TRACE_ID="${TRACE_ID:-signoz-backup-${BACKUP_ID_SEED}}"
+RUN_ID="${RUN_ID:-signoz-${BACKUP_ID_SEED}}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-SIGNOZ-BACKUP}"
+export TRACE_ID RUN_ID WORK_ITEM_ID
+
+COLLECTOR_ID_BEFORE=""
+COLLECTOR_STARTED_AT_BEFORE=""
+COLLECTOR_RESTARTS_BEFORE=""
+COLLECTOR_HEALTH_BEFORE=""
+COLLECTOR_OBSERVER_PID=""
+COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
CLEANUP_COMPLETE=0
FAILURE_NOTIFIED=0
@@ -35,27 +49,9 @@ run_collector_docker() {
"${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
}
-collector_running_state() {
- local running
- if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
- return 1
- fi
-
- case "${running}" in
- true)
- printf '%s\n' "running"
- ;;
- false)
- printf '%s\n' "stopped"
- ;;
- *)
- return 1
- ;;
- esac
-}
-
-collector_is_running() {
- [ "$(collector_running_state)" = "running" ]
+collector_identity_state() {
+ run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
+ "${COLLECTOR_NAME}" 2>/dev/null
}
notify_failure_once() {
@@ -67,50 +63,63 @@ notify_failure_once() {
notify_clawbot "failed" "${SERVICE}" "${detail}" || true
}
-restore_collector() {
- if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
- return 0
- fi
+observe_collector_continuity() {
+ local epoch observation
- while [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; do
- COLLECTOR_RESTORE_ATTEMPTS_USED=$((COLLECTOR_RESTORE_ATTEMPTS_USED + 1))
- log_info "重啟 ${COLLECTOR_NAME} (${COLLECTOR_RESTORE_ATTEMPTS_USED}/${COLLECTOR_RESTORE_MAX_ATTEMPTS})..."
-
- if run_collector_docker start "${COLLECTOR_NAME}" >/dev/null 2>&1 \
- && collector_is_running; then
- COLLECTOR_RESTORE_PENDING=0
- return 0
- fi
-
- log_warn "${COLLECTOR_NAME} 重啟或 running readback 失敗"
- if [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; then
- sleep "${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS}"
+ while true; do
+ epoch="$(date +%s)"
+ if observation="$(collector_identity_state)"; then
+ printf '%s\t%s\n' "${epoch}" "${observation}" >> "${COLLECTOR_OBSERVATIONS}"
+ else
+ printf '%s\tunknown\tunknown\tunknown\tunknown\tunknown\n' "${epoch}" >> "${COLLECTOR_OBSERVATIONS}"
fi
+ sleep "${COLLECTOR_CONTINUITY_POLL_SECONDS}"
done
-
- log_error "${COLLECTOR_NAME} 在有界重試後仍無法恢復"
- return 1
}
-verify_collector_final_state() {
- if [ "${COLLECTOR_WAS_RUNNING}" -ne 1 ]; then
- log_info "${COLLECTOR_NAME} 原本即為 stopped，不執行自動啟動"
- return 0
- fi
+start_collector_observer() {
+ : > "${COLLECTOR_OBSERVATIONS}"
+ observe_collector_continuity &
+ COLLECTOR_OBSERVER_PID=$!
+}
- if collector_is_running; then
- log_success "${COLLECTOR_NAME} final running verifier 通過"
- return 0
+stop_collector_observer() {
+ if [ -n "${COLLECTOR_OBSERVER_PID}" ] \
+ && kill -0 "${COLLECTOR_OBSERVER_PID}" 2>/dev/null; then
+ kill -TERM "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
+ wait "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
fi
+ COLLECTOR_OBSERVER_PID=""
+}
- log_warn "${COLLECTOR_NAME} final running verifier 首次失敗，使用剩餘有界還原額度"
- COLLECTOR_RESTORE_PENDING=1
- if ! restore_collector; then
+verify_collector_continuity() {
+ local observation final_id final_running final_started_at final_restarts final_health
+
+ stop_collector_observer
+ [ -s "${COLLECTOR_OBSERVATIONS}" ] || {
+ log_error "${COLLECTOR_NAME} continuity receipt 為空"
+ return 1
+ }
+ if awk -F '\t' -v expected_id="${COLLECTOR_ID_BEFORE}" \
+ -v expected_started="${COLLECTOR_STARTED_AT_BEFORE}" \
+ -v expected_restarts="${COLLECTOR_RESTARTS_BEFORE}" \
+ -v expected_health="${COLLECTOR_HEALTH_BEFORE}" \
+ '$2 == "unknown" || $3 != "true" || $2 != expected_id || \
+ $4 != expected_started || $5 != expected_restarts || $6 != expected_health { exit 1 }' \
+ "${COLLECTOR_OBSERVATIONS}"; then
+ :
+ else
+ log_error "${COLLECTOR_NAME} 在 native backup window 發生停止或 identity drift"
return 1
fi
- # 與 docker start 的回傳值與 restore 內部 readback 分離，作為 success 前獨立 verifier。
- collector_is_running
+ observation="$(collector_identity_state)" || return 1
+ IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${observation}"
+ [ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
+ && [ "${final_running}" = "true" ] \
+ && [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
+ && [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
+ && [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ]
}
cleanup() {
@@ -120,13 +129,9 @@ cleanup() {
return "${exit_code}"
fi
CLEANUP_COMPLETE=1
-
- # cleanup 期間忽略重複 signal，避免還原與刪除流程重入。
trap '' HUP INT TERM
- if ! restore_collector; then
- notify_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}"
- fi
- rm -rf "${DUMP_DIR}"
+ stop_collector_observer
+ rm -rf -- "${DUMP_DIR}"
return "${exit_code}"
}
@@ -135,166 +140,198 @@ on_signal() {
exit "$((128 + signal_number))"
}
-create_volume_archive() {
- local volume_name="$1"
- local output_file="$2"
- local extra_exclude="${3:-}"
- local partial_file="${output_file}.partial"
+run_native_clickhouse_backup() {
+ local mode="$1"
- log_info "備份 volume: ${volume_name}"
- rm -f "${partial_file}" "${output_file}"
-
- if [ -n "${extra_exclude}" ]; then
- if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
- "${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
- docker run --rm -v "${volume_name}:/data" alpine \
- tar czf - "${extra_exclude}" /data \
- > "${partial_file}" 2>/dev/null; then
- log_error " Volume ${volume_name} archive 建立失敗"
- rm -f "${partial_file}" "${output_file}"
+ [ -f "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
+ && [ -x "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
+ && [ ! -L "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
+ || {
+ log_error "ClickHouse native backup adapter 缺失、不可執行或為 symlink"
return 1
- fi
- else
- if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
- "${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
- docker run --rm -v "${volume_name}:/data" alpine \
- tar czf - /data \
- > "${partial_file}" 2>/dev/null; then
- log_error " Volume ${volume_name} archive 建立失敗"
- rm -f "${partial_file}" "${output_file}"
- return 1
- fi
- fi
+ }
- if [ ! -s "${partial_file}" ]; then
- log_error " Volume ${volume_name} 備份失敗 (空檔案)"
- rm -f "${partial_file}" "${output_file}"
- return 1
- fi
-
- mv "${partial_file}" "${output_file}"
+ CLICKHOUSE_NATIVE_OUTPUT_DIR="${DUMP_DIR}" \
+ TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+ "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" "${mode}"
}
-verify_volume_archive() {
- local volume_name="$1"
- local output_file="$2"
+write_metadata_gap_receipt() {
+ cat > "${DUMP_DIR}/signoz-metadata-gap.json" </dev/null 2>&1; then
- log_error " Volume ${volume_name} archive integrity verifier 失敗"
- rm -f "${output_file}" "${output_file}.partial"
- return 1
- fi
+write_restic_receipt() {
+ local snapshot_id="$1"
+ local artifact_hash="$2"
+ local manifest_hash="$3"
+ local inventory_hash="$4"
+ local receipt_dir receipt_tmp receipt_file
- local size
- size="$(du -h "${output_file}" | cut -f1)"
- log_success " Volume ${volume_name} 備份與 integrity verifier 完成 (${size})"
+ [[ "${RESTIC_RECEIPT_ROOT}" == /* ]] && [ ! -L "${RESTIC_RECEIPT_ROOT}" ] || return 1
+ mkdir -p "${RESTIC_RECEIPT_ROOT}"
+ receipt_dir="${RESTIC_RECEIPT_ROOT}/${RUN_ID}"
+ [ ! -L "${receipt_dir}" ] || return 1
+ mkdir -m 700 -p "${receipt_dir}"
+ receipt_file="${receipt_dir}/restic-snapshot.json"
+ receipt_tmp="${receipt_file}.partial.$$"
+ printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","service":"signoz","snapshot_id":"%s","artifact_sha256":"%s","manifest_sha256":"%s","source_inventory_sha256":"%s","repository_scope":"local_same_failure_domain","restic_check_read_data_subset":"1%%","offsite_verified":false,"metadata_sqlite_terminal":"pending_application_consistent_export","completion_claim":"clickhouse_native_only"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${snapshot_id}" \
+ "${artifact_hash}" "${manifest_hash}" "${inventory_hash}" > "${receipt_tmp}"
+ chmod 600 "${receipt_tmp}"
+ mv "${receipt_tmp}" "${receipt_file}"
}
main() {
- local start_time=$(date +%s)
+ local start_time end_time duration snapshot_id snapshot_json tags
+ local initial_observation initial_running final_id final_running
+ local final_started_at final_restarts final_health
+ local artifact_file manifest_file inventory_file artifact_hash manifest_hash inventory_hash
+ start_time="$(date +%s)"
trap cleanup EXIT
trap 'on_signal 1' HUP
trap 'on_signal 2' INT
trap 'on_signal 15' TERM
- log_info "========== 開始 SignOz 備份 =========="
+ log_info "========== 開始 SigNoz ClickHouse native 備份 =========="
+ [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]] \
+ && [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0" ]] \
+ && [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0.0" ]] || {
+ notify_failure_once "COLLECTOR_CONTINUITY_POLL_SECONDS 無效"
+ exit 1
+ }
+ for timeout_value in "${RESTIC_INIT_TIMEOUT_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
+ "${RESTIC_READBACK_TIMEOUT_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}"; do
+ [[ "${timeout_value}" =~ ^[0-9]+$ ]] && [ "${timeout_value}" -gt 0 ] || {
+ notify_failure_once "Restic timeout contract 無效"
+ exit 1
+ }
+ done
mkdir -p "${DUMP_DIR}"
- local timestamp=$(date "+%Y%m%d_%H%M%S")
-
- # Step 1: 只在 Collector 原本 running 時暫停，並在 stop 前 arm 還原。
- local collector_initial_state
- if ! collector_initial_state="$(collector_running_state)"; then
- log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態"
- notify_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態"
+ # Check-mode must pass before the backup window. It writes only durable
+ # receipts and does not submit BACKUP or create a server artifact.
+ if ! run_native_clickhouse_backup --check; then
+ log_error "ClickHouse native BACKUP check-mode 失敗；沒有 volume tar fallback"
+ notify_failure_once "SigNoz ClickHouse native BACKUP runtime contract 未就緒"
exit 1
fi
- if [ "${collector_initial_state}" = "running" ]; then
- COLLECTOR_WAS_RUNNING=1
- COLLECTOR_RESTORE_PENDING=1
- log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..."
- if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then
- log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止"
- notify_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}"
- exit 1
- fi
- else
- log_error "${COLLECTOR_NAME} 初始狀態為 stopped，停止備份且不自動啟動"
- notify_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止"
- exit 1
- fi
- docker stop signoz-telemetrystore-migrator 2>/dev/null || true
-
- # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
- local clickhouse_archive="${DUMP_DIR}/clickhouse_${timestamp}.tar.gz"
- local sqlite_archive="${DUMP_DIR}/sqlite_${timestamp}.tar.gz"
- create_volume_archive "signoz-clickhouse" "${clickhouse_archive}" "--exclude=/data/tmp" || {
- log_error "ClickHouse volume 備份失敗"
- notify_failure_once "SignOz ClickHouse 備份失敗"
+ initial_observation="$(collector_identity_state)" || {
+ notify_failure_once "無法讀取 ${COLLECTOR_NAME} 初始 identity/state"
exit 1
}
-
- # Step 3: 備份 SQLite volume (SignOz metadata)
- create_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
- log_error "SQLite volume 備份失敗"
- notify_failure_once "SignOz SQLite 備份失敗"
+ IFS=$'\t' read -r COLLECTOR_ID_BEFORE initial_running COLLECTOR_STARTED_AT_BEFORE \
+ COLLECTOR_RESTARTS_BEFORE COLLECTOR_HEALTH_BEFORE <<< "${initial_observation}"
+ [ -n "${COLLECTOR_ID_BEFORE}" ] && [ "${initial_running}" = "true" ] || {
+ notify_failure_once "${COLLECTOR_NAME} 初始狀態不是 running"
exit 1
}
+ [ -n "${COLLECTOR_STARTED_AT_BEFORE}" ] \
+ && [[ "${COLLECTOR_RESTARTS_BEFORE}" =~ ^[0-9]+$ ]] \
+ && [[ "${COLLECTOR_HEALTH_BEFORE}" =~ ^(healthy|none)$ ]] || {
+ notify_failure_once "${COLLECTOR_NAME} 初始 runtime metadata 不完整"
+ exit 1
+ }
+ start_collector_observer
- # Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。
- if ! restore_collector; then
- notify_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}"
+ # Online native backup: no collector stop and no live-volume read.
+ if ! run_native_clickhouse_backup --apply; then
+ notify_failure_once "SigNoz ClickHouse native BACKUP 失敗"
exit 1
fi
- # Step 5: Collector 恢復後才在 host 做有界 archive integrity verifier。
- verify_volume_archive "signoz-clickhouse" "${clickhouse_archive}" || {
- notify_failure_once "SignOz ClickHouse archive integrity verifier 失敗"
+ if ! verify_collector_continuity; then
+ notify_failure_once "${COLLECTOR_NAME} no-downtime continuity verifier 失敗"
exit 1
- }
- verify_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
- notify_failure_once "SignOz SQLite archive integrity verifier 失敗"
- exit 1
- }
+ fi
+ log_success "${COLLECTOR_NAME} no-downtime continuity verifier 通過"
+
+ write_metadata_gap_receipt
+
+ artifact_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip' -print)"
+ manifest_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip.manifest.json' -print)"
+ inventory_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.source-inventory.tsv' -print)"
+ [ "$(printf '%s\n' "${artifact_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
+ && [ "$(printf '%s\n' "${manifest_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
+ && [ "$(printf '%s\n' "${inventory_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] || {
+ log_error "ClickHouse native artifact set 不唯一或不完整"
+ exit 1
+ }
+ artifact_hash="$(sha256sum "${artifact_file}" | awk '{print $1}')"
+ manifest_hash="$(sha256sum "${manifest_file}" | awk '{print $1}')"
+ inventory_hash="$(sha256sum "${inventory_file}" | awk '{print $1}')"
- # Step 6: 初始化 Restic 倉庫
if [ ! -d "${LOCAL_REPO}/data" ]; then
log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
- restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
+ timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_INIT_TIMEOUT_SECONDS}" \
+ restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
log_error "Restic 倉庫初始化失敗"
exit 1
}
fi
- # Step 7: Restic 備份
- log_info "建立 Restic 備份..."
- local tags=$(build_tags "${SERVICE}")
- restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" --password-file "${RESTIC_PASSWORD_FILE}" ${tags} 2>&1
+ log_info "建立 ClickHouse native artifact 的 Restic 備份..."
+ tags="$(build_tags "${SERVICE}")"
+ timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
+ restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
+ --password-file "${RESTIC_PASSWORD_FILE}" ${tags} \
+ --tag "trace:${TRACE_ID}" --tag "run:${RUN_ID}" \
+ --tag "work-item:${WORK_ITEM_ID}" 2>&1
- local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1)
+ snapshot_json="$(timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
+ "${RESTIC_READBACK_TIMEOUT_SECONDS}" restic -r "${LOCAL_REPO}" snapshots \
+ --tag "run:${RUN_ID}" --latest 1 --json \
+ --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)"
+ snapshot_id="$(printf '%s\n' "${snapshot_json}" \
+ | sed -n 's/.*"short_id":"\([^"]*\)".*/\1/p' | head -1)"
+ [[ "${snapshot_id}" =~ ^[A-Za-z0-9]+$ ]] || {
+ log_error "Restic same-run snapshot durable readback 缺失"
+ exit 1
+ }
+ timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}" \
+ restic -r "${LOCAL_REPO}" check --read-data-subset=1% \
+ --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
+ log_error "Restic repository/data-subset verifier 失敗"
+ exit 1
+ }
+ write_restic_receipt "${snapshot_id}" "${artifact_hash}" \
+ "${manifest_hash}" "${inventory_hash}" || {
+ log_error "Restic durable receipt writeback 失敗"
+ exit 1
+ }
log_success "Restic 備份完成: ${snapshot_id}"
- # Step 8: GFS 清理；real canary 可明確跳過 destructive retention。
if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
else
cleanup_old_backups "${LOCAL_REPO}"
fi
- # Step 9: success 前獨立檢查 runtime state；失敗時不得發送 success。
- if ! verify_collector_final_state; then
- notify_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗"
+ # Final readback is separate from the continuity observer receipt.
+ initial_observation="$(collector_identity_state)" || {
+ notify_failure_once "${COLLECTOR_NAME} final readback 失敗"
exit 1
- fi
+ }
+ IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${initial_observation}"
+ [ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
+ && [ "${final_running}" = "true" ] \
+ && [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
+ && [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
+ && [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ] || {
+ notify_failure_once "${COLLECTOR_NAME} final identity/start/restart/health verifier 失敗"
+ exit 1
+ }
- local end_time=$(date +%s)
- local duration=$((end_time - start_time))
- log_success "========== SignOz 備份完成 (${duration}s) =========="
- notify_clawbot "success" "${SERVICE}" "SignOz 備份完成 (ClickHouse+SQLite)" "${duration}"
+ end_time="$(date +%s)"
+ duration=$((end_time - start_time))
+ log_success "========== SigNoz ClickHouse native 備份完成 (${duration}s) =========="
+ notify_clawbot "warning" "${SERVICE}" \
+ "ClickHouse native backup 完成；metadata/SQLite application-consistent export 仍 pending" \
+ "${duration}"
}
main "$@"
diff --git a/scripts/backup/clickhouse-native-backup.sh b/scripts/backup/clickhouse-native-backup.sh
new file mode 100755
index 000000000..256273a90
--- /dev/null
+++ b/scripts/backup/clickhouse-native-backup.sh
@@ -0,0 +1,628 @@
+#!/usr/bin/env bash
+# =============================================================================
+# P0-OBS-002 - ClickHouse native BACKUP adapter for SigNoz
+#
+# This adapter deliberately has no live-volume tar fallback. It requires a
+# ClickHouse Disk configured and allowed for BACKUP, records every bounded
+# command's stdout/stderr/exit status, and copies only the completed native
+# backup artifact into the caller's staging directory.
+# =============================================================================
+
+set -euo pipefail
+
+CONTAINER_NAME="${CLICKHOUSE_NATIVE_CONTAINER_NAME:-signoz-clickhouse}"
+BACKUP_DISK="${CLICKHOUSE_NATIVE_BACKUP_DISK:-backups}"
+EXPECTED_DISK_PATH="${CLICKHOUSE_NATIVE_EXPECTED_DISK_PATH:-/backups/}"
+OUTPUT_DIR="${CLICKHOUSE_NATIVE_OUTPUT_DIR:-}"
+RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}"
+COMMAND_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS:-30}"
+BACKUP_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS:-1800}"
+POLL_INTERVAL_SECONDS="${CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS:-5}"
+KILL_AFTER_SECONDS="${CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS:-30}"
+POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}"
+EXPECTED_DATABASES_CSV="signoz_analytics,signoz_logs,signoz_metadata,signoz_meter,signoz_metrics,signoz_traces"
+EXPECTED_DATABASES_JSON='["signoz_analytics","signoz_logs","signoz_metadata","signoz_meter","signoz_metrics","signoz_traces"]'
+BACKUP_SCOPE_SQL="DATABASE signoz_analytics,DATABASE signoz_logs,DATABASE signoz_metadata,DATABASE signoz_meter,DATABASE signoz_metrics,DATABASE signoz_traces"
+
+TRACE_ID="${TRACE_ID:-}"
+RUN_ID="${RUN_ID:-}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-}"
+
+MODE=""
+IDENTITY_HASH=""
+OPERATION_ID=""
+ARTIFACT_NAME=""
+SERVER_ARTIFACT_PATH=""
+OUTPUT_FILE=""
+MANIFEST_FILE=""
+HASH_FILE=""
+INVENTORY_FILE=""
+OUTPUT_PARTIAL=""
+MANIFEST_PARTIAL=""
+INVENTORY_PARTIAL=""
+RECEIPT_DIR=""
+RECEIPT_LOG=""
+IDENTITY_FILE=""
+INVOCATION_ID=""
+
+LAST_STDOUT=""
+LAST_STDERR=""
+LAST_STATUS_FILE=""
+LAST_EXIT_CODE=0
+COMMAND_SEQUENCE=0
+
+CONTAINER_ID=""
+CLICKHOUSE_VERSION=""
+DATABASES_HASH=""
+SOURCE_INVENTORY_RECEIPT=""
+SOURCE_INVENTORY_HASH=""
+STATUS_PRESENT=0
+BACKUP_NAME=""
+BACKUP_STATUS=""
+BACKUP_NUM_FILES="0"
+BACKUP_TOTAL_SIZE="0"
+BACKUP_UNCOMPRESSED_SIZE="0"
+BACKUP_COMPRESSED_SIZE="0"
+BACKUP_ERROR_HEX="none"
+BACKUP_ERROR_HASH=""
+
+RECEIPTS_READY=0
+CHECK_PASS=0
+APPLY_PASS=0
+OUTPUT_CREATED=0
+SERVER_ARTIFACT_ACTIVE=0
+
+log_info() { printf '[INFO] %s\n' "$*"; }
+log_error() { printf '[ERROR] %s\n' "$*" >&2; }
+
+valid_identity() {
+ [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
+}
+
+valid_name() {
+ [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]
+}
+
+valid_positive_integer() {
+ [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
+}
+
+emit_receipt() {
+ local stage="$1"
+ local terminal="$2"
+ local detail="$3"
+ local observed_at
+
+ [ "${RECEIPTS_READY}" -eq 1 ] || return 0
+ observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+ printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${INVOCATION_ID}" \
+ "${observed_at}" "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}"
+}
+
+fail_closed() {
+ local detail="$1"
+ log_error "${detail}"
+ emit_receipt "failure" "failed" "${detail}"
+ exit 1
+}
+
+require_command() {
+ command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
+}
+
+cleanup() {
+ local exit_code=$?
+ local terminal="failed"
+
+ trap - EXIT HUP INT TERM
+ set +e
+ [ -z "${OUTPUT_PARTIAL}" ] || rm -f -- "${OUTPUT_PARTIAL}"
+ [ -z "${MANIFEST_PARTIAL}" ] || rm -f -- "${MANIFEST_PARTIAL}"
+ [ -z "${INVENTORY_PARTIAL}" ] || rm -f -- "${INVENTORY_PARTIAL}"
+
+ if [ "${OUTPUT_CREATED}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then
+ rm -f -- "${OUTPUT_FILE}" "${MANIFEST_FILE}" "${HASH_FILE}" "${INVENTORY_FILE}"
+ emit_receipt "cleanup" "removed" "unverified_local_artifact_removed"
+ fi
+
+ if [ "${SERVER_ARTIFACT_ACTIVE}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then
+ emit_receipt "cleanup" "preserved" \
+ "active_or_unverified_server_artifact_preserved_for_safe_followup"
+ fi
+
+ if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_PASS}" -eq 1 ]; then
+ terminal="check_pass_no_write"
+ elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "apply" ] && [ "${APPLY_PASS}" -eq 1 ]; then
+ terminal="pass"
+ fi
+
+ emit_receipt "terminal" "${terminal}" "clickhouse_native_backup_${terminal}"
+ printf 'CLICKHOUSE_NATIVE_BACKUP_TERMINAL trace_id=%s run_id=%s work_item_id=%s mode=%s terminal=%s exit_code=%s\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" "${terminal}" "${exit_code}"
+ exit "${exit_code}"
+}
+
+on_signal() {
+ local signal_number="$1"
+ emit_receipt "signal" "failed" "signal_${signal_number}"
+ exit "$((128 + signal_number))"
+}
+
+setup() {
+ local identity_content identity_tmp
+
+ [ "$#" -eq 1 ] || { log_error "exactly one mode is required: --check or --apply"; exit 2; }
+ case "$1" in
+ --check) MODE="check" ;;
+ --apply) MODE="apply" ;;
+ *) log_error "unsupported mode: use --check or --apply"; exit 2 ;;
+ esac
+
+ valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and path-safe"; exit 2; }
+ valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and path-safe"; exit 2; }
+ valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and path-safe"; exit 2; }
+ valid_name "${CONTAINER_NAME}" || { log_error "invalid container name"; exit 2; }
+ valid_name "${BACKUP_DISK}" || { log_error "invalid backup disk name"; exit 2; }
+ valid_positive_integer "${COMMAND_TIMEOUT_SECONDS}" || { log_error "invalid command timeout"; exit 2; }
+ valid_positive_integer "${BACKUP_TIMEOUT_SECONDS}" || { log_error "invalid backup timeout"; exit 2; }
+ valid_positive_integer "${POLL_INTERVAL_SECONDS}" || { log_error "invalid poll interval"; exit 2; }
+ valid_positive_integer "${KILL_AFTER_SECONDS}" || { log_error "invalid kill-after timeout"; exit 2; }
+ [[ "${EXPECTED_DISK_PATH}" == /*/ ]] && [[ "${EXPECTED_DISK_PATH}" != *".."* ]] \
+ || { log_error "expected disk path must be an absolute trailing-slash path without dot-dot"; exit 2; }
+ [ -n "${OUTPUT_DIR}" ] && [[ "${OUTPUT_DIR}" == /* ]] \
+ || { log_error "CLICKHOUSE_NATIVE_OUTPUT_DIR must be absolute"; exit 2; }
+ [ -d "${OUTPUT_DIR}" ] && [ -w "${OUTPUT_DIR}" ] && [ ! -L "${OUTPUT_DIR}" ] \
+ || { log_error "output directory missing, unwritable, or symlink"; exit 2; }
+ if [ -n "${POST_VERIFY_HOOK}" ]; then
+ [[ "${POST_VERIFY_HOOK}" == /* ]] && [ -f "${POST_VERIFY_HOOK}" ] \
+ && [ -x "${POST_VERIFY_HOOK}" ] && [ ! -L "${POST_VERIFY_HOOK}" ] \
+ || { log_error "post verifier hook must be an absolute executable regular file"; exit 2; }
+ fi
+
+ for command_name in awk basename cat chmod cmp cp date docker env flock grep mkdir mktemp mv rm sha256sum sleep timeout touch tr unzip wc; do
+ command -v "${command_name}" >/dev/null 2>&1 \
+ || { log_error "required command missing: ${command_name}"; exit 2; }
+ done
+
+ IDENTITY_HASH="$(printf '%s\000%s\000%s' "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" | sha256sum | awk '{print $1}')"
+ OPERATION_ID="awoooi-${IDENTITY_HASH}"
+ ARTIFACT_NAME="signoz-${IDENTITY_HASH}.zip"
+ SERVER_ARTIFACT_PATH="${EXPECTED_DISK_PATH%/}/${ARTIFACT_NAME}"
+ OUTPUT_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.zip"
+ MANIFEST_FILE="${OUTPUT_FILE}.manifest.json"
+ HASH_FILE="${OUTPUT_FILE}.sha256"
+ INVENTORY_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.source-inventory.tsv"
+ OUTPUT_PARTIAL="${OUTPUT_FILE}.partial"
+ MANIFEST_PARTIAL="${MANIFEST_FILE}.partial"
+ INVENTORY_PARTIAL="${INVENTORY_FILE}.partial"
+
+ [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root symlink is unsafe"; exit 2; }
+ mkdir -p "${RECEIPT_ROOT}"
+ RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
+ [ ! -L "${RECEIPT_DIR}" ] || { log_error "receipt directory symlink is unsafe"; exit 2; }
+ mkdir -m 700 -p "${RECEIPT_DIR}"
+ RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
+ IDENTITY_FILE="${RECEIPT_DIR}/identity.json"
+ INVOCATION_ID="${MODE}-$(date -u '+%Y%m%dT%H%M%SZ')-$$"
+
+ identity_content="{\"trace_id\":\"${TRACE_ID}\",\"run_id\":\"${RUN_ID}\",\"work_item_id\":\"${WORK_ITEM_ID}\",\"identity_hash\":\"${IDENTITY_HASH}\",\"operation_id\":\"${OPERATION_ID}\",\"artifact_name\":\"${ARTIFACT_NAME}\"}"
+ if [ -e "${IDENTITY_FILE}" ]; then
+ [ -f "${IDENTITY_FILE}" ] && [ ! -L "${IDENTITY_FILE}" ] \
+ || { log_error "existing identity receipt is unsafe"; exit 2; }
+ [ "$(cat "${IDENTITY_FILE}")" = "${identity_content}" ] \
+ || { log_error "run identity conflicts with durable receipt"; exit 2; }
+ else
+ identity_tmp="$(mktemp "${RECEIPT_DIR}/.identity.XXXXXX")"
+ printf '%s\n' "${identity_content}" > "${identity_tmp}"
+ chmod 600 "${identity_tmp}"
+ mv "${identity_tmp}" "${IDENTITY_FILE}"
+ fi
+ touch "${RECEIPT_LOG}"
+ RECEIPTS_READY=1
+
+ [ ! -L "${RECEIPT_DIR}/run.lock" ] || fail_closed "run_lock_symlink_unsafe"
+ exec 9>>"${RECEIPT_DIR}/run.lock"
+ flock -n 9 || fail_closed "same_run_native_backup_is_active"
+
+ trap cleanup EXIT
+ trap 'on_signal 1' HUP
+ trap 'on_signal 2' INT
+ trap 'on_signal 15' TERM
+}
+
+run_captured() {
+ local step="$1"
+ local stdout_hash stderr_hash terminal
+ shift
+
+ valid_name "${step}" || fail_closed "invalid_command_step"
+ COMMAND_SEQUENCE=$((COMMAND_SEQUENCE + 1))
+ LAST_STDOUT="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stdout"
+ LAST_STDERR="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stderr"
+ LAST_STATUS_FILE="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.status"
+
+ set +e
+ "$@" >"${LAST_STDOUT}" 2>"${LAST_STDERR}"
+ LAST_EXIT_CODE=$?
+ set -e
+ printf '%s\n' "${LAST_EXIT_CODE}" > "${LAST_STATUS_FILE}"
+ stdout_hash="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')"
+ stderr_hash="$(sha256sum "${LAST_STDERR}" | awk '{print $1}')"
+ terminal="pass"
+ [ "${LAST_EXIT_CODE}" -eq 0 ] || terminal="failed"
+ emit_receipt "command" "${terminal}" \
+ "step_${step}_exit_${LAST_EXIT_CODE}_stdout_${stdout_hash}_stderr_${stderr_hash}"
+ return "${LAST_EXIT_CODE}"
+}
+
+run_sql() {
+ local step="$1"
+ local query="$2"
+ shift 2
+ run_captured "${step}" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+ clickhouse-client --format TSVRaw "$@" --query "${query}"
+}
+
+perform_check() {
+ local inspect_line running disk_path system_backups_count database_count backup_grant
+ local expected_databases_file
+
+ run_captured "container" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker inspect \
+ --format $'{{.Id}}\t{{.State.Running}}' "${CONTAINER_NAME}" \
+ || fail_closed "clickhouse_container_inspect_failed"
+ inspect_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ IFS=$'\t' read -r CONTAINER_ID running <<< "${inspect_line}"
+ [ -n "${CONTAINER_ID}" ] && [ "${running}" = "true" ] \
+ || fail_closed "clickhouse_container_not_running_or_identity_ambiguous"
+
+ run_sql "version" 'SELECT version()' || fail_closed "clickhouse_version_query_failed"
+ CLICKHOUSE_VERSION="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ -n "${CLICKHOUSE_VERSION}" ] || fail_closed "clickhouse_version_empty"
+
+ run_sql "system_backups" \
+ "SELECT count() FROM system.tables WHERE database='system' AND name='backups'" \
+ || fail_closed "system_backups_capability_query_failed"
+ system_backups_count="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${system_backups_count}" = "1" ] || fail_closed "native_backup_capability_absent"
+
+ run_sql "backup_grant" 'CHECK GRANT BACKUP ON *.*' \
+ || fail_closed "native_backup_grant_check_failed"
+ backup_grant="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${backup_grant}" = "1" ] || fail_closed "native_backup_grant_absent"
+
+ run_sql "backup_disk" \
+ 'SELECT path FROM system.disks WHERE name={disk:String}' \
+ --param_disk "${BACKUP_DISK}" || fail_closed "backup_disk_query_failed"
+ disk_path="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${disk_path}" = "${EXPECTED_DISK_PATH}" ] \
+ || fail_closed "backup_disk_missing_or_path_mismatch"
+
+ run_captured "disk_readable" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+ test -d "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_absent"
+ run_captured "disk_writable" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+ test -w "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_unwritable"
+
+ run_sql "databases" \
+ "SELECT name FROM system.databases WHERE name NOT IN ('system','information_schema','INFORMATION_SCHEMA','default') ORDER BY name" \
+ || fail_closed "user_database_inventory_query_failed"
+ database_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
+ expected_databases_file="${RECEIPT_DIR}/${INVOCATION_ID}-expected-databases.tsv"
+ printf '%s\n' signoz_analytics signoz_logs signoz_metadata signoz_meter signoz_metrics signoz_traces \
+ > "${expected_databases_file}"
+ cmp -s "${expected_databases_file}" "${LAST_STDOUT}" \
+ || fail_closed "signoz_database_allowlist_drift"
+ [ "${database_count}" -eq 6 ] || fail_closed "signoz_database_count_drift"
+ DATABASES_HASH="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')"
+
+ run_sql "table_inventory" \
+ "SELECT database,name,engine,toString(ifNull(total_rows,0)),toString(ifNull(total_bytes,0)),hex(SHA256(create_table_query)) FROM system.tables WHERE database IN ('signoz_analytics','signoz_logs','signoz_metadata','signoz_meter','signoz_metrics','signoz_traces') ORDER BY database,name" \
+ || fail_closed "source_table_inventory_query_failed"
+ [ "$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")" -gt 0 ] \
+ || fail_closed "source_table_inventory_empty"
+ SOURCE_INVENTORY_RECEIPT="${LAST_STDOUT}"
+ SOURCE_INVENTORY_HASH="$(sha256sum "${SOURCE_INVENTORY_RECEIPT}" | awk '{print $1}')"
+
+ emit_receipt "sensor_source" "pass" \
+ "container_${CONTAINER_ID}_version_${CLICKHOUSE_VERSION}_disk_${BACKUP_DISK}"
+ emit_receipt "normalized_asset_identity" "pass" \
+ "identity_${IDENTITY_HASH}_operation_${OPERATION_ID}_artifact_${ARTIFACT_NAME}"
+ emit_receipt "source_of_truth_diff" "pass" \
+ "configured_disk_path_exact_six_databases_hash_${DATABASES_HASH}_table_inventory_${SOURCE_INVENTORY_HASH}"
+ emit_receipt "risk_policy" "pass" \
+ "risk_medium_native_backup_async_bounded_poll_exact_allowlist_no_volume_tar_fallback"
+}
+
+query_backup_status() {
+ local line_count status_line
+
+ run_sql "backup_status" \
+ "SELECT name,toString(status),toString(num_files),toString(total_size),toString(uncompressed_size),toString(compressed_size),if(empty(error),'none',hex(substring(error,1,2048))),hex(SHA256(error)) FROM system.backups WHERE id={operation_id:String}" \
+ --param_operation_id "${OPERATION_ID}" || fail_closed "system_backups_status_query_failed"
+ line_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
+ if [ "${line_count}" -eq 0 ]; then
+ STATUS_PRESENT=0
+ return 0
+ fi
+ [ "${line_count}" -eq 1 ] || fail_closed "system_backups_operation_id_not_unique"
+ STATUS_PRESENT=1
+ status_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ IFS=$'\t' read -r BACKUP_NAME BACKUP_STATUS BACKUP_NUM_FILES BACKUP_TOTAL_SIZE \
+ BACKUP_UNCOMPRESSED_SIZE BACKUP_COMPRESSED_SIZE BACKUP_ERROR_HEX \
+ BACKUP_ERROR_HASH <<< "${status_line}"
+ [[ "${BACKUP_NUM_FILES}" =~ ^[0-9]+$ ]] \
+ && [[ "${BACKUP_TOTAL_SIZE}" =~ ^[0-9]+$ ]] \
+ && [[ "${BACKUP_UNCOMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \
+ && [[ "${BACKUP_COMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \
+ || fail_closed "system_backups_metrics_ambiguous"
+}
+
+server_artifact_absent() {
+ run_captured "artifact_absent" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+ test ! -e "${SERVER_ARTIFACT_PATH}"
+}
+
+cleanup_server_artifact() {
+ run_captured "artifact_cleanup" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
+ rm -f -- "${SERVER_ARTIFACT_PATH}" || return 1
+ server_artifact_absent || return 1
+ SERVER_ARTIFACT_ACTIVE=0
+ emit_receipt "cleanup" "pass" "exact_server_artifact_removed_and_absent"
+}
+
+poll_backup_terminal() {
+ local deadline now expected_name
+ expected_name="Disk('${BACKUP_DISK}', '${ARTIFACT_NAME}')"
+ deadline=$(( $(date +%s) + BACKUP_TIMEOUT_SECONDS ))
+
+ while true; do
+ query_backup_status
+ [ "${STATUS_PRESENT}" -eq 1 ] || fail_closed "async_backup_status_disappeared"
+ [ "${BACKUP_NAME}" = "${expected_name}" ] || fail_closed "async_backup_name_mismatch"
+ emit_receipt "backup_status" "observed" \
+ "status_${BACKUP_STATUS}_files_${BACKUP_NUM_FILES}_total_${BACKUP_TOTAL_SIZE}_error_hash_${BACKUP_ERROR_HASH}"
+
+ case "${BACKUP_STATUS}" in
+ BACKUP_CREATED)
+ [ "${BACKUP_NUM_FILES}" -gt 0 ] \
+ && [ "${BACKUP_TOTAL_SIZE}" -gt 0 ] \
+ && [ "${BACKUP_UNCOMPRESSED_SIZE}" -gt 0 ] \
+ && [ "${BACKUP_COMPRESSED_SIZE}" -gt 0 ] \
+ || fail_closed "native_backup_terminal_metrics_empty"
+ SERVER_ARTIFACT_ACTIVE=1
+ return 0
+ ;;
+ CREATING_BACKUP)
+ SERVER_ARTIFACT_ACTIVE=1
+ ;;
+ BACKUP_FAILED|BACKUP_CANCELLED)
+ printf '%s\n' "${BACKUP_ERROR_HEX}" \
+ > "${RECEIPT_DIR}/${INVOCATION_ID}-backup-error-bounded.hex"
+ emit_receipt "backup_error" "retained" \
+ "bounded_hex_2048_bytes_hash_${BACKUP_ERROR_HASH}"
+ SERVER_ARTIFACT_ACTIVE=1
+ if ! cleanup_server_artifact; then
+ emit_receipt "cleanup" "failed" "failed_terminal_exact_artifact_cleanup_failed"
+ fi
+ fail_closed "native_backup_terminal_${BACKUP_STATUS}"
+ ;;
+ *) fail_closed "unexpected_native_backup_status_${BACKUP_STATUS}" ;;
+ esac
+
+ now="$(date +%s)"
+ if [ "${now}" -ge "${deadline}" ]; then
+ fail_closed "native_backup_poll_timeout_artifact_preserved"
+ fi
+ sleep "${POLL_INTERVAL_SECONDS}"
+ done
+}
+
+verify_existing_local_artifact() {
+ local recorded_hash actual_hash inventory_hash artifact_size
+ local manifest_hash hook_manifest_hash hook_inventory_hash
+
+ if [ ! -e "${OUTPUT_FILE}" ] && [ ! -e "${MANIFEST_FILE}" ] \
+ && [ ! -e "${HASH_FILE}" ] && [ ! -e "${INVENTORY_FILE}" ]; then
+ return 1
+ fi
+ [ -f "${OUTPUT_FILE}" ] && [ ! -L "${OUTPUT_FILE}" ] \
+ && [ -f "${MANIFEST_FILE}" ] && [ ! -L "${MANIFEST_FILE}" ] \
+ && [ -f "${HASH_FILE}" ] && [ ! -L "${HASH_FILE}" ] \
+ && [ -f "${INVENTORY_FILE}" ] && [ ! -L "${INVENTORY_FILE}" ] \
+ || fail_closed "local_artifact_identity_incomplete_or_unsafe"
+ grep -F -q "\"identity_hash\":\"${IDENTITY_HASH}\"" "${MANIFEST_FILE}" \
+ || fail_closed "local_artifact_manifest_identity_mismatch"
+ recorded_hash="$(awk 'NR == 1 { print $1 }' "${HASH_FILE}")"
+ [[ "${recorded_hash}" =~ ^[0-9a-f]{64}$ ]] || fail_closed "local_artifact_hash_receipt_invalid"
+ actual_hash="$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')"
+ [ "${actual_hash}" = "${recorded_hash}" ] || fail_closed "local_artifact_hash_mismatch"
+ artifact_size="$(wc -c < "${OUTPUT_FILE}" | tr -d ' ')"
+ inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')"
+ manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')"
+ grep -F -q "\"operation_id\":\"${OPERATION_ID}\"" "${MANIFEST_FILE}" \
+ || fail_closed "local_artifact_manifest_operation_mismatch"
+ grep -F -q "\"artifact_name\":\"${ARTIFACT_NAME}\"" "${MANIFEST_FILE}" \
+ || fail_closed "local_artifact_manifest_name_mismatch"
+ grep -F -q "\"databases\":${EXPECTED_DATABASES_JSON}" "${MANIFEST_FILE}" \
+ || fail_closed "local_artifact_manifest_database_allowlist_mismatch"
+ grep -F -q "\"source_inventory_sha256\":\"${inventory_hash}\"" "${MANIFEST_FILE}" \
+ || fail_closed "local_source_inventory_hash_mismatch"
+ grep -F -q "\"sha256\":\"${actual_hash}\"" "${MANIFEST_FILE}" \
+ || fail_closed "local_artifact_manifest_hash_mismatch"
+ grep -F -q "\"size_bytes\":${artifact_size}" "${MANIFEST_FILE}" \
+ || fail_closed "local_artifact_manifest_size_mismatch"
+ grep -F -q '"status":"BACKUP_CREATED"' "${MANIFEST_FILE}" \
+ || fail_closed "local_artifact_manifest_terminal_mismatch"
+ run_captured "local_zip_reuse" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_FILE}" \
+ || fail_closed "local_native_archive_reuse_verifier_failed"
+ if [ -n "${POST_VERIFY_HOOK}" ]; then
+ run_captured "reuse_verify_hook_check" env \
+ TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+ CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \
+ CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \
+ CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \
+ CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
+ CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
+ CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \
+ "${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_reuse_check_failed"
+ run_captured "reuse_verify_hook" env \
+ TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+ CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \
+ CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \
+ CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \
+ CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
+ CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
+ CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \
+ "${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_reuse_failed"
+ [ "$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')" = "${actual_hash}" ] \
+ || fail_closed "post_verify_hook_modified_reused_artifact"
+ hook_inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')"
+ [ "${hook_inventory_hash}" = "${inventory_hash}" ] \
+ || fail_closed "post_verify_hook_modified_reused_source_inventory"
+ hook_manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')"
+ [ "${hook_manifest_hash}" = "${manifest_hash}" ] \
+ || fail_closed "post_verify_hook_modified_reused_manifest"
+ fi
+ emit_receipt "idempotency" "reused" "verified_local_artifact_hash_${actual_hash}"
+ APPLY_PASS=1
+ return 0
+}
+
+copy_and_verify_artifact() {
+ local artifact_hash artifact_size hook_hash inventory_hash hook_inventory_hash
+ local manifest_hash hook_manifest_hash
+
+ rm -f -- "${OUTPUT_PARTIAL}" "${MANIFEST_PARTIAL}" "${INVENTORY_PARTIAL}"
+ run_captured "artifact_copy" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${BACKUP_TIMEOUT_SECONDS}" docker cp \
+ "${CONTAINER_NAME}:${SERVER_ARTIFACT_PATH}" "${OUTPUT_PARTIAL}" \
+ || fail_closed "native_backup_artifact_copy_failed"
+ [ -s "${OUTPUT_PARTIAL}" ] || fail_closed "native_backup_artifact_copy_empty"
+
+ run_captured "local_zip_verify" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_PARTIAL}" \
+ || fail_closed "native_backup_archive_integrity_failed"
+ artifact_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')"
+ artifact_size="$(wc -c < "${OUTPUT_PARTIAL}" | tr -d ' ')"
+ [[ "${artifact_size}" =~ ^[0-9]+$ ]] && [ "${artifact_size}" -gt 0 ] \
+ || fail_closed "native_backup_local_size_invalid"
+
+ cp "${SOURCE_INVENTORY_RECEIPT}" "${INVENTORY_PARTIAL}"
+ inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')"
+ [ "${inventory_hash}" = "${SOURCE_INVENTORY_HASH}" ] \
+ || fail_closed "copied_source_inventory_hash_mismatch"
+ printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","identity_hash":"%s","operation_id":"%s","artifact_name":"%s","databases":%s,"source_inventory_sha256":"%s","sha256":"%s","size_bytes":%s,"status":"BACKUP_CREATED","num_files":%s,"total_size":%s,"uncompressed_size":%s,"compressed_size":%s}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${IDENTITY_HASH}" \
+ "${OPERATION_ID}" "${ARTIFACT_NAME}" "${EXPECTED_DATABASES_JSON}" \
+ "${SOURCE_INVENTORY_HASH}" "${artifact_hash}" "${artifact_size}" \
+ "${BACKUP_NUM_FILES}" "${BACKUP_TOTAL_SIZE}" "${BACKUP_UNCOMPRESSED_SIZE}" \
+ "${BACKUP_COMPRESSED_SIZE}" > "${MANIFEST_PARTIAL}"
+
+ if [ -n "${POST_VERIFY_HOOK}" ]; then
+ manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')"
+ run_captured "post_verify_hook_check" env \
+ TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+ CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \
+ CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \
+ CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \
+ CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
+ CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
+ CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \
+ "${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_check_failed"
+ run_captured "post_verify_hook" env \
+ TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+ CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \
+ CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \
+ CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \
+ CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
+ CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
+ CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \
+ "${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_failed"
+ hook_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')"
+ [ "${hook_hash}" = "${artifact_hash}" ] || fail_closed "post_verify_hook_modified_artifact"
+ hook_inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')"
+ [ "${hook_inventory_hash}" = "${inventory_hash}" ] \
+ || fail_closed "post_verify_hook_modified_source_inventory"
+ hook_manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')"
+ [ "${hook_manifest_hash}" = "${manifest_hash}" ] \
+ || fail_closed "post_verify_hook_modified_manifest"
+ fi
+
+ mv "${OUTPUT_PARTIAL}" "${OUTPUT_FILE}"
+ OUTPUT_CREATED=1
+ mv "${INVENTORY_PARTIAL}" "${INVENTORY_FILE}"
+ printf '%s %s\n' "${artifact_hash}" "$(basename "${OUTPUT_FILE}")" > "${HASH_FILE}"
+ mv "${MANIFEST_PARTIAL}" "${MANIFEST_FILE}"
+
+ cleanup_server_artifact || fail_closed "completed_server_artifact_cleanup_failed"
+ emit_receipt "independent_verifier" "pass" \
+ "system_status_archive_integrity_hash_${artifact_hash}_size_${artifact_size}"
+ APPLY_PASS=1
+}
+
+apply_backup() {
+ local submission_line submitted_id submitted_status
+
+ grep -F -q '"stage":"check","terminal":"check_pass_no_write"' "${RECEIPT_LOG}" \
+ || fail_closed "same_run_check_pass_receipt_required_before_apply"
+ perform_check
+ emit_receipt "check" "pass" "apply_revalidation_pass"
+
+ if verify_existing_local_artifact; then
+ return 0
+ fi
+
+ query_backup_status
+ if [ "${STATUS_PRESENT}" -eq 0 ]; then
+ server_artifact_absent || fail_closed "orphan_server_artifact_without_system_status"
+ emit_receipt "ai_decision" "candidate" \
+ "submit_native_async_backup_exact_six_database_allowlist_to_exact_artifact"
+ SERVER_ARTIFACT_ACTIVE=1
+ run_sql "backup_submit" \
+ "BACKUP ${BACKUP_SCOPE_SQL} TO Disk({disk:String},{artifact:String}) SETTINGS id='${OPERATION_ID}' ASYNC" \
+ --param_disk "${BACKUP_DISK}" --param_artifact "${ARTIFACT_NAME}" \
+ || fail_closed "native_backup_submit_failed"
+ submission_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ IFS=$'\t' read -r submitted_id submitted_status <<< "${submission_line}"
+ [ "${submitted_id}" = "${OPERATION_ID}" ] \
+ || fail_closed "native_backup_submission_id_mismatch"
+ case "${submitted_status}" in
+ CREATING_BACKUP|BACKUP_CREATED) ;;
+ *) fail_closed "native_backup_submission_status_${submitted_status:-empty}" ;;
+ esac
+ SERVER_ARTIFACT_ACTIVE=1
+ emit_receipt "execution" "submitted" "operation_${OPERATION_ID}_status_${submitted_status}"
+ else
+ emit_receipt "idempotency" "resumed" "existing_operation_status_${BACKUP_STATUS}"
+ fi
+
+ poll_backup_terminal
+ copy_and_verify_artifact
+}
+
+main() {
+ setup "$@"
+
+ if [ "${MODE}" = "check" ]; then
+ perform_check
+ emit_receipt "ai_decision" "candidate" \
+ "native_backup_apply_requires_same_run_check_receipt"
+ emit_receipt "check" "check_pass_no_write" \
+ "runtime_contract_valid_no_backup_or_artifact_write"
+ CHECK_PASS=1
+ log_info "ClickHouse native backup check passed without runtime write"
+ return 0
+ fi
+
+ apply_backup
+ log_info "ClickHouse native backup verified: ${OUTPUT_FILE}"
+}
+
+main "$@"
diff --git a/scripts/backup/clickhouse-native-restore-drill.sh b/scripts/backup/clickhouse-native-restore-drill.sh
new file mode 100755
index 000000000..afb54ef3c
--- /dev/null
+++ b/scripts/backup/clickhouse-native-restore-drill.sh
@@ -0,0 +1,1152 @@
+#!/usr/bin/env bash
+# =============================================================================
+# P0-OBS-002 - isolated ClickHouse native RESTORE drill
+#
+# This verifier never joins a production network and never restores into an
+# existing ClickHouse instance. Apply mode stages the read-only source artifact
+# through an exact-image, network-none container into a fresh writable volume,
+# then creates one Docker internal network, one exact local ZooKeeper container,
+# one exact local ClickHouse container and fresh ephemeral volumes. Every owned
+# resource is removed on every terminal.
+# =============================================================================
+
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-${ROOT_DIR}/scripts/backup/clickhouse-restore-inventory.py}"
+BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml}"
+ISOLATED_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml}"
+
+BACKUP_DISK="backups"
+CONTAINER_ARTIFACT_NAME="native-backup.zip"
+CONTAINER_ARTIFACT_PATH="/backups/${CONTAINER_ARTIFACT_NAME}"
+STAGING_SOURCE_ARTIFACT_PATH="/tmp/awoooi-source-${CONTAINER_ARTIFACT_NAME}"
+RESTORE_SCOPE_SQL="DATABASE signoz_analytics,DATABASE signoz_logs,DATABASE signoz_metadata,DATABASE signoz_meter,DATABASE signoz_metrics,DATABASE signoz_traces"
+DATABASE_IN_SQL="'signoz_analytics','signoz_logs','signoz_metadata','signoz_meter','signoz_metrics','signoz_traces'"
+EMPTY_SHA256="E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"
+
+COMMAND_TIMEOUT_SECONDS="${CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS:-60}"
+STARTUP_TIMEOUT_SECONDS="${CLICKHOUSE_RESTORE_STARTUP_TIMEOUT_SECONDS:-180}"
+RESTORE_TIMEOUT_SECONDS="${CLICKHOUSE_RESTORE_TIMEOUT_SECONDS:-1800}"
+POLL_INTERVAL_SECONDS="${CLICKHOUSE_RESTORE_POLL_INTERVAL_SECONDS:-2}"
+KILL_AFTER_SECONDS="${CLICKHOUSE_RESTORE_KILL_AFTER_SECONDS:-15}"
+MAX_LOG_BYTES="${CLICKHOUSE_RESTORE_MAX_LOG_BYTES:-262144}"
+LOG_TAIL_LINES="${CLICKHOUSE_RESTORE_LOG_TAIL_LINES:-200}"
+
+MODE=""
+TRACE_ID="${TRACE_ID:-}"
+RUN_ID="${RUN_ID:-}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-}"
+ARTIFACT_PATH="${CLICKHOUSE_NATIVE_ARTIFACT_PATH:-}"
+MANIFEST_PATH="${CLICKHOUSE_NATIVE_MANIFEST_PATH:-}"
+SOURCE_INVENTORY_PATH="${CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH:-}"
+CLICKHOUSE_IMAGE="${CLICKHOUSE_RESTORE_CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:25.5.6}"
+CLICKHOUSE_IMAGE_ID_EXPECTED="${CLICKHOUSE_RESTORE_CLICKHOUSE_IMAGE_ID:-sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6}"
+ZOOKEEPER_IMAGE="${CLICKHOUSE_RESTORE_ZOOKEEPER_IMAGE:-signoz/zookeeper:3.7.1}"
+ZOOKEEPER_IMAGE_ID_EXPECTED="${CLICKHOUSE_RESTORE_ZOOKEEPER_IMAGE_ID:-sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0}"
+RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-/backup/logs/clickhouse-native-restore-drill}"
+HOOK_ARTIFACT_SHA256="${CLICKHOUSE_NATIVE_ARTIFACT_SHA256:-}"
+HOOK_OPERATION_ID="${CLICKHOUSE_NATIVE_OPERATION_ID:-}"
+HOOK_EXPECTED_DATABASES="${CLICKHOUSE_NATIVE_EXPECTED_DATABASES:-}"
+
+IDENTITY_HASH=""
+SHORT_ID=""
+RESTORE_REPLICA=""
+RESTORE_OPERATION_ID=""
+NETWORK_NAME=""
+STAGING_CONTAINER=""
+CLICKHOUSE_CONTAINER=""
+ZOOKEEPER_CONTAINER=""
+BACKUP_VOLUME=""
+CLICKHOUSE_VOLUME=""
+ZOOKEEPER_DATA_VOLUME=""
+ZOOKEEPER_LOG_VOLUME=""
+
+RECEIPT_DIR=""
+RECEIPT_LOG=""
+STATUS_FILE=""
+PREFLIGHT_FILE=""
+PREFLIGHT_CANDIDATE=""
+LOCK_DIR=""
+INVOCATION_ID=""
+RECEIPTS_READY=0
+COMMAND_SEQUENCE=0
+LAST_STDOUT=""
+LAST_STDERR=""
+LAST_EXIT_CODE=0
+
+ARTIFACT_SHA256=""
+STAGED_ARTIFACT_SHA256=""
+CLICKHOUSE_ARTIFACT_SHA256=""
+SOURCE_INVENTORY_SHA256=""
+BACKUP_OPERATION_ID=""
+SOURCE_TABLE_COUNT=0
+SOURCE_DATABASE_COUNT=0
+REPLICATED_TABLE_COUNT=0
+DISTRIBUTED_TABLE_COUNT=0
+PHYSICAL_TABLE_COUNT=0
+CLICKHOUSE_IMAGE_ID=""
+ZOOKEEPER_IMAGE_ID=""
+BACKUP_CONFIG_SHA256=""
+CLUSTER_CONFIG_SHA256=""
+
+RESTORE_TERMINAL="not_started"
+RESTORE_NUM_FILES=0
+RESTORE_TOTAL_SIZE=0
+MANIFEST_PARITY=0
+RESTORED_DATABASE_COUNT=0
+RESTORED_TABLE_COUNT=0
+CRITICAL_NONZERO_COUNT=0
+CHECK_TABLE_COUNT=0
+CHECK_TABLE_PASS_COUNT=0
+SOURCE_TOTAL_ROWS=0
+RESTORED_TOTAL_ROWS=0
+ROW_DELTA=0
+SOURCE_TOTAL_BYTES=0
+RESTORED_TOTAL_BYTES=0
+BYTE_DELTA=0
+
+NETWORK_CREATED=0
+STAGING_CONTAINER_CREATED=0
+CLICKHOUSE_CONTAINER_CREATED=0
+ZOOKEEPER_CONTAINER_CREATED=0
+BACKUP_VOLUME_CREATED=0
+CLICKHOUSE_VOLUME_CREATED=0
+ZOOKEEPER_DATA_VOLUME_CREATED=0
+ZOOKEEPER_LOG_VOLUME_CREATED=0
+EPHEMERAL_RUNTIME_CREATED=0
+ARTIFACT_STAGING_VERIFIED=0
+CLICKHOUSE_ARTIFACT_READBACK_VERIFIED=0
+APPLY_RESOURCE_SCOPE=0
+CHECK_PASS=0
+APPLY_PASS=0
+CLEANUP_PASS=1
+FINAL_REASON="not_started"
+
+usage() {
+ cat <<'USAGE'
+Usage:
+ clickhouse-native-restore-drill.sh --check|--apply \
+ --trace-id ID --run-id ID --work-item-id ID \
+ --artifact ABSOLUTE_PATH --manifest ABSOLUTE_PATH \
+ --source-inventory ABSOLUTE_PATH \
+ [--clickhouse-image REF --clickhouse-image-id sha256:...] \
+ [--zookeeper-image signoz/zookeeper:3.7.1] \
+ [--zookeeper-image-id sha256:...] [--receipt-root ABSOLUTE_PATH]
+
+--check performs local artifact/config validation plus read-only Docker image
+inspection. It creates no container, volume or network. --apply requires the
+same run's check receipt, stages the source through an exact-image network-none
+container into a writable owned backup volume, then restores only into fresh
+ephemeral volumes on an internal-only Docker network. It never attaches to
+production and publishes no ports. All ephemeral resources are removed on
+every terminal.
+
+When invoked as CLICKHOUSE_NATIVE_POST_VERIFY_HOOK, TRACE_ID/RUN_ID/
+WORK_ITEM_ID and CLICKHOUSE_NATIVE_{ARTIFACT,MANIFEST,SOURCE_INVENTORY}_PATH
+are accepted directly from the adapter's bounded environment. Deployments may
+set CLICKHOUSE_RESTORE_{INVENTORY_HELPER,BACKUP_DISK_CONFIG,CLUSTER_CONFIG} to
+reviewed absolute host paths; no env file is read.
+USAGE
+}
+
+log_info() { printf '[INFO] %s\n' "$*"; }
+log_error() { printf '[ERROR] %s\n' "$*" >&2; }
+
+valid_identity() {
+ [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
+}
+
+valid_positive_integer() {
+ [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
+}
+
+valid_image_ref() {
+ [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._/@:-]{0,255}$ ]]
+}
+
+valid_image_id() {
+ [[ "$1" =~ ^sha256:[0-9a-f]{64}$ ]]
+}
+
+require_value() {
+ local option="$1"
+ local value="${2:-}"
+ [ -n "${value}" ] || { log_error "${option} requires a value"; exit 64; }
+}
+
+set_mode() {
+ local requested="$1"
+ [ -z "${MODE}" ] || { log_error "exactly one of --check or --apply is required"; exit 64; }
+ MODE="${requested}"
+}
+
+parse_args() {
+ while [ "$#" -gt 0 ]; do
+ case "$1" in
+ --check) set_mode check ;;
+ --apply) set_mode apply ;;
+ --trace-id)
+ require_value "$1" "${2:-}"
+ TRACE_ID="$2"
+ shift
+ ;;
+ --run-id)
+ require_value "$1" "${2:-}"
+ RUN_ID="$2"
+ shift
+ ;;
+ --work-item-id)
+ require_value "$1" "${2:-}"
+ WORK_ITEM_ID="$2"
+ shift
+ ;;
+ --artifact)
+ require_value "$1" "${2:-}"
+ ARTIFACT_PATH="$2"
+ shift
+ ;;
+ --manifest)
+ require_value "$1" "${2:-}"
+ MANIFEST_PATH="$2"
+ shift
+ ;;
+ --source-inventory)
+ require_value "$1" "${2:-}"
+ SOURCE_INVENTORY_PATH="$2"
+ shift
+ ;;
+ --clickhouse-image)
+ require_value "$1" "${2:-}"
+ CLICKHOUSE_IMAGE="$2"
+ shift
+ ;;
+ --clickhouse-image-id)
+ require_value "$1" "${2:-}"
+ CLICKHOUSE_IMAGE_ID_EXPECTED="$2"
+ shift
+ ;;
+ --zookeeper-image)
+ require_value "$1" "${2:-}"
+ ZOOKEEPER_IMAGE="$2"
+ shift
+ ;;
+ --zookeeper-image-id)
+ require_value "$1" "${2:-}"
+ ZOOKEEPER_IMAGE_ID_EXPECTED="$2"
+ shift
+ ;;
+ --receipt-root)
+ require_value "$1" "${2:-}"
+ RECEIPT_ROOT="$2"
+ shift
+ ;;
+ -h|--help)
+ usage
+ exit 0
+ ;;
+ *)
+ log_error "unsupported argument: $1"
+ usage >&2
+ exit 64
+ ;;
+ esac
+ shift
+ done
+}
+
+validate_regular_file() {
+ local label="$1"
+ local path="$2"
+ [ -n "${path}" ] && [[ "${path}" == /* ]] && [[ "${path}" != *$'\n'* ]] \
+ || { log_error "${label} must be an absolute single-line path"; exit 64; }
+ [ -f "${path}" ] && [ ! -L "${path}" ] \
+ || { log_error "${label} must be a non-symlink regular file"; exit 64; }
+}
+
+validate_contract() {
+ [ -n "${MODE}" ] || { log_error "exactly one of --check or --apply is required"; exit 64; }
+ valid_identity "${TRACE_ID}" || { log_error "--trace-id is required and path-safe"; exit 64; }
+ valid_identity "${RUN_ID}" || { log_error "--run-id is required and path-safe"; exit 64; }
+ valid_identity "${WORK_ITEM_ID}" || { log_error "--work-item-id is required and path-safe"; exit 64; }
+ valid_image_ref "${CLICKHOUSE_IMAGE}" || { log_error "invalid ClickHouse image reference"; exit 64; }
+ valid_image_id "${CLICKHOUSE_IMAGE_ID_EXPECTED}" || { log_error "invalid expected ClickHouse image ID"; exit 64; }
+ valid_image_ref "${ZOOKEEPER_IMAGE}" || { log_error "invalid ZooKeeper image reference"; exit 64; }
+ [ "${ZOOKEEPER_IMAGE}" = "signoz/zookeeper:3.7.1" ] \
+ || { log_error "ZooKeeper image must be the reviewed signoz/zookeeper:3.7.1 reference"; exit 64; }
+ valid_image_id "${ZOOKEEPER_IMAGE_ID_EXPECTED}" || { log_error "invalid expected ZooKeeper image ID"; exit 64; }
+ [[ "${RECEIPT_ROOT}" == /* ]] && [[ "${RECEIPT_ROOT}" != *$'\n'* ]] \
+ || { log_error "receipt root must be an absolute single-line path"; exit 64; }
+ [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root symlink is unsafe"; exit 64; }
+ validate_regular_file artifact "${ARTIFACT_PATH}"
+ validate_regular_file manifest "${MANIFEST_PATH}"
+ validate_regular_file source_inventory "${SOURCE_INVENTORY_PATH}"
+ validate_regular_file backup_disk_config "${BACKUP_DISK_CONFIG}"
+ validate_regular_file isolated_cluster_config "${ISOLATED_CLUSTER_CONFIG}"
+ validate_regular_file inventory_helper "${INVENTORY_HELPER}"
+ if [ -n "${HOOK_EXPECTED_DATABASES}" ]; then
+ [ "${HOOK_EXPECTED_DATABASES}" = "signoz_analytics,signoz_logs,signoz_metadata,signoz_meter,signoz_metrics,signoz_traces" ] \
+ || { log_error "hook database allowlist mismatch"; exit 64; }
+ fi
+
+ for value in "${COMMAND_TIMEOUT_SECONDS}" "${STARTUP_TIMEOUT_SECONDS}" \
+ "${RESTORE_TIMEOUT_SECONDS}" "${POLL_INTERVAL_SECONDS}" \
+ "${KILL_AFTER_SECONDS}" "${MAX_LOG_BYTES}" "${LOG_TAIL_LINES}"; do
+ valid_positive_integer "${value}" || { log_error "timeout/log limits must be positive integers"; exit 64; }
+ done
+ for command_name in awk basename cat cmp date docker grep head mkdir mktemp mv python3 rm \
+ rmdir sed sha256sum sleep timeout touch tr unzip wc; do
+ command -v "${command_name}" >/dev/null 2>&1 \
+ || { log_error "required command missing: ${command_name}"; exit 64; }
+ done
+
+ grep -F -q 'backups' "${BACKUP_DISK_CONFIG}" \
+ || { log_error "repo-owned backup disk config does not allow backups disk"; exit 64; }
+ grep -F -q '/backups/' "${BACKUP_DISK_CONFIG}" \
+ || { log_error "repo-owned backup disk config path mismatch"; exit 64; }
+ grep -F -q 'restore-zookeeper' "${ISOLATED_CLUSTER_CONFIG}" \
+ || { log_error "isolated ZooKeeper host contract missing"; exit 64; }
+ grep -F -q '' "${ISOLATED_CLUSTER_CONFIG}" \
+ || { log_error "isolated cluster name contract missing"; exit 64; }
+ grep -F -q '01' "${ISOLATED_CLUSTER_CONFIG}" \
+ || { log_error "isolated shard macro contract missing"; exit 64; }
+ grep -F -q 'from_env="CLICKHOUSE_RESTORE_REPLICA"' "${ISOLATED_CLUSTER_CONFIG}" \
+ || { log_error "run-derived replica macro contract missing"; exit 64; }
+}
+
+sha256_file() {
+ sha256sum "$1" | awk '{print $1}'
+}
+
+emit_receipt() {
+ local stage="$1"
+ local terminal="$2"
+ local detail="$3"
+ local observed_at
+ [ "${RECEIPTS_READY}" -eq 1 ] || return 0
+ observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+ printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${INVOCATION_ID}" \
+ "${observed_at}" "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}"
+}
+
+fail_closed() {
+ FINAL_REASON="$1"
+ log_error "${FINAL_REASON}"
+ emit_receipt failure failed "${FINAL_REASON}"
+ exit 1
+}
+
+setup_receipts() {
+ local identity_file identity_content identity_tmp
+
+ IDENTITY_HASH="$(printf '%s\000%s\000%s' "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" | sha256sum | awk '{print $1}')"
+ SHORT_ID="${IDENTITY_HASH:0:16}"
+ RESTORE_REPLICA="restore-${SHORT_ID}"
+ RESTORE_OPERATION_ID="restore-${IDENTITY_HASH}"
+ NETWORK_NAME="awoooi-ch-restore-${SHORT_ID}-net"
+ STAGING_CONTAINER="awoooi-ch-restore-${SHORT_ID}-stage"
+ CLICKHOUSE_CONTAINER="awoooi-ch-restore-${SHORT_ID}"
+ ZOOKEEPER_CONTAINER="awoooi-zk-restore-${SHORT_ID}"
+ BACKUP_VOLUME="awoooi-ch-restore-${SHORT_ID}-backup"
+ CLICKHOUSE_VOLUME="awoooi-ch-restore-${SHORT_ID}-data"
+ ZOOKEEPER_DATA_VOLUME="awoooi-zk-restore-${SHORT_ID}-data"
+ ZOOKEEPER_LOG_VOLUME="awoooi-zk-restore-${SHORT_ID}-log"
+
+ mkdir -p "${RECEIPT_ROOT}"
+ RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
+ [ ! -L "${RECEIPT_DIR}" ] || { log_error "receipt directory symlink is unsafe"; exit 64; }
+ mkdir -m 700 -p "${RECEIPT_DIR}"
+ LOCK_DIR="${RECEIPT_DIR}/active.lock"
+ mkdir "${LOCK_DIR}" 2>/dev/null \
+ || { log_error "same restore-drill run is active or has a stale lock"; exit 1; }
+
+ RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
+ STATUS_FILE="${RECEIPT_DIR}/status.json"
+ PREFLIGHT_FILE="${RECEIPT_DIR}/preflight.contract"
+ INVOCATION_ID="${MODE}-$(date -u '+%Y%m%dT%H%M%SZ')-$$"
+ PREFLIGHT_CANDIDATE="${RECEIPT_DIR}/${INVOCATION_ID}.preflight.candidate"
+ identity_file="${RECEIPT_DIR}/identity.json"
+ identity_content="{\"trace_id\":\"${TRACE_ID}\",\"run_id\":\"${RUN_ID}\",\"work_item_id\":\"${WORK_ITEM_ID}\",\"identity_hash\":\"${IDENTITY_HASH}\"}"
+ if [ -e "${identity_file}" ]; then
+ [ -f "${identity_file}" ] && [ ! -L "${identity_file}" ] \
+ || { rmdir "${LOCK_DIR}"; log_error "existing identity receipt is unsafe"; exit 1; }
+ [ "$(sed -n '1p' "${identity_file}")" = "${identity_content}" ] \
+ || { rmdir "${LOCK_DIR}"; log_error "run identity conflicts with durable receipt"; exit 1; }
+ else
+ identity_tmp="$(mktemp "${RECEIPT_DIR}/.identity.XXXXXX")"
+ printf '%s\n' "${identity_content}" > "${identity_tmp}"
+ mv "${identity_tmp}" "${identity_file}"
+ fi
+ touch "${RECEIPT_LOG}"
+ RECEIPTS_READY=1
+ trap cleanup EXIT
+ trap 'on_signal 1' HUP
+ trap 'on_signal 2' INT
+ trap 'on_signal 15' TERM
+}
+
+run_captured() {
+ local step="$1"
+ local stdout_tmp stderr_tmp stdout_size stderr_size stdout_hash stderr_hash truncated=0
+ shift
+ [[ "${step}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] || fail_closed invalid_command_step
+
+ COMMAND_SEQUENCE=$((COMMAND_SEQUENCE + 1))
+ LAST_STDOUT="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stdout"
+ LAST_STDERR="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stderr"
+ stdout_tmp="$(mktemp "${RECEIPT_DIR}/.stdout.XXXXXX")"
+ stderr_tmp="$(mktemp "${RECEIPT_DIR}/.stderr.XXXXXX")"
+
+ set +e
+ "$@" >"${stdout_tmp}" 2>"${stderr_tmp}"
+ LAST_EXIT_CODE=$?
+ set -e
+ stdout_size="$(wc -c < "${stdout_tmp}" | tr -d ' ')"
+ stderr_size="$(wc -c < "${stderr_tmp}" | tr -d ' ')"
+ head -c "${MAX_LOG_BYTES}" "${stdout_tmp}" > "${LAST_STDOUT}"
+ head -c "${MAX_LOG_BYTES}" "${stderr_tmp}" > "${LAST_STDERR}"
+ rm -f "${stdout_tmp}" "${stderr_tmp}"
+ if [ "${stdout_size}" -gt "${MAX_LOG_BYTES}" ] || [ "${stderr_size}" -gt "${MAX_LOG_BYTES}" ]; then
+ truncated=1
+ [ "${LAST_EXIT_CODE}" -ne 0 ] || LAST_EXIT_CODE=70
+ fi
+ stdout_hash="$(sha256_file "${LAST_STDOUT}")"
+ stderr_hash="$(sha256_file "${LAST_STDERR}")"
+ printf '%s\n' "${LAST_EXIT_CODE}" > "${LAST_STDOUT}.status"
+ emit_receipt command "$([ "${LAST_EXIT_CODE}" -eq 0 ] && printf pass || printf failed)" \
+ "step_${step}_exit_${LAST_EXIT_CODE}_stdout_bytes_${stdout_size}_stderr_bytes_${stderr_size}_truncated_${truncated}_stdout_${stdout_hash}_stderr_${stderr_hash}"
+ return "${LAST_EXIT_CODE}"
+}
+
+run_sql() {
+ local step="$1"
+ local query="$2"
+ shift 2
+ run_captured "${step}" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CLICKHOUSE_CONTAINER}" \
+ clickhouse-client --format TSVRaw "$@" --query "${query}"
+}
+
+read_key() {
+ local key="$1"
+ local file="$2"
+ awk -F= -v wanted="${key}" '$1 == wanted {sub(/^[^=]*=/, ""); print; exit}' "${file}"
+}
+
+validate_preflight() {
+ local image_line
+
+ run_captured artifact_zip timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" unzip -tq "${ARTIFACT_PATH}" \
+ || fail_closed native_backup_zip_integrity_failed
+ run_captured manifest_inventory python3 "${INVENTORY_HELPER}" preflight \
+ --manifest "${MANIFEST_PATH}" --artifact "${ARTIFACT_PATH}" \
+ --source-inventory "${SOURCE_INVENTORY_PATH}" \
+ || fail_closed native_backup_manifest_or_source_inventory_failed
+
+ ARTIFACT_SHA256="$(read_key artifact_sha256 "${LAST_STDOUT}")"
+ SOURCE_INVENTORY_SHA256="$(read_key source_inventory_sha256 "${LAST_STDOUT}")"
+ BACKUP_OPERATION_ID="$(read_key backup_operation_id "${LAST_STDOUT}")"
+ SOURCE_TABLE_COUNT="$(read_key source_table_count "${LAST_STDOUT}")"
+ SOURCE_DATABASE_COUNT="$(read_key source_database_count "${LAST_STDOUT}")"
+ REPLICATED_TABLE_COUNT="$(read_key replicated_table_count "${LAST_STDOUT}")"
+ DISTRIBUTED_TABLE_COUNT="$(read_key distributed_table_count "${LAST_STDOUT}")"
+ PHYSICAL_TABLE_COUNT="$(read_key physical_table_count "${LAST_STDOUT}")"
+ [[ "${ARTIFACT_SHA256}" =~ ^[0-9a-f]{64}$ ]] || fail_closed artifact_hash_receipt_invalid
+ [[ "${SOURCE_INVENTORY_SHA256}" =~ ^[0-9a-f]{64}$ ]] || fail_closed inventory_hash_receipt_invalid
+ valid_identity "${BACKUP_OPERATION_ID}" || fail_closed backup_operation_receipt_invalid
+ if [ -n "${HOOK_ARTIFACT_SHA256}" ]; then
+ [ "${HOOK_ARTIFACT_SHA256}" = "${ARTIFACT_SHA256}" ] \
+ || fail_closed hook_artifact_sha256_mismatch
+ fi
+ if [ -n "${HOOK_OPERATION_ID}" ]; then
+ [ "${HOOK_OPERATION_ID}" = "${BACKUP_OPERATION_ID}" ] \
+ || fail_closed hook_backup_operation_id_mismatch
+ fi
+
+ run_captured clickhouse_image timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker image inspect --format '{{.Id}}' "${CLICKHOUSE_IMAGE}" \
+ || fail_closed clickhouse_local_image_missing
+ image_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${image_line}" = "${CLICKHOUSE_IMAGE_ID_EXPECTED}" ] \
+ || fail_closed clickhouse_local_image_id_mismatch
+ CLICKHOUSE_IMAGE_ID="${image_line}"
+
+ run_captured zookeeper_image timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker image inspect --format '{{.Id}}' "${ZOOKEEPER_IMAGE}" \
+ || fail_closed zookeeper_local_image_missing
+ image_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${image_line}" = "${ZOOKEEPER_IMAGE_ID_EXPECTED}" ] \
+ || fail_closed zookeeper_local_image_id_mismatch
+ ZOOKEEPER_IMAGE_ID="${image_line}"
+
+ BACKUP_CONFIG_SHA256="$(sha256_file "${BACKUP_DISK_CONFIG}")"
+ CLUSTER_CONFIG_SHA256="$(sha256_file "${ISOLATED_CLUSTER_CONFIG}")"
+ {
+ printf 'identity_hash=%s\n' "${IDENTITY_HASH}"
+ printf 'artifact_path=%s\n' "${ARTIFACT_PATH}"
+ printf 'artifact_sha256=%s\n' "${ARTIFACT_SHA256}"
+ printf 'manifest_path=%s\n' "${MANIFEST_PATH}"
+ printf 'source_inventory_path=%s\n' "${SOURCE_INVENTORY_PATH}"
+ printf 'source_inventory_sha256=%s\n' "${SOURCE_INVENTORY_SHA256}"
+ printf 'backup_operation_id=%s\n' "${BACKUP_OPERATION_ID}"
+ printf 'clickhouse_image=%s\n' "${CLICKHOUSE_IMAGE}"
+ printf 'clickhouse_image_id=%s\n' "${CLICKHOUSE_IMAGE_ID}"
+ printf 'zookeeper_image=%s\n' "${ZOOKEEPER_IMAGE}"
+ printf 'zookeeper_image_id=%s\n' "${ZOOKEEPER_IMAGE_ID}"
+ printf 'backup_config_sha256=%s\n' "${BACKUP_CONFIG_SHA256}"
+ printf 'cluster_config_sha256=%s\n' "${CLUSTER_CONFIG_SHA256}"
+ printf 'restore_replica=%s\n' "${RESTORE_REPLICA}"
+ printf 'source_database_count=%s\n' "${SOURCE_DATABASE_COUNT}"
+ printf 'source_table_count=%s\n' "${SOURCE_TABLE_COUNT}"
+ printf 'replicated_table_count=%s\n' "${REPLICATED_TABLE_COUNT}"
+ printf 'distributed_table_count=%s\n' "${DISTRIBUTED_TABLE_COUNT}"
+ printf 'physical_table_count=%s\n' "${PHYSICAL_TABLE_COUNT}"
+ } > "${PREFLIGHT_CANDIDATE}"
+ emit_receipt sensor_source pass \
+ "backup_${BACKUP_OPERATION_ID}_artifact_${ARTIFACT_SHA256}_inventory_${SOURCE_INVENTORY_SHA256}"
+ emit_receipt normalized_asset_identity pass \
+ "restore_${IDENTITY_HASH}_clickhouse_${CLICKHOUSE_IMAGE_ID}_zookeeper_${ZOOKEEPER_IMAGE_ID}"
+ emit_receipt risk_policy pass \
+ "risk_high_isolated_internal_network_fresh_keeper_fresh_data_volumes_no_production_attachment"
+}
+
+persist_check_contract() {
+ if [ -e "${PREFLIGHT_FILE}" ]; then
+ [ -f "${PREFLIGHT_FILE}" ] && [ ! -L "${PREFLIGHT_FILE}" ] \
+ || fail_closed existing_preflight_contract_unsafe
+ cmp -s "${PREFLIGHT_CANDIDATE}" "${PREFLIGHT_FILE}" \
+ || fail_closed same_run_preflight_contract_drift
+ rm -f "${PREFLIGHT_CANDIDATE}"
+ else
+ mv "${PREFLIGHT_CANDIDATE}" "${PREFLIGHT_FILE}"
+ fi
+ emit_receipt source_of_truth_diff pass \
+ "exact_six_databases_tables_${SOURCE_TABLE_COUNT}_replicated_${REPLICATED_TABLE_COUNT}_distributed_${DISTRIBUTED_TABLE_COUNT}"
+ emit_receipt ai_decision candidate \
+ "isolated_keeper_clickhouse_restore_requires_same_run_apply"
+ emit_receipt check check_pass_no_runtime_write \
+ "local_artifact_config_and_exact_local_images_valid_no_container_volume_or_network_created"
+ CHECK_PASS=1
+ FINAL_REASON="check_pass_no_runtime_write"
+}
+
+require_check_contract() {
+ [ -f "${PREFLIGHT_FILE}" ] && [ ! -L "${PREFLIGHT_FILE}" ] \
+ || fail_closed same_run_check_contract_missing
+ cmp -s "${PREFLIGHT_CANDIDATE}" "${PREFLIGHT_FILE}" \
+ || fail_closed apply_preflight_contract_drift
+ rm -f "${PREFLIGHT_CANDIDATE}"
+ grep -F -q '"stage":"terminal","terminal":"check_pass_no_runtime_write"' "${RECEIPT_LOG}" \
+ || fail_closed same_run_check_terminal_receipt_missing
+ CHECK_PASS=1
+ emit_receipt check pass apply_revalidation_matches_same_run_check_contract
+}
+
+ensure_resources_absent() {
+ docker_daemon_ready || fail_closed docker_daemon_unavailable_before_restore_create
+ exact_container_absent "${CLICKHOUSE_CONTAINER}" \
+ || fail_closed isolated_clickhouse_container_name_collision_or_query_failed
+ exact_container_absent "${ZOOKEEPER_CONTAINER}" \
+ || fail_closed isolated_zookeeper_container_name_collision_or_query_failed
+ exact_container_absent "${STAGING_CONTAINER}" \
+ || fail_closed isolated_staging_container_name_collision_or_query_failed
+ exact_volume_absent "${BACKUP_VOLUME}" \
+ || fail_closed isolated_backup_volume_name_collision_or_query_failed
+ exact_volume_absent "${CLICKHOUSE_VOLUME}" \
+ || fail_closed isolated_clickhouse_volume_name_collision_or_query_failed
+ exact_volume_absent "${ZOOKEEPER_DATA_VOLUME}" \
+ || fail_closed isolated_zookeeper_data_volume_name_collision_or_query_failed
+ exact_volume_absent "${ZOOKEEPER_LOG_VOLUME}" \
+ || fail_closed isolated_zookeeper_log_volume_name_collision_or_query_failed
+ exact_network_absent "${NETWORK_NAME}" \
+ || fail_closed isolated_network_name_collision_or_query_failed
+ emit_receipt source_of_truth_diff pass exact_ephemeral_resource_names_absent
+}
+
+docker_daemon_ready() {
+ timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker info --format '{{.ServerVersion}}' >/dev/null 2>&1
+}
+
+exact_container_absent() {
+ local name="$1" observed
+ observed="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker container ls -a --format '{{.Names}}')" || return 1
+ ! grep -F -x -q -- "${name}" <<< "${observed}"
+}
+
+exact_volume_absent() {
+ local name="$1" observed
+ observed="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker volume ls -q)" || return 1
+ ! grep -F -x -q -- "${name}" <<< "${observed}"
+}
+
+exact_network_absent() {
+ local name="$1" observed
+ observed="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker network ls --format '{{.Name}}')" || return 1
+ ! grep -F -x -q -- "${name}" <<< "${observed}"
+}
+
+wait_for_container_running() {
+ local label="$1"
+ local container="$2"
+ local deadline state
+ deadline=$(( $(date +%s) + STARTUP_TIMEOUT_SECONDS ))
+ while true; do
+ if run_captured "${label}_running" timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.State.Running}}' "${container}"; then
+ state="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${state}" = "true" ] && return 0
+ fi
+ [ "$(date +%s)" -lt "${deadline}" ] || fail_closed "${label}_startup_timeout"
+ sleep "${POLL_INTERVAL_SECONDS}"
+ done
+}
+
+wait_for_clickhouse() {
+ local deadline
+ deadline=$(( $(date +%s) + STARTUP_TIMEOUT_SECONDS ))
+ while true; do
+ if run_sql clickhouse_ready 'SELECT 1'; then
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "1" ] && return 0
+ fi
+ [ "$(date +%s)" -lt "${deadline}" ] || fail_closed clickhouse_startup_timeout
+ sleep "${POLL_INTERVAL_SECONDS}"
+ done
+}
+
+stage_artifact_into_backup_volume() {
+ local staged_hash
+
+ run_captured artifact_stage timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker run --pull=never \
+ --name "${STAGING_CONTAINER}" --network none --restart=no --user 0:0 \
+ --label "com.awoooi.restore.identity=${IDENTITY_HASH}" \
+ --env "EXPECTED_ARTIFACT_SHA256=${ARTIFACT_SHA256}" \
+ --mount "type=bind,src=${ARTIFACT_PATH},dst=${STAGING_SOURCE_ARTIFACT_PATH},readonly" \
+ --mount "type=volume,src=${BACKUP_VOLUME},dst=/backups" \
+ --entrypoint /bin/sh "${CLICKHOUSE_IMAGE}" -ceu \
+ 'cp -- /tmp/awoooi-source-native-backup.zip /backups/native-backup.zip
+ sync /backups/native-backup.zip
+ set -- $(sha256sum /backups/native-backup.zip)
+ [ "$1" = "$EXPECTED_ARTIFACT_SHA256" ]
+ printf "%s\n" "$1"' \
+ || fail_closed isolated_artifact_staging_failed
+ STAGING_CONTAINER_CREATED=1
+ staged_hash="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [[ "${staged_hash}" =~ ^[0-9a-f]{64}$ ]] \
+ || fail_closed staged_artifact_hash_readback_invalid
+ [ "${staged_hash}" = "${ARTIFACT_SHA256}" ] \
+ || fail_closed staged_artifact_hash_mismatch
+ STAGED_ARTIFACT_SHA256="${staged_hash}"
+
+ run_captured staging_image_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.Image}}' "${STAGING_CONTAINER}" \
+ || fail_closed isolated_staging_image_readback_failed
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "${CLICKHOUSE_IMAGE_ID_EXPECTED}" ] \
+ || fail_closed isolated_staging_runtime_image_id_mismatch
+ run_captured staging_network_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.HostConfig.NetworkMode}}' \
+ "${STAGING_CONTAINER}" \
+ || fail_closed isolated_staging_network_readback_failed
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = none ] \
+ || fail_closed isolated_staging_network_not_none
+ ARTIFACT_STAGING_VERIFIED=1
+ emit_receipt artifact_staging pass \
+ "read_only_source_to_writable_owned_volume_sha256_${STAGED_ARTIFACT_SHA256}_network_none_exact_clickhouse_image"
+}
+
+create_isolated_runtime() {
+ local clickhouse_artifact_hash
+
+ # From this point onward cleanup inspects every exact resource name even if
+ # a collision or Docker CLI timeout occurs before a *_CREATED flag is set.
+ APPLY_RESOURCE_SCOPE=1
+ ensure_resources_absent
+
+ run_captured network_create timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker network create --internal \
+ --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${NETWORK_NAME}" \
+ || fail_closed isolated_internal_network_create_failed
+ NETWORK_CREATED=1
+ EPHEMERAL_RUNTIME_CREATED=1
+
+ run_captured backup_volume timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker volume create \
+ --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${BACKUP_VOLUME}" \
+ || fail_closed isolated_backup_volume_create_failed
+ BACKUP_VOLUME_CREATED=1
+ run_captured clickhouse_volume timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker volume create \
+ --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${CLICKHOUSE_VOLUME}" \
+ || fail_closed isolated_clickhouse_volume_create_failed
+ CLICKHOUSE_VOLUME_CREATED=1
+ run_captured zookeeper_data_volume timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker volume create \
+ --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${ZOOKEEPER_DATA_VOLUME}" \
+ || fail_closed isolated_zookeeper_data_volume_create_failed
+ ZOOKEEPER_DATA_VOLUME_CREATED=1
+ run_captured zookeeper_log_volume timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker volume create \
+ --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${ZOOKEEPER_LOG_VOLUME}" \
+ || fail_closed isolated_zookeeper_log_volume_create_failed
+ ZOOKEEPER_LOG_VOLUME_CREATED=1
+
+ stage_artifact_into_backup_volume
+
+ run_captured zookeeper_run timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker run --detach --pull=never \
+ --name "${ZOOKEEPER_CONTAINER}" --network "${NETWORK_NAME}" \
+ --network-alias restore-zookeeper --restart=no \
+ --label "com.awoooi.restore.identity=${IDENTITY_HASH}" \
+ --env "ALLOW_ANONYMOUS_LOGIN=yes" \
+ --mount "type=volume,src=${ZOOKEEPER_DATA_VOLUME},dst=/data" \
+ --mount "type=volume,src=${ZOOKEEPER_LOG_VOLUME},dst=/datalog" \
+ "${ZOOKEEPER_IMAGE}" \
+ || fail_closed isolated_zookeeper_start_failed
+ ZOOKEEPER_CONTAINER_CREATED=1
+ wait_for_container_running zookeeper "${ZOOKEEPER_CONTAINER}"
+
+ run_captured clickhouse_run timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker run --detach --pull=never \
+ --name "${CLICKHOUSE_CONTAINER}" --network "${NETWORK_NAME}" \
+ --network-alias restore-clickhouse --restart=no \
+ --label "com.awoooi.restore.identity=${IDENTITY_HASH}" \
+ --env "CLICKHOUSE_RESTORE_REPLICA=${RESTORE_REPLICA}" \
+ --mount "type=volume,src=${CLICKHOUSE_VOLUME},dst=/var/lib/clickhouse" \
+ --mount "type=volume,src=${BACKUP_VOLUME},dst=/backups" \
+ --mount "type=bind,src=${BACKUP_DISK_CONFIG},dst=/etc/clickhouse-server/config.d/awoooi-backup-disk.xml,readonly" \
+ --mount "type=bind,src=${ISOLATED_CLUSTER_CONFIG},dst=/etc/clickhouse-server/config.d/awoooi-restore-isolation.xml,readonly" \
+ "${CLICKHOUSE_IMAGE}" \
+ || fail_closed isolated_clickhouse_start_failed
+ CLICKHOUSE_CONTAINER_CREATED=1
+ wait_for_container_running clickhouse "${CLICKHOUSE_CONTAINER}"
+ wait_for_clickhouse
+
+ run_captured zookeeper_image_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.Image}}' "${ZOOKEEPER_CONTAINER}" \
+ || fail_closed isolated_zookeeper_image_readback_failed
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "${ZOOKEEPER_IMAGE_ID_EXPECTED}" ] \
+ || fail_closed isolated_zookeeper_runtime_image_id_mismatch
+ run_captured clickhouse_image_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}" \
+ || fail_closed isolated_clickhouse_image_readback_failed
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "${CLICKHOUSE_IMAGE_ID_EXPECTED}" ] \
+ || fail_closed isolated_clickhouse_runtime_image_id_mismatch
+ run_captured clickhouse_artifact_hash_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CLICKHOUSE_CONTAINER}" \
+ sha256sum "${CONTAINER_ARTIFACT_PATH}" \
+ || fail_closed isolated_clickhouse_artifact_hash_readback_failed
+ clickhouse_artifact_hash="$(awk 'NR == 1 {print $1}' "${LAST_STDOUT}")"
+ [[ "${clickhouse_artifact_hash}" =~ ^[0-9a-f]{64}$ ]] \
+ || fail_closed isolated_clickhouse_artifact_hash_readback_invalid
+ CLICKHOUSE_ARTIFACT_SHA256="${clickhouse_artifact_hash}"
+ [ "${CLICKHOUSE_ARTIFACT_SHA256}" = "${ARTIFACT_SHA256}" ] \
+ || fail_closed isolated_clickhouse_artifact_hash_mismatch
+ CLICKHOUSE_ARTIFACT_READBACK_VERIFIED=1
+ emit_receipt execution created \
+ "artifact_${ARTIFACT_SHA256}_staged_read_only_source_writable_backup_volume_internal_network_ephemeral_keeper_and_clickhouse_started_pull_never_exact_image_ids_no_published_ports"
+}
+
+verify_isolation_and_empty_target() {
+ local observed
+
+ run_captured network_internal timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker network inspect --format '{{.Internal}}' "${NETWORK_NAME}" \
+ || fail_closed isolated_network_inspect_failed
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "true" ] \
+ || fail_closed restore_network_not_internal
+ for container in "${ZOOKEEPER_CONTAINER}" "${CLICKHOUSE_CONTAINER}"; do
+ run_captured container_network timeout --kill-after="${KILL_AFTER_SECONDS}" \
+ "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.HostConfig.NetworkMode}}' "${container}" \
+ || fail_closed isolated_container_network_inspect_failed
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "${NETWORK_NAME}" ] \
+ || fail_closed isolated_container_has_unexpected_network
+ done
+
+ run_sql disk_contract \
+ "SELECT path FROM system.disks WHERE name={disk:String}" --param_disk "${BACKUP_DISK}" \
+ || fail_closed isolated_backup_disk_missing
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "/backups/" ] \
+ || fail_closed isolated_backup_disk_path_mismatch
+
+ run_sql macro_contract \
+ 'SELECT macro,substitution FROM system.macros ORDER BY macro' \
+ || fail_closed isolated_macro_query_failed
+ grep -F -x -q $'replica\t'"${RESTORE_REPLICA}" "${LAST_STDOUT}" \
+ || fail_closed isolated_replica_macro_mismatch
+ grep -F -x -q $'shard\t01' "${LAST_STDOUT}" \
+ || fail_closed isolated_shard_macro_mismatch
+
+ run_sql cluster_contract \
+ "SELECT cluster,host_name,toString(port) FROM system.clusters WHERE cluster='cluster' ORDER BY shard_num,replica_num" \
+ || fail_closed isolated_cluster_query_failed
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = $'cluster\trestore-clickhouse\t9000' ] \
+ || fail_closed isolated_cluster_contract_mismatch
+
+ run_sql keeper_contract "SELECT count() > 0 FROM system.zookeeper WHERE path='/'" \
+ || fail_closed isolated_keeper_connection_failed
+ [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "1" ] \
+ || fail_closed isolated_keeper_connection_not_ready
+
+ run_sql empty_databases \
+ "SELECT count() FROM system.databases WHERE name IN (${DATABASE_IN_SQL})" \
+ || fail_closed isolated_database_empty_check_failed
+ observed="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${observed}" = "0" ] || fail_closed isolated_restore_target_databases_not_empty
+ run_sql empty_tables \
+ "SELECT count() FROM system.tables WHERE database IN (${DATABASE_IN_SQL})" \
+ || fail_closed isolated_table_empty_check_failed
+ observed="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${observed}" = "0" ] || fail_closed isolated_restore_target_tables_not_empty
+ emit_receipt independent_pre_verifier pass \
+ "artifact_${STAGED_ARTIFACT_SHA256}_readback_internal_network_only_fresh_keeper_exact_macros_cluster_and_empty_signoz_target"
+}
+
+query_restore_status() {
+ local line_count status_line restore_name error_hash
+ run_sql restore_status \
+ "SELECT name,toString(status),toString(num_files),toString(total_size),hex(SHA256(error)) FROM system.backups WHERE id={operation_id:String}" \
+ --param_operation_id "${RESTORE_OPERATION_ID}" \
+ || fail_closed restore_system_backups_query_failed
+ line_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
+ [ "${line_count}" -eq 1 ] || fail_closed restore_system_backups_terminal_not_unique
+ status_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ IFS=$'\t' read -r restore_name RESTORE_TERMINAL RESTORE_NUM_FILES RESTORE_TOTAL_SIZE error_hash <<< "${status_line}"
+ [ "${restore_name}" = "Disk('${BACKUP_DISK}', '${CONTAINER_ARTIFACT_NAME}')" ] \
+ || fail_closed restore_system_backups_artifact_mismatch
+ [[ "${RESTORE_NUM_FILES}" =~ ^[0-9]+$ ]] && [[ "${RESTORE_TOTAL_SIZE}" =~ ^[0-9]+$ ]] \
+ || fail_closed restore_system_backups_metrics_invalid
+ [ "${error_hash}" = "${EMPTY_SHA256}" ] \
+ || fail_closed restore_system_backups_error_present
+}
+
+restore_native_backup() {
+ local submission_line submitted_id submitted_status deadline
+ emit_receipt ai_decision candidate \
+ "restore_exact_six_signoz_databases_to_empty_isolated_clickhouse_with_fresh_keeper"
+ run_sql restore_submit \
+ "RESTORE ${RESTORE_SCOPE_SQL} FROM Disk({disk:String},{artifact:String}) SETTINGS id='${RESTORE_OPERATION_ID}' ASYNC" \
+ --param_disk "${BACKUP_DISK}" --param_artifact "${CONTAINER_ARTIFACT_NAME}" \
+ || fail_closed native_restore_submission_failed
+ submission_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ IFS=$'\t' read -r submitted_id submitted_status <<< "${submission_line}"
+ [ "${submitted_id}" = "${RESTORE_OPERATION_ID}" ] \
+ || fail_closed native_restore_submission_id_mismatch
+ case "${submitted_status}" in
+ RESTORING|RESTORED) ;;
+ *) fail_closed "native_restore_submission_status_${submitted_status:-empty}" ;;
+ esac
+ emit_receipt execution submitted \
+ "restore_operation_${RESTORE_OPERATION_ID}_status_${submitted_status}"
+
+ deadline=$(( $(date +%s) + RESTORE_TIMEOUT_SECONDS ))
+ while true; do
+ query_restore_status
+ emit_receipt restore_status observed \
+ "status_${RESTORE_TERMINAL}_files_${RESTORE_NUM_FILES}_total_${RESTORE_TOTAL_SIZE}"
+ case "${RESTORE_TERMINAL}" in
+ RESTORED) return 0 ;;
+ RESTORING) ;;
+ RESTORE_FAILED|RESTORE_CANCELLED) fail_closed "native_restore_terminal_${RESTORE_TERMINAL}" ;;
+ *) fail_closed "native_restore_terminal_unexpected_${RESTORE_TERMINAL}" ;;
+ esac
+ [ "$(date +%s)" -lt "${deadline}" ] || fail_closed native_restore_poll_timeout
+ sleep "${POLL_INTERVAL_SECONDS}"
+ done
+}
+
+verify_restored_inventory_and_tables() {
+ local restored_inventory compare_receipt check_list database table output step
+
+ run_sql restored_inventory \
+ "SELECT database,name,engine,toString(ifNull(total_rows,0)),toString(ifNull(total_bytes,0)),hex(SHA256(create_table_query)) FROM system.tables WHERE database IN (${DATABASE_IN_SQL}) ORDER BY database,name" \
+ || fail_closed restored_table_inventory_query_failed
+ restored_inventory="${LAST_STDOUT}"
+ [ -s "${restored_inventory}" ] || fail_closed restored_table_inventory_empty
+
+ run_captured inventory_compare python3 "${INVENTORY_HELPER}" compare \
+ --source-inventory "${SOURCE_INVENTORY_PATH}" \
+ --restored-inventory "${restored_inventory}" \
+ || fail_closed restored_database_table_engine_manifest_mismatch
+ compare_receipt="${LAST_STDOUT}"
+ MANIFEST_PARITY="$(read_key manifest_parity "${compare_receipt}")"
+ RESTORED_DATABASE_COUNT="$(read_key restored_database_count "${compare_receipt}")"
+ RESTORED_TABLE_COUNT="$(read_key restored_table_count "${compare_receipt}")"
+ PHYSICAL_TABLE_COUNT="$(read_key physical_table_count "${compare_receipt}")"
+ CRITICAL_NONZERO_COUNT="$(read_key critical_nonzero_count "${compare_receipt}")"
+ SOURCE_TOTAL_ROWS="$(read_key source_total_rows "${compare_receipt}")"
+ RESTORED_TOTAL_ROWS="$(read_key restored_total_rows "${compare_receipt}")"
+ ROW_DELTA="$(read_key row_delta "${compare_receipt}")"
+ SOURCE_TOTAL_BYTES="$(read_key source_total_bytes "${compare_receipt}")"
+ RESTORED_TOTAL_BYTES="$(read_key restored_total_bytes "${compare_receipt}")"
+ BYTE_DELTA="$(read_key byte_delta "${compare_receipt}")"
+ [ "${MANIFEST_PARITY}" = "1" ] || fail_closed restored_manifest_parity_not_proven
+
+ run_captured physical_table_list python3 "${INVENTORY_HELPER}" check-list \
+ --inventory "${restored_inventory}" \
+ || fail_closed restored_physical_table_list_failed
+ check_list="${LAST_STDOUT}"
+ CHECK_TABLE_COUNT="$(awk 'NF { count++ } END { print count + 0 }' "${check_list}")"
+ [ "${CHECK_TABLE_COUNT}" -eq "${PHYSICAL_TABLE_COUNT}" ] \
+ || fail_closed restored_physical_table_count_mismatch
+
+ while IFS=$'\t' read -r database table; do
+ [ -n "${database}" ] && [ -n "${table}" ] || continue
+ step="check_table_$(printf '%03d' "$((CHECK_TABLE_PASS_COUNT + 1))")"
+ run_sql "${step}" \
+ "CHECK TABLE \`${database}\`.\`${table}\` SETTINGS check_query_single_value_result=1" \
+ || fail_closed "check_table_command_failed_${database}_${table}"
+ output="$(tr -d '\r\n' < "${LAST_STDOUT}")"
+ [ "${output}" = "1" ] || fail_closed "check_table_not_passed_${database}_${table}"
+ CHECK_TABLE_PASS_COUNT=$((CHECK_TABLE_PASS_COUNT + 1))
+ done < "${check_list}"
+ [ "${CHECK_TABLE_PASS_COUNT}" -eq "${CHECK_TABLE_COUNT}" ] \
+ || fail_closed check_table_pass_count_mismatch
+
+ emit_receipt independent_post_verifier pass \
+ "restore_RESTORED_manifest_parity_${MANIFEST_PARITY}_tables_${RESTORED_TABLE_COUNT}_check_tables_${CHECK_TABLE_PASS_COUNT}_critical_nonzero_${CRITICAL_NONZERO_COUNT}_row_delta_${ROW_DELTA}_byte_delta_${BYTE_DELTA}"
+ APPLY_PASS=1
+ FINAL_REASON="isolated_native_restore_verified"
+}
+
+capture_container_log() {
+ local container="$1"
+ local label="$2"
+ local raw target
+ raw="$(mktemp "${RECEIPT_DIR}/.${label}.XXXXXX")"
+ target="${RECEIPT_DIR}/${INVOCATION_ID}-${label}.log"
+ set +e
+ timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker logs --tail "${LOG_TAIL_LINES}" "${container}" >"${raw}" 2>&1
+ set -e
+ head -c "${MAX_LOG_BYTES}" "${raw}" > "${target}"
+ rm -f "${raw}"
+}
+
+cleanup_container() {
+ local container="$1"
+ local label inspect_status remove_status final_status
+ set +e
+ label="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker container inspect --format '{{index .Config.Labels "com.awoooi.restore.identity"}}' \
+ "${container}" 2>/dev/null)"
+ inspect_status=$?
+ set -e
+ if [ "${inspect_status}" -ne 0 ]; then
+ exact_container_absent "${container}"
+ return $?
+ fi
+ [ "${label}" = "${IDENTITY_HASH}" ] || return 1
+ capture_container_log "${container}" "$(basename "${container}")"
+ set +e
+ timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker rm -f "${container}" >/dev/null 2>&1
+ remove_status=$?
+ exact_container_absent "${container}"
+ final_status=$?
+ set -e
+ [ "${remove_status}" -eq 0 ] && [ "${final_status}" -eq 0 ] \
+ && docker_daemon_ready
+}
+
+cleanup_volume() {
+ local volume="$1"
+ local label inspect_status remove_status final_status
+ set +e
+ label="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker volume inspect --format '{{index .Labels "com.awoooi.restore.identity"}}' \
+ "${volume}" 2>/dev/null)"
+ inspect_status=$?
+ set -e
+ if [ "${inspect_status}" -ne 0 ]; then
+ exact_volume_absent "${volume}"
+ return $?
+ fi
+ [ "${label}" = "${IDENTITY_HASH}" ] || return 1
+ set +e
+ timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker volume rm "${volume}" >/dev/null 2>&1
+ remove_status=$?
+ exact_volume_absent "${volume}"
+ final_status=$?
+ set -e
+ [ "${remove_status}" -eq 0 ] && [ "${final_status}" -eq 0 ] \
+ && docker_daemon_ready
+}
+
+cleanup_network() {
+ local label inspect_status remove_status final_status
+ set +e
+ label="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker network inspect --format '{{index .Labels "com.awoooi.restore.identity"}}' \
+ "${NETWORK_NAME}" 2>/dev/null)"
+ inspect_status=$?
+ set -e
+ if [ "${inspect_status}" -ne 0 ]; then
+ exact_network_absent "${NETWORK_NAME}"
+ return $?
+ fi
+ [ "${label}" = "${IDENTITY_HASH}" ] || return 1
+ set +e
+ timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \
+ docker network rm "${NETWORK_NAME}" >/dev/null 2>&1
+ remove_status=$?
+ exact_network_absent "${NETWORK_NAME}"
+ final_status=$?
+ set -e
+ [ "${remove_status}" -eq 0 ] && [ "${final_status}" -eq 0 ] \
+ && docker_daemon_ready
+}
+
+verify_all_resources_absent() {
+ docker_daemon_ready || return 1
+ exact_container_absent "${STAGING_CONTAINER}" \
+ && exact_container_absent "${CLICKHOUSE_CONTAINER}" \
+ && exact_container_absent "${ZOOKEEPER_CONTAINER}" \
+ && exact_volume_absent "${BACKUP_VOLUME}" \
+ && exact_volume_absent "${CLICKHOUSE_VOLUME}" \
+ && exact_volume_absent "${ZOOKEEPER_DATA_VOLUME}" \
+ && exact_volume_absent "${ZOOKEEPER_LOG_VOLUME}" \
+ && exact_network_absent "${NETWORK_NAME}" \
+ && docker_daemon_ready
+}
+
+write_status() {
+ local terminal="$1"
+ local exit_code="$2"
+ local cleanup_json runtime_json staging_json clickhouse_hash_json check_json apply_json status_tmp
+ cleanup_json=false
+ runtime_json=false
+ staging_json=false
+ clickhouse_hash_json=false
+ check_json=false
+ apply_json=false
+ [ "${CLEANUP_PASS}" -eq 1 ] && cleanup_json=true
+ [ "${EPHEMERAL_RUNTIME_CREATED}" -eq 1 ] && runtime_json=true
+ [ "${ARTIFACT_STAGING_VERIFIED}" -eq 1 ] && staging_json=true
+ [ "${CLICKHOUSE_ARTIFACT_READBACK_VERIFIED}" -eq 1 ] && clickhouse_hash_json=true
+ [ "${CHECK_PASS}" -eq 1 ] && check_json=true
+ [ "${APPLY_PASS}" -eq 1 ] && apply_json=true
+ status_tmp="${STATUS_FILE}.tmp.$$"
+ printf '%s\n' \
+ '{' \
+ ' "schema_version":"clickhouse_native_restore_drill_status_v1",' \
+ " \"trace_id\":\"${TRACE_ID}\"," \
+ " \"run_id\":\"${RUN_ID}\"," \
+ " \"work_item_id\":\"${WORK_ITEM_ID}\"," \
+ " \"identity_hash\":\"${IDENTITY_HASH}\"," \
+ " \"mode\":\"${MODE}\"," \
+ " \"terminal\":\"${terminal}\"," \
+ " \"reason\":\"${FINAL_REASON}\"," \
+ " \"exit_code\":${exit_code}," \
+ " \"artifact_sha256\":\"${ARTIFACT_SHA256}\"," \
+ " \"staged_artifact_sha256\":\"${STAGED_ARTIFACT_SHA256}\"," \
+ " \"clickhouse_artifact_sha256\":\"${CLICKHOUSE_ARTIFACT_SHA256}\"," \
+ " \"source_inventory_sha256\":\"${SOURCE_INVENTORY_SHA256}\"," \
+ " \"clickhouse_image_id\":\"${CLICKHOUSE_IMAGE_ID}\"," \
+ " \"zookeeper_image_id\":\"${ZOOKEEPER_IMAGE_ID}\"," \
+ ' "network_mode":"docker_internal_only",' \
+ ' "staging_network_mode":"none",' \
+ ' "production_network_attached":false,' \
+ ' "production_restore_performed":false,' \
+ ' "allow_non_empty_tables":false,' \
+ ' "isolated_keeper":true,' \
+ ' "artifact_source_mount":"read_only_staging_only",' \
+ ' "backup_volume_mount":"writable_ephemeral",' \
+ " \"artifact_staging_verified\":${staging_json}," \
+ " \"clickhouse_artifact_readback_verified\":${clickhouse_hash_json}," \
+ " \"ephemeral_runtime_created\":${runtime_json}," \
+ " \"check_pass\":${check_json}," \
+ " \"apply_pass\":${apply_json}," \
+ " \"restore_terminal\":\"${RESTORE_TERMINAL}\"," \
+ " \"manifest_parity\":${MANIFEST_PARITY}," \
+ " \"source_database_count\":${SOURCE_DATABASE_COUNT}," \
+ " \"restored_database_count\":${RESTORED_DATABASE_COUNT}," \
+ " \"source_table_count\":${SOURCE_TABLE_COUNT}," \
+ " \"restored_table_count\":${RESTORED_TABLE_COUNT}," \
+ " \"check_table_count\":${CHECK_TABLE_COUNT}," \
+ " \"check_table_pass_count\":${CHECK_TABLE_PASS_COUNT}," \
+ " \"critical_nonzero_count\":${CRITICAL_NONZERO_COUNT}," \
+ " \"source_total_rows\":${SOURCE_TOTAL_ROWS}," \
+ " \"restored_total_rows\":${RESTORED_TOTAL_ROWS}," \
+ " \"row_delta\":${ROW_DELTA}," \
+ " \"source_total_bytes\":${SOURCE_TOTAL_BYTES}," \
+ " \"restored_total_bytes\":${RESTORED_TOTAL_BYTES}," \
+ " \"byte_delta\":${BYTE_DELTA}," \
+ " \"cleanup_verified\":${cleanup_json}," \
+ ' "secret_values_collected":false' \
+ '}' > "${status_tmp}"
+ mv "${status_tmp}" "${STATUS_FILE}"
+}
+
+cleanup() {
+ local exit_code=$?
+ local terminal="failed"
+ trap - EXIT HUP INT TERM
+ set +e
+
+ [ -z "${PREFLIGHT_CANDIDATE}" ] || rm -f "${PREFLIGHT_CANDIDATE}"
+ if [ "${APPLY_RESOURCE_SCOPE}" -eq 1 ]; then
+ cleanup_container "${CLICKHOUSE_CONTAINER}" || CLEANUP_PASS=0
+ cleanup_container "${ZOOKEEPER_CONTAINER}" || CLEANUP_PASS=0
+ cleanup_container "${STAGING_CONTAINER}" || CLEANUP_PASS=0
+ cleanup_volume "${BACKUP_VOLUME}" || CLEANUP_PASS=0
+ cleanup_volume "${CLICKHOUSE_VOLUME}" || CLEANUP_PASS=0
+ cleanup_volume "${ZOOKEEPER_DATA_VOLUME}" || CLEANUP_PASS=0
+ cleanup_volume "${ZOOKEEPER_LOG_VOLUME}" || CLEANUP_PASS=0
+ cleanup_network || CLEANUP_PASS=0
+ verify_all_resources_absent || CLEANUP_PASS=0
+ fi
+ [ -z "${LOCK_DIR}" ] || rmdir "${LOCK_DIR}" 2>/dev/null || CLEANUP_PASS=0
+
+ if [ "${CLEANUP_PASS}" -ne 1 ]; then
+ FINAL_REASON="ephemeral_resource_cleanup_failed"
+ exit_code=1
+ elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = check ] && [ "${CHECK_PASS}" -eq 1 ]; then
+ terminal="check_pass_no_runtime_write"
+ elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = apply ] && [ "${APPLY_PASS}" -eq 1 ]; then
+ terminal="verified"
+ fi
+ emit_receipt cleanup "$([ "${CLEANUP_PASS}" -eq 1 ] && printf pass || printf failed)" \
+ "staging_keeper_clickhouse_containers_backup_data_volumes_and_internal_network_absence_verified_${CLEANUP_PASS}"
+ emit_receipt terminal "${terminal}" "${FINAL_REASON}"
+ write_status "${terminal}" "${exit_code}"
+ printf 'CLICKHOUSE_NATIVE_RESTORE_DRILL_TERMINAL trace_id=%s run_id=%s work_item_id=%s mode=%s terminal=%s cleanup=%s exit_code=%s\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" "${terminal}" \
+ "${CLEANUP_PASS}" "${exit_code}"
+ exit "${exit_code}"
+}
+
+on_signal() {
+ local signal_number="$1"
+ FINAL_REASON="signal_${signal_number}"
+ emit_receipt signal failed "${FINAL_REASON}"
+ exit "$((128 + signal_number))"
+}
+
+main() {
+ parse_args "$@"
+ validate_contract
+ setup_receipts
+ validate_preflight
+
+ if [ "${MODE}" = check ]; then
+ persist_check_contract
+ log_info "isolated native RESTORE drill check passed without runtime mutation"
+ return 0
+ fi
+
+ require_check_contract
+ create_isolated_runtime
+ verify_isolation_and_empty_target
+ restore_native_backup
+ verify_restored_inventory_and_tables
+ log_info "isolated native RESTORE drill verified; ephemeral runtime will be removed"
+}
+
+main "$@"
diff --git a/scripts/backup/clickhouse-restore-inventory.py b/scripts/backup/clickhouse-restore-inventory.py
new file mode 100755
index 000000000..42b970b1f
--- /dev/null
+++ b/scripts/backup/clickhouse-restore-inventory.py
@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""Validate SigNoz ClickHouse native-backup inventories without data access.
+
+The shell restore-drill driver uses this helper for two narrow jobs:
+
+* prove that a local artifact is a completed, hashed native BACKUP with the
+ exact six-database source inventory expected by P0-OBS-002; and
+* compare the restored database/table/engine set with that source inventory.
+
+Row and byte values are recorded as bounded aggregate evidence, not exact
+parity gates, because the source inventory is captured while online ingestion
+can still advance before the native BACKUP snapshot is committed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+EXPECTED_DATABASES = (
+ "signoz_analytics",
+ "signoz_logs",
+ "signoz_metadata",
+ "signoz_meter",
+ "signoz_metrics",
+ "signoz_traces",
+)
+CRITICAL_TABLES = (
+ ("signoz_logs", "logs_v2"),
+ ("signoz_metrics", "time_series_v4"),
+ ("signoz_metrics", "samples_v4"),
+ ("signoz_traces", "signoz_index_v3"),
+)
+SAFE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+SAFE_OPERATION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
+SHA256 = re.compile(r"^[0-9a-f]{64}$")
+SCHEMA_SHA256 = re.compile(r"^[0-9A-F]{64}$")
+
+
+class ContractError(ValueError):
+ """A bounded, non-secret backup or inventory contract failure."""
+
+
+@dataclass(frozen=True)
+class TableReceipt:
+ database: str
+ table: str
+ engine: str
+ rows: int
+ bytes: int
+ schema_hash: str
+
+
+def sha256_file(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def parse_inventory(path: Path) -> dict[tuple[str, str], TableReceipt]:
+ receipts: dict[tuple[str, str], TableReceipt] = {}
+ try:
+ lines = path.read_text(encoding="utf-8").splitlines()
+ except (OSError, UnicodeError) as exc:
+ raise ContractError("inventory_read_failed") from exc
+
+ if not lines:
+ raise ContractError("inventory_empty")
+
+ for line_number, line in enumerate(lines, start=1):
+ fields = line.split("\t")
+ if len(fields) != 6:
+ raise ContractError(f"inventory_field_count_invalid_line_{line_number}")
+ database, table, engine, rows_text, bytes_text, schema_hash = fields
+ if not SAFE_IDENTIFIER.fullmatch(database):
+ raise ContractError(
+ f"inventory_database_identifier_unsafe_line_{line_number}"
+ )
+ if not SAFE_IDENTIFIER.fullmatch(table):
+ raise ContractError(f"inventory_table_identifier_unsafe_line_{line_number}")
+ if not SAFE_IDENTIFIER.fullmatch(engine):
+ raise ContractError(
+ f"inventory_engine_identifier_unsafe_line_{line_number}"
+ )
+ if not rows_text.isdecimal() or not bytes_text.isdecimal():
+ raise ContractError(f"inventory_counter_invalid_line_{line_number}")
+ if not SCHEMA_SHA256.fullmatch(schema_hash):
+ raise ContractError(f"inventory_schema_hash_invalid_line_{line_number}")
+ key = (database, table)
+ if key in receipts:
+ raise ContractError(f"inventory_duplicate_table_line_{line_number}")
+ receipts[key] = TableReceipt(
+ database=database,
+ table=table,
+ engine=engine,
+ rows=int(rows_text),
+ bytes=int(bytes_text),
+ schema_hash=schema_hash,
+ )
+ return receipts
+
+
+def require_database_set(receipts: dict[tuple[str, str], TableReceipt]) -> None:
+ observed = tuple(sorted({receipt.database for receipt in receipts.values()}))
+ if observed != EXPECTED_DATABASES:
+ raise ContractError("signoz_database_set_mismatch")
+
+
+def require_critical_tables(
+ receipts: dict[tuple[str, str], TableReceipt], *, require_nonzero: bool
+) -> None:
+ for key in CRITICAL_TABLES:
+ receipt = receipts.get(key)
+ if receipt is None:
+ raise ContractError(f"critical_table_missing_{key[0]}_{key[1]}")
+ if require_nonzero and receipt.rows <= 0:
+ raise ContractError(f"critical_table_empty_{key[0]}_{key[1]}")
+
+
+def emit(values: list[tuple[str, object]]) -> None:
+ for key, value in values:
+ print(f"{key}={value}")
+
+
+def preflight(args: argparse.Namespace) -> None:
+ manifest_path = Path(args.manifest)
+ artifact_path = Path(args.artifact)
+ inventory_path = Path(args.source_inventory)
+ try:
+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+ raise ContractError("backup_manifest_read_failed") from exc
+ if not isinstance(manifest, dict):
+ raise ContractError("backup_manifest_not_object")
+ if manifest.get("status") != "BACKUP_CREATED":
+ raise ContractError("backup_terminal_not_BACKUP_CREATED")
+
+ artifact_sha256 = sha256_file(artifact_path)
+ inventory_sha256 = sha256_file(inventory_path)
+ artifact_size = artifact_path.stat().st_size
+ manifest_sha256 = manifest.get("sha256")
+ manifest_inventory_sha256 = manifest.get("source_inventory_sha256")
+ manifest_size = manifest.get("size_bytes")
+ if not isinstance(manifest_sha256, str) or not SHA256.fullmatch(manifest_sha256):
+ raise ContractError("backup_manifest_sha256_invalid")
+ if manifest_sha256 != artifact_sha256:
+ raise ContractError("backup_artifact_sha256_mismatch")
+ if manifest_inventory_sha256 != inventory_sha256:
+ raise ContractError("source_inventory_sha256_mismatch")
+ if not isinstance(manifest_size, int) or manifest_size <= 0:
+ raise ContractError("backup_manifest_size_invalid")
+ if manifest_size != artifact_size:
+ raise ContractError("backup_artifact_size_mismatch")
+ if tuple(manifest.get("databases", ())) != EXPECTED_DATABASES:
+ raise ContractError("backup_manifest_database_allowlist_mismatch")
+
+ operation_id = manifest.get("operation_id")
+ if not isinstance(operation_id, str) or not SAFE_OPERATION_ID.fullmatch(
+ operation_id
+ ):
+ raise ContractError("backup_operation_id_invalid")
+
+ receipts = parse_inventory(inventory_path)
+ require_database_set(receipts)
+ require_critical_tables(receipts, require_nonzero=True)
+ replicated_count = sum(
+ receipt.engine.startswith("Replicated") for receipt in receipts.values()
+ )
+ distributed_count = sum(
+ receipt.engine == "Distributed" for receipt in receipts.values()
+ )
+ physical_count = sum("MergeTree" in receipt.engine for receipt in receipts.values())
+ if replicated_count <= 0:
+ raise ContractError("replicated_table_precondition_missing")
+ if distributed_count <= 0:
+ raise ContractError("distributed_table_precondition_missing")
+ if physical_count <= 0:
+ raise ContractError("physical_table_precondition_missing")
+
+ emit(
+ [
+ ("artifact_sha256", artifact_sha256),
+ ("artifact_size_bytes", artifact_size),
+ ("source_inventory_sha256", inventory_sha256),
+ ("source_database_count", len(EXPECTED_DATABASES)),
+ ("source_table_count", len(receipts)),
+ ("replicated_table_count", replicated_count),
+ ("distributed_table_count", distributed_count),
+ ("physical_table_count", physical_count),
+ ("backup_operation_id", operation_id),
+ ]
+ )
+
+
+def compare(args: argparse.Namespace) -> None:
+ source = parse_inventory(Path(args.source_inventory))
+ restored = parse_inventory(Path(args.restored_inventory))
+ require_database_set(source)
+ require_database_set(restored)
+ require_critical_tables(source, require_nonzero=True)
+ require_critical_tables(restored, require_nonzero=True)
+
+ source_keys = set(source)
+ restored_keys = set(restored)
+ if source_keys != restored_keys:
+ raise ContractError("database_table_manifest_set_mismatch")
+ for key in sorted(source_keys):
+ if source[key].engine != restored[key].engine:
+ raise ContractError(f"table_engine_mismatch_{key[0]}_{key[1]}")
+ if source[key].schema_hash != restored[key].schema_hash:
+ raise ContractError(f"table_schema_mismatch_{key[0]}_{key[1]}")
+
+ source_rows = sum(receipt.rows for receipt in source.values())
+ restored_rows = sum(receipt.rows for receipt in restored.values())
+ source_bytes = sum(receipt.bytes for receipt in source.values())
+ restored_bytes = sum(receipt.bytes for receipt in restored.values())
+ physical_count = sum("MergeTree" in receipt.engine for receipt in restored.values())
+ emit(
+ [
+ ("manifest_parity", 1),
+ ("restored_database_count", len(EXPECTED_DATABASES)),
+ ("restored_table_count", len(restored)),
+ ("physical_table_count", physical_count),
+ ("critical_nonzero_count", len(CRITICAL_TABLES)),
+ ("source_total_rows", source_rows),
+ ("restored_total_rows", restored_rows),
+ ("row_delta", restored_rows - source_rows),
+ ("source_total_bytes", source_bytes),
+ ("restored_total_bytes", restored_bytes),
+ ("byte_delta", restored_bytes - source_bytes),
+ ]
+ )
+
+
+def check_list(args: argparse.Namespace) -> None:
+ receipts = parse_inventory(Path(args.inventory))
+ require_database_set(receipts)
+ for key in sorted(receipts):
+ receipt = receipts[key]
+ if "MergeTree" in receipt.engine:
+ print(f"{receipt.database}\t{receipt.table}")
+
+
+def parser() -> argparse.ArgumentParser:
+ root = argparse.ArgumentParser()
+ commands = root.add_subparsers(dest="command", required=True)
+
+ preflight_parser = commands.add_parser("preflight")
+ preflight_parser.add_argument("--manifest", required=True)
+ preflight_parser.add_argument("--artifact", required=True)
+ preflight_parser.add_argument("--source-inventory", required=True)
+ preflight_parser.set_defaults(handler=preflight)
+
+ compare_parser = commands.add_parser("compare")
+ compare_parser.add_argument("--source-inventory", required=True)
+ compare_parser.add_argument("--restored-inventory", required=True)
+ compare_parser.set_defaults(handler=compare)
+
+ check_parser = commands.add_parser("check-list")
+ check_parser.add_argument("--inventory", required=True)
+ check_parser.set_defaults(handler=check_list)
+ return root
+
+
+def main() -> int:
+ args = parser().parse_args()
+ try:
+ args.handler(args)
+ except (ContractError, OSError) as exc:
+ print(f"ERROR={exc}", file=sys.stderr)
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/backup/run-signoz-backup-canary.sh b/scripts/backup/run-signoz-backup-canary.sh
new file mode 100755
index 000000000..c6bdddfe4
--- /dev/null
+++ b/scripts/backup/run-signoz-backup-canary.sh
@@ -0,0 +1,665 @@
+#!/usr/bin/env bash
+# =============================================================================
+# P0-OBS-002 - SigNoz online backup canary and legacy maintenance lease verifier
+#
+# The default native-backup path never stops the collector and therefore does
+# not depend on, read, or mutate the legacy monitor/cooldown contract. The
+# bounded lease path remains available only when an explicit legacy stop-mode
+# canary is requested. Neither mode reads or sources a secret/.env file.
+# =============================================================================
+
+set -euo pipefail
+
+COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}"
+BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}"
+MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}"
+MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}"
+MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}"
+COOLDOWN_FILE="${MONITOR_COOLDOWN_DIR}/${COLLECTOR_NAME}.cooldown"
+RECEIPT_ROOT="${SIGNOZ_CANARY_RECEIPT_ROOT:-/backup/logs/signoz-canary}"
+LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}"
+
+CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}"
+TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}"
+LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}"
+MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}"
+STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}"
+EXPECT_COLLECTOR_STOP="${SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP:-0}"
+
+TRACE_ID="${TRACE_ID:-}"
+RUN_ID="${RUN_ID:-}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-}"
+
+RECEIPT_DIR=""
+RECEIPT_LOG=""
+BACKUP_OUTPUT=""
+MONITOR_WINDOW=""
+STATE_OBSERVATIONS=""
+ROLLBACK_FILE=""
+ROLLBACK_METADATA=""
+
+RECEIPTS_READY=0
+LEASE_ROLLBACK_READY=0
+LEASE_MUTATION_STARTED=0
+LEASE_ARMED=0
+PRIOR_LEASE_PRESENT=0
+PRIOR_LEASE_MODE=""
+PRIOR_LEASE_UID=""
+PRIOR_LEASE_GID=""
+PRIOR_LEASE_HASH=""
+PRIOR_LEASE_VALUE=""
+LEASE_TMP=""
+BACKUP_PID=""
+OBSERVER_PID=""
+CANARY_VERIFIED=0
+CHECK_VERIFIED=0
+MODE=""
+
+log_info() { printf '[INFO] %s\n' "$*"; }
+log_error() { printf '[ERROR] %s\n' "$*" >&2; }
+
+valid_identity() {
+ [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
+}
+
+valid_positive_integer() {
+ [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
+}
+
+valid_poll_interval() {
+ [[ "$1" =~ ^[0-9]+([.][0-9]+)?$ ]] && [[ "$1" != "0" ]] && [[ "$1" != "0.0" ]]
+}
+
+emit_receipt() {
+ local stage="$1"
+ local terminal="$2"
+ local detail="$3"
+ local observed_at
+
+ [ "${RECEIPTS_READY}" -eq 1 ] || return 0
+ observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+ printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${observed_at}" \
+ "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}"
+}
+
+fail_closed() {
+ local detail="$1"
+ log_error "${detail}"
+ emit_receipt "failure" "failed" "${detail}"
+ exit 1
+}
+
+require_command() {
+ command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
+}
+
+require_monitor_fragment() {
+ local fragment="$1"
+ local count
+ count="$(grep -F -c -- "${fragment}" "${MONITOR_SCRIPT}" || true)"
+ [ "${count}" -eq 1 ] || fail_closed "monitor_contract_ambiguous_fragment_count_${count}"
+}
+
+require_monitor_function_fragment() {
+ local function_name="$1"
+ local fragment="$2"
+ local count
+
+ count="$(awk -v header="${function_name}() {" -v needle="${fragment}" '
+ $0 == header {
+ inside = 1
+ depth = 0
+ }
+ inside {
+ original = $0
+ if (index(original, needle)) {
+ count++
+ }
+ open_line = original
+ close_line = original
+ depth += gsub(/\{/, "", open_line)
+ depth -= gsub(/\}/, "", close_line)
+ if (depth == 0) {
+ print count + 0
+ printed = 1
+ exit
+ }
+ }
+ END {
+ if (!printed) {
+ print count + 0
+ }
+ }
+ ' "${MONITOR_SCRIPT}")"
+ [ "${count}" -eq 1 ] \
+ || fail_closed "monitor_${function_name}_contract_ambiguous_fragment_count_${count}"
+}
+
+collector_running_state() {
+ local running
+ if ! running="$(docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
+ return 1
+ fi
+ case "${running}" in
+ true) printf '%s\n' "running" ;;
+ false) printf '%s\n' "stopped" ;;
+ *) return 1 ;;
+ esac
+}
+
+collector_runtime_metadata() {
+ docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
+ "${COLLECTOR_NAME}" 2>/dev/null
+}
+
+listener_is_up() {
+ local port="$1"
+ ss -H -lnt | awk -v port="${port}" '
+ $4 ~ (":" port "$") { found = 1 }
+ END { exit(found ? 0 : 1) }
+ '
+}
+
+verify_collector_runtime() {
+ local expected_id="$1"
+ local expected_started_at="$2"
+ local expected_restart_count="$3"
+ local expected_health="$4"
+ local metadata actual_id actual_running actual_started_at actual_restart_count actual_health
+
+ metadata="$(collector_runtime_metadata)" || return 1
+ IFS=$'\t' read -r actual_id actual_running actual_started_at actual_restart_count actual_health <<< "${metadata}"
+ [ "${actual_id}" = "${expected_id}" ] || return 1
+ [ "${actual_running}" = "true" ] || return 1
+ [ "${actual_started_at}" = "${expected_started_at}" ] || return 1
+ [ "${actual_restart_count}" = "${expected_restart_count}" ] || return 1
+ [ "${actual_health}" = "${expected_health}" ] || return 1
+ listener_is_up 4317 || return 1
+ listener_is_up 4318 || return 1
+}
+
+restore_lease() {
+ local restore_tmp restore_hash current_value
+
+ [ "${LEASE_MUTATION_STARTED}" -eq 1 ] || return 0
+ [ "${LEASE_ROLLBACK_READY}" -eq 1 ] || return 1
+
+ if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then
+ [ -f "${ROLLBACK_FILE}" ] || return 1
+ restore_tmp="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.restore.${RUN_ID}.XXXXXX")" || return 1
+ cp -p "${ROLLBACK_FILE}" "${restore_tmp}" || return 1
+ chmod "${PRIOR_LEASE_MODE}" "${restore_tmp}" || return 1
+ mv -f "${restore_tmp}" "${COOLDOWN_FILE}" || return 1
+ restore_tmp=""
+
+ restore_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" || return 1
+ [ "${restore_hash}" = "${PRIOR_LEASE_HASH}" ] || return 1
+ [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] || return 1
+ [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] || return 1
+ [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] || return 1
+ current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
+ [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] || return 1
+ else
+ rm -f "${COOLDOWN_FILE}" || return 1
+ [ ! -e "${COOLDOWN_FILE}" ] || return 1
+ fi
+
+ LEASE_ARMED=0
+ LEASE_MUTATION_STARTED=0
+ return 0
+}
+
+terminate_children() {
+ if [ -n "${BACKUP_PID}" ] && kill -0 "${BACKUP_PID}" 2>/dev/null; then
+ kill -TERM "${BACKUP_PID}" 2>/dev/null || true
+ wait "${BACKUP_PID}" 2>/dev/null || true
+ fi
+ if [ -n "${OBSERVER_PID}" ] && kill -0 "${OBSERVER_PID}" 2>/dev/null; then
+ kill -TERM "${OBSERVER_PID}" 2>/dev/null || true
+ wait "${OBSERVER_PID}" 2>/dev/null || true
+ fi
+}
+
+cleanup() {
+ local exit_code=$?
+ local lease_terminal="not_armed"
+ local overall_terminal="failed"
+
+ trap - EXIT HUP INT TERM
+ set +e
+ terminate_children
+ rm -f "${LEASE_TMP}" 2>/dev/null || true
+
+ if [ "${LEASE_MUTATION_STARTED}" -eq 1 ]; then
+ if restore_lease; then
+ lease_terminal="restored"
+ else
+ lease_terminal="restore_failed"
+ exit_code=91
+ fi
+ elif [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then
+ lease_terminal="no_write"
+ elif [ "${MODE}" = "apply" ] && [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
+ lease_terminal="no_write"
+ fi
+
+ if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then
+ overall_terminal="check_pass_no_write"
+ elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then
+ overall_terminal="pass"
+ fi
+
+ emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}"
+ if [ "${overall_terminal}" = "pass" ]; then
+ emit_receipt "closure_writeback" "pass" \
+ "no_incident_notification_or_learning_delta_local_durable_receipt_ack"
+ elif [ "${overall_terminal}" = "check_pass_no_write" ]; then
+ emit_receipt "closure_writeback" "not_applicable" \
+ "check_mode_no_runtime_change_local_durable_receipt_ack"
+ else
+ emit_receipt "closure_writeback" "deferred" \
+ "failed_terminal_requires_followup_no_completion_claim"
+ fi
+ emit_receipt "terminal" "${overall_terminal}" "backup_canary_${overall_terminal}"
+ printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \
+ "${lease_terminal}" "${exit_code}"
+ exit "${exit_code}"
+}
+
+on_signal() {
+ local signal_number="$1"
+ emit_receipt "signal" "failed" "signal_${signal_number}"
+ exit "$((128 + signal_number))"
+}
+
+setup_receipts() {
+ valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and must be path-safe"; exit 2; }
+ valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and must be path-safe"; exit 2; }
+ valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and must be path-safe"; exit 2; }
+
+ [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root must not be a symlink"; exit 2; }
+ mkdir -p "${RECEIPT_ROOT}"
+ RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
+ [ ! -e "${RECEIPT_DIR}" ] || { log_error "receipt directory already exists: ${RECEIPT_DIR}"; exit 2; }
+ mkdir -m 700 "${RECEIPT_DIR}"
+
+ RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
+ BACKUP_OUTPUT="${RECEIPT_DIR}/backup-output.log"
+ MONITOR_WINDOW="${RECEIPT_DIR}/monitor-window.log"
+ STATE_OBSERVATIONS="${RECEIPT_DIR}/collector-state.tsv"
+ ROLLBACK_FILE="${RECEIPT_DIR}/cooldown.rollback"
+ ROLLBACK_METADATA="${RECEIPT_DIR}/cooldown.rollback.json"
+ : > "${RECEIPT_LOG}"
+ RECEIPTS_READY=1
+
+ trap cleanup EXIT
+ trap 'on_signal 1' HUP
+ trap 'on_signal 2' INT
+ trap 'on_signal 15' TERM
+}
+
+verify_monitor_contract() {
+ local cron_output cron_count cron_line exclude_count exclude_line
+
+ [ -f "${MONITOR_SCRIPT}" ] && [ -r "${MONITOR_SCRIPT}" ] && [ ! -L "${MONITOR_SCRIPT}" ] \
+ || fail_closed "monitor_script_missing_unreadable_or_symlink"
+ [ -f "${MONITOR_LOG}" ] && [ -r "${MONITOR_LOG}" ] && [ ! -L "${MONITOR_LOG}" ] \
+ || fail_closed "monitor_log_missing_unreadable_or_symlink"
+ [ -d "${MONITOR_COOLDOWN_DIR}" ] && [ -w "${MONITOR_COOLDOWN_DIR}" ] && [ ! -L "${MONITOR_COOLDOWN_DIR}" ] \
+ || fail_closed "monitor_cooldown_directory_missing_unwritable_or_symlink"
+
+ require_monitor_fragment "COOLDOWN_DIR:=${MONITOR_COOLDOWN_DIR}"
+ require_monitor_fragment 'local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown"'
+ require_monitor_fragment 'last_sent=$(cat "$cooldown_file")'
+ # The live monitor has another cooldown path with the same elapsed
+ # calculation. Bind this assertion to is_in_cooldown instead of requiring
+ # a globally unique string.
+ require_monitor_function_fragment is_in_cooldown 'elapsed=$(( now - last_sent ))'
+ require_monitor_fragment 'if (( elapsed < ACTION_COOLDOWN_SECONDS )); then'
+ require_monitor_fragment 'COOLDOWN: ${container}'
+ require_monitor_fragment 'is_in_cooldown "$container_name" && continue'
+ require_monitor_fragment 'set_cooldown "$container_name"'
+ require_monitor_fragment 'log "AUTO_REPAIR: docker restart ${container}"'
+ require_monitor_fragment 'if docker restart "$container"'
+
+ exclude_count="$(grep -F -c 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}" || true)"
+ [ "${exclude_count}" -eq 1 ] || fail_closed "monitor_exclude_contract_ambiguous"
+ exclude_line="$(grep -F 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}")"
+ [[ "${exclude_line}" != *"${COLLECTOR_NAME}"* ]] || fail_closed "collector_already_excluded_monitor_contract_changed"
+ [ "$(grep -F -c "${COLLECTOR_NAME}" "${MONITOR_SCRIPT}" || true)" -eq 0 ] \
+ || fail_closed "collector_literal_present_monitor_contract_changed"
+
+ cron_output="$(crontab -l 2>/dev/null)" || fail_closed "monitor_crontab_unreadable"
+ cron_count="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" '
+ $0 !~ /^[[:space:]]*#/ && index($0, script) { count++ }
+ END { print count + 0 }
+ ')"
+ [ "${cron_count}" -eq 1 ] || fail_closed "monitor_cron_contract_ambiguous_count_${cron_count}"
+ cron_line="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" '
+ $0 !~ /^[[:space:]]*#/ && index($0, script) { print }
+ ')"
+ [[ "${cron_line}" == "*/5 * * * * "* ]] || fail_closed "monitor_cron_schedule_not_every_five_minutes"
+ [[ "${cron_line}" == *">> ${MONITOR_LOG}"* ]] || fail_closed "monitor_cron_log_route_mismatch"
+ [[ "${cron_line}" != *"COOLDOWN_DIR="* ]] || fail_closed "monitor_cron_cooldown_override_ambiguous"
+
+}
+
+inspect_prior_lease() {
+ local current_uid current_gid
+ current_uid="$(id -u)"
+ current_gid="$(id -g)"
+
+ if [ -e "${COOLDOWN_FILE}" ]; then
+ [ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \
+ || fail_closed "existing_cooldown_receipt_unsafe"
+ PRIOR_LEASE_PRESENT=1
+ PRIOR_LEASE_MODE="$(stat -c '%a' "${COOLDOWN_FILE}")"
+ PRIOR_LEASE_UID="$(stat -c '%u' "${COOLDOWN_FILE}")"
+ PRIOR_LEASE_GID="$(stat -c '%g' "${COOLDOWN_FILE}")"
+ [ "${PRIOR_LEASE_UID}" = "${current_uid}" ] && [ "${PRIOR_LEASE_GID}" = "${current_gid}" ] \
+ || fail_closed "existing_cooldown_receipt_owner_mismatch"
+ PRIOR_LEASE_VALUE="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
+ [[ "${PRIOR_LEASE_VALUE}" =~ ^[0-9]+$ ]] || fail_closed "existing_cooldown_receipt_not_epoch"
+ PRIOR_LEASE_HASH="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
+ else
+ PRIOR_LEASE_PRESENT=0
+ PRIOR_LEASE_MODE="664"
+ PRIOR_LEASE_UID="${current_uid}"
+ PRIOR_LEASE_GID="${current_gid}"
+ PRIOR_LEASE_HASH="absent"
+ PRIOR_LEASE_VALUE="absent"
+ fi
+
+}
+
+capture_prior_lease() {
+ local current_hash current_value
+
+ if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then
+ [ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \
+ || fail_closed "cooldown_changed_between_check_and_rollback_capture"
+ current_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
+ current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
+ [ "${current_hash}" = "${PRIOR_LEASE_HASH}" ] \
+ && [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] \
+ && [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] \
+ && [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] \
+ && [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] \
+ || fail_closed "cooldown_changed_between_check_and_rollback_capture"
+ cp -p "${COOLDOWN_FILE}" "${ROLLBACK_FILE}"
+ [ "$(sha256sum "${ROLLBACK_FILE}" | awk '{print $1}')" = "${PRIOR_LEASE_HASH}" ] \
+ || fail_closed "cooldown_changed_between_check_and_rollback_capture"
+ else
+ [ ! -e "${COOLDOWN_FILE}" ] \
+ || fail_closed "cooldown_appeared_between_check_and_rollback_capture"
+ fi
+
+ printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","prior_present":%s,"prior_mode":"%s","prior_uid":"%s","prior_gid":"%s","prior_hash":"%s","prior_value":"%s"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${PRIOR_LEASE_PRESENT}" \
+ "${PRIOR_LEASE_MODE}" "${PRIOR_LEASE_UID}" "${PRIOR_LEASE_GID}" \
+ "${PRIOR_LEASE_HASH}" "${PRIOR_LEASE_VALUE}" > "${ROLLBACK_METADATA}"
+ LEASE_ROLLBACK_READY=1
+ emit_receipt "rollback_capture" "pass" "prior_cooldown_receipt_captured"
+}
+
+arm_lease() {
+ local now lease_epoch lease_hash
+ now="$(date +%s)"
+ lease_epoch=$((now + CANARY_TIMEOUT_SECONDS + LEASE_MARGIN_SECONDS))
+ LEASE_TMP="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.lease.${RUN_ID}.XXXXXX")"
+ printf '%s\n' "${lease_epoch}" > "${LEASE_TMP}"
+ chmod "${PRIOR_LEASE_MODE}" "${LEASE_TMP}"
+
+ LEASE_MUTATION_STARTED=1
+ mv -f "${LEASE_TMP}" "${COOLDOWN_FILE}"
+ LEASE_TMP=""
+ LEASE_ARMED=1
+
+ [ "$(tr -d '\r\n' < "${COOLDOWN_FILE}")" = "${lease_epoch}" ] \
+ || fail_closed "cooldown_lease_epoch_readback_mismatch"
+ [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "$(id -u)" ] \
+ || fail_closed "cooldown_lease_owner_readback_mismatch"
+ [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "$(id -g)" ] \
+ || fail_closed "cooldown_lease_group_readback_mismatch"
+ lease_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
+ emit_receipt "lease" "armed" "future_epoch_${lease_epoch}_hash_${lease_hash}"
+}
+
+observe_collector() {
+ local epoch state
+ while kill -0 "${BACKUP_PID}" 2>/dev/null; do
+ epoch="$(date +%s)"
+ if state="$(collector_running_state)"; then
+ printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}"
+ else
+ printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}"
+ fi
+ sleep "${STATE_POLL_SECONDS}"
+ done
+
+ epoch="$(date +%s)"
+ if state="$(collector_running_state)"; then
+ printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}"
+ else
+ printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}"
+ fi
+}
+
+verify_monitor_window() {
+ local log_identity_before="$1"
+ local log_size_before="$2"
+ local log_identity_after log_size_after start_byte
+ local auto_repair_count detected_count cooldown_count
+ local stop_epoch restore_epoch next_cron_epoch cron_crossed=0
+
+ log_identity_after="$(stat -c '%d:%i' "${MONITOR_LOG}")"
+ log_size_after="$(stat -c '%s' "${MONITOR_LOG}")"
+ [ "${log_identity_after}" = "${log_identity_before}" ] \
+ || fail_closed "monitor_log_rotated_during_canary"
+ [ "${log_size_after}" -ge "${log_size_before}" ] \
+ || fail_closed "monitor_log_shrank_during_canary"
+
+ start_byte=$((log_size_before + 1))
+ tail -c "+${start_byte}" "${MONITOR_LOG}" > "${MONITOR_WINDOW}"
+ auto_repair_count="$(grep -a -F -c "AUTO_REPAIR: docker restart ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)"
+ detected_count="$(grep -a -F -c "DETECTED: ${COLLECTOR_NAME} state=exited" "${MONITOR_WINDOW}" || true)"
+ cooldown_count="$(grep -a -F -c "COOLDOWN: ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)"
+
+ [ "${auto_repair_count}" -eq 0 ] || fail_closed "monitor_auto_repair_contaminated_canary"
+ [ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
+ || fail_closed "collector_state_observer_ambiguous"
+
+ if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
+ [ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
+ || fail_closed "online_native_backup_collector_continuity_failed"
+ [ "${detected_count}" -eq 0 ] \
+ || fail_closed "online_native_backup_monitor_detected_collector"
+ emit_receipt "monitor_verifier" "pass" \
+ "online_native_collector_continuous_auto_repair_0_detected_0"
+ return 0
+ fi
+
+ stop_epoch="$(awk '$2 == "stopped" { print $1; exit }' "${STATE_OBSERVATIONS}")"
+ [ -n "${stop_epoch}" ] || fail_closed "collector_stop_transition_not_observed"
+ restore_epoch="$(awk '
+ $2 == "stopped" { stopped_seen = 1; next }
+ stopped_seen && $2 == "running" { print $1; exit }
+ ' "${STATE_OBSERVATIONS}")"
+ [ -n "${restore_epoch}" ] || fail_closed "collector_restore_transition_not_observed"
+
+ next_cron_epoch=$(( ((stop_epoch / MONITOR_CRON_INTERVAL_SECONDS) + 1) * MONITOR_CRON_INTERVAL_SECONDS ))
+ if [ "${next_cron_epoch}" -le "${restore_epoch}" ]; then
+ cron_crossed=1
+ fi
+
+ if [ "${detected_count}" -gt 0 ]; then
+ [ "${cooldown_count}" -ge "${detected_count}" ] \
+ || fail_closed "monitor_detected_without_matching_cooldown"
+ fi
+ if [ "${cron_crossed}" -eq 1 ]; then
+ [ "${detected_count}" -gt 0 ] && [ "${cooldown_count}" -gt 0 ] \
+ || fail_closed "cron_crossed_without_collector_cooldown_receipt"
+ fi
+
+ emit_receipt "monitor_verifier" "pass" \
+ "auto_repair_${auto_repair_count}_detected_${detected_count}_cooldown_${cooldown_count}_cron_crossed_${cron_crossed}"
+}
+
+verify_online_collector_window() {
+ [ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
+ || fail_closed "collector_state_observer_ambiguous"
+ [ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
+ || fail_closed "online_native_backup_collector_continuity_failed"
+ emit_receipt "monitor_verifier" "pass" \
+ "online_native_collector_continuous_monitor_contract_not_applicable"
+}
+
+main() {
+ local container_id_before container_started_at_before container_restart_count_before
+ local container_health_before collector_metadata_before
+ local monitor_log_identity="" monitor_log_size=""
+ local source_diff_detail ai_decision_detail
+ local backup_status
+
+ setup_receipts
+
+ [ "$#" -eq 1 ] || fail_closed "exactly_one_mode_required_use_check_or_apply"
+ case "$1" in
+ --check) MODE="check" ;;
+ --apply) MODE="apply" ;;
+ *) fail_closed "unsupported_mode_use_check_or_apply" ;;
+ esac
+
+ for command_name in awk date docker env flock grep kill pgrep sha256sum sleep ss timeout tr; do
+ require_command "${command_name}"
+ done
+ valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout"
+ valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after"
+ valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval"
+ case "${EXPECT_COLLECTOR_STOP}" in
+ 0|1) ;;
+ *) fail_closed "invalid_expect_collector_stop_mode" ;;
+ esac
+
+ [ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \
+ || fail_closed "backup_script_missing_unexecutable_or_symlink"
+ if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+ for command_name in cp crontab id mktemp mv rm stat tail; do
+ require_command "${command_name}"
+ done
+ valid_positive_integer "${LEASE_MARGIN_SECONDS}" || fail_closed "invalid_lease_margin"
+ valid_positive_integer "${MONITOR_CRON_INTERVAL_SECONDS}" || fail_closed "invalid_monitor_interval"
+ [ "${MONITOR_CRON_INTERVAL_SECONDS}" -le 300 ] || fail_closed "monitor_interval_exceeds_cron_contract"
+ verify_monitor_contract
+ fi
+
+ [ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe"
+ if [ -e "${LOCK_FILE}" ]; then
+ [ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file"
+ fi
+ exec 9>>"${LOCK_FILE}"
+ flock -n 9 || fail_closed "another_signoz_canary_holds_lock"
+ if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
+ fail_closed "another_signoz_backup_is_running"
+ fi
+
+ collector_metadata_before="$(collector_runtime_metadata)" \
+ || fail_closed "collector_runtime_metadata_unreadable"
+ IFS=$'\t' read -r container_id_before _container_running_before \
+ container_started_at_before container_restart_count_before container_health_before \
+ <<< "${collector_metadata_before}"
+ [ -n "${container_id_before}" ] \
+ && [ -n "${container_started_at_before}" ] \
+ && [[ "${container_restart_count_before}" =~ ^[0-9]+$ ]] \
+ && [[ "${container_health_before}" =~ ^(healthy|none)$ ]] \
+ || fail_closed "collector_runtime_metadata_invalid"
+ verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \
+ "${container_restart_count_before}" "${container_health_before}" \
+ || fail_closed "collector_preflight_runtime_failed"
+
+ if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+ monitor_log_identity="$(stat -c '%d:%i' "${MONITOR_LOG}")"
+ monitor_log_size="$(stat -c '%s' "${MONITOR_LOG}")"
+ inspect_prior_lease
+ emit_receipt "sensor_source" "pass" \
+ "monitor_hash_$(sha256sum "${MONITOR_SCRIPT}" | awk '{print $1}')_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
+ source_diff_detail="legacy_monitor_and_cron_contract_exact_cooldown_prior_present_${PRIOR_LEASE_PRESENT}_hash_${PRIOR_LEASE_HASH}"
+ ai_decision_detail="arm_exact_container_future_epoch_run_legacy_stop_backup_then_restore"
+ else
+ PRIOR_LEASE_PRESENT=0
+ PRIOR_LEASE_HASH="not_applicable"
+ emit_receipt "sensor_source" "pass" \
+ "online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
+ source_diff_detail="online_native_backup_monitor_and_cooldown_contract_not_applicable"
+ ai_decision_detail="run_online_native_backup_without_collector_or_monitor_mutation"
+ fi
+ emit_receipt "normalized_asset_identity" "pass" \
+ "asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker"
+ emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}"
+ emit_receipt "ai_decision" "candidate" "${ai_decision_detail}"
+ emit_receipt "risk_policy" "pass" \
+ "risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_independent_verifier"
+ emit_receipt "check" "pass" \
+ "mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent"
+
+ if [ "${MODE}" = "check" ]; then
+ CHECK_VERIFIED=1
+ log_info "SigNoz backup canary check passed without arming lease or launching backup"
+ return 0
+ fi
+
+ if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+ capture_prior_lease
+ arm_lease
+ else
+ emit_receipt "lease" "not_required" \
+ "online_native_backup_has_no_collector_stop_or_cooldown_write"
+ fi
+
+ : > "${STATE_OBSERVATIONS}"
+ set +e
+ timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \
+ env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
+ BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \
+ > "${BACKUP_OUTPUT}" 2>&1 &
+ BACKUP_PID=$!
+ observe_collector &
+ OBSERVER_PID=$!
+ wait "${BACKUP_PID}"
+ backup_status=$?
+ BACKUP_PID=""
+ wait "${OBSERVER_PID}"
+ OBSERVER_PID=""
+ set -e
+
+ emit_receipt "execution" "$([ "${backup_status}" -eq 0 ] && printf pass || printf failed)" \
+ "backup_exit_${backup_status}"
+ [ "${backup_status}" -eq 0 ] || fail_closed "backup_terminal_nonzero_${backup_status}"
+ grep -a -F -q '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)' "${BACKUP_OUTPUT}" \
+ || fail_closed "retention_skip_receipt_missing"
+ if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+ grep -a -F -q '========== SignOz 備份完成 (' "${BACKUP_OUTPUT}" \
+ || fail_closed "backup_success_terminal_missing"
+ else
+ grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \
+ || fail_closed "native_backup_success_terminal_missing"
+ fi
+
+ if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
+ verify_monitor_window "${monitor_log_identity}" "${monitor_log_size}"
+ else
+ verify_online_collector_window
+ fi
+ verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \
+ "${container_restart_count_before}" "${container_health_before}" \
+ || fail_closed "collector_post_backup_runtime_failed"
+ emit_receipt "post_verifier" "pass" \
+ "collector_running_same_identity_started_at_restart_count_health_listeners_4317_4318"
+
+ CANARY_VERIFIED=1
+ log_info "SigNoz backup canary verified; cooldown lease will now roll back"
+}
+
+main "$@"
diff --git a/scripts/backup/tests/test_backup_signoz_collector_restore.py b/scripts/backup/tests/test_backup_signoz_collector_restore.py
index 01d9635e8..71113b96c 100644
--- a/scripts/backup/tests/test_backup_signoz_collector_restore.py
+++ b/scripts/backup/tests/test_backup_signoz_collector_restore.py
@@ -5,6 +5,7 @@ import shutil
import subprocess
from pathlib import Path
+
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh"
@@ -20,27 +21,24 @@ def _run_backup(
tmp_path: Path,
*,
initial_running: bool = True,
- docker_run_mode: str = "success",
- docker_run_target: str = "signoz-clickhouse",
- archive_timeout_target: str = "",
- tar_verify_mode: str = "success",
- tar_verify_target: str = "clickhouse_",
- tar_timeout_target: str = "",
- docker_start_failures: int = 0,
+ native_check_status: int = 0,
+ native_apply_status: int = 0,
+ continuity_mode: str = "stable",
restic_backup_status: int = 0,
- restore_max_attempts: int = 3,
+ skip_retention: bool = False,
) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
harness = tmp_path / "backup"
fake_bin = tmp_path / "bin"
backup_base = tmp_path / "repository"
event_log = tmp_path / "events.log"
+ collector_id = tmp_path / "collector.id"
collector_state = tmp_path / "collector.state"
- start_count = tmp_path / "start.count"
harness.mkdir()
fake_bin.mkdir()
(backup_base / "signoz" / "data").mkdir(parents=True)
shutil.copy2(SCRIPT, harness / SCRIPT.name)
event_log.touch()
+ collector_id.write_text("sha256:collector-stable\n", encoding="utf-8")
collector_state.write_text(
"true\n" if initial_running else "false\n",
encoding="utf-8",
@@ -50,37 +48,62 @@ def _run_backup(
"""\
BACKUP_BASE="${TEST_BACKUP_BASE:?}"
RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password"
-log_info() { :; }
-log_warn() { :; }
-log_error() { :; }
-log_success() { :; }
+log_info() { printf 'info %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
+log_warn() { printf 'warn %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
+log_error() { printf 'error %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
+log_success() { printf 'success %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
-build_tags() { :; }
-cleanup_old_backups() { :; }
+build_tags() { printf '%s' '--tag signoz'; }
+cleanup_old_backups() { printf 'retention %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
""",
encoding="utf-8",
)
-
+ _write_executable(
+ harness / "clickhouse-native-backup.sh",
+ """\
+#!/bin/bash
+set -eu
+printf 'native %s trace=%s run=%s work=%s\\n' \
+ "${1:-}" "${TRACE_ID:-}" "${RUN_ID:-}" "${WORK_ITEM_ID:-}" \
+ >> "${TEST_EVENT_LOG:?}"
+case "${1:-}" in
+ --check)
+ exit "${FAKE_NATIVE_CHECK_STATUS:-0}"
+ ;;
+ --apply)
+ status="${FAKE_NATIVE_APPLY_STATUS:-0}"
+ [ "$status" -eq 0 ] || exit "$status"
+ case "${FAKE_CONTINUITY_MODE:-stable}" in
+ stopped)
+ printf 'false\\n' > "${TEST_COLLECTOR_STATE:?}"
+ sleep 0.08
+ printf 'true\\n' > "${TEST_COLLECTOR_STATE:?}"
+ ;;
+ identity)
+ printf 'sha256:collector-replaced\\n' > "${TEST_COLLECTOR_ID:?}"
+ sleep 0.08
+ ;;
+ *) sleep 0.08 ;;
+ esac
+ printf 'native-zip\\n' \
+ > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip"
+ printf '{}\\n' \
+ > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip.manifest.json"
+ printf 'db\\ttable\\tengine\\trows\\tbytes\\t%s\\n' \
+ "$(printf 'A%.0s' {1..64})" \
+ > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.source-inventory.tsv"
+ ;;
+ *) exit 64 ;;
+esac
+""",
+ )
_write_executable(
fake_bin / "timeout",
"""\
#!/bin/bash
set -eu
-printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
-while [[ "${1:-}" == --kill-after=* ]]; do
- shift
-done
+while [[ "${1:-}" == --kill-after=* ]]; do shift; done
shift
-if [ -n "${FAKE_ARCHIVE_TIMEOUT_TARGET:-}" ] \
- && [ "${1:-}" = "docker" ] \
- && [[ " $* " == *"${FAKE_ARCHIVE_TIMEOUT_TARGET}"* ]]; then
- exit 124
-fi
-if [ -n "${FAKE_TAR_TIMEOUT_TARGET:-}" ] \
- && [ "${1:-}" = "tar" ] \
- && [[ " $* " == *"${FAKE_TAR_TIMEOUT_TARGET}"* ]]; then
- exit 124
-fi
exec "$@"
""",
)
@@ -89,61 +112,12 @@ exec "$@"
"""\
#!/bin/bash
set -eu
-printf 'docker %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
-case "${1:-}" in
- inspect)
- cat "${TEST_COLLECTOR_STATE:?}"
- ;;
- stop)
- if [ "${2:-}" = "signoz-otel-collector" ]; then
- printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
- fi
- ;;
- start)
- count=0
- if [ -e "${TEST_START_COUNT:?}" ]; then
- count=$(cat "${TEST_START_COUNT}")
- fi
- count=$((count + 1))
- printf '%s\n' "$count" > "${TEST_START_COUNT}"
- if [ "$count" -le "${FAKE_DOCKER_START_FAILURES:-0}" ]; then
- exit 1
- fi
- printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
- ;;
- run)
- mode=success
- if [[ " $* " == *"${FAKE_DOCKER_RUN_TARGET:-signoz-clickhouse}"* ]]; then
- mode="${FAKE_DOCKER_RUN_MODE:-success}"
- fi
- if [ "$mode" = "signal" ]; then
- kill -TERM "${PPID}"
- sleep 0.1
- exit 0
- fi
- if [ "$mode" = "empty" ]; then
- exit 0
- fi
- if [ "$mode" = "partial_nonzero" ]; then
- printf 'partial-archive-data\n'
- exit 23
- fi
- printf 'archive-data\n'
- ;;
-esac
-""",
- )
- _write_executable(
- fake_bin / "tar",
- """\
-#!/bin/bash
-set -eu
-printf 'tar %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
-if [[ " $* " == *"${FAKE_TAR_VERIFY_TARGET:-signoz-clickhouse}"* ]] \
- && [ "${FAKE_TAR_VERIFY_MODE:-success}" = "corrupt" ]; then
- exit 2
-fi
-exit 0
+printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
+[ "${1:-}" = "inspect" ] || exit 70
+printf '%s\\t%s\\t%s\\t%s\\t%s\\n' \
+ "$(tr -d '\\r\\n' < "${TEST_COLLECTOR_ID:?}")" \
+ "$(tr -d '\\r\\n' < "${TEST_COLLECTOR_STATE:?}")" \
+ '2026-07-15T00:00:00Z' '0' 'healthy'
""",
)
_write_executable(
@@ -151,18 +125,20 @@ exit 0
"""\
#!/bin/bash
set -eu
-printf 'restic %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
+printf 'restic %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
case " $* " in
- *" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
- *" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
+ *" backup "*)
+ dump_dir="${4:?}"
+ if [ -f "$dump_dir/signoz-metadata-gap.json" ]; then
+ printf 'gap %s\\n' "$(cat "$dump_dir/signoz-metadata-gap.json")" \
+ >> "${TEST_EVENT_LOG:?}"
+ fi
+ exit "${FAKE_RESTIC_BACKUP_STATUS:-0}"
+ ;;
+ *" snapshots "*)
+ printf '[{"short_id":"native123"}]\\n'
+ ;;
esac
-""",
- )
- _write_executable(
- fake_bin / "du",
- """\
-#!/bin/bash
-printf '4K\t%s\n' "${2:-unknown}"
""",
)
@@ -171,22 +147,20 @@ printf '4K\t%s\n' "${2:-unknown}"
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"TEST_BACKUP_BASE": str(backup_base),
"TEST_EVENT_LOG": str(event_log),
+ "TEST_COLLECTOR_ID": str(collector_id),
"TEST_COLLECTOR_STATE": str(collector_state),
- "TEST_START_COUNT": str(start_count),
- "FAKE_DOCKER_RUN_MODE": docker_run_mode,
- "FAKE_DOCKER_RUN_TARGET": docker_run_target,
- "FAKE_ARCHIVE_TIMEOUT_TARGET": archive_timeout_target,
- "FAKE_TAR_VERIFY_MODE": tar_verify_mode,
- "FAKE_TAR_VERIFY_TARGET": tar_verify_target,
- "FAKE_TAR_TIMEOUT_TARGET": tar_timeout_target,
- "FAKE_DOCKER_START_FAILURES": str(docker_start_failures),
+ "FAKE_NATIVE_CHECK_STATUS": str(native_check_status),
+ "FAKE_NATIVE_APPLY_STATUS": str(native_apply_status),
+ "FAKE_CONTINUITY_MODE": continuity_mode,
"FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
- "COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts),
- "COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0",
+ "TRACE_ID": "trace-P0-OBS-002",
+ "RUN_ID": f"run-{tmp_path.name}",
+ "WORK_ITEM_ID": "P0-OBS-002",
+ "SIGNOZ_BACKUP_RECEIPT_ROOT": str(tmp_path / "receipts"),
+ "COLLECTOR_CONTINUITY_POLL_SECONDS": "0.01",
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
- "ARCHIVE_CREATE_TIMEOUT_SECONDS": "1",
- "ARCHIVE_VERIFY_TIMEOUT_SECONDS": "1",
"TIMEOUT_KILL_AFTER_SECONDS": "1",
+ "BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0",
}
process = subprocess.Popen(
["bash", str(harness / SCRIPT.name)],
@@ -210,274 +184,155 @@ def _events(events: list[str], prefix: str) -> list[str]:
return [event for event in events if event.startswith(prefix)]
-def _run_backup_all_with_signoz_failure(
+def test_source_sunsets_raw_volumes_and_collector_mutation() -> None:
+ source = SCRIPT.read_text(encoding="utf-8")
+
+ assert "signoz-clickhouse" not in source
+ assert "signoz-sqlite" not in source
+ assert "docker run" not in source
+ assert "docker stop" not in source
+ assert "docker start" not in source
+ assert "create_volume_archive" not in source
+ assert "pending_application_consistent_export" in source
+ assert '"raw_volume_read":false' in source
+ assert "{{.State.StartedAt}}" in source
+ assert "{{.RestartCount}}" in source
+ assert '{{if (index .State "Health")}}' in source
+ assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
+ assert 'notify_clawbot "success"' not in source
+ assert "no-downtime continuity verifier" in source
+
+
+def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
tmp_path: Path,
-) -> tuple[subprocess.CompletedProcess[str], list[str]]:
- harness = tmp_path / "backup-all"
- fake_scripts = tmp_path / "deployed-scripts"
- event_log = tmp_path / "backup-all-events.log"
- harness.mkdir()
- fake_scripts.mkdir()
- event_log.touch()
+) -> None:
+ result, events, dump_dir = _run_backup(tmp_path)
- source = BACKUP_ALL.read_text(encoding="utf-8").replace(
- "/backup/scripts",
- str(fake_scripts),
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert len(_events(events, "native --check")) == 1
+ assert len(_events(events, "native --apply")) == 1
+ check_identity = _events(events, "native --check")[0].split(" trace=", 1)[1]
+ apply_identity = _events(events, "native --apply")[0].split(" trace=", 1)[1]
+ assert check_identity == apply_identity
+ assert not _events(events, "docker stop")
+ assert not _events(events, "docker start")
+ assert not _events(events, "docker run")
+ assert any("pending_application_consistent_export" in event for event in events)
+ assert len(_events(events, "notify warning signoz ")) == 1
+ assert not _events(events, "notify success ")
+ assert any("SigNoz ClickHouse native 備份完成" in event for event in events)
+ assert any(
+ "--tag run:run-" in event for event in events if event.startswith("restic ")
)
- backup_all = harness / "backup-all.sh"
- _write_executable(backup_all, source)
- (harness / "common.sh").write_text(
- """\
-log_info() { :; }
-log_warn() { :; }
-log_error() { :; }
-log_success() { :; }
-notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
-""",
- encoding="utf-8",
- )
-
- for name in (
- "backup-gitea.sh",
- "backup-momo.sh",
- "backup-harbor.sh",
- "backup-awoooi.sh",
- "backup-langfuse.sh",
- "backup-monitoring.sh",
- "backup-signoz.sh",
- "backup-open-webui.sh",
- "backup-clawbot.sh",
- ):
- status = 17 if name == "backup-signoz.sh" else 0
- _write_executable(
- fake_scripts / name,
- f"#!/bin/bash\nprintf 'run {name}\\n' >> \"${{TEST_EVENT_LOG:?}}\"\nexit {status}\n",
- )
-
- result = subprocess.run(
- ["bash", str(backup_all)],
- env={**os.environ, "TEST_EVENT_LOG": str(event_log)},
- text=True,
- capture_output=True,
- timeout=10,
- )
- return result, event_log.read_text(encoding="utf-8").splitlines()
+ assert any("restic -r" in event and " check " in event for event in events)
+ receipts = list((tmp_path / "receipts").glob("*/restic-snapshot.json"))
+ assert len(receipts) == 1
+ receipt = receipts[0].read_text(encoding="utf-8")
+ assert '"snapshot_id":"native123"' in receipt
+ assert '"offsite_verified":false' in receipt
+ assert not dump_dir.exists()
-def test_backup_jobs_deploys_the_guarded_signoz_script() -> None:
- playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
-
- assert 'dest: "/backup/scripts/{{ item }}"' in playbook
- assert "- backup-signoz.sh" in playbook
- assert "tags: backup_jobs" in playbook
-
-
-def test_restore_policy_is_bounded_timed_and_independently_verified() -> None:
- source = SCRIPT.read_text(encoding="utf-8")
-
- assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source
- assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source
- assert 'ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"' in source
- assert 'ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"' in source
- assert 'TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"' in source
- assert 'timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}"' in source
- assert 'tar -tzf "${output_file}"' in source
- assert "docker run" in source
- assert "|| true" not in source.split("create_volume_archive()", 1)[1].split(
- "verify_volume_archive()",
- 1,
- )[0]
- assert "verify_collector_final_state" in source
- assert source.index("verify_collector_final_state") < source.rindex(
- 'notify_clawbot "success"'
- )
-
-
-def test_canary_retention_skip_is_explicit_and_defaults_off() -> None:
- source = SCRIPT.read_text(encoding="utf-8")
-
- assert 'BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"' in source
- assert 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' in source
- assert source.index('restic -r "${LOCAL_REPO}" backup') < source.index(
- 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]'
- )
- assert source.index('if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]') < source.index(
- "if ! verify_collector_final_state"
- )
-
-
-def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None:
- result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal")
+def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None:
+ result, events, dump_dir = _run_backup(tmp_path, native_check_status=19)
assert result.returncode != 0
- assert _events(events, "docker stop signoz-otel-collector")
- assert len(_events(events, "docker start signoz-otel-collector")) == 1
- assert not _events(events, "notify success ")
+ assert _events(events, "native --check")
+ assert not _events(events, "native --apply")
+ assert not _events(events, "restic ")
+ assert len(_events(events, "notify failed signoz ")) == 1
assert not dump_dir.exists()
-def test_collector_is_restored_when_clickhouse_backup_fails(tmp_path: Path) -> None:
- result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="empty")
+def test_native_apply_failure_never_reports_scoped_success(tmp_path: Path) -> None:
+ result, events, dump_dir = _run_backup(tmp_path, native_apply_status=23)
- assert result.returncode == 1
- assert len(_events(events, "docker start signoz-otel-collector")) == 1
+ assert result.returncode != 0
+ assert _events(events, "native --apply")
+ assert not _events(events, "restic ")
+ assert not _events(events, "notify warning ")
assert not _events(events, "notify success ")
+ assert len(_events(events, "notify failed signoz ")) == 1
assert not dump_dir.exists()
-def test_exit_cleanup_does_not_restart_an_already_restored_collector(
+def test_initially_stopped_collector_fails_without_apply(tmp_path: Path) -> None:
+ result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
+
+ assert result.returncode != 0
+ assert _events(events, "native --check")
+ assert not _events(events, "native --apply")
+ assert not _events(events, "restic ")
+ assert not dump_dir.exists()
+
+
+def test_continuity_observer_rejects_stopped_sample(tmp_path: Path) -> None:
+ result, events, dump_dir = _run_backup(tmp_path, continuity_mode="stopped")
+
+ assert result.returncode != 0
+ assert any("continuity verifier 失敗" in event for event in events)
+ assert not _events(events, "restic ")
+ assert not _events(events, "notify warning ")
+ assert not dump_dir.exists()
+
+
+def test_continuity_observer_rejects_container_identity_drift(
+ tmp_path: Path,
+) -> None:
+ result, events, dump_dir = _run_backup(tmp_path, continuity_mode="identity")
+
+ assert result.returncode != 0
+ assert any("continuity verifier 失敗" in event for event in events)
+ assert not _events(events, "restic ")
+ assert not dump_dir.exists()
+
+
+def test_restic_failure_is_nonzero_and_no_completion_notification(
tmp_path: Path,
) -> None:
result, events, dump_dir = _run_backup(tmp_path, restic_backup_status=17)
assert result.returncode == 17
- assert len(_events(events, "docker start signoz-otel-collector")) == 1
+ assert _events(events, "restic ")
+ assert not _events(events, "notify warning ")
assert not _events(events, "notify success ")
assert not dump_dir.exists()
-def test_fail_once_restores_within_budget_and_succeeds(tmp_path: Path) -> None:
- result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=1)
+def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None:
+ result, events, _dump_dir = _run_backup(tmp_path, skip_retention=True)
assert result.returncode == 0
- assert len(_events(events, "docker start signoz-otel-collector")) == 2
- assert len(_events(events, "notify success ")) == 1
- assert not _events(events, "notify failed ")
- assert not dump_dir.exists()
+ assert any("BACKUP_SKIP_RETENTION_CLEANUP=1" in event for event in events)
+ assert not _events(events, "retention ")
-def test_permanent_restore_failure_is_nonzero_and_never_notifies_success(
- tmp_path: Path,
-) -> None:
- result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=99)
+def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None:
+ playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
- assert result.returncode != 0
- assert len(_events(events, "docker start signoz-otel-collector")) == 3
- assert len(_events(events, "notify failed ")) == 1
- assert not _events(events, "notify success ")
- assert not dump_dir.exists()
-
-
-def test_initially_stopped_fails_closed_without_stop_start_or_success(
- tmp_path: Path,
-) -> None:
- result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
-
- assert result.returncode != 0
- assert not _events(events, "docker stop signoz-otel-collector")
- assert not _events(events, "docker start signoz-otel-collector")
- assert len(_events(events, "notify failed ")) == 1
- assert not _events(events, "notify success ")
- assert not dump_dir.exists()
-
-
-def test_success_notification_follows_independent_final_running_readback(
- tmp_path: Path,
-) -> None:
- result, events, _dump_dir = _run_backup(tmp_path)
-
- assert result.returncode == 0
- inspect_positions = [
- index
- for index, event in enumerate(events)
- if event.startswith("docker inspect --format")
- ]
- success_position = next(
- index
- for index, event in enumerate(events)
- if event.startswith("notify success ")
+ assert 'dest: "/backup/scripts/{{ item }}"' in playbook
+ assert "- backup-signoz.sh" in playbook
+ assert "- clickhouse-native-backup.sh" in playbook
+ assert "- clickhouse-native-restore-drill.sh" in playbook
+ assert "- clickhouse-restore-inventory.py" in playbook
+ assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in playbook
+ assert (
+ "/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
+ in playbook
)
- assert len(inspect_positions) >= 3
- assert inspect_positions[-1] < success_position
+ assert "tags: backup_jobs" in playbook
- start_position = next(
- index
- for index, event in enumerate(events)
- if event.startswith("docker start signoz-otel-collector")
+
+def test_scripts_are_valid_bash() -> None:
+ result = subprocess.run(
+ [
+ "bash",
+ "-n",
+ str(SCRIPT),
+ str(ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"),
+ ],
+ text=True,
+ capture_output=True,
+ check=False,
)
- tar_positions = [
- index for index, event in enumerate(events) if event.startswith("tar -tzf")
- ]
- assert len(tar_positions) == 2
- assert start_position < tar_positions[0]
-
-
-def test_archive_create_timeout_is_bounded_restores_and_notifies_once(
- tmp_path: Path,
-) -> None:
- result, events, dump_dir = _run_backup(
- tmp_path,
- archive_timeout_target="signoz-clickhouse",
- )
-
- assert result.returncode != 0
- assert len(_events(events, "docker start signoz-otel-collector")) == 1
- assert len(_events(events, "notify failed ")) == 1
- assert not _events(events, "notify success ")
- assert not dump_dir.exists()
-
-
-def test_partial_sqlite_archive_nonzero_fails_overall_and_removes_output(
- tmp_path: Path,
-) -> None:
- result, events, dump_dir = _run_backup(
- tmp_path,
- docker_run_mode="partial_nonzero",
- docker_run_target="signoz-sqlite",
- )
-
- assert result.returncode != 0
- assert len(_events(events, "docker start signoz-otel-collector")) == 1
- assert len(_events(events, "notify failed ")) == 1
- assert not _events(events, "notify success ")
- assert not dump_dir.exists()
-
-
-def test_corrupt_archive_fails_after_collector_restore_and_notifies_once(
- tmp_path: Path,
-) -> None:
- result, events, dump_dir = _run_backup(
- tmp_path,
- tar_verify_mode="corrupt",
- tar_verify_target="clickhouse_",
- )
-
- assert result.returncode != 0
- start_position = next(
- index
- for index, event in enumerate(events)
- if event.startswith("docker start signoz-otel-collector")
- )
- tar_position = next(
- index for index, event in enumerate(events) if event.startswith("tar -tzf")
- )
- assert start_position < tar_position
- assert len(_events(events, "notify failed ")) == 1
- assert not _events(events, "notify success ")
- assert not dump_dir.exists()
-
-
-def test_integrity_verifier_timeout_is_bounded_and_never_reports_success(
- tmp_path: Path,
-) -> None:
- result, events, dump_dir = _run_backup(
- tmp_path,
- tar_timeout_target="clickhouse_",
- )
-
- assert result.returncode != 0
- assert len(_events(events, "docker start signoz-otel-collector")) == 1
- assert len(_events(events, "notify failed ")) == 1
- assert not _events(events, "notify success ")
- assert not dump_dir.exists()
-
-
-def test_backup_all_propagates_signoz_failure_to_nonzero_terminal(
- tmp_path: Path,
-) -> None:
- result, events = _run_backup_all_with_signoz_failure(tmp_path)
-
- assert result.returncode == 1
- assert "run backup-signoz.sh" in events
- assert any(event.startswith("notify warning all ") for event in events)
- assert not any(event.startswith("notify success all ") for event in events)
+ assert result.returncode == 0, result.stderr
diff --git a/scripts/backup/tests/test_clickhouse_native_backup.py b/scripts/backup/tests/test_clickhouse_native_backup.py
new file mode 100644
index 000000000..39e1ed922
--- /dev/null
+++ b/scripts/backup/tests/test_clickhouse_native_backup.py
@@ -0,0 +1,478 @@
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"
+PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
+
+
+def _write_executable(path: Path, text: str) -> None:
+ path.write_text(text, encoding="utf-8")
+ path.chmod(0o755)
+
+
+def _install_fakes(fake_bin: Path) -> None:
+ _write_executable(
+ fake_bin / "timeout",
+ """\
+#!/bin/bash
+set -eu
+while [[ "${1:-}" == --kill-after=* ]]; do shift; done
+shift
+exec "$@"
+""",
+ )
+ _write_executable(fake_bin / "flock", "#!/bin/bash\nexit 0\n")
+ _write_executable(
+ fake_bin / "sha256sum",
+ """\
+#!/usr/bin/env python3
+import hashlib
+import sys
+
+if len(sys.argv) == 1:
+ payload = sys.stdin.buffer.read()
+ print(f"{hashlib.sha256(payload).hexdigest()} -")
+else:
+ with open(sys.argv[1], "rb") as source:
+ payload = source.read()
+ print(f"{hashlib.sha256(payload).hexdigest()} {sys.argv[1]}")
+""",
+ )
+ _write_executable(
+ fake_bin / "unzip",
+ """\
+#!/bin/bash
+set -eu
+printf 'unzip %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
+exit "${FAKE_UNZIP_STATUS:-0}"
+""",
+ )
+ _write_executable(
+ fake_bin / "docker",
+ """\
+#!/bin/bash
+set -eu
+printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
+
+find_arg() {
+ local wanted="$1"
+ shift
+ while [ "$#" -gt 0 ]; do
+ if [ "$1" = "$wanted" ]; then
+ printf '%s\\n' "${2:-}"
+ return 0
+ fi
+ shift
+ done
+ return 1
+}
+
+case "${1:-}" in
+ inspect)
+ printf 'sha256:clickhouse-test-id\\ttrue\\n'
+ ;;
+ cp)
+ [ -e "${TEST_SERVER_ARTIFACT:?}" ] || exit 44
+ printf 'clickhouse-native-zip-bytes\\n' > "${3:?}"
+ ;;
+ exec)
+ shift
+ container="${1:?}"
+ shift
+ case "${1:-}" in
+ test)
+ shift
+ if [ "${1:-}" = "!" ]; then
+ [ ! -e "${TEST_SERVER_ARTIFACT:?}" ]
+ elif [ "${1:-}" = "-d" ] || [ "${1:-}" = "-w" ]; then
+ [ "${FAKE_DISK_READY:-1}" = "1" ]
+ [ "${2:-}" = "${FAKE_DISK_PATH:-/backups/}" ]
+ else
+ exit 45
+ fi
+ ;;
+ rm)
+ rm -f "${TEST_SERVER_ARTIFACT:?}"
+ ;;
+ clickhouse-client)
+ query="$(find_arg --query "$@")"
+ operation_id="$(find_arg --param_operation_id "$@" || true)"
+ artifact="$(find_arg --param_artifact "$@" || true)"
+ disk="$(find_arg --param_disk "$@" || true)"
+ case "$query" in
+ 'SELECT version()')
+ printf '25.5.1.1\\n'
+ ;;
+ *"database='system' AND name='backups'"*)
+ printf '%s\\n' "${FAKE_SYSTEM_BACKUPS_COUNT:-1}"
+ ;;
+ 'CHECK GRANT BACKUP ON *.*')
+ printf '%s\\n' "${FAKE_BACKUP_GRANT:-1}"
+ ;;
+ *"FROM system.disks"*)
+ printf '%s\\n' "${FAKE_DISK_PATH:-/backups/}"
+ ;;
+ *"FROM system.databases"*)
+ printf '%s\\n' signoz_analytics signoz_logs signoz_metadata \
+ signoz_meter signoz_metrics signoz_traces
+ if [ "${FAKE_EXTRA_DATABASE:-0}" = "1" ]; then
+ printf 'unexpected_db\\n'
+ fi
+ ;;
+ *"FROM system.tables WHERE database IN"*)
+ schema_hash="$(printf 'A%.0s' {1..64})"
+ printf 'signoz_analytics\\tanalytics\\tReplicatedMergeTree\\t10\\t1000\\t%s\\n' "$schema_hash"
+ printf 'signoz_logs\\tlogs_v2\\tReplicatedMergeTree\\t20\\t2000\\t%s\\n' "$schema_hash"
+ printf 'signoz_metadata\\tmetadata\\tMergeTree\\t3\\t300\\t%s\\n' "$schema_hash"
+ printf 'signoz_meter\\tmeter\\tDistributed\\t0\\t0\\t%s\\n' "$schema_hash"
+ printf 'signoz_metrics\\tsamples_v4\\tReplicatedMergeTree\\t40\\t4000\\t%s\\n' "$schema_hash"
+ printf 'signoz_traces\\tsignoz_index_v3\\tReplicatedMergeTree\\t50\\t5000\\t%s\\n' "$schema_hash"
+ ;;
+ BACKUP*)
+ case "$query" in
+ *" SETTINGS id='${EXPECTED_FAKE_OPERATION_ID:?}' ASYNC")
+ operation_id="${EXPECTED_FAKE_OPERATION_ID}"
+ ;;
+ *) exit 49 ;;
+ esac
+ printf '%s\\n' "$operation_id" > "${TEST_OPERATION_ID:?}"
+ printf '%s\\n' "$artifact" > "${TEST_ARTIFACT_NAME:?}"
+ printf '%s\\n' "$disk" > "${TEST_DISK_NAME:?}"
+ : > "${TEST_SUBMITTED:?}"
+ : > "${TEST_SERVER_ARTIFACT:?}"
+ printf '%s\\tCREATING_BACKUP\\n' "$operation_id"
+ ;;
+ *"FROM system.backups WHERE id="*)
+ mode="${FAKE_STATUS_MODE:-success}"
+ if [ "$mode" = "existing_success" ] || [ -e "${TEST_SUBMITTED:?}" ]; then
+ artifact_name="$(cat "${TEST_ARTIFACT_NAME:?}" 2>/dev/null || true)"
+ [ -n "$artifact_name" ] || artifact_name="${EXPECTED_FAKE_ARTIFACT:?}"
+ backup_name="Disk('backups', '$artifact_name')"
+ case "$mode" in
+ failed)
+ printf '%s\\tBACKUP_FAILED\\t0\\t0\\t0\\t0\\t434F44453A20353938\\t%s\\n' \
+ "$backup_name" "$(printf '1%.0s' {1..64})"
+ ;;
+ *)
+ : > "${TEST_SERVER_ARTIFACT:?}"
+ printf '%s\\tBACKUP_CREATED\\t12\\t1200\\t2400\\t1100\\tnone\\t%s\\n' \
+ "$backup_name" "$(printf '0%.0s' {1..64})"
+ ;;
+ esac
+ fi
+ ;;
+ *) exit 46 ;;
+ esac
+ ;;
+ *) exit 47 ;;
+ esac
+ ;;
+ *) exit 48 ;;
+esac
+""",
+ )
+
+
+def _harness(
+ tmp_path: Path,
+ *,
+ status_mode: str = "success",
+ extra_database: bool = False,
+ disk_ready: bool = True,
+ unzip_status: int = 0,
+ with_hook: bool = False,
+) -> tuple[dict[str, str], dict[str, Path]]:
+ fake_bin = tmp_path / "bin"
+ output = tmp_path / "output"
+ receipts = tmp_path / "receipts"
+ fake_bin.mkdir()
+ output.mkdir()
+ receipts.mkdir()
+ _install_fakes(fake_bin)
+
+ event_log = tmp_path / "events.log"
+ event_log.touch()
+ paths = {
+ "output": output,
+ "receipts": receipts,
+ "events": event_log,
+ "submitted": tmp_path / "submitted",
+ "server_artifact": tmp_path / "server-artifact",
+ "operation_id": tmp_path / "operation-id",
+ "artifact_name": tmp_path / "artifact-name",
+ "disk_name": tmp_path / "disk-name",
+ "hook_events": tmp_path / "hook-events.log",
+ }
+ identity_hash = (
+ subprocess.check_output(
+ [str(fake_bin / "sha256sum")],
+ input=b"trace-native\0run-native\0P0-OBS-002",
+ )
+ .decode()
+ .split()[0]
+ )
+ expected_artifact = f"signoz-{identity_hash}.zip"
+
+ env = {
+ **os.environ,
+ "PATH": f"{fake_bin}:{os.environ['PATH']}",
+ "TRACE_ID": "trace-native",
+ "RUN_ID": "run-native",
+ "WORK_ITEM_ID": "P0-OBS-002",
+ "CLICKHOUSE_NATIVE_OUTPUT_DIR": str(output),
+ "CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(receipts),
+ "CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS": "2",
+ "CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS": "2",
+ "CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS": "1",
+ "CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS": "1",
+ "TEST_EVENT_LOG": str(event_log),
+ "TEST_SUBMITTED": str(paths["submitted"]),
+ "TEST_SERVER_ARTIFACT": str(paths["server_artifact"]),
+ "TEST_OPERATION_ID": str(paths["operation_id"]),
+ "TEST_ARTIFACT_NAME": str(paths["artifact_name"]),
+ "TEST_DISK_NAME": str(paths["disk_name"]),
+ "EXPECTED_FAKE_ARTIFACT": expected_artifact,
+ "EXPECTED_FAKE_OPERATION_ID": f"awoooi-{identity_hash}",
+ "FAKE_STATUS_MODE": status_mode,
+ "FAKE_EXTRA_DATABASE": "1" if extra_database else "0",
+ "FAKE_DISK_READY": "1" if disk_ready else "0",
+ "FAKE_UNZIP_STATUS": str(unzip_status),
+ }
+ if with_hook:
+ hook = tmp_path / "restore-verifier-hook.sh"
+ _write_executable(
+ hook,
+ """\
+#!/bin/bash
+set -eu
+if [ "$1" = "--apply" ]; then
+ test -f "$CLICKHOUSE_NATIVE_ARTIFACT_PATH"
+ test -f "$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH"
+ test -f "$CLICKHOUSE_NATIVE_MANIFEST_PATH"
+ grep -q '"status":"BACKUP_CREATED"' "$CLICKHOUSE_NATIVE_MANIFEST_PATH"
+fi
+printf '%s trace=%s run=%s work=%s dbs=%s inventory=%s manifest=%s\\n' \
+ "$1" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
+ "$CLICKHOUSE_NATIVE_EXPECTED_DATABASES" \
+ "$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH" \
+ "${CLICKHOUSE_NATIVE_MANIFEST_PATH:-none}" \
+ >> "${TEST_HOOK_EVENTS:?}"
+""",
+ )
+ env["CLICKHOUSE_NATIVE_POST_VERIFY_HOOK"] = str(hook)
+ env["TEST_HOOK_EVENTS"] = str(paths["hook_events"])
+ return env, paths
+
+
+def _run(env: dict[str, str], mode: str) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ ["bash", str(SCRIPT), mode],
+ env=env,
+ text=True,
+ capture_output=True,
+ timeout=15,
+ check=False,
+ )
+
+
+def _receipts(path: Path) -> list[dict[str, object]]:
+ receipt_file = path / "run-native" / "receipts.jsonl"
+ return [
+ json.loads(line)
+ for line in receipt_file.read_text(encoding="utf-8").splitlines()
+ ]
+
+
+def test_check_validates_exact_scope_without_backup_or_artifact_write(
+ tmp_path: Path,
+) -> None:
+ env, paths = _harness(tmp_path)
+ result = _run(env, "--check")
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ events = paths["events"].read_text(encoding="utf-8")
+ assert "BACKUP DATABASE" not in events
+ assert "docker cp" not in events
+ assert "docker exec signoz-clickhouse rm" not in events
+ assert not list(paths["output"].iterdir())
+ receipts = _receipts(paths["receipts"])
+ assert receipts[-1]["terminal"] == "check_pass_no_write"
+
+
+def test_apply_requires_same_run_check_receipt(tmp_path: Path) -> None:
+ env, paths = _harness(tmp_path)
+ result = _run(env, "--apply")
+
+ assert result.returncode != 0
+ assert "same_run_check_pass_receipt_required_before_apply" in result.stderr
+ assert paths["events"].read_text(encoding="utf-8") == ""
+
+
+def test_apply_submits_exact_six_database_backup_and_verifies_zip(
+ tmp_path: Path,
+) -> None:
+ env, paths = _harness(tmp_path)
+ assert _run(env, "--check").returncode == 0
+ result = _run(env, "--apply")
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ events = paths["events"].read_text(encoding="utf-8")
+ assert "BACKUP DATABASE signoz_analytics,DATABASE signoz_logs" in events
+ assert "DATABASE signoz_metadata,DATABASE signoz_meter" in events
+ assert "DATABASE signoz_metrics,DATABASE signoz_traces" in events
+ assert "BACKUP ALL" not in events
+ backup_line = next(
+ line for line in events.splitlines() if "--query BACKUP " in line
+ )
+ assert " SETTINGS id='awoooi-" in backup_line
+ assert "id={operation_id:String}" not in backup_line
+ assert "--param_operation_id" not in backup_line
+ assert "docker cp signoz-clickhouse:/backups/signoz-" in events
+ assert "unzip -tq" in events
+ assert "docker exec signoz-clickhouse rm -f -- /backups/signoz-" in events
+ assert not paths["server_artifact"].exists()
+ outputs = list(paths["output"].glob("clickhouse-native-*.zip"))
+ assert len(outputs) == 1
+ manifest = json.loads(Path(str(outputs[0]) + ".manifest.json").read_text())
+ assert manifest["databases"] == [
+ "signoz_analytics",
+ "signoz_logs",
+ "signoz_metadata",
+ "signoz_meter",
+ "signoz_metrics",
+ "signoz_traces",
+ ]
+ inventory = list(paths["output"].glob("*.source-inventory.tsv"))
+ assert len(inventory) == 1
+ assert "ReplicatedMergeTree" in inventory[0].read_text(encoding="utf-8")
+ assert "Distributed" in inventory[0].read_text(encoding="utf-8")
+
+
+def test_database_allowlist_drift_fails_check(tmp_path: Path) -> None:
+ env, _paths = _harness(tmp_path, extra_database=True)
+ result = _run(env, "--check")
+ assert result.returncode != 0
+ assert "signoz_database_allowlist_drift" in result.stderr
+
+
+def test_missing_backup_grant_fails_check_before_submission(tmp_path: Path) -> None:
+ env, paths = _harness(tmp_path)
+ env["FAKE_BACKUP_GRANT"] = "0"
+
+ result = _run(env, "--check")
+
+ assert result.returncode != 0
+ assert "native_backup_grant_absent" in result.stderr
+ assert not paths["submitted"].exists()
+
+
+def test_missing_runtime_backup_disk_fails_check(tmp_path: Path) -> None:
+ env, _paths = _harness(tmp_path, disk_ready=False)
+ result = _run(env, "--check")
+ assert result.returncode != 0
+ assert "backup_disk_directory_absent" in result.stderr
+
+
+def test_failed_async_terminal_has_exact_cleanup_and_no_local_artifact(
+ tmp_path: Path,
+) -> None:
+ env, paths = _harness(tmp_path, status_mode="failed")
+ assert _run(env, "--check").returncode == 0
+ result = _run(env, "--apply")
+
+ assert result.returncode != 0
+ assert "native_backup_terminal_BACKUP_FAILED" in result.stderr
+ assert not paths["server_artifact"].exists()
+ assert not list(paths["output"].iterdir())
+ receipt_dir = paths["receipts"] / "run-native"
+ assert list(receipt_dir.glob("*.stderr"))
+ assert list(receipt_dir.glob("*.status"))
+ bounded_errors = list(receipt_dir.glob("*-backup-error-bounded.hex"))
+ assert len(bounded_errors) == 1
+ assert bounded_errors[0].read_text(encoding="utf-8") == ("434F44453A20353938\n")
+
+
+def test_zip_failure_removes_local_partial_and_preserves_server_evidence(
+ tmp_path: Path,
+) -> None:
+ env, paths = _harness(tmp_path, unzip_status=7)
+ assert _run(env, "--check").returncode == 0
+ result = _run(env, "--apply")
+
+ assert result.returncode != 0
+ assert "native_backup_archive_integrity_failed" in result.stderr
+ assert paths["server_artifact"].exists()
+ assert not list(paths["output"].iterdir())
+ receipts = _receipts(paths["receipts"])
+ assert any(
+ item["stage"] == "cleanup" and item["terminal"] == "preserved"
+ for item in receipts
+ )
+
+
+def test_restore_verifier_hook_uses_check_and_apply_contract(tmp_path: Path) -> None:
+ env, paths = _harness(tmp_path, with_hook=True)
+ assert _run(env, "--check").returncode == 0
+ result = _run(env, "--apply")
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ hook_events = paths["hook_events"].read_text(encoding="utf-8").splitlines()
+ assert sum(event.startswith("--check ") for event in hook_events) == 1
+ assert sum(event.startswith("--apply ") for event in hook_events) == 1
+ assert all("signoz_analytics,signoz_logs" in event for event in hook_events)
+ assert any(
+ "manifest=" in event and "manifest=none" not in event
+ for event in hook_events
+ if event.startswith("--apply ")
+ )
+
+
+def test_verified_local_artifact_is_idempotently_reused(tmp_path: Path) -> None:
+ env, paths = _harness(tmp_path)
+ assert _run(env, "--check").returncode == 0
+ assert _run(env, "--apply").returncode == 0
+ submissions_before = (
+ paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE")
+ )
+
+ result = _run(env, "--apply")
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ submissions_after = (
+ paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE")
+ )
+ assert submissions_after == submissions_before
+ receipts = _receipts(paths["receipts"])
+ assert any(
+ item["stage"] == "idempotency" and item["terminal"] == "reused"
+ for item in receipts
+ )
+
+
+def test_source_contract_has_no_raw_volume_fallback_and_is_deployed() -> None:
+ source = SCRIPT.read_text(encoding="utf-8")
+ playbook = PLAYBOOK.read_text(encoding="utf-8")
+
+ assert "BACKUP ${BACKUP_SCOPE_SQL}" in source
+ assert "SETTINGS id='${OPERATION_ID}' ASYNC" in source
+ assert "SETTINGS id={operation_id:String}" not in source
+ assert "BACKUP ALL" not in source
+ assert ".zip" in source
+ assert "unzip -tq" in source
+ assert "--format $'{{.Id}}\\t{{.State.Running}}'" in source
+ assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
+ assert "docker run" not in source
+ assert "clickhouse-native-backup.sh" in playbook
+ result = subprocess.run(
+ ["bash", "-n", str(SCRIPT)],
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ assert result.returncode == 0, result.stderr
diff --git a/scripts/backup/tests/test_clickhouse_native_restore_drill.py b/scripts/backup/tests/test_clickhouse_native_restore_drill.py
new file mode 100644
index 000000000..cd5a298f7
--- /dev/null
+++ b/scripts/backup/tests/test_clickhouse_native_restore_drill.py
@@ -0,0 +1,848 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import zipfile
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "clickhouse-native-restore-drill.sh"
+INVENTORY_HELPER = ROOT / "clickhouse-restore-inventory.py"
+REPO_ROOT = ROOT.parents[1]
+CLICKHOUSE_IMAGE_ID = (
+ "sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6"
+)
+ZOOKEEPER_IMAGE_ID = (
+ "sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0"
+)
+EMPTY_SHA256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"
+EXPECTED_DATABASES = [
+ "signoz_analytics",
+ "signoz_logs",
+ "signoz_metadata",
+ "signoz_meter",
+ "signoz_metrics",
+ "signoz_traces",
+]
+
+
+def _inventory(
+ *,
+ restored_delta: int = 0,
+ engine_mismatch: bool = False,
+ schema_mismatch: bool = False,
+) -> str:
+ rows = [
+ ("signoz_analytics", "events", "ReplicatedMergeTree", 10, 100),
+ ("signoz_analytics", "events_all", "Distributed", 10, 100),
+ ("signoz_logs", "logs_v2", "ReplicatedMergeTree", 20, 200),
+ ("signoz_metadata", "resource_attrs", "ReplicatedMergeTree", 2, 20),
+ ("signoz_meter", "meter", "ReplicatedMergeTree", 3, 30),
+ ("signoz_metrics", "samples_v4", "ReplicatedMergeTree", 30, 300),
+ ("signoz_metrics", "time_series_v4", "ReplicatedMergeTree", 12, 120),
+ ("signoz_traces", "signoz_index_v3", "ReplicatedMergeTree", 40, 400),
+ ]
+ rendered: list[str] = []
+ for index, (database, table, engine, row_count, byte_count) in enumerate(rows):
+ if engine_mismatch and table == "meter":
+ engine = "MergeTree"
+ schema_hash = (
+ hashlib.sha256(f"{database}.{table}:{engine}".encode("utf-8"))
+ .hexdigest()
+ .upper()
+ )
+ if schema_mismatch and table == "meter":
+ schema_hash = "F" * 64
+ rendered.append(
+ "\t".join(
+ (
+ database,
+ table,
+ engine,
+ str(row_count + (restored_delta if index == 0 else 0)),
+ str(byte_count + (restored_delta if index == 0 else 0)),
+ schema_hash,
+ )
+ )
+ )
+ return "\n".join(rendered) + "\n"
+
+
+def _write_backup_fixture(tmp_path: Path) -> tuple[Path, Path, Path]:
+ artifact = tmp_path / "clickhouse-native.zip.partial"
+ with zipfile.ZipFile(artifact, "w") as archive:
+ archive.writestr(".backup", "bounded native backup fixture\n")
+ inventory = tmp_path / "source-inventory.tsv.partial"
+ inventory.write_text(_inventory(), encoding="utf-8")
+ artifact_sha = hashlib.sha256(artifact.read_bytes()).hexdigest()
+ inventory_sha = hashlib.sha256(inventory.read_bytes()).hexdigest()
+ manifest = tmp_path / "manifest.json.partial"
+ manifest.write_text(
+ json.dumps(
+ {
+ "trace_id": "backup-trace",
+ "run_id": "backup-run",
+ "work_item_id": "P0-OBS-002",
+ "operation_id": "awoooi-backup-operation",
+ "databases": EXPECTED_DATABASES,
+ "source_inventory_sha256": inventory_sha,
+ "sha256": artifact_sha,
+ "size_bytes": artifact.stat().st_size,
+ "status": "BACKUP_CREATED",
+ },
+ separators=(",", ":"),
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ return artifact, manifest, inventory
+
+
+def _write_fake_docker(tmp_path: Path) -> tuple[Path, Path, Path]:
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ state_dir = tmp_path / "docker-state"
+ state_dir.mkdir()
+ log_path = tmp_path / "docker.log"
+ docker = bin_dir / "docker"
+ docker.write_text(
+ r"""#!/usr/bin/env bash
+set -euo pipefail
+printf '%s\n' "$*" >> "${FAKE_DOCKER_LOG:?}"
+state="${FAKE_DOCKER_STATE:?}"
+cmd="${1:?}"
+shift
+last_arg() { eval "printf '%s' \"\${$#}\""; }
+
+case "$cmd" in
+ info)
+ [ "${FAKE_DOCKER_INFO_FAIL:-0}" = 0 ] || exit 78
+ printf '25.0.0\\n'
+ ;;
+ image)
+ sub="${1:?}"
+ shift
+ [ "$sub" = inspect ] || exit 90
+ image="$(last_arg "$@")"
+ case "$image" in
+ clickhouse/clickhouse-server:25.5.6) printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" ;;
+ signoz/zookeeper:3.7.1) printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" ;;
+ *) exit 1 ;;
+ esac
+ ;;
+ container)
+ sub="${1:?}"
+ shift
+ case "$sub" in
+ inspect)
+ format=""
+ if [ "${1:-}" = --format ]; then
+ format="$2"
+ shift 2
+ fi
+ name="${1:?}"
+ [ -f "$state/container-$name" ] || exit 1
+ if [[ "$format" == *Labels* ]]; then
+ if [ "${FAKE_LABEL_INSPECT_HANG:-0}" = 1 ] && [[ "$name" == *ch-restore* ]]; then
+ sleep 10
+ fi
+ cat "$state/container-label-$name"
+ fi
+ ;;
+ ls)
+ for item in "$state"/container-*; do
+ [ -e "$item" ] || continue
+ case "$item" in
+ *-label-*|*-network-*|*-image-*) continue ;;
+ esac
+ basename "$item" | sed 's/^container-//'
+ done
+ ;;
+ *) exit 90 ;;
+ esac
+ ;;
+ inspect)
+ format=""
+ if [ "${1:-}" = --format ]; then
+ format="$2"
+ shift 2
+ fi
+ name="${1:?}"
+ [ -f "$state/container-$name" ] || exit 1
+ case "$format" in
+ *State.Running*) printf 'true\n' ;;
+ *HostConfig.NetworkMode*) cat "$state/container-network-$name" ;;
+ *.Image*) cat "$state/container-image-$name" ;;
+ *) printf '%s\n' "fake-$name" ;;
+ esac
+ ;;
+ network)
+ sub="${1:?}"
+ shift
+ case "$sub" in
+ create)
+ name="$(last_arg "$@")"
+ label=""
+ while [ "$#" -gt 0 ]; do
+ if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi
+ done
+ touch "$state/network-$name"
+ printf '%s\n' "$label" > "$state/network-label-$name"
+ printf '%s\n' "$name" > "$state/network-name"
+ printf '%s\n' "$name"
+ ;;
+ inspect)
+ format=""
+ if [ "${1:-}" = --format ]; then
+ format="$2"
+ shift 2
+ fi
+ name="${1:?}"
+ [ -f "$state/network-$name" ] || exit 1
+ if [[ "$format" == *Internal* ]]; then printf 'true\n'; fi
+ if [[ "$format" == *Labels* ]]; then cat "$state/network-label-$name"; fi
+ ;;
+ ls)
+ for item in "$state"/network-*; do
+ [ -e "$item" ] || continue
+ case "$item" in *-label-*|*/network-name) continue ;; esac
+ basename "$item" | sed 's/^network-//'
+ done
+ ;;
+ rm)
+ name="${1:?}"
+ rm -f "$state/network-$name" "$state/network-label-$name" "$state/network-name"
+ printf '%s\n' "$name"
+ ;;
+ *) exit 90 ;;
+ esac
+ ;;
+ volume)
+ sub="${1:?}"
+ shift
+ case "$sub" in
+ create)
+ name="$(last_arg "$@")"
+ label=""
+ while [ "$#" -gt 0 ]; do
+ if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi
+ done
+ touch "$state/volume-$name"
+ printf '%s\n' "$label" > "$state/volume-label-$name"
+ printf '%s\n' "$name"
+ ;;
+ inspect)
+ format=""
+ if [ "${1:-}" = --format ]; then format="$2"; shift 2; fi
+ name="${1:?}"
+ [ -f "$state/volume-$name" ] || exit 1
+ if [[ "$format" == *Labels* ]]; then cat "$state/volume-label-$name"; fi
+ ;;
+ ls)
+ for item in "$state"/volume-*; do
+ [ -e "$item" ] || continue
+ case "$item" in *-label-*) continue ;; esac
+ basename "$item" | sed 's/^volume-//'
+ done
+ ;;
+ rm)
+ name="${1:?}"
+ rm -f "$state/volume-$name" "$state/volume-label-$name"
+ if [[ "$name" == *-backup ]]; then
+ rm -f "$state/staged-artifact-sha256"
+ fi
+ printf '%s\n' "$name"
+ ;;
+ *) exit 90 ;;
+ esac
+ ;;
+ run)
+ name=""
+ replica=""
+ label=""
+ network=""
+ image=""
+ expected_artifact_sha256=""
+ while [ "$#" -gt 0 ]; do
+ case "$1" in
+ --name)
+ name="$2"
+ shift 2
+ ;;
+ --env)
+ if [[ "$2" == CLICKHOUSE_RESTORE_REPLICA=* ]]; then
+ replica="${2#*=}"
+ fi
+ if [[ "$2" == EXPECTED_ARTIFACT_SHA256=* ]]; then
+ expected_artifact_sha256="${2#*=}"
+ fi
+ shift 2
+ ;;
+ --label)
+ label="${2#*=}"
+ shift 2
+ ;;
+ --network)
+ network="$2"
+ shift 2
+ ;;
+ --network-alias|--restart|--mount|--user|--entrypoint)
+ shift 2
+ ;;
+ --detach|--pull=never)
+ shift
+ ;;
+ clickhouse/clickhouse-server:25.5.6|signoz/zookeeper:3.7.1)
+ image="$1"
+ shift
+ ;;
+ *)
+ shift
+ ;;
+ esac
+ done
+ [ -n "$name" ] || exit 91
+ touch "$state/container-$name"
+ printf '%s\n' "$label" > "$state/container-label-$name"
+ printf '%s\n' "$network" > "$state/container-network-$name"
+ case "$image" in
+ clickhouse/clickhouse-server:25.5.6)
+ printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" > "$state/container-image-$name"
+ ;;
+ signoz/zookeeper:3.7.1)
+ printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" > "$state/container-image-$name"
+ ;;
+ *) exit 96 ;;
+ esac
+ if [ -n "$replica" ]; then printf '%s\n' "$replica" > "$state/replica"; fi
+ if [ "$network" = none ]; then
+ if [ "${FAKE_STAGING_HANG:-0}" = 1 ]; then sleep 10; fi
+ [ "${FAKE_STAGING_FAIL:-0}" = 0 ] || exit 94
+ staged_hash="$expected_artifact_sha256"
+ if [ "${FAKE_STAGING_HASH_MISMATCH:-0}" = 1 ]; then
+ staged_hash="$(printf '0%.0s' {1..64})"
+ fi
+ printf '%s\n' "$staged_hash" > "$state/staged-artifact-sha256"
+ printf '%s\n' "$staged_hash"
+ else
+ printf 'fake-container-%s\n' "$name"
+ fi
+ ;;
+ exec)
+ container="${1:?}"
+ shift
+ [ -f "$state/container-$container" ] || exit 1
+ if [ "${1:-}" = sha256sum ]; then
+ [ -f "$state/staged-artifact-sha256" ] || exit 97
+ artifact_hash="$(cat "$state/staged-artifact-sha256")"
+ if [ "${FAKE_CLICKHOUSE_HASH_MISMATCH:-0}" = 1 ]; then
+ artifact_hash="$(printf 'f%.0s' {1..64})"
+ fi
+ printf '%s /backups/native-backup.zip\n' "$artifact_hash"
+ exit 0
+ fi
+ query=""
+ operation=""
+ while [ "$#" -gt 0 ]; do
+ case "$1" in
+ --query)
+ query="$2"
+ shift 2
+ ;;
+ --param_operation_id)
+ operation="$2"
+ shift 2
+ ;;
+ *) shift ;;
+ esac
+ done
+ case "$query" in
+ CHECK\ TABLE*)
+ if [ "${FAKE_CHECK_TABLE_FAIL:-0}" = 1 ]; then printf '0\n'; else printf '1\n'; fi
+ ;;
+ RESTORE\ *)
+ case "$query" in
+ *" SETTINGS id='${EXPECTED_FAKE_RESTORE_OPERATION_ID:?}' ASYNC")
+ operation="${EXPECTED_FAKE_RESTORE_OPERATION_ID}"
+ ;;
+ *) exit 93 ;;
+ esac
+ touch "$state/restored"
+ printf '%s\tRESTORING\n' "$operation"
+ ;;
+ *FROM\ system.backups*)
+ printf "Disk('backups', 'native-backup.zip')\tRESTORED\t100\t1000\t%s\n" "${EMPTY_SHA256:?}"
+ ;;
+ *FROM\ system.disks*) printf '/backups/\n' ;;
+ *FROM\ system.macros*)
+ printf 'replica\t%s\nshard\t01\n' "$(cat "$state/replica")"
+ ;;
+ *FROM\ system.clusters*) printf 'cluster\trestore-clickhouse\t9000\n' ;;
+ *FROM\ system.zookeeper*) printf '1\n' ;;
+ *"SELECT count() FROM system.databases"*) printf '0\n' ;;
+ *"SELECT count() FROM system.tables"*) printf '0\n' ;;
+ *"SELECT database,name,engine"*) cat "${FAKE_RESTORED_INVENTORY:?}" ;;
+ "SELECT 1") printf '1\n' ;;
+ *) printf 'UNHANDLED QUERY: %s\n' "$query" >&2; exit 92 ;;
+ esac
+ ;;
+ logs)
+ printf 'bounded fake container log\n'
+ ;;
+ rm)
+ if [ "${1:-}" = -f ]; then shift; fi
+ name="${1:?}"
+ rm -f "$state/container-$name" "$state/container-label-$name" \
+ "$state/container-network-$name" "$state/container-image-$name"
+ printf '%s\n' "$name"
+ ;;
+ *) exit 99 ;;
+esac
+""",
+ encoding="utf-8",
+ )
+ docker.chmod(0o755)
+ return bin_dir, state_dir, log_path
+
+
+def _environment(
+ tmp_path: Path,
+ artifact: Path,
+ manifest: Path,
+ inventory: Path,
+ restored_inventory: Path,
+) -> dict[str, str]:
+ bin_dir, state_dir, log_path = _write_fake_docker(tmp_path)
+ return os.environ | {
+ "PATH": f"{bin_dir}:{os.environ['PATH']}",
+ "TRACE_ID": "trace-restore-test",
+ "RUN_ID": "run-restore-test",
+ "WORK_ITEM_ID": "P0-OBS-002",
+ "CLICKHOUSE_NATIVE_ARTIFACT_PATH": str(artifact),
+ "CLICKHOUSE_NATIVE_MANIFEST_PATH": str(manifest),
+ "CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH": str(inventory),
+ "CLICKHOUSE_NATIVE_ARTIFACT_SHA256": hashlib.sha256(
+ artifact.read_bytes()
+ ).hexdigest(),
+ "CLICKHOUSE_NATIVE_OPERATION_ID": "awoooi-backup-operation",
+ "CLICKHOUSE_NATIVE_EXPECTED_DATABASES": ",".join(EXPECTED_DATABASES),
+ "CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(tmp_path / "receipts"),
+ "CLICKHOUSE_RESTORE_POLL_INTERVAL_SECONDS": "1",
+ "CLICKHOUSE_RESTORE_STARTUP_TIMEOUT_SECONDS": "5",
+ "CLICKHOUSE_RESTORE_TIMEOUT_SECONDS": "5",
+ "CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS": "5",
+ "FAKE_DOCKER_LOG": str(log_path),
+ "FAKE_DOCKER_STATE": str(state_dir),
+ "FAKE_CLICKHOUSE_IMAGE_ID": CLICKHOUSE_IMAGE_ID,
+ "FAKE_ZOOKEEPER_IMAGE_ID": ZOOKEEPER_IMAGE_ID,
+ "FAKE_RESTORED_INVENTORY": str(restored_inventory),
+ "EXPECTED_FAKE_RESTORE_OPERATION_ID": "restore-"
+ + hashlib.sha256(
+ b"trace-restore-test\0run-restore-test\0P0-OBS-002"
+ ).hexdigest(),
+ "EMPTY_SHA256": EMPTY_SHA256,
+ }
+
+
+def _run(mode: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
+ return subprocess.run(
+ ["bash", str(SCRIPT), mode],
+ text=True,
+ capture_output=True,
+ env=env,
+ )
+
+
+def _status(tmp_path: Path) -> dict[str, object]:
+ return json.loads(
+ (tmp_path / "receipts" / "run-restore-test" / "status.json").read_text(
+ encoding="utf-8"
+ )
+ )
+
+
+def _assert_ephemeral_resources_absent(state_dir: Path) -> None:
+ assert not list(state_dir.glob("container-*"))
+ assert not list(state_dir.glob("volume-*"))
+ assert not list(state_dir.glob("network-*"))
+ assert not (state_dir / "staged-artifact-sha256").exists()
+
+
+def test_hook_check_uses_env_contract_without_runtime_creation(tmp_path: Path) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(restored_delta=3), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+
+ result = _run("--check", env)
+
+ assert result.returncode == 0, result.stderr
+ assert "terminal=check_pass_no_runtime_write" in result.stdout
+ docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8")
+ assert "image inspect" in docker_log
+ assert "network create" not in docker_log
+ assert "volume create" not in docker_log
+ assert not any(line.startswith("run ") for line in docker_log.splitlines())
+ status = _status(tmp_path)
+ assert status["check_pass"] is True
+ assert status["ephemeral_runtime_created"] is False
+ assert status["production_network_attached"] is False
+ assert status["production_restore_performed"] is False
+
+
+def test_apply_restores_on_internal_pair_and_cleans_every_resource(
+ tmp_path: Path,
+) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(restored_delta=3), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ check = _run("--check", env)
+ assert check.returncode == 0, check.stderr
+
+ result = _run("--apply", env)
+
+ assert result.returncode == 0, result.stderr
+ assert "terminal=verified" in result.stdout
+ docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines()
+ network_create = next(
+ line for line in docker_lines if line.startswith("network create")
+ )
+ assert "--internal" in network_create
+ run_lines = [line for line in docker_lines if line.startswith("run ")]
+ assert len(run_lines) == 3
+ assert all("--pull=never" in line for line in run_lines)
+ assert all("--publish" not in line and " -p " not in line for line in run_lines)
+
+ staging_run = next(line for line in run_lines if "--network none" in line)
+ zookeeper_run = next(line for line in run_lines if "signoz/zookeeper:3.7.1" in line)
+ clickhouse_run = next(
+ line for line in run_lines if "--network-alias restore-clickhouse" in line
+ )
+ assert "--user 0:0" in staging_run
+ assert "--entrypoint /bin/sh" in staging_run
+ assert (
+ f"type=bind,src={artifact},dst=/tmp/awoooi-source-native-backup.zip,readonly"
+ in staging_run
+ )
+ assert sum(f"src={artifact}" in line for line in run_lines) == 1
+ assert "type=volume,src=awoooi-ch-restore-" in staging_run
+ assert "-backup,dst=/backups" in staging_run
+ assert "ALLOW_ANONYMOUS_LOGIN=yes" in zookeeper_run
+ assert "--network awoooi-ch-restore-" in zookeeper_run
+ assert "--network awoooi-ch-restore-" in clickhouse_run
+ assert str(artifact) not in clickhouse_run
+ assert "type=volume,src=awoooi-ch-restore-" in clickhouse_run
+ assert "-backup,dst=/backups" in clickhouse_run
+ assert "dst=/backups,readonly" not in clickhouse_run
+ assert (
+ "dst=/etc/clickhouse-server/config.d/awoooi-backup-disk.xml,readonly"
+ in clickhouse_run
+ )
+ assert (
+ "dst=/etc/clickhouse-server/config.d/awoooi-restore-isolation.xml,readonly"
+ in clickhouse_run
+ )
+ hash_readback = next(
+ line
+ for line in docker_lines
+ if line.startswith("exec ") and "sha256sum /backups/native-backup.zip" in line
+ )
+ assert "exec awoooi-ch-restore-" in hash_readback
+ restore_line = next(line for line in docker_lines if "RESTORE DATABASE" in line)
+ assert "allow_non_empty_tables" not in restore_line
+ assert " SETTINGS id='restore-" in restore_line
+ assert "id={operation_id:String}" not in restore_line
+ macro_line = next(line for line in docker_lines if "FROM system.macros" in line)
+ assert "SELECT macro,substitution" in macro_line
+ assert "SELECT name,value" not in macro_line
+ state_dir = Path(env["FAKE_DOCKER_STATE"])
+ _assert_ephemeral_resources_absent(state_dir)
+ status = _status(tmp_path)
+ assert status["terminal"] == "verified"
+ assert status["check_pass"] is True
+ assert status["restore_terminal"] == "RESTORED"
+ assert status["manifest_parity"] == 1
+ assert status["check_table_count"] == status["check_table_pass_count"] == 7
+ assert status["critical_nonzero_count"] == 4
+ assert status["row_delta"] == 3
+ expected_hash = hashlib.sha256(artifact.read_bytes()).hexdigest()
+ assert status["artifact_sha256"] == expected_hash
+ assert status["staged_artifact_sha256"] == expected_hash
+ assert status["clickhouse_artifact_sha256"] == expected_hash
+ assert status["artifact_staging_verified"] is True
+ assert status["clickhouse_artifact_readback_verified"] is True
+ assert status["staging_network_mode"] == "none"
+ assert status["artifact_source_mount"] == "read_only_staging_only"
+ assert status["backup_volume_mount"] == "writable_ephemeral"
+ assert status["cleanup_verified"] is True
+
+
+def test_apply_failure_still_cleans_pair_network_and_volumes(tmp_path: Path) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ assert _run("--check", env).returncode == 0
+ env["FAKE_CHECK_TABLE_FAIL"] = "1"
+
+ result = _run("--apply", env)
+
+ assert result.returncode == 1
+ assert "terminal=failed" in result.stdout
+ state_dir = Path(env["FAKE_DOCKER_STATE"])
+ _assert_ephemeral_resources_absent(state_dir)
+ status = _status(tmp_path)
+ assert status["terminal"] == "failed"
+ assert status["cleanup_verified"] is True
+ assert status["apply_pass"] is False
+
+
+def test_artifact_staging_failure_cleans_exact_resources(tmp_path: Path) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ assert _run("--check", env).returncode == 0
+ env["FAKE_STAGING_FAIL"] = "1"
+
+ result = _run("--apply", env)
+
+ assert result.returncode == 1
+ _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
+ status = _status(tmp_path)
+ assert status["terminal"] == "failed"
+ assert status["reason"] == "isolated_artifact_staging_failed"
+ assert status["artifact_staging_verified"] is False
+ assert status["cleanup_verified"] is True
+
+
+def test_artifact_staging_hang_is_bounded_and_cleans_exact_resources(
+ tmp_path: Path,
+) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1"
+ assert _run("--check", env).returncode == 0
+ env["FAKE_STAGING_HANG"] = "1"
+
+ result = subprocess.run(
+ ["bash", str(SCRIPT), "--apply"],
+ text=True,
+ capture_output=True,
+ env=env,
+ timeout=12,
+ )
+
+ assert result.returncode == 1
+ _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
+ status = _status(tmp_path)
+ assert status["terminal"] == "failed"
+ assert status["reason"] == "isolated_artifact_staging_failed"
+ assert status["artifact_staging_verified"] is False
+ assert status["cleanup_verified"] is True
+
+
+def test_staged_artifact_hash_mismatch_fails_before_isolated_services(
+ tmp_path: Path,
+) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ assert _run("--check", env).returncode == 0
+ env["FAKE_STAGING_HASH_MISMATCH"] = "1"
+
+ result = _run("--apply", env)
+
+ assert result.returncode == 1
+ docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines()
+ assert not any(
+ "signoz/zookeeper:3.7.1" in line
+ for line in docker_lines
+ if line.startswith("run ")
+ )
+ _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
+ status = _status(tmp_path)
+ assert status["reason"] == "staged_artifact_hash_mismatch"
+ assert status["cleanup_verified"] is True
+
+
+def test_clickhouse_artifact_hash_readback_mismatch_fails_and_cleans(
+ tmp_path: Path,
+) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ assert _run("--check", env).returncode == 0
+ env["FAKE_CLICKHOUSE_HASH_MISMATCH"] = "1"
+
+ result = _run("--apply", env)
+
+ assert result.returncode == 1
+ _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
+ status = _status(tmp_path)
+ assert status["reason"] == "isolated_clickhouse_artifact_hash_mismatch"
+ assert status["artifact_staging_verified"] is True
+ assert status["clickhouse_artifact_readback_verified"] is False
+ assert status["clickhouse_artifact_sha256"] == "f" * 64
+ assert status["cleanup_verified"] is True
+
+
+def test_docker_daemon_failure_cannot_claim_cleanup_verified(tmp_path: Path) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ assert _run("--check", env).returncode == 0
+ env["FAKE_DOCKER_INFO_FAIL"] = "1"
+
+ result = _run("--apply", env)
+
+ assert result.returncode == 1
+ status = _status(tmp_path)
+ assert status["terminal"] == "failed"
+ assert status["cleanup_verified"] is False
+ assert status["reason"] == "ephemeral_resource_cleanup_failed"
+
+
+def test_cleanup_label_inspect_hang_is_bounded_and_fails_closed(tmp_path: Path) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1"
+ assert _run("--check", env).returncode == 0
+ env["FAKE_CHECK_TABLE_FAIL"] = "1"
+ env["FAKE_LABEL_INSPECT_HANG"] = "1"
+
+ result = subprocess.run(
+ ["bash", str(SCRIPT), "--apply"],
+ text=True,
+ capture_output=True,
+ env=env,
+ timeout=12,
+ )
+
+ assert result.returncode == 1
+ status = _status(tmp_path)
+ assert status["terminal"] == "failed"
+ assert status["cleanup_verified"] is False
+ assert status["reason"] == "ephemeral_resource_cleanup_failed"
+
+
+def test_check_fails_closed_on_local_image_id_drift(tmp_path: Path) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ env["FAKE_CLICKHOUSE_IMAGE_ID"] = "sha256:" + "0" * 64
+
+ result = _run("--check", env)
+
+ assert result.returncode == 1
+ assert "clickhouse_local_image_id_mismatch" in result.stderr
+ docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8")
+ assert "network create" not in docker_log
+ assert "volume create" not in docker_log
+
+
+def test_flat_host_deploy_accepts_reviewed_absolute_helper_and_config_paths(
+ tmp_path: Path,
+) -> None:
+ artifact, manifest, inventory = _write_backup_fixture(tmp_path)
+ restored = tmp_path / "restored.tsv"
+ restored.write_text(_inventory(), encoding="utf-8")
+ env = _environment(tmp_path, artifact, manifest, inventory, restored)
+ host_root = tmp_path / "host-backup"
+ host_root.mkdir()
+ helper = host_root / "clickhouse-restore-inventory.py"
+ backup_config = host_root / "backup_disk.xml"
+ cluster_config = host_root / "isolated-cluster.xml"
+ shutil.copyfile(INVENTORY_HELPER, helper)
+ shutil.copyfile(
+ REPO_ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml", backup_config
+ )
+ shutil.copyfile(
+ REPO_ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
+ cluster_config,
+ )
+ env.update(
+ {
+ "CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(helper),
+ "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_config),
+ "CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config),
+ }
+ )
+
+ result = _run("--check", env)
+
+ assert result.returncode == 0, result.stderr
+ assert "terminal=check_pass_no_runtime_write" in result.stdout
+
+
+def test_inventory_compare_allows_online_row_delta_but_not_engine_drift(
+ tmp_path: Path,
+) -> None:
+ source = tmp_path / "source.tsv"
+ restored = tmp_path / "restored.tsv"
+ source.write_text(_inventory(), encoding="utf-8")
+ restored.write_text(_inventory(restored_delta=9), encoding="utf-8")
+
+ accepted = subprocess.run(
+ [
+ "python3",
+ str(INVENTORY_HELPER),
+ "compare",
+ "--source-inventory",
+ str(source),
+ "--restored-inventory",
+ str(restored),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ assert accepted.returncode == 0, accepted.stderr
+ assert "manifest_parity=1" in accepted.stdout
+ assert "row_delta=9" in accepted.stdout
+
+ restored.write_text(_inventory(engine_mismatch=True), encoding="utf-8")
+ rejected = subprocess.run(
+ [
+ "python3",
+ str(INVENTORY_HELPER),
+ "compare",
+ "--source-inventory",
+ str(source),
+ "--restored-inventory",
+ str(restored),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ assert rejected.returncode == 1
+ assert "table_engine_mismatch_signoz_meter_meter" in rejected.stderr
+
+ restored.write_text(_inventory(schema_mismatch=True), encoding="utf-8")
+ schema_rejected = subprocess.run(
+ [
+ "python3",
+ str(INVENTORY_HELPER),
+ "compare",
+ "--source-inventory",
+ str(source),
+ "--restored-inventory",
+ str(restored),
+ ],
+ text=True,
+ capture_output=True,
+ )
+ assert schema_rejected.returncode == 1
+ assert "table_schema_mismatch_signoz_meter_meter" in schema_rejected.stderr
diff --git a/scripts/backup/tests/test_signoz_backup_canary_wrapper.py b/scripts/backup/tests/test_signoz_backup_canary_wrapper.py
new file mode 100644
index 000000000..f2ba902af
--- /dev/null
+++ b/scripts/backup/tests/test_signoz_backup_canary_wrapper.py
@@ -0,0 +1,528 @@
+from __future__ import annotations
+
+import json
+import os
+import stat
+import subprocess
+import time
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh"
+PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
+ANSIBLE_VALIDATE = ROOT / "scripts" / "ops" / "ansible-validate.sh"
+READINESS_AUDIT = (
+ ROOT / "scripts" / "reboot-recovery" / "reboot-recovery-readiness-audit.sh"
+)
+
+
+def _write_executable(path: Path, text: str) -> None:
+ path.write_text(text, encoding="utf-8")
+ path.chmod(0o755)
+
+
+def _monitor_source(cooldown_dir: Path, *, omit_contract: bool = False) -> str:
+ cooldown_log = (
+ 'log "COOLDOWN: ${container} skipped"'
+ if not omit_contract
+ else 'log "maintenance skip for ${container}"'
+ )
+ return f"""\
+#!/bin/bash
+: "${{ACTION_COOLDOWN_SECONDS:=${{SEND_COOLDOWN_SECONDS}}}}"
+: "${{COOLDOWN_DIR:={cooldown_dir}}}"
+: "${{EXCLUDE_CONTAINERS:=signoz-clickhouse}}"
+is_in_cooldown() {{
+ local container="$1"
+ local cooldown_file="${{COOLDOWN_DIR}}/${{container}}.cooldown"
+ local last_sent now elapsed
+ last_sent=$(cat "$cooldown_file")
+ now=$(date +%s)
+ elapsed=$(( now - last_sent ))
+ if (( elapsed < ACTION_COOLDOWN_SECONDS )); then
+ {cooldown_log}
+ return 0
+ fi
+ return 1
+}}
+set_cooldown() {{ :; }}
+# A second monitor path may use the same elapsed expression. The wrapper must
+# validate the expression inside is_in_cooldown, not require global uniqueness.
+# elapsed=$(( now - last_sent ))
+while read -r container_name; do
+ is_in_cooldown "$container_name" && continue
+ set_cooldown "$container_name"
+ container="$container_name"
+ log "AUTO_REPAIR: docker restart ${{container}}"
+ if docker restart "$container"; then
+ :
+ fi
+done
+"""
+
+
+def _install_fake_commands(fake_bin: Path) -> None:
+ _write_executable(
+ fake_bin / "crontab",
+ """\
+#!/bin/bash
+set -eu
+[ "${1:-}" = "-l" ]
+printf '*/5 * * * * %s >> %s 2>&1\n' \
+ "${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
+if [ "${FAKE_DUPLICATE_CRON:-0}" = "1" ]; then
+ printf '*/5 * * * * %s >> %s 2>&1\n' \
+ "${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
+fi
+""",
+ )
+ _write_executable(
+ fake_bin / "docker",
+ """\
+#!/bin/bash
+set -eu
+[ "${1:-}" = "inspect" ] || exit 70
+case " $* " in
+ *"{{.State.StartedAt}}"*)
+ printf 'sha256:collector-test-id\\t%s\\t2026-07-15T00:00:00Z\\t0\\thealthy\\n' \
+ "$(cat "${TEST_COLLECTOR_STATE:?}")"
+ ;;
+ *"{{.Id}}"*) printf 'sha256:collector-test-id\n' ;;
+ *"{{.State.Running}}"*) cat "${TEST_COLLECTOR_STATE:?}" ;;
+ *) exit 71 ;;
+esac
+""",
+ )
+ _write_executable(
+ fake_bin / "ss",
+ """\
+#!/bin/bash
+set -eu
+while IFS= read -r port; do
+ [ -n "$port" ] || continue
+ printf 'LISTEN 0 4096 *:%s *:*\n' "$port"
+done < "${TEST_LISTENERS:?}"
+""",
+ )
+ _write_executable(
+ fake_bin / "timeout",
+ """\
+#!/bin/bash
+set -eu
+while [[ "${1:-}" == --kill-after=* ]]; do
+ shift
+done
+[ "$#" -gt 1 ]
+shift
+exec "$@"
+""",
+ )
+ _write_executable(
+ fake_bin / "flock",
+ """\
+#!/bin/bash
+exit 0
+""",
+ )
+ _write_executable(
+ fake_bin / "pgrep",
+ """\
+#!/bin/bash
+exit 1
+""",
+ )
+ _write_executable(
+ fake_bin / "stat",
+ """\
+#!/usr/bin/env python3
+import os
+import stat
+import sys
+
+if len(sys.argv) != 4 or sys.argv[1] != "-c":
+ raise SystemExit(64)
+value = os.stat(sys.argv[3])
+formats = {
+ "%a": format(stat.S_IMODE(value.st_mode), "o"),
+ "%u": str(value.st_uid),
+ "%g": str(value.st_gid),
+ "%s": str(value.st_size),
+ "%d:%i": f"{value.st_dev}:{value.st_ino}",
+}
+if sys.argv[2] not in formats:
+ raise SystemExit(65)
+print(formats[sys.argv[2]])
+""",
+ )
+ _write_executable(
+ fake_bin / "sha256sum",
+ """\
+#!/usr/bin/env python3
+import hashlib
+import sys
+
+if len(sys.argv) != 2:
+ raise SystemExit(64)
+with open(sys.argv[1], "rb") as source:
+ digest = hashlib.sha256(source.read()).hexdigest()
+print(f"{digest} {sys.argv[1]}")
+""",
+ )
+
+
+def _backup_source() -> str:
+ return """\
+#!/bin/bash
+set -eu
+[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80
+[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ]
+printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
+ > "${TEST_BACKUP_IDENTITIES:?}"
+if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
+ cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}"
+ printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
+ case "${FAKE_MONITOR_MODE:-cooldown}" in
+ cooldown)
+ printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
+ >> "${TEST_MONITOR_LOG:?}"
+ printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
+ >> "${TEST_MONITOR_LOG:?}"
+ ;;
+ auto_repair)
+ printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
+ >> "${TEST_MONITOR_LOG:?}"
+ printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
+ >> "${TEST_MONITOR_LOG:?}"
+ printf '[test] AUTO_REPAIR: docker restart signoz-otel-collector\n' \
+ >> "${TEST_MONITOR_LOG:?}"
+ ;;
+ none) : ;;
+ *) exit 81 ;;
+ esac
+ sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
+ printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
+else
+ sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
+fi
+if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then
+ printf '4317\n' > "${TEST_LISTENERS:?}"
+fi
+printf '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)\n'
+if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
+ printf '[SUCCESS] ========== SignOz 備份完成 (1s) ==========\n'
+else
+ printf '[SUCCESS] ========== SigNoz ClickHouse native 備份完成 (1s) ==========\n'
+fi
+"""
+
+
+def _run_canary(
+ tmp_path: Path,
+ *,
+ mode: str = "apply",
+ prior_lease: bool = True,
+ monitor_mode: str = "cooldown",
+ backup_sleep_seconds: float = 0.2,
+ monitor_interval_seconds: int = 300,
+ omit_monitor_contract: bool = False,
+ duplicate_cron: bool = False,
+ drop_listener: bool = False,
+ expect_collector_stop: bool = True,
+) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]:
+ fake_bin = tmp_path / "bin"
+ cooldown_dir = tmp_path / "cooldown"
+ receipt_root = tmp_path / "receipts"
+ fake_bin.mkdir()
+ cooldown_dir.mkdir()
+ receipt_root.mkdir()
+
+ monitor_script = tmp_path / "docker-health-monitor.sh"
+ monitor_log = tmp_path / "monitor.log"
+ backup_script = tmp_path / "backup-signoz.sh"
+ collector_state = tmp_path / "collector.state"
+ listeners = tmp_path / "listeners"
+ identities = tmp_path / "backup-identities.tsv"
+ lease_observed = tmp_path / "lease-observed.txt"
+ cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown"
+ run_id = f"canary-{tmp_path.name}"
+
+ _write_executable(
+ monitor_script,
+ _monitor_source(cooldown_dir, omit_contract=omit_monitor_contract),
+ )
+ _write_executable(backup_script, _backup_source())
+ _install_fake_commands(fake_bin)
+ monitor_log.write_text("[test] monitor seed\n", encoding="utf-8")
+ collector_state.write_text("true\n", encoding="utf-8")
+ listeners.write_text("4317\n4318\n", encoding="utf-8")
+ if prior_lease:
+ cooldown_file.write_text("123\n", encoding="utf-8")
+ cooldown_file.chmod(0o640)
+
+ env = {
+ **os.environ,
+ "PATH": f"{fake_bin}:{os.environ['PATH']}",
+ "TRACE_ID": "trace-P0-OBS-002",
+ "RUN_ID": run_id,
+ "WORK_ITEM_ID": "P0-OBS-002",
+ "SIGNOZ_CANARY_BACKUP_SCRIPT": str(backup_script),
+ "SIGNOZ_CANARY_MONITOR_SCRIPT": str(monitor_script),
+ "SIGNOZ_CANARY_MONITOR_LOG": str(monitor_log),
+ "SIGNOZ_CANARY_COOLDOWN_DIR": str(cooldown_dir),
+ "SIGNOZ_CANARY_RECEIPT_ROOT": str(receipt_root),
+ "SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"),
+ "SIGNOZ_CANARY_TIMEOUT_SECONDS": "5",
+ "SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2",
+ "SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2",
+ "SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds),
+ "SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05",
+ "SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"),
+ "TEST_MONITOR_SCRIPT": str(monitor_script),
+ "TEST_MONITOR_LOG": str(monitor_log),
+ "TEST_COLLECTOR_STATE": str(collector_state),
+ "TEST_LISTENERS": str(listeners),
+ "TEST_BACKUP_IDENTITIES": str(identities),
+ "TEST_COOLDOWN_FILE": str(cooldown_file),
+ "TEST_LEASE_OBSERVED": str(lease_observed),
+ "FAKE_MONITOR_MODE": monitor_mode,
+ "FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds),
+ "FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0",
+ "FAKE_DROP_LISTENER": "1" if drop_listener else "0",
+ "FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0",
+ }
+ started_at = int(time.time())
+ result = subprocess.run(
+ ["bash", str(WRAPPER), f"--{mode}"],
+ env=env,
+ text=True,
+ capture_output=True,
+ timeout=15,
+ check=False,
+ )
+ paths = {
+ "cooldown": cooldown_file,
+ "receipts": receipt_root / run_id / "receipts.jsonl",
+ "rollback_metadata": receipt_root / run_id / "cooldown.rollback.json",
+ "identities": identities,
+ "lease_observed": lease_observed,
+ "collector_state": collector_state,
+ "listeners": listeners,
+ }
+ return result, paths, str(started_at)
+
+
+def _receipts(path: Path) -> list[dict[str, object]]:
+ return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
+
+
+def _terminal(receipts: list[dict[str, object]], stage: str) -> str:
+ return str([item for item in receipts if item["stage"] == stage][-1]["terminal"])
+
+
+def test_existing_lease_is_atomically_replaced_and_exactly_restored(
+ tmp_path: Path,
+) -> None:
+ result, paths, started_at = _run_canary(tmp_path)
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+ assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
+ assert int(paths["lease_observed"].read_text(encoding="utf-8")) >= (
+ int(started_at) + 7
+ )
+ identities = paths["identities"].read_text(encoding="utf-8").split("\t")
+ assert identities[0] == "trace-P0-OBS-002"
+ assert identities[1].startswith("canary-")
+ assert identities[2] == "P0-OBS-002\n"
+ receipts = _receipts(paths["receipts"])
+ assert _terminal(receipts, "rollback") == "restored"
+ assert _terminal(receipts, "terminal") == "pass"
+ metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8"))
+ assert metadata["trace_id"] == "trace-P0-OBS-002"
+ assert metadata["prior_present"] == 1
+
+ stages = [str(item["stage"]) for item in receipts]
+ ordered = [
+ "sensor_source",
+ "normalized_asset_identity",
+ "source_of_truth_diff",
+ "ai_decision",
+ "risk_policy",
+ "check",
+ "execution",
+ "post_verifier",
+ "rollback",
+ "closure_writeback",
+ "terminal",
+ ]
+ assert [stages.index(stage) for stage in ordered] == sorted(
+ stages.index(stage) for stage in ordered
+ )
+
+
+def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None:
+ result, paths, _ = _run_canary(tmp_path, prior_lease=False)
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert not paths["cooldown"].exists()
+ receipts = _receipts(paths["receipts"])
+ assert _terminal(receipts, "rollback") == "restored"
+ assert _terminal(receipts, "terminal") == "pass"
+
+
+def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
+ tmp_path: Path,
+) -> None:
+ result, paths, _ = _run_canary(tmp_path, expect_collector_stop=False)
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+ assert not paths["lease_observed"].exists()
+ assert paths["collector_state"].read_text(encoding="utf-8") == "true\n"
+ receipts = _receipts(paths["receipts"])
+ assert any(
+ item["stage"] == "lease" and item["terminal"] == "not_required"
+ for item in receipts
+ )
+ assert _terminal(receipts, "rollback") == "no_write"
+ assert _terminal(receipts, "terminal") == "pass"
+
+
+def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
+ tmp_path: Path,
+) -> None:
+ result, paths, _ = _run_canary(
+ tmp_path,
+ expect_collector_stop=False,
+ omit_monitor_contract=True,
+ duplicate_cron=True,
+ )
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert not paths["lease_observed"].exists()
+ receipts = _receipts(paths["receipts"])
+ source_diff = [
+ item for item in receipts if item["stage"] == "source_of_truth_diff"
+ ][-1]
+ assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"]
+ assert _terminal(receipts, "terminal") == "pass"
+
+
+def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None:
+ result, paths, _ = _run_canary(tmp_path, monitor_mode="auto_repair")
+
+ assert result.returncode != 0
+ assert "monitor_auto_repair_contaminated_canary" in result.stderr
+ assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+ receipts = _receipts(paths["receipts"])
+ assert _terminal(receipts, "rollback") == "restored"
+ assert _terminal(receipts, "terminal") == "failed"
+
+
+def test_cron_crossing_requires_monitor_cooldown_receipt(tmp_path: Path) -> None:
+ result, paths, _ = _run_canary(
+ tmp_path,
+ monitor_mode="none",
+ backup_sleep_seconds=1.2,
+ monitor_interval_seconds=1,
+ )
+
+ assert result.returncode != 0
+ assert "cron_crossed_without_collector_cooldown_receipt" in result.stderr
+ assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+
+
+def test_cron_crossing_with_detected_and_cooldown_receipts_passes(
+ tmp_path: Path,
+) -> None:
+ result, paths, _ = _run_canary(
+ tmp_path,
+ monitor_mode="cooldown",
+ backup_sleep_seconds=1.2,
+ monitor_interval_seconds=1,
+ )
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ receipts = _receipts(paths["receipts"])
+ monitor_receipt = [
+ item for item in receipts if item["stage"] == "monitor_verifier"
+ ][-1]
+ assert monitor_receipt["terminal"] == "pass"
+ assert "cron_crossed_1" in str(monitor_receipt["detail"])
+
+
+def test_missing_monitor_contract_fails_before_lease_or_backup(tmp_path: Path) -> None:
+ result, paths, _ = _run_canary(tmp_path, omit_monitor_contract=True)
+
+ assert result.returncode != 0
+ assert "monitor_contract_ambiguous_fragment_count_0" in result.stderr
+ assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+ assert not paths["lease_observed"].exists()
+ receipts = _receipts(paths["receipts"])
+ assert _terminal(receipts, "rollback") == "not_armed"
+ assert _terminal(receipts, "terminal") == "failed"
+
+
+def test_ambiguous_cron_contract_fails_before_lease_or_backup(
+ tmp_path: Path,
+) -> None:
+ result, paths, _ = _run_canary(tmp_path, duplicate_cron=True)
+
+ assert result.returncode != 0
+ assert "monitor_cron_contract_ambiguous_count_2" in result.stderr
+ assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+ assert not paths["lease_observed"].exists()
+
+
+def test_post_backup_listener_failure_fails_and_restores_lease(
+ tmp_path: Path,
+) -> None:
+ result, paths, _ = _run_canary(tmp_path, drop_listener=True)
+
+ assert result.returncode != 0
+ assert "collector_post_backup_runtime_failed" in result.stderr
+ assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+ receipts = _receipts(paths["receipts"])
+ assert _terminal(receipts, "rollback") == "restored"
+ assert _terminal(receipts, "terminal") == "failed"
+
+
+def test_wrapper_deployment_and_static_contracts_are_owned() -> None:
+ wrapper = WRAPPER.read_text(encoding="utf-8")
+ assert "BACKUP_SKIP_RETENTION_CLEANUP=1" in wrapper
+ assert 'TRACE_ID="${TRACE_ID:-}"' in wrapper
+ assert 'RUN_ID="${RUN_ID:-}"' in wrapper
+ assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper
+ assert "monitor_auto_repair_contaminated_canary" in wrapper
+ assert "collector_post_backup_runtime_failed" in wrapper
+ assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8")
+ assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text(
+ encoding="utf-8"
+ )
+ assert str(WRAPPER.relative_to(ROOT)) in READINESS_AUDIT.read_text(encoding="utf-8")
+
+
+def test_check_mode_proves_contract_without_lease_or_backup_write(
+ tmp_path: Path,
+) -> None:
+ result, paths, _ = _run_canary(tmp_path, mode="check")
+
+ assert result.returncode == 0, result.stdout + result.stderr
+ assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
+ assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
+ assert not paths["identities"].exists()
+ assert not paths["lease_observed"].exists()
+ check_receipts = _receipts(paths["receipts"])
+ assert _terminal(check_receipts, "rollback") == "no_write"
+ assert _terminal(check_receipts, "terminal") == "check_pass_no_write"
+
+
+def test_wrapper_is_valid_bash() -> None:
+ result = subprocess.run(
+ ["bash", "-n", str(WRAPPER)],
+ text=True,
+ capture_output=True,
+ check=False,
+ )
+ assert result.returncode == 0, result.stderr
diff --git a/scripts/ops/ansible-validate.sh b/scripts/ops/ansible-validate.sh
index c84cd6d90..36a1ef359 100755
--- a/scripts/ops/ansible-validate.sh
+++ b/scripts/ops/ansible-validate.sh
@@ -54,6 +54,9 @@ bash -n \
scripts/backup/backup-langfuse.sh \
scripts/backup/backup-monitoring.sh \
scripts/backup/backup-signoz.sh \
+ scripts/backup/clickhouse-native-backup.sh \
+ scripts/backup/clickhouse-native-restore-drill.sh \
+ scripts/backup/run-signoz-backup-canary.sh \
scripts/backup/backup-open-webui.sh \
scripts/backup/backup-clawbot.sh \
scripts/backup/backup-configs.sh \
@@ -81,6 +84,7 @@ python3 -m py_compile \
scripts/ops/backup-alert-label-contract-check.py \
scripts/ops/backup-alert-live-visibility-check.py \
scripts/ops/recovery-scorecard-contract-check.py \
+ scripts/backup/clickhouse-restore-inventory.py \
scripts/ops/doc-secrets-sanity-check.py
echo "Python 語法 OK"
diff --git a/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh
new file mode 100755
index 000000000..22a79e705
--- /dev/null
+++ b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh
@@ -0,0 +1,594 @@
+#!/usr/bin/env bash
+# P0-OBS-002 - controlled deployment of the host110 ClickHouse native BACKUP disk.
+#
+# This script owns only an additive Compose override and one ClickHouse config
+# fragment. It never runs BACKUP/RESTORE, reads a dotenv/secret source, pulls an
+# image, or recreates another service.
+
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml"
+LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
+
+TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
+REMOTE_DIR="/home/wooo/signoz/deploy/docker"
+REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
+REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
+REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
+REMOTE_PROJECT="signoz"
+REMOTE_SERVICE="clickhouse"
+CLICKHOUSE_CONTAINER="signoz-clickhouse"
+COLLECTOR_CONTAINER="signoz-otel-collector"
+SIGNOZ_CONTAINER="signoz"
+ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
+HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
+CONTAINER_BACKUP_DIR="/backups"
+CLICKHOUSE_DISK="backups"
+CLICKHOUSE_UID="101"
+CLICKHOUSE_GID="101"
+RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
+API_URL="http://127.0.0.1:8080/api/v1/health"
+SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+SAFE_HOME="/home/wooo"
+
+SSH_OPTS=(
+ -o BatchMode=yes
+ -o ConnectTimeout=8
+ -o ConnectionAttempts=1
+ -o ServerAliveInterval=5
+ -o ServerAliveCountMax=2
+)
+
+usage() {
+ cat <<'USAGE'
+Usage: deploy-signoz-clickhouse-backup-disk.sh [--check|--apply]
+
+Required environment:
+ TRACE_ID Path-safe controlled-apply trace identifier
+ RUN_ID Unique path-safe execution identifier
+ WORK_ITEM_ID Path-safe work-item identifier
+
+--check performs local source validation and remote read-only Compose/runtime
+checks. --apply atomically installs the two repo-owned artifacts, recreates only
+the ClickHouse service with --pull never, independently verifies runtime state,
+and restores the exact prior managed Compose state on failure.
+USAGE
+}
+
+MODE="${1:---check}"
+case "${MODE}" in
+ --check|--apply) ;;
+ -h|--help) usage; exit 0 ;;
+ *) usage >&2; exit 64 ;;
+esac
+[ "$#" -le 1 ] || { usage >&2; exit 64; }
+
+TRACE_ID="${TRACE_ID:-}"
+RUN_ID="${RUN_ID:-}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-}"
+
+valid_identity() {
+ [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
+}
+
+for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
+ valid_identity "${value}" || {
+ printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
+ exit 64
+ }
+done
+
+for command_name in python3 scp ssh; do
+ command -v "${command_name}" >/dev/null 2>&1 || {
+ printf 'required local command missing: %s\n' "${command_name}" >&2
+ exit 69
+ }
+done
+
+hash_file() {
+ if command -v sha256sum >/dev/null 2>&1; then
+ sha256sum "$1" | awk '{print $1}'
+ else
+ shasum -a 256 "$1" | awk '{print $1}'
+ fi
+}
+
+validate_local_sources() {
+ [ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ]
+ [ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ]
+
+ python3 - "${LOCAL_CONFIG}" <<'PY'
+from pathlib import Path
+import sys
+import xml.etree.ElementTree as ET
+
+path = Path(sys.argv[1])
+root = ET.parse(path).getroot()
+assert root.tag == "clickhouse"
+disk = root.find("./storage_configuration/disks/backups")
+assert disk is not None
+assert disk.findtext("type") == "local"
+assert disk.findtext("path") == "/backups/"
+backups = root.find("./backups")
+assert backups is not None
+assert backups.findtext("allowed_disk") == "backups"
+assert backups.findtext("allowed_path") == "/backups/"
+assert backups.findtext("allow_concurrent_backups") == "false"
+assert backups.findtext("allow_concurrent_restores") == "false"
+PY
+
+ [ "$(grep -F -c ' clickhouse:' "${LOCAL_OVERRIDE}")" -eq 1 ]
+ [ "$(grep -F -c 'source: /backup/staging/signoz-clickhouse' "${LOCAL_OVERRIDE}")" -eq 1 ]
+ [ "$(grep -F -c 'target: /backups' "${LOCAL_OVERRIDE}")" -eq 1 ]
+ [ "$(grep -F -c 'target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml' "${LOCAL_OVERRIDE}")" -eq 1 ]
+ ! grep -Eq '^[[:space:]]*(image|build|command|environment):' "${LOCAL_OVERRIDE}"
+}
+
+validate_local_sources
+OVERRIDE_SHA="$(hash_file "${LOCAL_OVERRIDE}")"
+CONFIG_SHA="$(hash_file "${LOCAL_CONFIG}")"
+
+local_receipt() {
+ local phase="$1" result="$2" detail="$3"
+ printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
+}
+
+remote_candidate_compose_check() {
+ ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" \
+ "env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
+ < "${LOCAL_OVERRIDE}"
+}
+
+remote_read_only_check() {
+ ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
+ "${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK'
+set -euo pipefail
+
+TRACE_ID="$1"
+RUN_ID="$2"
+WORK_ITEM_ID="$3"
+EXPECTED_OVERRIDE_SHA="$4"
+EXPECTED_CONFIG_SHA="$5"
+REMOTE_DIR="/home/wooo/signoz/deploy/docker"
+REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
+REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
+REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
+HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
+SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+SAFE_HOME="/home/wooo"
+
+file_hash_or_absent() {
+ if [ -f "$1" ] && [ ! -L "$1" ]; then
+ sha256sum "$1" | awk '{print $1}'
+ elif [ ! -e "$1" ]; then
+ printf 'absent\n'
+ else
+ printf 'unsafe\n'
+ fi
+}
+
+[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
+for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
+ docker inspect "${container}" >/dev/null
+done
+env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+ docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
+ config -q >/dev/null
+env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+ docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \
+ config --services | grep -qx clickhouse
+
+override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")"
+config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")"
+override_match=false
+config_match=false
+[ "${override_hash}" = "${EXPECTED_OVERRIDE_SHA}" ] && override_match=true
+[ "${config_hash}" = "${EXPECTED_CONFIG_SHA}" ] && config_match=true
+
+host_dir=absent
+if [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]; then
+ host_dir=present
+elif [ -e "${HOST_BACKUP_DIR}" ]; then
+ host_dir=unsafe
+fi
+
+disk_count="$(docker exec signoz-clickhouse clickhouse-client --query \
+ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
+mount_count="$(docker inspect --format \
+ '{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \
+ signoz-clickhouse | tr -cd '1' | wc -c | xargs)"
+collector_running="$(docker inspect --format '{{.State.Running}}' signoz-otel-collector)"
+collector_restarts="$(docker inspect --format '{{.RestartCount}}' signoz-otel-collector)"
+
+printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \
+ "${config_match}" "${host_dir}" "${disk_count}" "${mount_count}" \
+ "${collector_running}" "${collector_restarts}"
+REMOTE_CHECK
+}
+
+local_receipt sensor_source pass "override_sha_${OVERRIDE_SHA}_config_sha_${CONFIG_SHA}"
+local_receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_disk_backups"
+remote_candidate_compose_check
+remote_read_only_check
+local_receipt check pass "compose_config_q_no_output_no_write"
+
+if [ "${MODE}" = "--check" ]; then
+ local_receipt terminal check_pass_no_write "source_and_remote_preflight_valid"
+ exit 0
+fi
+
+STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}"
+STAGE_CREATED=0
+
+cleanup_stage() {
+ if [ "${STAGE_CREATED}" -eq 1 ]; then
+ ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
+set -u
+stage_dir="$1"
+rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml"
+rmdir "${stage_dir}" 2>/dev/null || true
+REMOTE_CLEANUP
+ fi
+}
+trap cleanup_stage EXIT
+
+ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
+set -euo pipefail
+stage_dir="$1"
+[ ! -e "${stage_dir}" ]
+umask 077
+mkdir -m 0700 "${stage_dir}"
+REMOTE_STAGE
+STAGE_CREATED=1
+
+scp -q "${SSH_OPTS[@]}" "${LOCAL_OVERRIDE}" \
+ "${TARGET_HOST}:${STAGE_DIR}/override.yaml"
+scp -q "${SSH_OPTS[@]}" "${LOCAL_CONFIG}" \
+ "${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml"
+
+ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
+ "${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY'
+set -euo pipefail
+
+TRACE_ID="$1"
+RUN_ID="$2"
+WORK_ITEM_ID="$3"
+EXPECTED_OVERRIDE_SHA="$4"
+EXPECTED_CONFIG_SHA="$5"
+STAGE_DIR="$6"
+
+REMOTE_DIR="/home/wooo/signoz/deploy/docker"
+REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
+REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
+REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
+STAGE_OVERRIDE="${STAGE_DIR}/override.yaml"
+STAGE_CONFIG="${STAGE_DIR}/backup_disk.xml"
+REMOTE_PROJECT="signoz"
+REMOTE_SERVICE="clickhouse"
+CLICKHOUSE_CONTAINER="signoz-clickhouse"
+COLLECTOR_CONTAINER="signoz-otel-collector"
+SIGNOZ_CONTAINER="signoz"
+ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
+HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
+RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
+RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
+RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
+API_URL="http://127.0.0.1:8080/api/v1/health"
+SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+SAFE_HOME="/home/wooo"
+LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock"
+
+APPLY_STARTED=0
+DEPLOY_VERIFIED=0
+HOST_DIR_CREATED=0
+PRIOR_OVERRIDE_PRESENT=0
+PRIOR_CONFIG_PRESENT=0
+PRIOR_DISK_COUNT=0
+BEFORE_IMAGE_ID=""
+BEFORE_DATA_VOLUME=""
+BEFORE_COLLECTOR_ID=""
+BEFORE_COLLECTOR_RESTARTS=""
+BEFORE_SIGNOZ_ID=""
+BEFORE_SIGNOZ_RESTARTS=""
+BEFORE_ZOOKEEPER_ID=""
+BEFORE_ZOOKEEPER_RESTARTS=""
+
+receipt() {
+ local phase="$1" result="$2" detail="$3"
+ local line
+ line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
+ printf '%s\n' "${line}"
+ printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
+}
+
+compose_base_config() {
+ env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+ docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
+ -f "${REMOTE_BASE}" config -q >/dev/null 2>&1
+}
+
+compose_managed_config() {
+ env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+ docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
+ -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" config -q >/dev/null 2>&1
+}
+
+compose_stage_config() {
+ env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+ docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
+ -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config -q >/dev/null 2>&1
+}
+
+recreate_managed() {
+ timeout --signal=TERM --kill-after=30 300 \
+ env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+ docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
+ -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
+ up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
+ >/dev/null 2>&1
+}
+
+recreate_base() {
+ timeout --signal=TERM --kill-after=30 300 \
+ env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+ docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
+ -f "${REMOTE_BASE}" \
+ up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
+ >/dev/null 2>&1
+}
+
+wait_clickhouse() {
+ local attempt running health
+ for attempt in $(seq 1 36); do
+ running="$(docker inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
+ health="$(docker inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
+ "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
+ if [ "${running}" = true ] && [ "${health}" = healthy ] \
+ && timeout --kill-after=5 10 docker exec "${CLICKHOUSE_CONTAINER}" \
+ clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then
+ return 0
+ fi
+ sleep 5
+ done
+ return 1
+}
+
+listener_up() {
+ local port="$1"
+ ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
+}
+
+wait_api_and_listeners() {
+ local attempt api_code
+ for attempt in $(seq 1 30); do
+ api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}" 2>/dev/null || true)"
+ if [ "${api_code}" = 200 ] && listener_up 4317 && listener_up 4318 && listener_up 8080; then
+ return 0
+ fi
+ sleep 5
+ done
+ return 1
+}
+
+restore_managed_files() {
+ local tmp
+ if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
+ tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}"
+ install -m 0644 "${RECEIPT_DIR}/prior-override.yaml" "${tmp}"
+ mv -f "${tmp}" "${REMOTE_OVERRIDE}"
+ else
+ rm -f "${REMOTE_OVERRIDE}"
+ fi
+
+ if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then
+ tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}"
+ install -m 0644 "${RECEIPT_DIR}/prior-backup-disk.xml" "${tmp}"
+ mv -f "${tmp}" "${REMOTE_CONFIG}"
+ else
+ rm -f "${REMOTE_CONFIG}"
+ fi
+}
+
+rollback_deploy() {
+ local disk_count rollback_dir_terminal=retained
+ restore_managed_files || return 1
+
+ if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
+ compose_managed_config || return 1
+ recreate_managed || return 1
+ else
+ compose_base_config || return 1
+ recreate_base || return 1
+ fi
+ wait_clickhouse || return 1
+ wait_api_and_listeners || return 1
+
+ disk_count="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1
+ [ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1
+ [ "$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
+ [ "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
+ [ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
+ [ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1
+
+ if [ "${HOST_DIR_CREATED}" -eq 1 ]; then
+ if sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
+ rollback_dir_terminal=removed_empty
+ else
+ rollback_dir_terminal=retained_nonempty_no_delete
+ fi
+ fi
+ receipt rollback pass "prior_compose_state_restored_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
+}
+
+on_exit() {
+ local rc=$? rollback_rc=0 terminal=failed
+ trap - EXIT HUP INT TERM
+ set +e
+ if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
+ rollback_deploy
+ rollback_rc=$?
+ if [ "${rollback_rc}" -ne 0 ]; then
+ receipt rollback failed "prior_compose_state_restore_failed"
+ rc=91
+ fi
+ fi
+ if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
+ terminal=pass
+ fi
+ receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}"
+ exit "${rc}"
+}
+
+mkdir -p "${RECEIPT_ROOT}"
+[ ! -e "${RECEIPT_DIR}" ]
+mkdir -m 0700 "${RECEIPT_DIR}"
+: > "${RECEIPT_LOG}"
+trap on_exit EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+exec 9>>"${LOCK_FILE}"
+flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; }
+
+for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss sudo timeout; do
+ command -v "${command_name}" >/dev/null 2>&1 \
+ || { receipt check failed "required_command_missing_${command_name}"; exit 69; }
+done
+
+[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
+[ -f "${STAGE_OVERRIDE}" ] && [ ! -L "${STAGE_OVERRIDE}" ]
+[ -f "${STAGE_CONFIG}" ] && [ ! -L "${STAGE_CONFIG}" ]
+[ "$(sha256sum "${STAGE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
+[ "$(sha256sum "${STAGE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
+[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
+[ -d "${REMOTE_DIR}" ] && [ ! -L "${REMOTE_DIR}" ] && [ -w "${REMOTE_DIR}" ]
+[ -d /backup ] && [ ! -L /backup ]
+
+compose_base_config
+compose_stage_config
+env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
+ docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \
+ -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
+ | grep -qx "${REMOTE_SERVICE}"
+
+if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
+ receipt check failed active_signoz_backup_detected
+ exit 75
+fi
+active_native_backups="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+ "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
+[ "${active_native_backups}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
+
+for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do
+ [ "$(docker inspect --format '{{.State.Running}}' "${container}")" = true ]
+done
+wait_clickhouse
+wait_api_and_listeners
+
+if [ -e "${REMOTE_OVERRIDE}" ]; then
+ [ -f "${REMOTE_OVERRIDE}" ] && [ ! -L "${REMOTE_OVERRIDE}" ]
+ PRIOR_OVERRIDE_PRESENT=1
+fi
+if [ -e "${REMOTE_CONFIG}" ]; then
+ [ -f "${REMOTE_CONFIG}" ] && [ ! -L "${REMOTE_CONFIG}" ]
+ PRIOR_CONFIG_PRESENT=1
+fi
+[ "${PRIOR_OVERRIDE_PRESENT}" -eq "${PRIOR_CONFIG_PRESENT}" ] \
+ || { receipt check failed partial_managed_file_drift; exit 78; }
+if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
+ compose_managed_config
+ cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
+ cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
+fi
+
+PRIOR_DISK_COUNT="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
+BEFORE_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
+BEFORE_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
+[ -n "${BEFORE_DATA_VOLUME}" ]
+BEFORE_COLLECTOR_ID="$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
+BEFORE_COLLECTOR_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
+BEFORE_SIGNOZ_ID="$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
+BEFORE_SIGNOZ_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
+BEFORE_ZOOKEEPER_ID="$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
+BEFORE_ZOOKEEPER_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"
+
+receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid"
+receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}"
+receipt source_of_truth_diff pass "prior_override_${PRIOR_OVERRIDE_PRESENT}_prior_config_${PRIOR_CONFIG_PRESENT}_prior_disk_count_${PRIOR_DISK_COUNT}"
+receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only"
+receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback"
+receipt check pass "compose_config_q_stage_hashes_runtime_preflight_backup_idle"
+
+APPLY_STARTED=1
+if [ -e "${HOST_BACKUP_DIR}" ]; then
+ [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]
+ [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
+else
+ sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
+ HOST_DIR_CREATED=1
+fi
+
+install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
+install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
+mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
+mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
+[ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
+[ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
+compose_managed_config
+
+recreate_managed
+wait_clickhouse
+
+AFTER_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
+AFTER_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
+[ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ]
+[ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ]
+
+backup_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
+config_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
+[ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ]
+[ "${config_mount}" = false ]
+[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
+server_uid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
+ awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)"
+server_gid="$(docker exec "${CLICKHOUSE_CONTAINER}" \
+ awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)"
+[ "${server_uid}" = 101 ]
+[ "${server_gid}" = 101 ]
+docker exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
+ 'test -d /backups && test -w /backups'
+
+disk_state="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+ "SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
+[ "${disk_state}" = "Local|0|0|0|1" ]
+[ "$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
+ "EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ]
+docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
+ | grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)'
+
+[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
+[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
+[ "$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
+[ "$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
+[ "$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
+[ "$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
+wait_api_and_listeners
+
+receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged"
+receipt post_verifier pass "disk_backups_local_rw_healthy_server_uid101_write_access_concurrency_disabled_collector_unchanged_listeners_4317_4318_8080_api_200"
+receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_executed"
+DEPLOY_VERIFIED=1
+REMOTE_APPLY
+
+trap - EXIT
+cleanup_stage
+local_receipt terminal pass "remote_controlled_apply_and_postverify_complete"
diff --git a/scripts/ops/deploy-signoz-native-backup-toolchain.sh b/scripts/ops/deploy-signoz-native-backup-toolchain.sh
new file mode 100755
index 000000000..02ae66a68
--- /dev/null
+++ b/scripts/ops/deploy-signoz-native-backup-toolchain.sh
@@ -0,0 +1,801 @@
+#!/usr/bin/env bash
+# P0-OBS-002 - atomically deploy the host110 SigNoz native backup toolchain.
+#
+# This deployer owns seven repo-sourced files only. It does not run a backup or
+# restore, restart a container, pull an image, or inspect a secret-bearing
+# runtime source.
+
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+TARGET_HOST="wooo@192.168.0.110"
+SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+SAFE_HOME="/home/wooo"
+API_URL="http://127.0.0.1:8080/api/v1/health"
+
+REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"
+REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}"
+RECEIPT_ROOT="${SIGNOZ_TOOLCHAIN_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-native-backup-toolchain}"
+STAGE_ROOT="${SIGNOZ_TOOLCHAIN_STAGE_ROOT:-/tmp}"
+
+LABELS=(
+ backup-signoz.sh
+ clickhouse-native-backup.sh
+ clickhouse-native-restore-drill.sh
+ clickhouse-restore-inventory.py
+ run-signoz-backup-canary.sh
+ backup_disk.xml
+ isolated-cluster.xml
+)
+LOCAL_SOURCES=(
+ "${ROOT_DIR}/scripts/backup/backup-signoz.sh"
+ "${ROOT_DIR}/scripts/backup/clickhouse-native-backup.sh"
+ "${ROOT_DIR}/scripts/backup/clickhouse-native-restore-drill.sh"
+ "${ROOT_DIR}/scripts/backup/clickhouse-restore-inventory.py"
+ "${ROOT_DIR}/scripts/backup/run-signoz-backup-canary.sh"
+ "${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
+ "${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
+)
+MODES=(0755 0755 0755 0755 0755 0644 0644)
+
+SSH_OPTS=(
+ -o BatchMode=yes
+ -o ConnectTimeout=8
+ -o ConnectionAttempts=1
+ -o ServerAliveInterval=5
+ -o ServerAliveCountMax=2
+)
+
+usage() {
+ cat <<'USAGE'
+Usage: deploy-signoz-native-backup-toolchain.sh [--check|--apply]
+
+Required environment:
+ TRACE_ID Path-safe controlled-apply trace identifier
+ RUN_ID Unique path-safe execution identifier
+ WORK_ITEM_ID Path-safe work-item identifier
+
+Optional safe absolute path environment:
+ SIGNOZ_TOOLCHAIN_SCRIPT_DIR
+ SIGNOZ_TOOLCHAIN_CONFIG_ROOT
+ SIGNOZ_TOOLCHAIN_RECEIPT_ROOT
+ SIGNOZ_TOOLCHAIN_STAGE_ROOT
+
+--check validates local sources and performs a remote read-only target diff and
+runtime readiness check. --apply stages all seven sources, validates hashes and
+syntax, atomically replaces exact targets, and rolls every target back to its
+prior hash/mode/owner if any bounded verifier fails.
+
+Flat restore-driver invocation must set:
+ CLICKHOUSE_RESTORE_INVENTORY_HELPER=/clickhouse-restore-inventory.py
+ CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG=/config.d/backup_disk.xml
+ CLICKHOUSE_RESTORE_CLUSTER_CONFIG=/restore-drill/config.d/isolated-cluster.xml
+USAGE
+}
+
+MODE="${1:---check}"
+case "${MODE}" in
+ --check|--apply) ;;
+ -h|--help) usage; exit 0 ;;
+ *) usage >&2; exit 64 ;;
+esac
+[ "$#" -le 1 ] || { usage >&2; exit 64; }
+
+TRACE_ID="${TRACE_ID:-}"
+RUN_ID="${RUN_ID:-}"
+WORK_ITEM_ID="${WORK_ITEM_ID:-}"
+
+valid_identity() {
+ [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
+}
+
+safe_absolute_path() {
+ local path="$1"
+ [[ "${path}" == /* ]] || return 1
+ [[ "${path}" != / && "${path}" != */ ]] || return 1
+ [[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1
+ case "/${path#/}/" in
+ *'//'*|*'/../'*|*'/./'*) return 1 ;;
+ esac
+}
+
+paths_overlap() {
+ local left="$1" right="$2"
+ [[ "${left}" = "${right}" || "${left}" == "${right}"/* || "${right}" == "${left}"/* ]]
+}
+
+for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
+ valid_identity "${value}" || {
+ printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
+ exit 64
+ }
+done
+
+for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
+ safe_absolute_path "${path}" || {
+ printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2
+ exit 64
+ }
+done
+case "${REMOTE_SCRIPT_DIR}" in /backup/*) ;; *) printf 'script dir must stay under /backup\n' >&2; exit 64 ;; esac
+case "${REMOTE_CONFIG_ROOT}" in /backup/*) ;; *) printf 'config root must stay under /backup\n' >&2; exit 64 ;; esac
+case "${RECEIPT_ROOT}" in /backup/*) ;; *) printf 'receipt root must stay under /backup\n' >&2; exit 64 ;; esac
+case "${STAGE_ROOT}" in /tmp|/tmp/*|/backup/deploy-staging|/backup/deploy-staging/*) ;;
+ *) printf 'stage root must stay under /tmp or /backup/deploy-staging\n' >&2; exit 64 ;;
+esac
+if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \
+ || paths_overlap "${REMOTE_SCRIPT_DIR}" "${RECEIPT_ROOT}" \
+ || paths_overlap "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" \
+ || paths_overlap "${STAGE_ROOT}" "${REMOTE_SCRIPT_DIR}" \
+ || paths_overlap "${STAGE_ROOT}" "${REMOTE_CONFIG_ROOT}" \
+ || paths_overlap "${STAGE_ROOT}" "${RECEIPT_ROOT}"; then
+ printf 'toolchain target, receipt, and stage paths must not overlap\n' >&2
+ exit 64
+fi
+
+for command_name in bash python3 scp ssh; do
+ command -v "${command_name}" >/dev/null 2>&1 || {
+ printf 'required local command missing: %s\n' "${command_name}" >&2
+ exit 69
+ }
+done
+
+hash_file() {
+ if command -v sha256sum >/dev/null 2>&1; then
+ sha256sum "$1" | awk '{print $1}'
+ else
+ shasum -a 256 "$1" | awk '{print $1}'
+ fi
+}
+
+validate_xml_pair() {
+ python3 - "$1" "$2" <<'PY'
+from pathlib import Path
+import sys
+import xml.etree.ElementTree as ET
+
+backup_path = Path(sys.argv[1])
+cluster_path = Path(sys.argv[2])
+backup = ET.parse(backup_path).getroot()
+cluster = ET.parse(cluster_path).getroot()
+assert backup.tag == "clickhouse"
+assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
+assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
+assert backup.findtext("./backups/allowed_disk") == "backups"
+assert backup.findtext("./backups/allowed_path") == "/backups/"
+assert backup.findtext("./backups/allow_concurrent_backups") == "false"
+assert backup.findtext("./backups/allow_concurrent_restores") == "false"
+assert cluster.tag == "clickhouse"
+assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
+assert cluster.findtext("./zookeeper/node/port") == "2181"
+assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
+assert cluster.findtext("./remote_servers/cluster/shard/replica/port") == "9000"
+assert cluster.findtext("./macros/shard") == "01"
+replica = cluster.find("./macros/replica")
+assert replica is not None
+assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
+PY
+}
+
+validate_local_sources() {
+ local index source
+ for index in "${!LOCAL_SOURCES[@]}"; do
+ source="${LOCAL_SOURCES[index]}"
+ [ -f "${source}" ] && [ ! -L "${source}" ]
+ [ -s "${source}" ]
+ done
+
+ for index in 0 1 2 4; do
+ bash -n "${LOCAL_SOURCES[index]}"
+ done
+ python3 - "${LOCAL_SOURCES[3]}" <<'PY'
+from pathlib import Path
+import sys
+
+path = Path(sys.argv[1])
+compile(path.read_text(encoding="utf-8"), str(path), "exec")
+PY
+ validate_xml_pair "${LOCAL_SOURCES[5]}" "${LOCAL_SOURCES[6]}"
+
+ python3 - "${LOCAL_SOURCES[2]}" <<'PY'
+from pathlib import Path
+import sys
+
+text = Path(sys.argv[1]).read_text(encoding="utf-8")
+for token in (
+ "CLICKHOUSE_RESTORE_INVENTORY_HELPER",
+ "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG",
+ "CLICKHOUSE_RESTORE_CLUSTER_CONFIG",
+):
+ assert token in text
+PY
+}
+
+validate_local_sources
+SOURCE_HASHES=()
+SOURCE_DETAIL=""
+for index in "${!LOCAL_SOURCES[@]}"; do
+ source_hash="$(hash_file "${LOCAL_SOURCES[index]}")"
+ SOURCE_HASHES+=("${source_hash}")
+ SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_"
+done
+SOURCE_DETAIL="${SOURCE_DETAIL%_}"
+
+CONFIG_DIR="${REMOTE_CONFIG_ROOT}/config.d"
+RESTORE_CONFIG_DIR="${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
+TARGET_PATHS=(
+ "${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
+ "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
+ "${CONFIG_DIR}/backup_disk.xml"
+ "${RESTORE_CONFIG_DIR}/isolated-cluster.xml"
+)
+STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-native-backup-toolchain.${RUN_ID}"
+
+local_receipt() {
+ local phase="$1" result="$2" detail="$3"
+ printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
+}
+
+remote_read_only_check() {
+ ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
+ "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \
+ "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
+set -euo pipefail
+
+TRACE_ID="$1"
+RUN_ID="$2"
+WORK_ITEM_ID="$3"
+REMOTE_SCRIPT_DIR="$4"
+REMOTE_CONFIG_ROOT="$5"
+RECEIPT_ROOT="$6"
+STAGE_ROOT="$7"
+shift 7
+EXPECTED_HASHES=("$@")
+LABELS=(
+ backup-signoz.sh
+ clickhouse-native-backup.sh
+ clickhouse-native-restore-drill.sh
+ clickhouse-restore-inventory.py
+ run-signoz-backup-canary.sh
+ backup_disk.xml
+ isolated-cluster.xml
+)
+MODES=(0755 0755 0755 0755 0755 0644 0644)
+TARGETS=(
+ "${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
+ "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
+ "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
+ "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
+)
+API_URL="http://127.0.0.1:8080/api/v1/health"
+
+[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
+[ -d /backup ] && [ ! -L /backup ]
+for command_name in awk curl docker pgrep sha256sum ss stat; do
+ command -v "${command_name}" >/dev/null 2>&1
+done
+
+listener_up() {
+ local port="$1"
+ ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
+}
+
+validate_existing_ancestors() {
+ local requested="$1" current="" component
+ local -a components
+ IFS='/' read -r -a components <<<"${requested#/}"
+ for component in "${components[@]}"; do
+ [ -n "${component}" ] || continue
+ current="${current}/${component}"
+ if [ -e "${current}" ] || [ -L "${current}" ]; then
+ [ -d "${current}" ] && [ ! -L "${current}" ]
+ else
+ break
+ fi
+ done
+}
+
+runtime_line() {
+ local container="$1" value running health
+ value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
+ IFS='|' read -r _ _ _ running health <<<"${value}"
+ [ "${running}" = true ]
+ case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
+ printf '%s' "${value}"
+}
+
+if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
+ printf 'active SigNoz backup toolchain process detected\n' >&2
+ exit 75
+fi
+validate_existing_ancestors "${REMOTE_SCRIPT_DIR}"
+validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/config.d"
+validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
+validate_existing_ancestors "${RECEIPT_ROOT}"
+validate_existing_ancestors "${STAGE_ROOT}"
+[ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ]
+active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
+ "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
+[ "${active_operations}" = 0 ]
+
+runtime_summary=""
+for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
+ runtime_summary+="${container}_$(runtime_line "${container}")_"
+done
+for port in 4317 4318 8080; do listener_up "${port}"; done
+api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
+[ "${api_code}" = 200 ]
+
+matching=0
+drift=0
+absent=0
+for index in "${!TARGETS[@]}"; do
+ target="${TARGETS[index]}"
+ if [ -e "${target}" ] || [ -L "${target}" ]; then
+ [ -f "${target}" ] && [ ! -L "${target}" ]
+ current_hash="$(sha256sum "${target}" | awk '{print $1}')"
+ current_mode="$(stat -c '%a' "${target}")"
+ if [ "${current_hash}" = "${EXPECTED_HASHES[index]}" ] && [ "0${current_mode}" = "${MODES[index]}" ]; then
+ matching=$((matching + 1))
+ else
+ drift=$((drift + 1))
+ fi
+ else
+ absent=$((absent + 1))
+ fi
+done
+
+printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"source_of_truth_diff","result":"pass","detail":"matching_%s_drift_%s_absent_%s_active_operations_0_api_200_ports_4317_4318_8080"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${matching}" "${drift}" "${absent}"
+printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"runtime_readiness","result":"pass","detail":"container_identity_started_at_restart_count_health_captured_%s"}\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${runtime_summary%_}"
+REMOTE_CHECK
+}
+
+local_receipt sensor_source pass "${SOURCE_DETAIL}"
+local_receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
+remote_read_only_check
+local_receipt check pass "local_syntax_xml_python_compile_and_remote_read_only_preflight"
+
+if [ "${MODE}" = --check ]; then
+ local_receipt terminal check_pass_no_write "remote_target_unchanged_no_stage_no_candidate"
+ exit 0
+fi
+
+STAGE_CREATED=0
+cleanup_stage() {
+ if [ "${STAGE_CREATED}" -eq 1 ]; then
+ ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
+set -u
+stage_dir="$1"
+if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
+ rm -rf "${stage_dir}/pycache" "${stage_dir}/candidate-pycache"
+ rm -f \
+ "${stage_dir}/backup-signoz.sh" \
+ "${stage_dir}/clickhouse-native-backup.sh" \
+ "${stage_dir}/clickhouse-native-restore-drill.sh" \
+ "${stage_dir}/clickhouse-restore-inventory.py" \
+ "${stage_dir}/run-signoz-backup-canary.sh" \
+ "${stage_dir}/backup_disk.xml" \
+ "${stage_dir}/isolated-cluster.xml"
+ rmdir "${stage_dir}" 2>/dev/null || true
+fi
+REMOTE_CLEANUP
+ fi
+}
+trap cleanup_stage EXIT
+
+ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
+set -euo pipefail
+stage_dir="$1"
+[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
+stage_root="${stage_dir%/*}"
+[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ]
+umask 077
+mkdir -m 0700 "${stage_dir}"
+REMOTE_STAGE
+STAGE_CREATED=1
+
+for index in "${!LOCAL_SOURCES[@]}"; do
+ scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
+ "${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"
+done
+
+ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
+ "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \
+ "${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
+set -euo pipefail
+
+TRACE_ID="$1"
+RUN_ID="$2"
+WORK_ITEM_ID="$3"
+REMOTE_SCRIPT_DIR="$4"
+REMOTE_CONFIG_ROOT="$5"
+RECEIPT_ROOT="$6"
+STAGE_DIR="$7"
+shift 7
+EXPECTED_HASHES=("$@")
+LABELS=(
+ backup-signoz.sh
+ clickhouse-native-backup.sh
+ clickhouse-native-restore-drill.sh
+ clickhouse-restore-inventory.py
+ run-signoz-backup-canary.sh
+ backup_disk.xml
+ isolated-cluster.xml
+)
+MODES=(0755 0755 0755 0755 0755 0644 0644)
+STAGED=(
+ "${STAGE_DIR}/backup-signoz.sh"
+ "${STAGE_DIR}/clickhouse-native-backup.sh"
+ "${STAGE_DIR}/clickhouse-native-restore-drill.sh"
+ "${STAGE_DIR}/clickhouse-restore-inventory.py"
+ "${STAGE_DIR}/run-signoz-backup-canary.sh"
+ "${STAGE_DIR}/backup_disk.xml"
+ "${STAGE_DIR}/isolated-cluster.xml"
+)
+TARGETS=(
+ "${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
+ "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
+ "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
+ "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
+ "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
+)
+TARGET_DIRS=(
+ "${REMOTE_SCRIPT_DIR}"
+ "${REMOTE_CONFIG_ROOT}/config.d"
+ "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
+)
+CONTAINERS=(signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1)
+API_URL="http://127.0.0.1:8080/api/v1/health"
+RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
+RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
+PRIOR_DIR="${RECEIPT_DIR}/prior"
+PRIOR_MANIFEST="${RECEIPT_DIR}/prior-manifest.tsv"
+DEPLOYED_MANIFEST="${RECEIPT_DIR}/deployed-manifest.tsv"
+RUNTIME_BEFORE="${RECEIPT_DIR}/runtime-before.tsv"
+RUNTIME_AFTER="${RECEIPT_DIR}/runtime-after.tsv"
+LOCK_FILE="/tmp/awoooi-signoz-native-backup-toolchain.lock"
+
+APPLY_STARTED=0
+DEPLOY_VERIFIED=0
+RUNTIME_BASELINE_CAPTURED=0
+LOCK_HELD=0
+REMOTE_UID="$(id -u)"
+REMOTE_GID="$(id -g)"
+CREATED_DIRS=()
+PRIOR_PRESENT=()
+PRIOR_MODES=()
+PRIOR_UIDS=()
+PRIOR_GIDS=()
+PRIOR_HASHES=()
+
+for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat; do
+ command -v "${command_name}" >/dev/null 2>&1 || {
+ printf 'required remote command missing: %s\n' "${command_name}" >&2
+ exit 69
+ }
+done
+if [ "${REMOTE_UID}" -ne 0 ]; then
+ command -v sudo >/dev/null 2>&1 || { printf 'required remote command missing: sudo\n' >&2; exit 69; }
+ sudo -n true
+fi
+[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
+[ -d /backup ] && [ ! -L /backup ]
+
+run_root() {
+ if [ "${REMOTE_UID}" -eq 0 ]; then
+ "$@"
+ else
+ sudo -n "$@"
+ fi
+}
+
+root_hash() {
+ if [ "${REMOTE_UID}" -eq 0 ]; then
+ sha256sum "$1" | awk '{print $1}'
+ else
+ sudo -n sha256sum "$1" | awk '{print $1}'
+ fi
+}
+
+ensure_dir_tree() {
+ local requested="$1" record_created="$2" current="" component
+ local -a components
+ IFS='/' read -r -a components <<<"${requested#/}"
+ for component in "${components[@]}"; do
+ [ -n "${component}" ] || continue
+ current="${current}/${component}"
+ if [ -e "${current}" ] || [ -L "${current}" ]; then
+ [ -d "${current}" ] && [ ! -L "${current}" ]
+ else
+ run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0755 "${current}"
+ if [ "${record_created}" = true ]; then
+ CREATED_DIRS+=("${current}")
+ fi
+ fi
+ done
+}
+
+ensure_dir_tree "${RECEIPT_ROOT}" false
+[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]
+run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0700 "${RECEIPT_DIR}"
+install -d -m 0700 "${PRIOR_DIR}"
+: > "${RECEIPT_LOG}"
+
+receipt() {
+ local phase="$1" result="$2" detail="$3" line
+ line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
+ printf '%s\n' "${line}"
+ printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
+}
+
+listener_up() {
+ local port="$1"
+ ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
+}
+
+runtime_snapshot() {
+ local container value running health port api_code
+ for container in "${CONTAINERS[@]}"; do
+ value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
+ IFS='|' read -r _ _ _ running health <<<"${value}"
+ [ "${running}" = true ]
+ case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
+ printf 'container\t%s\t%s\n' "${container}" "${value}"
+ done
+ for port in 4317 4318 8080; do
+ listener_up "${port}"
+ printf 'listener\t%s\tup\n' "${port}"
+ done
+ api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
+ [ "${api_code}" = 200 ]
+ printf 'api\thealth\t%s\n' "${api_code}"
+}
+
+validate_xml_pair() {
+ python3 - "$1" "$2" <<'PY'
+from pathlib import Path
+import sys
+import xml.etree.ElementTree as ET
+
+backup_path = Path(sys.argv[1])
+cluster_path = Path(sys.argv[2])
+backup = ET.parse(backup_path).getroot()
+cluster = ET.parse(cluster_path).getroot()
+assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
+assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
+assert backup.findtext("./backups/allowed_disk") == "backups"
+assert backup.findtext("./backups/allowed_path") == "/backups/"
+assert backup.findtext("./backups/allow_concurrent_backups") == "false"
+assert backup.findtext("./backups/allow_concurrent_restores") == "false"
+assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
+assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
+assert cluster.findtext("./macros/shard") == "01"
+replica = cluster.find("./macros/replica")
+assert replica is not None
+assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
+PY
+}
+
+cleanup_transients() {
+ local index rc=0
+ for index in "${!TARGETS[@]}"; do
+ run_root rm -f \
+ "${TARGETS[index]}.candidate.${RUN_ID}" \
+ "${TARGETS[index]}.rollback.${RUN_ID}" || rc=1
+ done
+ if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then
+ rm -rf "${STAGE_DIR}/pycache" "${STAGE_DIR}/candidate-pycache" || rc=1
+ rm -f "${STAGED[@]}" || rc=1
+ rmdir "${STAGE_DIR}" 2>/dev/null || rc=1
+ else
+ rc=1
+ fi
+ return "${rc}"
+}
+
+verify_residue_zero() {
+ local index
+ [ ! -e "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
+ for index in "${!TARGETS[@]}"; do
+ [ ! -e "${TARGETS[index]}.candidate.${RUN_ID}" ] \
+ && [ ! -L "${TARGETS[index]}.candidate.${RUN_ID}" ] || return 1
+ [ ! -e "${TARGETS[index]}.rollback.${RUN_ID}" ] \
+ && [ ! -L "${TARGETS[index]}.rollback.${RUN_ID}" ] || return 1
+ done
+}
+
+rollback_targets() (
+ set -euo pipefail
+ local index target rollback_candidate actual_mode actual_hash
+ for index in "${!TARGETS[@]}"; do
+ target="${TARGETS[index]}"
+ if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
+ rollback_candidate="${target}.rollback.${RUN_ID}"
+ run_root install \
+ -o "${PRIOR_UIDS[index]}" -g "${PRIOR_GIDS[index]}" \
+ -m "${PRIOR_MODES[index]}" \
+ "${PRIOR_DIR}/${LABELS[index]}" "${rollback_candidate}"
+ [ "$(root_hash "${rollback_candidate}")" = "${PRIOR_HASHES[index]}" ]
+ run_root mv -f "${rollback_candidate}" "${target}"
+ actual_mode="$(stat -c '%a' "${target}")"
+ actual_hash="$(root_hash "${target}")"
+ [ "${actual_mode}" = "${PRIOR_MODES[index]}" ]
+ [ "${actual_hash}" = "${PRIOR_HASHES[index]}" ]
+ else
+ run_root rm -f "${target}"
+ [ ! -e "${target}" ] && [ ! -L "${target}" ]
+ fi
+ done
+
+ for ((index=${#CREATED_DIRS[@]} - 1; index >= 0; index--)); do
+ run_root rmdir "${CREATED_DIRS[index]}"
+ done
+
+ if [ "${RUNTIME_BASELINE_CAPTURED}" -eq 1 ]; then
+ runtime_snapshot > "${RECEIPT_DIR}/runtime-rollback.tsv"
+ cmp -s "${RUNTIME_BEFORE}" "${RECEIPT_DIR}/runtime-rollback.tsv"
+ fi
+ receipt rollback pass "all_prior_files_modes_hashes_owners_and_runtime_restored"
+)
+
+on_exit() {
+ local rc=$? rollback_rc=0 cleanup_rc=0 residue_rc=0 terminal=failed
+ trap - EXIT HUP INT TERM
+ set +e
+ if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
+ rollback_targets
+ rollback_rc=$?
+ if [ "${rollback_rc}" -ne 0 ]; then
+ receipt rollback failed "prior_state_restore_or_runtime_verifier_failed"
+ rc=91
+ fi
+ elif [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 0 ]; then
+ receipt rollback no_write "bounded_execution_not_started_targets_unchanged"
+ fi
+ cleanup_transients
+ cleanup_rc=$?
+ verify_residue_zero
+ residue_rc=$?
+ if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then
+ rc=92
+ fi
+ if [ "${LOCK_HELD}" -eq 1 ]; then
+ flock -u 9 >/dev/null 2>&1
+ exec 9>&-
+ rm -f "${LOCK_FILE}"
+ fi
+ if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
+ terminal=pass
+ fi
+ receipt terminal "${terminal}" "exit_${rc}_stage_candidate_residue_0_receipt_ref_${RECEIPT_LOG}"
+ exit "${rc}"
+}
+
+trap on_exit EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+exec 9>>"${LOCK_FILE}"
+flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; }
+LOCK_HELD=1
+
+if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
+ receipt check failed active_signoz_backup_toolchain_process
+ exit 75
+fi
+active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \
+ "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
+[ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
+
+runtime_snapshot > "${RUNTIME_BEFORE}"
+RUNTIME_BASELINE_CAPTURED=1
+
+[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
+for index in "${!STAGED[@]}"; do
+ [ -f "${STAGED[index]}" ] && [ ! -L "${STAGED[index]}" ]
+ [ "$(sha256sum "${STAGED[index]}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
+done
+for index in 0 1 2 4; do bash -n "${STAGED[index]}"; done
+PYTHONPYCACHEPREFIX="${STAGE_DIR}/pycache" python3 -m py_compile "${STAGED[3]}"
+validate_xml_pair "${STAGED[5]}" "${STAGED[6]}"
+
+: > "${PRIOR_MANIFEST}"
+for index in "${!TARGETS[@]}"; do
+ target="${TARGETS[index]}"
+ [ ! -e "${target}.candidate.${RUN_ID}" ] && [ ! -L "${target}.candidate.${RUN_ID}" ]
+ [ ! -e "${target}.rollback.${RUN_ID}" ] && [ ! -L "${target}.rollback.${RUN_ID}" ]
+ if [ -e "${target}" ] || [ -L "${target}" ]; then
+ [ -f "${target}" ] && [ ! -L "${target}" ]
+ PRIOR_PRESENT[index]=1
+ PRIOR_MODES[index]="$(stat -c '%a' "${target}")"
+ PRIOR_UIDS[index]="$(stat -c '%u' "${target}")"
+ PRIOR_GIDS[index]="$(stat -c '%g' "${target}")"
+ PRIOR_HASHES[index]="$(root_hash "${target}")"
+ run_root cp -p "${target}" "${PRIOR_DIR}/${LABELS[index]}"
+ [ "$(root_hash "${PRIOR_DIR}/${LABELS[index]}")" = "${PRIOR_HASHES[index]}" ]
+ else
+ PRIOR_PRESENT[index]=0
+ PRIOR_MODES[index]=absent
+ PRIOR_UIDS[index]=absent
+ PRIOR_GIDS[index]=absent
+ PRIOR_HASHES[index]=absent
+ fi
+ printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+ "${LABELS[index]}" "${target}" "${PRIOR_PRESENT[index]}" \
+ "${PRIOR_MODES[index]}" "${PRIOR_UIDS[index]}" "${PRIOR_GIDS[index]}" \
+ "${PRIOR_HASHES[index]}" >> "${PRIOR_MANIFEST}"
+done
+
+receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources"
+receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
+receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}"
+receipt ai_decision candidate "atomic_install_native_backup_toolchain_only"
+receipt risk_policy pass "risk_medium_bounded_files_only_no_runtime_execution_no_container_change_full_rollback"
+receipt check pass "runtime_baseline_active_operations_0_hash_bash_xml_py_compile"
+
+APPLY_STARTED=1
+for directory in "${TARGET_DIRS[@]}"; do
+ ensure_dir_tree "${directory}" true
+done
+
+for index in "${!TARGETS[@]}"; do
+ target="${TARGETS[index]}"
+ if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
+ target_uid="${PRIOR_UIDS[index]}"
+ target_gid="${PRIOR_GIDS[index]}"
+ else
+ target_uid="${REMOTE_UID}"
+ target_gid="${REMOTE_GID}"
+ fi
+ run_root install -o "${target_uid}" -g "${target_gid}" -m "${MODES[index]}" \
+ "${STAGED[index]}" "${target}.candidate.${RUN_ID}"
+ [ "$(root_hash "${target}.candidate.${RUN_ID}")" = "${EXPECTED_HASHES[index]}" ]
+ [ "0$(stat -c '%a' "${target}.candidate.${RUN_ID}")" = "${MODES[index]}" ]
+done
+
+for index in "${!TARGETS[@]}"; do
+ run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"
+done
+
+for index in "${!TARGETS[@]}"; do
+ [ -f "${TARGETS[index]}" ] && [ ! -L "${TARGETS[index]}" ]
+ [ "$(root_hash "${TARGETS[index]}")" = "${EXPECTED_HASHES[index]}" ]
+ [ "0$(stat -c '%a' "${TARGETS[index]}")" = "${MODES[index]}" ]
+done
+for index in 0 1 2 4; do bash -n "${TARGETS[index]}"; done
+PYTHONPYCACHEPREFIX="${STAGE_DIR}/candidate-pycache" python3 -m py_compile "${TARGETS[3]}"
+validate_xml_pair "${TARGETS[5]}" "${TARGETS[6]}"
+
+: > "${DEPLOYED_MANIFEST}"
+for index in "${!TARGETS[@]}"; do
+ printf '%s\t%s\t%s\t%s\n' \
+ "${LABELS[index]}" "${TARGETS[index]}" "${MODES[index]}" \
+ "${EXPECTED_HASHES[index]}" >> "${DEPLOYED_MANIFEST}"
+done
+
+runtime_snapshot > "${RUNTIME_AFTER}"
+cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"
+receipt execution pass "seven_candidates_atomically_replaced_expected_hashes_and_modes"
+receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged"
+receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull"
+DEPLOY_VERIFIED=1
+REMOTE_APPLY
+
+trap - EXIT
+cleanup_stage
+local_receipt terminal pass "remote_controlled_apply_postverify_and_stage_candidate_residue_0"
diff --git a/scripts/ops/deploy-signoz-prometheus-route.sh b/scripts/ops/deploy-signoz-prometheus-route.sh
new file mode 100755
index 000000000..570b79190
--- /dev/null
+++ b/scripts/ops/deploy-signoz-prometheus-route.sh
@@ -0,0 +1,218 @@
+#!/usr/bin/env bash
+# Deploy only the canonical SigNoz blackbox route to the host110 Prometheus.
+# The full Prometheus file intentionally remains untouched because production
+# contains unrelated, independently governed scrape targets.
+
+set -euo pipefail
+
+TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
+RUN_ID="${RUN_ID:?RUN_ID is required}"
+WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
+TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
+PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
+PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
+REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
+MODE="${1:-apply}"
+
+case "${MODE}" in
+ apply|--dry-run) ;;
+ *) printf 'usage: %s [apply|--dry-run]\n' "$0" >&2; exit 64 ;;
+esac
+
+for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
+ case "${value}" in
+ *[!A-Za-z0-9._-]*|'')
+ printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
+ exit 64
+ ;;
+ esac
+done
+
+ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
+ bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
+ "${PROMETHEUS_URL}" "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
+set -euo pipefail
+
+TRACE_ID="$1"
+RUN_ID="$2"
+WORK_ITEM_ID="$3"
+MODE="$4"
+PROMETHEUS_URL="$5"
+PROMETHEUS_CONTAINER="$6"
+REMOTE_CONFIG="$7"
+REMOTE_CANDIDATE="/tmp/awoooi-prometheus-signoz.${RUN_ID}.yml"
+RUNTIME_CANDIDATE="/etc/prometheus/.awoooi-prometheus-signoz.${RUN_ID}.yml"
+BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
+APPLY_STARTED=0
+DEPLOY_VERIFIED=0
+
+cleanup_candidate() {
+ rm -f "${REMOTE_CANDIDATE}"
+ docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
+ rm -f "${RUNTIME_CANDIDATE}" >/dev/null 2>&1 || true
+}
+
+rollback_deploy() {
+ cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}"
+ curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
+ curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
+ printf 'rollback_verifier trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
+}
+
+on_exit() {
+ local rc=$?
+ trap - EXIT
+ set +e
+ cleanup_candidate
+ if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
+ && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
+ rollback_deploy
+ fi
+ exit "${rc}"
+}
+trap on_exit EXIT
+
+test -f "${REMOTE_CONFIG}"
+test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
+curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
+
+python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
+from pathlib import Path
+import sys
+
+source = Path(sys.argv[1])
+candidate = Path(sys.argv[2])
+text = source.read_text(encoding="utf-8")
+
+canonical = " - http://192.168.0.110:8080/api/v1/health\n"
+legacy = " - 192.168.0.188:3301\n"
+anchor = " - http://192.168.0.110:3001\n"
+
+
+def block_bounds(payload: str, job: str) -> tuple[int, int]:
+ marker = f' - job_name: "{job}"\n'
+ start = payload.find(marker)
+ if start < 0 or payload.find(marker, start + 1) >= 0:
+ raise SystemExit(f"expected exactly one {job} job")
+ end = payload.find("\n - job_name:", start + len(marker))
+ return start, len(payload) if end < 0 else end + 1
+
+
+http_start, http_end = block_bounds(text, "blackbox-http")
+http_block = text[http_start:http_end]
+if http_block.count(canonical) > 1 or http_block.count(anchor) != 1:
+ raise SystemExit("canonical HTTP target or insertion anchor is ambiguous")
+if canonical not in http_block:
+ http_block = http_block.replace(anchor, anchor + canonical, 1)
+text = text[:http_start] + http_block + text[http_end:]
+
+tcp_start, tcp_end = block_bounds(text, "blackbox-tcp")
+tcp_block = text[tcp_start:tcp_end]
+if tcp_block.count(legacy) > 1:
+ raise SystemExit("legacy TCP target is ambiguous")
+tcp_block = tcp_block.replace(legacy, "", 1)
+text = text[:tcp_start] + tcp_block + text[tcp_end:]
+
+http_start, http_end = block_bounds(text, "blackbox-http")
+tcp_start, tcp_end = block_bounds(text, "blackbox-tcp")
+if text[http_start:http_end].count(canonical) != 1:
+ raise SystemExit("canonical HTTP target was not normalized exactly once")
+if legacy in text[tcp_start:tcp_end]:
+ raise SystemExit("legacy TCP target remains after normalization")
+
+candidate.write_text(text, encoding="utf-8")
+PY
+
+docker cp "${REMOTE_CANDIDATE}" \
+ "${PROMETHEUS_CONTAINER}:${RUNTIME_CANDIDATE}" >/dev/null
+docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${RUNTIME_CANDIDATE}"
+
+SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
+CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
+printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
+ "${SOURCE_SHA}" "${CANDIDATE_SHA}"
+
+if [ "${MODE}" = "--dry-run" ]; then
+ printf 'terminal trace_id=%s run_id=%s status=check_pass_no_write\n' \
+ "${TRACE_ID}" "${RUN_ID}"
+ exit 0
+fi
+
+cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
+APPLY_STARTED=1
+cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
+
+ACTIVE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
+RUNTIME_SHA="$(docker exec "${PROMETHEUS_CONTAINER}" \
+ sha256sum /etc/prometheus/prometheus.yml | awk '{print $1}')"
+test "${ACTIVE_SHA}" = "${CANDIDATE_SHA}"
+test "${RUNTIME_SHA}" = "${CANDIDATE_SHA}"
+curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
+curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
+printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=target_first_config_apply active_sha=%s runtime_sha=%s ready=true backup=%s\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${ACTIVE_SHA}" \
+ "${RUNTIME_SHA}" "${BACKUP_CONFIG}"
+
+target_state() {
+ curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
+import json, sys
+payload = json.load(sys.stdin)
+targets = payload["data"]["activeTargets"]
+canonical = [t for t in targets if t.get("labels", {}).get("job") == "blackbox-http" and t.get("labels", {}).get("instance") == "http://192.168.0.110:8080/api/v1/health"]
+legacy = [t for t in targets if t.get("labels", {}).get("instance") == "192.168.0.188:3301"]
+if len(canonical) == 1:
+ target = canonical[0]
+ print("{}|{}|{}|{}".format(
+ 1,
+ target.get("health", "unknown"),
+ target.get("lastScrape", "never"),
+ len(legacy),
+ ))
+else:
+ print(f"{len(canonical)}|missing|never|{len(legacy)}")
+'
+}
+
+FIRST_SCRAPE=""
+for attempt in $(seq 1 9); do
+ IFS='|' read -r count health last_scrape legacy_count <<<"$(target_state)"
+ printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s legacy_count=%s\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
+ "${last_scrape}" "${legacy_count}"
+ if [ "${count}" = 1 ] && [ "${health}" = up ] \
+ && [ "${legacy_count}" = 0 ] && [ "${last_scrape}" != never ]; then
+ FIRST_SCRAPE="${last_scrape}"
+ break
+ fi
+ sleep 10
+done
+test -n "${FIRST_SCRAPE}"
+
+sleep 16
+IFS='|' read -r count health SECOND_SCRAPE legacy_count <<<"$(target_state)"
+printf 'target_probe trace_id=%s run_id=%s sample=post_cooldown count=%s health=%s last_scrape=%s legacy_count=%s\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${count}" "${health}" \
+ "${SECOND_SCRAPE}" "${legacy_count}"
+test "${count}" = 1
+test "${health}" = up
+test "${legacy_count}" = 0
+test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
+
+PROBE_VALUE="$(curl -fsS --get \
+ --data-urlencode 'query=probe_success{job="blackbox-http",instance="http://192.168.0.110:8080/api/v1/health"}' \
+ "${PROMETHEUS_URL}/api/v1/query" | python3 -c '
+import json, sys
+result = json.load(sys.stdin)["data"]["result"]
+print(result[0]["value"][1] if len(result) == 1 else "invalid")
+')"
+test "${PROBE_VALUE}" = 1
+
+DEPLOY_VERIFIED=1
+printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=target_up_two_scrapes first=%s second=%s probe_success=%s legacy_target_count=0\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
+ "${SECOND_SCRAPE}" "${PROBE_VALUE}"
+printf 'terminal trace_id=%s run_id=%s status=completed_target_first backup=%s\n' \
+ "${TRACE_ID}" "${RUN_ID}" "${BACKUP_CONFIG}"
+REMOTE
diff --git a/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
new file mode 100644
index 000000000..504e2ca11
--- /dev/null
+++ b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import subprocess
+import xml.etree.ElementTree as ET
+
+
+ROOT = Path(__file__).resolve().parents[3]
+CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
+OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
+DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
+
+
+def executable_text() -> str:
+ return "\n".join(
+ line
+ for line in DEPLOYER.read_text(encoding="utf-8").splitlines()
+ if not line.lstrip().startswith("#")
+ )
+
+
+def test_clickhouse_config_declares_one_allowed_local_backup_disk() -> None:
+ root = ET.parse(CONFIG).getroot()
+ assert root.tag == "clickhouse"
+
+ disks = root.findall("./storage_configuration/disks/*")
+ assert [disk.tag for disk in disks] == ["backups"]
+ assert disks[0].findtext("type") == "local"
+ assert disks[0].findtext("path") == "/backups/"
+
+ backups = root.find("./backups")
+ assert backups is not None
+ assert backups.findtext("allowed_disk") == "backups"
+ assert backups.findtext("allowed_path") == "/backups/"
+ assert backups.findtext("allow_concurrent_backups") == "false"
+ assert backups.findtext("allow_concurrent_restores") == "false"
+ assert CONFIG.read_text(encoding="utf-8").count("") == 1
+
+
+def test_compose_override_is_additive_clickhouse_only_and_uses_dedicated_mounts() -> (
+ None
+):
+ text = OVERRIDE.read_text(encoding="utf-8")
+ assert text.count(" clickhouse:") == 1
+ assert "otel-collector:" not in text
+ assert "signoz:" not in text
+ assert "zookeeper:" not in text
+ assert "source: /backup/staging/signoz-clickhouse" in text
+ assert "target: /backups" in text
+ assert (
+ "source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml"
+ ) in text
+ assert ("target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml") in text
+ assert text.count("read_only: true") == 1
+ assert text.count("read_only: false") == 1
+ for forbidden in ("image:", "build:", "command:", "environment:"):
+ assert forbidden not in text
+
+
+def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
+ syntax = subprocess.run(
+ ["bash", "-n", str(DEPLOYER)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ assert syntax.returncode == 0, syntax.stderr
+
+ help_result = subprocess.run(
+ ["bash", str(DEPLOYER), "--help"],
+ check=False,
+ capture_output=True,
+ text=True,
+ env={
+ key: value
+ for key, value in os.environ.items()
+ if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+ },
+ )
+ assert help_result.returncode == 0
+ assert "--check" in help_result.stdout
+ assert "--apply" in help_result.stdout
+
+
+def test_deployer_requires_all_three_controlled_apply_identifiers_before_ssh() -> None:
+ env = {
+ key: value
+ for key, value in os.environ.items()
+ if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+ }
+ result = subprocess.run(
+ ["bash", str(DEPLOYER), "--check"],
+ check=False,
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert result.returncode == 64
+ assert "must be path-safe" in result.stderr
+ assert "ssh:" not in result.stderr
+
+
+def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> None:
+ text = DEPLOYER.read_text(encoding="utf-8")
+ check_terminal = text.index('if [ "${MODE}" = "--check" ]')
+ stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
+ first_scp = text.index("scp -q")
+
+ assert check_terminal < stage_assignment < first_scp
+ assert "-f '${REMOTE_BASE}' -f - config -q" in text
+ assert '< "${LOCAL_OVERRIDE}"' in text
+ assert "check_pass_no_write" in text
+
+
+def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
+ text = DEPLOYER.read_text(encoding="utf-8")
+ executable = executable_text()
+
+ assert 'REMOTE_SERVICE="clickhouse"' in text
+ assert "--env-file /dev/null" in text
+ assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
+ assert "timeout --signal=TERM --kill-after=30 300" in text
+ assert "up -d --no-deps --pull never --force-recreate" in text
+ assert '"${REMOTE_SERVICE}"' in text
+ assert "docker compose down" not in executable
+ assert "docker pull" not in executable
+ assert "docker volume" not in executable
+ assert "docker system prune" not in executable
+ assert "github.com" not in text.lower()
+ assert "ghcr.io" not in text.lower()
+
+
+def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
+ text = DEPLOYER.read_text(encoding="utf-8")
+ assert "active_signoz_backup_detected" in text
+ assert "active_native_backup_or_restore" in text
+ assert "partial_managed_file_drift" in text
+ assert "another_deploy_holds_lock" in text
+ assert "backup-signoz[.]sh" in text
+ assert "CREATING_BACKUP" in text
+ assert "RESTORING" in text
+
+
+def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
+ text = DEPLOYER.read_text(encoding="utf-8")
+ assert "BEFORE_IMAGE_ID" in text
+ assert "AFTER_IMAGE_ID" in text
+ assert "BEFORE_DATA_VOLUME" in text
+ assert "AFTER_DATA_VOLUME" in text
+ assert 'eq .Destination "/var/lib/clickhouse"' in text
+ for prefix in ("COLLECTOR", "SIGNOZ", "ZOOKEEPER"):
+ assert f"BEFORE_{prefix}_ID" in text
+ assert f"BEFORE_{prefix}_RESTARTS" in text
+ assert "image_and_data_volume_unchanged" in text
+
+
+def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
+ text = DEPLOYER.read_text(encoding="utf-8")
+ assert "system.disks WHERE name='backups'" in text
+ assert "Local|0|0|0|1" in text
+ assert "EXISTS TABLE system.backups" in text
+ assert "SHOW GRANTS" in text
+ assert "BACKUP([,[:space:]]|$)" in text
+ assert 'eq .Destination "/backups"' in text
+ assert "101:101:750" in text
+ assert "server_uid" in text
+ assert "server_gid" in text
+ assert "docker exec -u 101:101" in text
+ assert "test -w /backups" in text
+ assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
+ assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
+ identity_start = text.index('server_uid="')
+ identity_block = text[
+ identity_start : text.index("docker exec -u 101:101", identity_start)
+ ]
+ assert "sh -c" not in identity_block
+ assert "concurrency_disabled" in text
+ for port in (4317, 4318, 8080):
+ assert f"listener_up {port}" in text
+ assert "api_code" in text
+ assert "disk_backups_local_rw_healthy" in text
+
+
+def test_failed_apply_restores_prior_managed_compose_state() -> None:
+ text = DEPLOYER.read_text(encoding="utf-8")
+ assert "restore_managed_files" in text
+ assert "rollback_deploy" in text
+ assert "PRIOR_OVERRIDE_PRESENT" in text
+ assert "PRIOR_CONFIG_PRESENT" in text
+ assert "prior-override.yaml" in text
+ assert "prior-backup-disk.xml" in text
+ assert "prior_compose_state_restored" in text
+ assert "retained_nonempty_no_delete" in text
+ assert "rc=91" in text
+ assert "trap on_exit EXIT" in text
+
+
+def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:
+ executable = executable_text()
+ lowered = executable.lower()
+ assert ".config.env" not in lowered
+ assert "config.env" not in lowered
+ assert ".env" not in lowered
+ assert "docker inspect --format '{{json .config.env}}'" not in lowered
+ assert " backup all" not in lowered
+ assert " restore all" not in lowered
+ assert "clickhouse-client --query 'backup" not in lowered
+ assert "clickhouse-client --query 'restore" not in lowered
+ assert "no_backup_or_restore_executed" in executable
+
+
+def test_receipts_cover_controlled_apply_and_durable_terminal() -> None:
+ text = DEPLOYER.read_text(encoding="utf-8")
+ for phase in (
+ "sensor_source",
+ "normalized_asset_identity",
+ "source_of_truth_diff",
+ "ai_decision",
+ "risk_policy",
+ "check",
+ "execution",
+ "post_verifier",
+ "rollback",
+ "closure_writeback",
+ "terminal",
+ ):
+ assert phase in text
+ assert "receipts.jsonl" in text
+ assert "override_sha_" in text
+ assert "config_sha_" in text
diff --git a/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
new file mode 100644
index 000000000..9d105ae82
--- /dev/null
+++ b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
@@ -0,0 +1,339 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+import subprocess
+
+
+ROOT = Path(__file__).resolve().parents[3]
+DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
+
+
+def deployer_text() -> str:
+ return DEPLOYER.read_text(encoding="utf-8")
+
+
+def executable_text() -> str:
+ return "\n".join(
+ line
+ for line in deployer_text().splitlines()
+ if not line.lstrip().startswith("#")
+ )
+
+
+def base_env() -> dict[str, str]:
+ env = os.environ.copy()
+ env.update(
+ {
+ "TRACE_ID": "P0-OBS-002",
+ "RUN_ID": "toolchain-test-run",
+ "WORK_ITEM_ID": "P0-OBS-002",
+ }
+ )
+ return env
+
+
+def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
+ syntax = subprocess.run(
+ ["bash", "-n", str(DEPLOYER)],
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+ assert syntax.returncode == 0, syntax.stderr
+
+ help_result = subprocess.run(
+ ["bash", str(DEPLOYER), "--help"],
+ check=False,
+ capture_output=True,
+ text=True,
+ env={
+ key: value
+ for key, value in os.environ.items()
+ if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+ },
+ )
+ assert help_result.returncode == 0
+ assert "--check" in help_result.stdout
+ assert "--apply" in help_result.stdout
+ assert "CLICKHOUSE_RESTORE_INVENTORY_HELPER" in help_result.stdout
+ assert "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" in help_result.stdout
+ assert "CLICKHOUSE_RESTORE_CLUSTER_CONFIG" in help_result.stdout
+
+
+def test_deployer_requires_controlled_apply_ids_before_network_access() -> None:
+ env = {
+ key: value
+ for key, value in os.environ.items()
+ if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
+ }
+ result = subprocess.run(
+ ["bash", str(DEPLOYER), "--check"],
+ check=False,
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert result.returncode == 64
+ assert "must be path-safe" in result.stderr
+ assert "ssh:" not in result.stderr
+
+
+def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> None:
+ env = base_env()
+ env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/scripts/../escape"
+ result = subprocess.run(
+ ["bash", str(DEPLOYER), "--check"],
+ check=False,
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert result.returncode == 64
+ assert "safe absolute paths" in result.stderr
+ assert "ssh:" not in result.stderr
+
+ env = base_env()
+ env["SIGNOZ_TOOLCHAIN_RECEIPT_ROOT"] = "/var/tmp/receipts"
+ result = subprocess.run(
+ ["bash", str(DEPLOYER), "--check"],
+ check=False,
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert result.returncode == 64
+ assert "receipt root must stay under /backup" in result.stderr
+
+ env = base_env()
+ env["SIGNOZ_TOOLCHAIN_STAGE_ROOT"] = "/backup/deploy-staging"
+ env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/deploy-staging/scripts"
+ result = subprocess.run(
+ ["bash", str(DEPLOYER), "--check"],
+ check=False,
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert result.returncode == 64
+ assert "must not overlap" in result.stderr
+
+
+def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
+ text = deployer_text()
+ assert 'TARGET_HOST="wooo@192.168.0.110"' in text
+ assert "TARGET_HOST:-" not in text
+ assert "TARGET_HOST=" not in text.replace('TARGET_HOST="wooo@192.168.0.110"', "", 1)
+
+
+def test_exact_seven_artifacts_and_target_modes_are_declared() -> None:
+ text = deployer_text()
+ for source in (
+ "scripts/backup/backup-signoz.sh",
+ "scripts/backup/clickhouse-native-backup.sh",
+ "scripts/backup/clickhouse-native-restore-drill.sh",
+ "scripts/backup/clickhouse-restore-inventory.py",
+ "scripts/backup/run-signoz-backup-canary.sh",
+ "ops/signoz/clickhouse/config.d/backup_disk.xml",
+ "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
+ ):
+ assert source in text
+
+ for target in (
+ "${REMOTE_SCRIPT_DIR}/backup-signoz.sh",
+ "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh",
+ "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh",
+ "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py",
+ "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh",
+ "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml",
+ "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml",
+ ):
+ assert target in text
+ assert "MODES=(0755 0755 0755 0755 0755 0644 0644)" in text
+ assert 'REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"' in text
+ assert (
+ 'REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-'
+ '/backup/config/signoz/clickhouse}"'
+ ) in text
+
+
+def test_local_and_remote_stage_validation_cover_hash_syntax_python_and_xml() -> None:
+ text = deployer_text()
+ assert "validate_local_sources" in text
+ assert "SOURCE_HASHES" in text
+ assert "bash -n" in text
+ assert "python3 -m py_compile" in text
+ assert "validate_xml_pair" in text
+ for expected in (
+ '"/backups/"',
+ '"restore-zookeeper"',
+ '"restore-clickhouse"',
+ '"CLICKHOUSE_RESTORE_REPLICA"',
+ ):
+ assert expected in text
+
+
+def test_check_mode_exits_before_remote_stage_or_scp() -> None:
+ text = deployer_text()
+ check_exit = text.index('if [ "${MODE}" = --check ]')
+ stage_create = text.index("STAGE_CREATED=0")
+ first_scp = text.index("scp -q")
+ assert check_exit < stage_create < first_scp
+ assert "check_pass_no_write" in text
+ assert "remote_target_unchanged_no_stage_no_candidate" in text
+
+ remote_check = text[
+ text.index("remote_read_only_check() {") : text.index(
+ "local_receipt sensor_source", text.index("remote_read_only_check() {")
+ )
+ ]
+ for forbidden in ("mkdir ", "install ", "mv ", "rm ", "scp "):
+ assert forbidden not in remote_check
+
+
+def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
+ fake_bin = tmp_path / "bin"
+ fake_bin.mkdir()
+ log = tmp_path / "calls.log"
+ ssh = fake_bin / "ssh"
+ ssh.write_text(
+ "#!/bin/sh\n"
+ 'printf "ssh %s\\n" "$*" >> "$FAKE_CALL_LOG"\n'
+ "cat >/dev/null\n"
+ 'printf \'{"phase":"source_of_truth_diff","result":"pass"}\\n\'\n',
+ encoding="utf-8",
+ )
+ scp = fake_bin / "scp"
+ scp.write_text(
+ '#!/bin/sh\nprintf "scp %s\\n" "$*" >> "$FAKE_CALL_LOG"\nexit 99\n',
+ encoding="utf-8",
+ )
+ ssh.chmod(0o755)
+ scp.chmod(0o755)
+
+ env = base_env()
+ env["PATH"] = f"{fake_bin}:{env['PATH']}"
+ env["FAKE_CALL_LOG"] = str(log)
+ result = subprocess.run(
+ ["bash", str(DEPLOYER), "--check"],
+ check=False,
+ capture_output=True,
+ text=True,
+ env=env,
+ )
+ assert result.returncode == 0, result.stderr
+ calls = log.read_text(encoding="utf-8").splitlines()
+ assert len(calls) == 1
+ assert calls[0].startswith("ssh ")
+ assert "wooo@192.168.0.110" in calls[0]
+ assert "check_pass_no_write" in result.stdout
+
+
+def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
+ text = deployer_text()
+ for token in (
+ "prior-manifest.tsv",
+ "PRIOR_PRESENT",
+ "PRIOR_MODES",
+ "PRIOR_UIDS",
+ "PRIOR_GIDS",
+ "PRIOR_HASHES",
+ "cp -p",
+ "rollback_targets",
+ "all_prior_files_modes_hashes_owners_and_runtime_restored",
+ ):
+ assert token in text
+ assert "trap on_exit EXIT" in text
+ assert "rc=91" in text
+ assert 'run_root rm -f "${target}"' in text
+ assert 'run_root rmdir "${CREATED_DIRS[index]}"' in text
+ assert "bounded_execution_not_started_targets_unchanged" in text
+
+
+def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None:
+ text = deployer_text()
+ assert '"${target}.candidate.${RUN_ID}"' in text
+ assert (
+ 'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
+ ) in text
+ assert "deployed-manifest.tsv" in text
+ assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text
+ assert "stage_candidate_residue_0" in text
+ assert "verify_residue_zero" in text
+
+
+def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
+ text = deployer_text()
+ for container in (
+ "signoz-clickhouse",
+ "signoz-otel-collector",
+ "signoz",
+ "signoz-zookeeper-1",
+ ):
+ assert container in text
+ for field in (
+ ".Id",
+ ".State.StartedAt",
+ ".RestartCount",
+ ".State.Running",
+ 'index .State "Health"',
+ ):
+ assert field in text
+ assert "runtime-before.tsv" in text
+ assert "runtime-after.tsv" in text
+ assert 'cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"' in text
+ for port in (4317, 4318, 8080):
+ assert "for port in 4317 4318 8080" in text
+ assert str(port) in text
+ assert "api_200_unchanged" in text
+
+
+def test_apply_fails_closed_while_backup_or_restore_is_active() -> None:
+ text = deployer_text()
+ assert "active_signoz_backup_toolchain_process" in text
+ assert "active_native_backup_or_restore" in text
+ assert "CREATING_BACKUP" in text
+ assert "RESTORING" in text
+ assert "another_toolchain_deploy_holds_lock" in text
+
+
+def test_deployer_never_executes_runtime_work_or_uses_forbidden_supply_chain() -> None:
+ executable = executable_text().lower()
+ for forbidden in (
+ "docker restart",
+ "docker compose",
+ "docker pull",
+ "docker run",
+ "backup all",
+ "restore all",
+ "ansible-playbook",
+ "github.com",
+ "api.github.com",
+ "raw.githubusercontent.com",
+ "codeload.github.com",
+ "ghcr.io",
+ " gh ",
+ ):
+ assert forbidden not in executable
+ assert "no_backup_restore_restart_or_pull" in executable
+
+
+def test_receipts_cover_complete_controlled_apply_contract() -> None:
+ text = deployer_text()
+ for phase in (
+ "sensor_source",
+ "normalized_asset_identity",
+ "source_of_truth_diff",
+ "ai_decision",
+ "risk_policy",
+ "check",
+ "execution",
+ "post_verifier",
+ "rollback",
+ "closure_writeback",
+ "terminal",
+ ):
+ assert phase in text
+ assert "receipts.jsonl" in text
+ assert '"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"' in text
+ assert "durable_manifests_and_receipts_ack" in text
diff --git a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh
index ecdcc095b..9b0527662 100755
--- a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh
+++ b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh
@@ -265,6 +265,12 @@ require_file scripts/backup/backup-awoooi-frequent.sh "AWOOOI high-frequency DB
require_file scripts/backup/backup-langfuse.sh "Langfuse backup script"
require_file scripts/backup/backup-monitoring.sh "Monitoring backup script"
require_file scripts/backup/backup-signoz.sh "SignOz backup script"
+require_file scripts/backup/clickhouse-native-backup.sh "ClickHouse native backup adapter"
+require_file scripts/backup/clickhouse-native-restore-drill.sh "ClickHouse isolated native restore drill"
+require_file scripts/backup/clickhouse-restore-inventory.py "ClickHouse restore inventory verifier"
+require_file scripts/backup/run-signoz-backup-canary.sh "Guarded SignOz backup canary wrapper"
+require_file ops/signoz/clickhouse/config.d/backup_disk.xml "ClickHouse backup disk config"
+require_file ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml "ClickHouse isolated restore config"
require_file scripts/backup/backup-open-webui.sh "Open-WebUI backup script"
require_file scripts/backup/backup-clawbot.sh "ClawBot backup script"
require_file scripts/backup/backup-configs.sh "Host/service config backup"
diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py
index 3e6010ca9..a24b7e37f 100644
--- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py
+++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py
@@ -2,8 +2,8 @@
"""No-write source preflight for P0-OBS-002 Wave A.
The checker intentionally does not access credentials, databases, runtime
-volumes, Docker, Kubernetes, or remote hosts. Runtime evidence belongs to a
-separate verifier receipt.
+volumes, Docker, Kubernetes, or remote hosts. It validates runtime evidence
+only from the committed post-closure receipt.
"""
from __future__ import annotations
@@ -16,17 +16,24 @@ from typing import Any
import yaml
SCHEMA = "awoooi_signoz_canonical_route_preflight_v1"
+POST_CLOSURE_SCHEMA = "awoooi_signoz_post_closure_regression_v2"
+POST_CLOSURE_SNAPSHOT = "ops/signoz/p0-obs-002-post-closure-regression.yaml"
EXPECTED_BLOCKERS = [
"organization_not_initialized",
"direct_ingress_policy_not_closed",
"http_bridge_not_supervised",
"filelog_normalization_degraded",
- "api_runtime_cooldown_pending",
- "cd_probe_safety_fix_not_deployed",
- "backup_collector_restore_guard_not_deployed",
- "signoz_control_plane_health_route_not_deployed",
"wave_b_not_authorized",
]
+EXPECTED_POST_CLOSURE_BLOCKERS = [
+ "signoz_sqlite_application_consistent_backup_pending",
+ "signoz_backup_offsite_dr_pending",
+ "clickhouse_native_failed_artifact_retention_pending",
+ "clickhouse_native_resume_submitted_inventory_pending",
+ "service_registry_runtime_mirror_reconciliation_pending",
+ "github_freeze_legacy_asset_retirement_pending",
+ "monitoring_generator_duplicate_awoooi_api_identity_pending",
+]
def _load_yaml(path: Path) -> Any:
@@ -40,6 +47,16 @@ def _read(root: Path, relative: str) -> str:
def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
contract = _load_yaml(contract_path)
+ post_closure = contract.get("runtime_observations", {}).get(
+ "post_closure_regression", {}
+ )
+ snapshot_path = str(post_closure.get("snapshot", ""))
+ regression_path = root / POST_CLOSURE_SNAPSHOT
+ regression = (
+ _load_yaml(regression_path)
+ if snapshot_path == POST_CLOSURE_SNAPSHOT and regression_path.is_file()
+ else {}
+ )
checks: dict[str, bool] = {}
checks["contract_schema"] = (
@@ -66,8 +83,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
config = _read(root, "apps/api/src/core/config.py")
checks["dual_route_application_allow"] = (
'default=["192.168.0.188", "192.168.0.110"]' in config
- and 'default=["192.168.0.112", "192.168.0.120", "192.168.0.121"]'
- in config
+ and 'default=["192.168.0.112", "192.168.0.120", "192.168.0.121"]' in config
)
network_documents = list(
@@ -82,18 +98,14 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
)
canonical_ports: set[int] = set()
for rule in egress_policy.get("spec", {}).get("egress", []):
- cidrs = {
- item.get("ipBlock", {}).get("cidr") for item in rule.get("to", [])
- }
+ cidrs = {item.get("ipBlock", {}).get("cidr") for item in rule.get("to", [])}
if "192.168.0.110/32" in cidrs:
canonical_ports.update(
int(item["port"])
for item in rule.get("ports", [])
if isinstance(item.get("port"), int)
)
- checks["dual_route_networkpolicy_allow"] = {4317, 4318}.issubset(
- canonical_ports
- )
+ checks["dual_route_networkpolicy_allow"] = {4317, 4318}.issubset(canonical_ports)
unit = _read(root, "ops/systemd/user/awoooi-signoz-otlp-bridge.service")
checks["bridge_unit_hardened"] = all(
@@ -112,14 +124,15 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
root, "scripts/reboot-recovery/verify-signoz-otel-signals.sh"
)
checks["route_specific_verifier"] = (
- "OTLP_ENDPOINT" in verifier and "ROUTE_ID" in verifier
+ "OTLP_ENDPOINT" in verifier
+ and "ROUTE_ID" in verifier
and all(
value in signal_verifier
for value in (
- 'send_signal logs',
- 'send_signal metrics',
- 'send_signal traces',
- '/v1/$signal',
+ "send_signal logs",
+ "send_signal metrics",
+ "send_signal traces",
+ "/v1/$signal",
"signoz_logs.logs_v2",
"signoz_metrics.time_series_v4",
"signoz_metrics.samples_v4",
@@ -127,57 +140,54 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
)
)
)
- grpc_bridge = _read(
- root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh"
- )
- grpc_unit = _read(
- root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service"
- )
+ grpc_bridge = _read(root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh")
+ grpc_unit = _read(root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service")
grpc_verifier = _read(
root, "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
)
_, grpc_apply_marker, grpc_after_apply = grpc_bridge.partition(" --apply)")
- grpc_apply_block, grpc_status_marker, _ = grpc_after_apply.partition(
- " --status)"
- )
- checks["grpc_transport_recovery_source_ready"] = all(
- value in grpc_bridge
- for value in (
- "LISTEN_PORT:-24317",
- "UPSTREAM_PORT:-4317",
- "SOURCE_RANGE:-192.168.0.120/31",
- "systemctl --user enable --now",
- "--rollback",
+ grpc_apply_block, grpc_status_marker, _ = grpc_after_apply.partition(" --status)")
+ checks["grpc_transport_recovery_source_ready"] = (
+ all(
+ value in grpc_bridge
+ for value in (
+ "LISTEN_PORT:-24317",
+ "UPSTREAM_PORT:-4317",
+ "SOURCE_RANGE:-192.168.0.120/31",
+ "systemctl --user enable --now",
+ "--rollback",
+ )
)
- ) and all(
- value in grpc_unit
- for value in (
- "LISTEN_PORT=24317",
- "UPSTREAM_PORT=4317",
- "SOURCE_RANGE=192.168.0.120/31",
- "Restart=on-failure",
- "NoNewPrivileges=true",
+ and all(
+ value in grpc_unit
+ for value in (
+ "LISTEN_PORT=24317",
+ "UPSTREAM_PORT=4317",
+ "SOURCE_RANGE=192.168.0.120/31",
+ "Restart=on-failure",
+ "NoNewPrivileges=true",
+ )
)
- ) and all(
- value in grpc_verifier
- for value in (
- "WINDOW_SECONDS:-600",
- "serviceName={service:String}",
- "signoz_metrics.samples_v4",
- "Failed to export traces",
- "restart_delta_0",
+ and all(
+ value in grpc_verifier
+ for value in (
+ "WINDOW_SECONDS:-600",
+ "serviceName={service:String}",
+ "signoz_metrics.samples_v4",
+ "Failed to export traces",
+ "restart_delta_0",
+ )
)
)
checks["grpc_apply_waits_for_independent_post_verifier"] = (
bool(grpc_apply_marker)
and bool(grpc_status_marker)
and (
- 'receipt learning partial '
- '"application_trace_metric_10m_post_verifier_pending"'
- in grpc_apply_block
+ "receipt learning partial "
+ '"application_trace_metric_10m_post_verifier_pending"' in grpc_apply_block
)
and (
- 'receipt terminal partial '
+ "receipt terminal partial "
'"grpc_transport_restored_independent_600s_post_verifier_pending"'
in grpc_apply_block
)
@@ -185,7 +195,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
and grpc_apply_block.index("receipt learning partial")
< grpc_apply_block.index("receipt terminal partial")
and (
- 'receipt terminal pass '
+ "receipt terminal pass "
'"real_application_otlp_grpc_trace_metric_freshness_closed"'
in grpc_verifier
)
@@ -202,14 +212,11 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
and contract.get("runtime_observations", {}).get("grpc_bridge_supervisor")
== "user_systemd_enabled_active"
and grpc_runtime.get("runtime_status") == "restored_verified"
- and grpc_source_fix.get("runtime_apply_status")
- == "applied_post_verifier_pass"
+ and grpc_source_fix.get("runtime_apply_status") == "applied_post_verifier_pass"
and grpc_source_fix.get("runtime_verifier_terminal") == "pass"
and int(grpc_source_fix.get("runtime_verifier_window_seconds") or 0) >= 600
)
- static_recovery = _read(
- root, "scripts/reboot-recovery/signoz-110-static-ingest.sh"
- )
+ static_recovery = _read(root, "scripts/reboot-recovery/signoz-110-static-ingest.sh")
checks["managed_promotion_fail_closed"] = all(
value in static_recovery
for value in (
@@ -219,6 +226,453 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
)
)
+ checks["post_closure_snapshot_path"] = snapshot_path == POST_CLOSURE_SNAPSHOT
+ checks["post_closure_schema_v2"] = (
+ regression.get("schema") == POST_CLOSURE_SCHEMA
+ and post_closure.get("snapshot_schema") == POST_CLOSURE_SCHEMA
+ and regression.get("work_item_id") == "P0-OBS-002"
+ and regression.get("trace_id") == "P0-OBS-002"
+ )
+
+ asset_reconciliation = regression.get("asset_reconciliation", {})
+ source_registry = _load_yaml(root / "ops/config/service-registry.yaml")
+ runtime_configmap = _load_yaml(
+ root / "k8s/awoooi-prod/15-service-registry-configmap.yaml"
+ )
+ runtime_registry = yaml.safe_load(
+ runtime_configmap.get("data", {}).get("service-registry.yaml", "")
+ )
+ source_services = {
+ item["name"]: item for item in source_registry.get("services", [])
+ }
+ runtime_services = {
+ item["name"]: item for item in runtime_registry.get("services", [])
+ }
+ shared_services = set(source_services) & set(runtime_services)
+ exact_shared_services = {
+ name
+ for name in shared_services
+ if source_services[name] == runtime_services[name]
+ }
+ source_only_services = sorted(set(source_services) - set(runtime_services))
+ checks["service_registry_runtime_mirror_drift_explicit"] = (
+ asset_reconciliation.get("drift_work_item_id") == "P0-OBS-002-ASSET-DRIFT-001"
+ and asset_reconciliation.get("source_service_count")
+ == len(source_services)
+ == 27
+ and asset_reconciliation.get("runtime_mirror_service_count")
+ == len(runtime_services)
+ == 24
+ and asset_reconciliation.get("exact_shared_service_count")
+ == len(exact_shared_services)
+ == 24
+ and asset_reconciliation.get("shared_service_drift_count") == 0
+ and asset_reconciliation.get("source_only_services")
+ == source_only_services
+ == ["bitan-app", "sentry", "signoz"]
+ and source_services.get("signoz-clickhouse")
+ == runtime_services.get("signoz-clickhouse")
+ and asset_reconciliation.get("signoz_clickhouse_exact_match") is True
+ and asset_reconciliation.get("live_signoz_query_host") == "192.168.0.110"
+ and asset_reconciliation.get("source_signoz_host") == "192.168.0.188"
+ and asset_reconciliation.get("terminal") == "partial_degraded"
+ )
+ github_legacy_drift = asset_reconciliation.get(
+ "github_freeze_legacy_asset_drift", {}
+ )
+ checks["github_freeze_legacy_asset_drift_explicit"] = (
+ github_legacy_drift.get("drift_work_item_id") == "P0-OBS-002-ASSET-DRIFT-002"
+ and github_legacy_drift.get("superseded_by") == "global_product_governance_v2"
+ and github_legacy_drift.get("status") == "pending_controlled_retirement"
+ and github_legacy_drift.get("active_github_use_in_this_run") is False
+ and len(github_legacy_drift.get("source_references", [])) == 3
+ and github_legacy_drift.get("replacement")
+ == "gitea_runner_readiness_and_queue_receipts"
+ and github_legacy_drift.get("p0_p1_github_recovery_scheduled") is False
+ )
+ duplicate_identity_drift = asset_reconciliation.get(
+ "monitoring_generator_duplicate_identity_drift", {}
+ )
+ checks["monitoring_generator_duplicate_identity_drift_explicit"] = (
+ duplicate_identity_drift.get("drift_work_item_id")
+ == "P0-OBS-002-ASSET-DRIFT-003"
+ and duplicate_identity_drift.get("status") == "pending_source_deduplication"
+ and duplicate_identity_drift.get("registry_service_count") == 26
+ and duplicate_identity_drift.get("awoooi_api_registry_entry_count") == 2
+ and duplicate_identity_drift.get("generated_scrape_config_count") == 24
+ and duplicate_identity_drift.get("generated_awoooi_api_job_count") == 2
+ and duplicate_identity_drift.get("generated_blackbox_target_count") == 20
+ and duplicate_identity_drift.get("runtime_apply_performed") is False
+ )
+
+ cd = regression.get("release_receipts", {}).get("gitea_cd_5140", {})
+ cd_probe = cd.get("cd_probe", {})
+ checks["cd_5140_receipt_consistent"] = (
+ cd.get("gitea_run_id") == 5140
+ and cd.get("source_sha") == "e4da6570a645e504cc93647d396f45f167b3172a"
+ and cd.get("workflow_terminal") == "success"
+ and cd.get("unit_tests", {}).get("passed") == 4817
+ and cd.get("integration_tests_passed") == 5
+ and cd.get("production_readback", {}).get("build_tag_matched") is True
+ and cd.get("production_readback", {}).get("desired_tag_matched") is True
+ and cd_probe.get("receipt_provenance") == "gitea_run_5140_job_log"
+ and cd_probe.get("passed") is True
+ and cd_probe.get("workers") == 4
+ and cd_probe.get("safety_reserve_connections") == 4
+ and cd_probe.get("restart_free") is True
+ )
+
+ applies = regression.get("controlled_applies", {})
+ config_route = applies.get("config_route_patch", {})
+ checks["config_route_patch_receipt_consistent"] = (
+ config_route.get("run_id")
+ == "P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1"
+ and config_route.get("source_commit")
+ == "9a9d1464a586ba172d1613db5bc285def43eb3e8"
+ and config_route.get("included_in_release_sha") == cd.get("source_sha")
+ and config_route.get("normalized_asset", {}).get("internal_url")
+ == "http://192.168.0.110:8080"
+ and config_route.get("source_truth_diff", {}).get("otlp_producer_route_changed")
+ is False
+ and config_route.get("execution_receipt", {}).get("deployment_status")
+ == "deployed_verified"
+ and config_route.get("terminal") == "applied_post_verifier_pass"
+ )
+
+ prometheus = applies.get("prometheus_target_route", {})
+ prometheus_check = prometheus.get("check_receipt", {})
+ prometheus_execution = prometheus.get("execution_receipt", {})
+ prometheus_post = prometheus.get("post_verifier", {})
+ prometheus_independent = prometheus.get("independent_verifier", {})
+ checks["prometheus_target_first_receipt_consistent"] = (
+ prometheus.get("run_id")
+ == "P0-OBS-002-prometheus-route-20260714T215106Z-apply1"
+ and prometheus_check.get("promtool") == "pass"
+ and prometheus_check.get("candidate_sha")
+ == prometheus_execution.get("active_sha")
+ == prometheus_execution.get("runtime_sha")
+ and prometheus_execution.get("action") == "target_first_config_apply"
+ and prometheus_post.get("terminal") == "target_up_two_scrapes"
+ and prometheus_post.get("target_count") == 1
+ and prometheus_post.get("target_health") == "up"
+ and prometheus_post.get("probe_success") == 1
+ and prometheus_post.get("legacy_target_count") == 0
+ and prometheus_independent.get("active_sha_matches_runtime") is True
+ and prometheus_independent.get("target_count") == 1
+ and prometheus_independent.get("target_health") == "up"
+ and prometheus_independent.get("legacy_target_count") == 0
+ and prometheus_independent.get("terminal") == "pass"
+ and prometheus.get("terminal") == "completed_target_first"
+ )
+
+ alert_rules = applies.get("alert_rules", {})
+ alert_check = alert_rules.get("check_receipt", {})
+ alert_execution = alert_rules.get("execution_receipt", {})
+ alert_post = alert_rules.get("post_verifier", {})
+ checks["alert_rules_receipt_consistent"] = (
+ alert_rules.get("run_id") == "P0-OBS-002-alert-rules-20260714T215328Z-apply1"
+ and alert_check.get("promtool") == "pass"
+ and alert_check.get("source_sha")
+ == alert_execution.get("remote_sha")
+ == alert_execution.get("canonical_sha")
+ == alert_execution.get("runtime_sha")
+ and alert_execution.get("hashes_match") is True
+ and alert_post.get("window_seconds") == 130
+ and alert_post.get("terminal") == "pass"
+ and alert_post.get("signoz_rule_count") == 1
+ and alert_post.get("signoz_rule_health") == "ok"
+ and alert_rules.get("terminal") == "completed_verified"
+ )
+
+ native_disk = applies.get("clickhouse_native_backup_disk", {})
+ native_disk_attempts = native_disk.get("attempts", [])
+ disk_apply1 = native_disk_attempts[0] if len(native_disk_attempts) == 2 else {}
+ disk_retry2 = native_disk_attempts[1] if len(native_disk_attempts) == 2 else {}
+ checks["clickhouse_native_backup_disk_receipt_consistent"] = (
+ native_disk.get("source", {}).get("override_sha256")
+ == "cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488"
+ and native_disk.get("source", {}).get("config_sha256")
+ == "c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9"
+ and native_disk.get("check_receipt", {}).get("terminal")
+ == "check_pass_no_write"
+ and disk_apply1.get("terminal") == "failed_rolled_back"
+ and disk_apply1.get("exit_code") == 2
+ and disk_apply1.get("rollback", {}).get("terminal") == "pass"
+ and disk_apply1.get("rollback", {}).get("original_data_volume_preserved")
+ is True
+ and disk_retry2.get("run_id")
+ == "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
+ and disk_retry2.get("terminal") == "pass"
+ and disk_retry2.get("execution", {}).get("image_digest_unchanged") is True
+ and disk_retry2.get("execution", {}).get("data_volume_unchanged") is True
+ and disk_retry2.get("post_verifier", {}).get("independent") is True
+ and disk_retry2.get("post_verifier", {}).get("server_write_access") is True
+ and disk_retry2.get("post_verifier", {}).get("dependent_restart_count_delta")
+ == 0
+ and disk_retry2.get("post_verifier", {}).get("api_http_status") == 200
+ and disk_retry2.get("post_verifier", {}).get("active_backup_or_restore_count")
+ == 0
+ and native_disk.get("terminal") == "deployed_verified"
+ )
+
+ native_canary = applies.get("clickhouse_native_backup_restore_canary", {})
+ native_attempts = native_canary.get("attempts", [])
+ native_apply2 = native_attempts[1] if len(native_attempts) == 4 else {}
+ native_apply3 = native_attempts[2] if len(native_attempts) == 4 else {}
+ native_apply4 = native_attempts[3] if len(native_attempts) == 4 else {}
+ native_backup = native_apply4.get("backup", {})
+ native_restore = native_apply4.get("isolated_restore", {})
+ native_restic = native_apply4.get("restic", {})
+ native_post = native_apply4.get("independent_post_verifier", {})
+ toolchain_apply = native_canary.get("toolchain_controlled_apply", {})
+ checks["native_backup_restore_canary_closed"] = (
+ native_canary.get("completion_scope") == "clickhouse_native_only"
+ and native_canary.get("source", {}).get("native_backup_sha256")
+ == "754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a"
+ and native_canary.get("source", {}).get("isolated_restore_sha256")
+ == "57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7"
+ and native_canary.get("source", {}).get("post_canary_focused_tests_passed")
+ == 94
+ and native_canary.get("source", {}).get("post_format_focused_tests_passed")
+ == 94
+ and native_canary.get("source", {}).get("ruff_format") == "pass"
+ and toolchain_apply.get("apply_run_id")
+ == "P0-OBS-002-native-toolchain-20260715T001252Z-apply4"
+ and toolchain_apply.get("apply_terminal") == "pass"
+ and toolchain_apply.get("postcheck_terminal") == "check_pass_no_write"
+ and toolchain_apply.get("postcheck_diff")
+ == "matching_7_drift_0_absent_0_active_operations_0"
+ and native_apply2.get("artifact_retention") == "preserved_failed_run_evidence"
+ and native_apply2.get("cleanup_verified") is True
+ and native_apply3.get("artifact_retention") == "preserved_failed_run_evidence"
+ and native_apply3.get("cleanup_verified") is True
+ and native_apply4.get("run_id")
+ == "P0-OBS-002-native-canary-20260715T001329Z-apply4"
+ and native_apply4.get("check_terminal") == "check_pass_no_write"
+ and native_apply4.get("terminal") == "pass"
+ and native_backup.get("terminal") == "BACKUP_CREATED"
+ and native_backup.get("error") == ""
+ and native_backup.get("artifact_sha256")
+ == "4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc"
+ and native_restore.get("terminal") == "verified"
+ and native_restore.get("clickhouse_restore_terminal") == "RESTORED"
+ and native_restore.get("production_network_attached") is False
+ and native_restore.get("production_restore_performed") is False
+ and native_restore.get("source_staged_clickhouse_artifact_sha_match") is True
+ and native_restore.get("database_count")
+ == native_restore.get("restored_database_count")
+ == 6
+ and native_restore.get("table_count")
+ == native_restore.get("restored_table_count")
+ == 100
+ and native_restore.get("table_engine_schema_manifest_parity") is True
+ and native_restore.get("check_table_count")
+ == native_restore.get("check_table_pass_count")
+ == 44
+ and native_restore.get("critical_nonzero_count") == 4
+ and native_restore.get("aggregate_counter_exact_parity_gate") is False
+ and native_restic.get("snapshot_id") == "574e2a1a"
+ and native_restic.get("snapshot_readback") == "pass"
+ and native_restic.get("repository_scope") == "local_same_failure_domain"
+ and native_restic.get("offsite_verified") is False
+ and native_post.get("production_restart_count_delta") == 0
+ and native_post.get("api_http_status") == 200
+ and native_post.get("new_server_artifact_removed") is True
+ and native_post.get("apply2_apply3_preserved_artifact_hashes_unchanged") is True
+ and native_post.get(
+ "exact_ephemeral_container_volume_network_process_tmp_residue_count"
+ )
+ == 0
+ and native_post.get("cleanup_verified") is True
+ and native_post.get("secret_values_collected") is False
+ and native_canary.get("terminal") == "production_closed_clickhouse_native_scope"
+ )
+
+ backup = applies.get("backup_canary", {})
+ backup_attempts = backup.get("attempts", [])
+ backup_attempt = backup_attempts[0] if len(backup_attempts) == 2 else {}
+ retry2 = backup_attempts[1] if len(backup_attempts) == 2 else {}
+ backup_rollback = backup_attempt.get("rollback", {})
+ retry2_rollback = retry2.get("rollback", {})
+ checks["backup_canary_failure_and_rollback_consistent"] = (
+ backup.get("guard_source", {}).get("deployment_status") == "deployed_verified"
+ and backup_attempt.get("run_id")
+ == "P0-OBS-002-backup-canary-20260714T213210Z-retry1"
+ and backup_attempt.get("terminal") == "failed_rolled_back"
+ and backup_attempt.get("classification")
+ == "failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention"
+ and backup_attempt.get("archive_container_exit_code") == 137
+ and backup_attempt.get("root_cause")
+ == "docker_health_monitor_restarted_collector_during_intentional_backup_stop"
+ and backup_attempt.get("clickhouse_archive_committed") is False
+ and backup_attempt.get("sqlite_stage_reached") is False
+ and backup_attempt.get("restic_stage_reached") is False
+ and backup_attempt.get("retention_stage_reached") is False
+ and backup_attempt.get("success_stage_reached") is False
+ and backup_rollback.get("collector_running") is True
+ and backup_rollback.get("collector_restart_count") == 0
+ and backup_rollback.get("grpc_4317_listening") is True
+ and backup_rollback.get("http_4318_listening") is True
+ and backup_rollback.get("current_temp_dir_absent") is True
+ and backup_rollback.get("partial_artifacts_absent") is True
+ and retry2.get("run_id") == "P0-OBS-002-backup-canary-20260714T221230Z-retry2"
+ and retry2.get("terminal") == "failed_rolled_back"
+ and retry2.get("backup_script_exit_code") == 1
+ and retry2.get("archive_container_exit_code") == "unknown_not_durably_captured"
+ and retry2.get("root_cause")
+ == "raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract"
+ and retry2.get("exact_tar_error_proven") is False
+ and retry2.get("clickhouse_archive_committed") is False
+ and retry2.get("sqlite_stage_reached") is False
+ and retry2.get("restic_stage_reached") is False
+ and retry2.get("retention_stage_reached") is False
+ and retry2.get("success_stage_reached") is False
+ and retry2.get("monitor_window", {}).get("auto_repair_count") == 0
+ and retry2_rollback.get("cooldown_lease_restored") is True
+ and retry2_rollback.get("collector_running") is True
+ and retry2_rollback.get("collector_restart_count") == 0
+ and retry2_rollback.get("grpc_4317_listening") is True
+ and retry2_rollback.get("http_4318_listening") is True
+ and retry2_rollback.get("current_temp_dir_absent") is True
+ and retry2_rollback.get("partial_artifacts_absent") is True
+ )
+
+ post_release = regression.get("post_release_runtime_verifier", {})
+ checks["post_release_600s_verifier_consistent"] = (
+ post_release.get("run_id") == "P0-OBS-002-post-release-20260714T215500Z"
+ and post_release.get("window_seconds") == 600
+ and post_release.get("terminal") == "pass"
+ and post_release.get("restart_delta") == 0
+ and post_release.get("exporter_trace_errors") == 0
+ and post_release.get("exporter_metric_errors") == 0
+ and post_release.get("trace_count") == 4026
+ and post_release.get("metric_sample_count") == 76369
+ and post_release.get("api_runtime_cooldown_closed") is True
+ and regression.get("regressions", {})
+ .get("api_runtime", {})
+ .get("last_observed_cooldown_closed")
+ is True
+ )
+
+ native_migration = regression.get("completed_verifiers", {}).get(
+ "clickhouse_native_backup_migration", {}
+ )
+ pending_verifiers = regression.get("pending_verifiers", {})
+ sqlite_backup = pending_verifiers.get(
+ "signoz_sqlite_application_consistent_backup", {}
+ )
+ offsite_backup = pending_verifiers.get("signoz_backup_offsite_dr", {})
+ failed_artifacts = pending_verifiers.get(
+ "clickhouse_native_failed_artifact_retention", {}
+ )
+ resume_inventory = pending_verifiers.get(
+ "clickhouse_native_resume_submitted_inventory", {}
+ )
+ checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = (
+ native_migration.get("prior_run_id") == retry2.get("run_id")
+ and native_migration.get("run_id") == native_apply4.get("run_id")
+ and native_migration.get("status")
+ == "production_closed_clickhouse_native_scope"
+ and native_migration.get("native_backup_terminal") == "BACKUP_CREATED"
+ and native_migration.get("isolated_restore_terminal") == "verified"
+ and native_migration.get("restic_snapshot_id") == "574e2a1a"
+ and native_migration.get("independent_verifier") == "pass"
+ and native_migration.get("exact_cleanup") == "pass"
+ and sqlite_backup.get("status")
+ == "pending_supported_non_raw_export_or_coordinated_snapshot"
+ and sqlite_backup.get("raw_sqlite_read_or_sync_allowed") is False
+ and sqlite_backup.get("sqlite_cli_present_in_runtime") is False
+ and offsite_backup.get("status")
+ == "pending_offsite_snapshot_and_restore_verifier"
+ and offsite_backup.get("current_repository_scope")
+ == "local_same_failure_domain"
+ and offsite_backup.get("offsite_verified") is False
+ and failed_artifacts.get("status") == "pending_bounded_retention_decision"
+ and failed_artifacts.get("destructive_cleanup_authorized") is False
+ and failed_artifacts.get("active_backup_or_restore_count") == 0
+ and len(failed_artifacts.get("preserved_artifacts", [])) == 2
+ and resume_inventory.get("status")
+ == "pending_durable_submitted_inventory_contract"
+ and resume_inventory.get("same_run_resume_safe") is False
+ and len(resume_inventory.get("required_preconditions", [])) == 4
+ )
+
+ checks["post_closure_contract_mirror_consistent"] = (
+ post_closure.get("gitea_cd_run_id") == cd.get("gitea_run_id")
+ and post_closure.get("release_source_sha") == cd.get("source_sha")
+ and post_closure.get("api_runtime_cooldown_closed") is True
+ and post_closure.get("cd_probe_safety_fix_deployed") is True
+ and post_closure.get("config_route_patch_deployed") is True
+ and post_closure.get("signoz_control_plane_health_route_deployed") is True
+ and post_closure.get("backup_collector_restore_guard_deployed") is True
+ and post_closure.get("backup_canary_scope") == "clickhouse_native_only"
+ and post_closure.get("backup_canary_verified") is True
+ and post_closure.get("backup_canary_last_run_id") == native_apply4.get("run_id")
+ and post_closure.get("backup_canary_last_terminal") == "pass"
+ and post_closure.get("backup_canary_cooldown_retry_status")
+ == "superseded_by_online_native_no_stop_contract"
+ and post_closure.get("clickhouse_native_backup_migration_status")
+ == "production_closed"
+ and post_closure.get("clickhouse_native_backup_disk_run_id")
+ == disk_retry2.get("run_id")
+ and post_closure.get("clickhouse_native_backup_disk_status")
+ == "deployed_verified"
+ and post_closure.get("clickhouse_native_backup_toolchain_run_id")
+ == toolchain_apply.get("apply_run_id")
+ and post_closure.get("clickhouse_native_backup_toolchain_postcheck_run_id")
+ == toolchain_apply.get("postcheck_run_id")
+ and post_closure.get("clickhouse_native_restore_terminal") == "verified"
+ and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a"
+ and post_closure.get("clickhouse_native_offsite_status") == "pending"
+ and post_closure.get("clickhouse_native_failed_artifact_retention_status")
+ == "pending_bounded_retention_decision"
+ and post_closure.get("clickhouse_native_resume_inventory_status")
+ == "pending_durable_submitted_inventory_contract"
+ and post_closure.get("signoz_sqlite_application_consistent_backup_status")
+ == "pending"
+ and post_closure.get("service_registry_runtime_mirror_status")
+ == "partial_degraded"
+ and post_closure.get("service_registry_runtime_mirror_work_item_id")
+ == "P0-OBS-002-ASSET-DRIFT-001"
+ and post_closure.get("github_freeze_legacy_asset_status")
+ == "pending_controlled_retirement"
+ and post_closure.get("github_freeze_legacy_asset_work_item_id")
+ == "P0-OBS-002-ASSET-DRIFT-002"
+ and post_closure.get("monitoring_generator_duplicate_identity_status")
+ == "pending_source_deduplication"
+ and post_closure.get("monitoring_generator_duplicate_identity_work_item_id")
+ == "P0-OBS-002-ASSET-DRIFT-003"
+ and post_closure.get("post_release_otlp_grpc_600s_verifier_status") == "pass"
+ )
+
+ regression_terminal = regression.get("terminal", {})
+ checks["post_closure_terminal_blocker_consistent"] = (
+ regression_terminal.get("status") == "partial_degraded"
+ and list(regression_terminal.get("blockers", []))
+ == EXPECTED_POST_CLOSURE_BLOCKERS
+ )
+
+ regression_text = _read(root, POST_CLOSURE_SNAPSHOT).lower()
+ checks["post_closure_no_secret_or_github_dependency"] = (
+ regression.get("scope_boundaries", {}).get("secret_read") is False
+ and regression.get("scope_boundaries", {}).get("github_used") is False
+ and all(
+ forbidden not in regression_text
+ for forbidden in (
+ "github.com",
+ "api.github.com",
+ "raw.githubusercontent.com",
+ "codeload.github.com",
+ "ghcr.io",
+ "authorization:",
+ "password:",
+ "private_key:",
+ "cookie:",
+ "session:",
+ )
+ )
+ )
+
terminal = contract.get("terminal", {})
blockers = list(terminal.get("blockers", []))
checks["expected_blockers_explicit"] = blockers == EXPECTED_BLOCKERS
@@ -240,7 +694,9 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]:
def main() -> int:
parser = argparse.ArgumentParser()
- parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2])
+ parser.add_argument(
+ "--root", type=Path, default=Path(__file__).resolve().parents[2]
+ )
parser.add_argument(
"--contract",
type=Path,
diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
index ef0400221..831e82f3d 100644
--- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
+++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py
@@ -27,7 +27,9 @@ POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.
def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
- assert payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
+ assert (
+ payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
+ )
assert payload["work_item_id"] == "P0-OBS-002"
assert payload["phase"] == "wave_a_dual_allow"
assert payload["routes"]["active_route"] == "incumbent"
@@ -43,14 +45,33 @@ def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() ->
)
-def test_contract_contains_no_credential_values_or_destructive_db_plan() -> None:
- text = CONTRACT.read_text(encoding="utf-8").lower()
- assert "signoz_user_root_password:" not in text
- assert "signoz-api-key:" not in text
- assert "authorization:" not in text
- assert "raw_relational_database_update" in text
- assert "stateful_volume_restore" in text
- assert "apply_allowed_in_wave_a: false" in text
+def test_contract_and_receipt_contain_no_secret_values_or_github_dependency() -> None:
+ contract_text = CONTRACT.read_text(encoding="utf-8").lower()
+ regression_text = POST_CLOSURE_REGRESSION.read_text(encoding="utf-8").lower()
+ forbidden = (
+ "signoz_user_root_password:",
+ "signoz-api-key:",
+ "authorization:",
+ "password:",
+ "private_key:",
+ "cookie:",
+ "session:",
+ "github.com",
+ "api.github.com",
+ "raw.githubusercontent.com",
+ "codeload.github.com",
+ "ghcr.io",
+ )
+ for text in (contract_text, regression_text):
+ assert all(value not in text for value in forbidden)
+
+ assert "raw_relational_database_update" in contract_text
+ assert "stateful_volume_restore" in contract_text
+ assert "apply_allowed_in_wave_a: false" in contract_text
+
+ regression = yaml.safe_load(regression_text)
+ assert regression["scope_boundaries"]["secret_read"] is False
+ assert regression["scope_boundaries"]["github_used"] is False
def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
@@ -69,9 +90,31 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
assert payload["checks"] and all(payload["checks"].values())
assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True
assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True
+ assert payload["checks"]["post_closure_schema_v2"] is True
+ assert payload["checks"]["post_release_600s_verifier_consistent"] is True
+ assert (
+ payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"]
+ is True
+ )
+ assert payload["checks"]["service_registry_runtime_mirror_drift_explicit"]
+ assert payload["checks"]["github_freeze_legacy_asset_drift_explicit"]
+ assert payload["checks"]["monitoring_generator_duplicate_identity_drift_explicit"]
assert "organization_not_initialized" in payload["blockers"]
- assert "signoz_control_plane_health_route_not_deployed" in payload["blockers"]
assert "wave_b_not_authorized" in payload["blockers"]
+ assert payload["blockers"] == [
+ "organization_not_initialized",
+ "direct_ingress_policy_not_closed",
+ "http_bridge_not_supervised",
+ "filelog_normalization_degraded",
+ "wave_b_not_authorized",
+ ]
+ for closed_blocker in (
+ "api_runtime_cooldown_pending",
+ "cd_probe_safety_fix_not_deployed",
+ "backup_collector_restore_guard_not_deployed",
+ "signoz_control_plane_health_route_not_deployed",
+ ):
+ assert closed_blocker not in payload["blockers"]
def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
@@ -129,7 +172,10 @@ def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion()
assert snapshot["trace_id"] == parity["trace_id"]
assert snapshot["route_parity_terminal"] == "pass"
assert snapshot["program_terminal"] == "partial_degraded"
- assert snapshot["controlled_apply_contract"]["learning_writeback"]["status"] == "partial"
+ assert (
+ snapshot["controlled_apply_contract"]["learning_writeback"]["status"]
+ == "partial"
+ )
assert (
snapshot["controlled_apply_contract"]["learning_writeback"][
"durable_gitea_feature_acknowledgement"
@@ -151,13 +197,13 @@ def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() ->
text = OTEL_DAEMONSET.read_text(encoding="utf-8")
assert "id: cri_parser" in text
assert "on_error: send_quiet" not in text
- assert 'if: \'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"\'' in text
+ assert "if: 'body matches \"^[^ ]+ (stdout|stderr) [^ ]* .*$\"'" in text
assert "id: mark_cri_parsed" in text
- assert 'value: parsed_cri' in text
+ assert "value: parsed_cri" in text
assert "id: move_cri_body" in text
assert "if: attributes.log != nil" in text
assert "id: mark_cri_fallback" in text
- assert 'value: fallback_unparsed' in text
+ assert "value: fallback_unparsed" in text
assert 'if: attributes["awoooi.log.parse_status"] == nil' in text
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
@@ -231,11 +277,10 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier()
apply_block = bridge.split(" --apply)", 1)[1].split(" --status)", 1)[0]
learning = (
- 'receipt learning partial '
- '"application_trace_metric_10m_post_verifier_pending"'
+ 'receipt learning partial "application_trace_metric_10m_post_verifier_pending"'
)
terminal = (
- 'receipt terminal partial '
+ "receipt terminal partial "
'"grpc_transport_restored_independent_600s_post_verifier_pending"'
)
assert learning in apply_block
@@ -243,9 +288,8 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier()
assert apply_block.index(learning) < apply_block.index(terminal)
assert "receipt terminal pass" not in apply_block
assert (
- 'receipt terminal pass '
- '"real_application_otlp_grpc_trace_metric_freshness_closed"'
- in verifier
+ "receipt terminal pass "
+ '"real_application_otlp_grpc_trace_metric_freshness_closed"' in verifier
)
@@ -266,7 +310,7 @@ def test_contract_grpc_route_state_matches_verified_runtime_evidence() -> None:
assert grpc["source_fix"]["runtime_verifier_window_seconds"] >= 600
-def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None:
+def test_grpc_runtime_verifier_uses_real_aggregates_and_ten_minute_gate() -> None:
text = GRPC_VERIFIER.read_text(encoding="utf-8")
for expected in (
'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"',
@@ -293,10 +337,7 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
assert grpc["recent_awoooi_api_metric_sample_count"] == 60180
assert grpc["http_only_verifier_blind_spot"] is True
assert grpc["source_fix"]["check_terminal"] == "pass"
- assert (
- grpc["source_fix"]["runtime_apply_status"]
- == "applied_post_verifier_pass"
- )
+ assert grpc["source_fix"]["runtime_apply_status"] == "applied_post_verifier_pass"
assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass"
receipt = yaml.safe_load(GRPC_RUNTIME_RECEIPT.read_text(encoding="utf-8"))
@@ -312,13 +353,431 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
assert regression["baseline"]["verifier_terminal"] == "pass"
- assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False
+ assert (
+ regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"]
+ is True
+ )
assert regression["controlled_recovery"]["stateful_services_touched"] == 0
assert regression["controlled_recovery"]["grpc_4317_listening"] is True
assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False
assert regression["terminal"]["status"] == "partial_degraded"
+def test_post_closure_v2_records_cd_5140_and_config_route_patch() -> None:
+ receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+ assert receipt["schema"] == "awoooi_signoz_post_closure_regression_v2"
+
+ cd = receipt["release_receipts"]["gitea_cd_5140"]
+ assert cd["gitea_run_id"] == 5140
+ assert cd["source_sha"] == "e4da6570a645e504cc93647d396f45f167b3172a"
+ assert cd["workflow_terminal"] == "success"
+ assert cd["focused_tests_passed"] == 6
+ assert cd["unit_tests"] == {"passed": 4817, "skipped": 23, "warnings": 69}
+ assert cd["integration_tests_passed"] == 5
+ assert cd["production_readback"]["build_tag_matched"] is True
+ assert cd["production_readback"]["desired_tag_matched"] is True
+ assert cd["production_readback"]["rollout_terminal"] == "success"
+ assert cd["postdeploy"]["alert_chain_terminal"] == "pass"
+ assert cd["postdeploy"]["playwright_passed"] == 5
+ assert cd["cd_probe"]["receipt_provenance"] == "gitea_run_5140_job_log"
+ assert cd["cd_probe"]["restart_free"] is True
+ assert cd["cd_probe"]["workers"] == 4
+ assert cd["cd_probe"]["safety_reserve_connections"] == 4
+
+ route = receipt["controlled_applies"]["config_route_patch"]
+ assert route["run_id"] == ("P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1")
+ assert route["source_commit"] == "9a9d1464a586ba172d1613db5bc285def43eb3e8"
+ assert route["included_in_release_sha"] == cd["source_sha"]
+ assert route["normalized_asset"]["internal_url"] == "http://192.168.0.110:8080"
+ assert route["normalized_asset"]["public_url"] == "https://signoz.wooo.work"
+ assert route["source_truth_diff"]["otlp_producer_route_changed"] is False
+ assert route["execution_receipt"]["deployment_status"] == "deployed_verified"
+ assert route["terminal"] == "applied_post_verifier_pass"
+
+
+def test_prometheus_target_first_receipt_has_independent_verifier() -> None:
+ receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+ route = receipt["controlled_applies"]["prometheus_target_route"]
+ check = route["check_receipt"]
+ execution = route["execution_receipt"]
+ post = route["post_verifier"]
+ independent = route["independent_verifier"]
+
+ assert route["run_id"] == ("P0-OBS-002-prometheus-route-20260714T215106Z-apply1")
+ assert check["promtool"] == "pass"
+ assert check["candidate_sha"] == execution["active_sha"] == execution["runtime_sha"]
+ assert execution["action"] == "target_first_config_apply"
+ assert execution["backup_sha"] == check["source_sha"]
+ assert post["terminal"] == "target_up_two_scrapes"
+ assert post["first_scrape_at"] == "2026-07-15T05:52:38.146935853+08:00"
+ assert post["second_scrape_at"] == "2026-07-15T05:52:53.146935853+08:00"
+ assert post["first_scrape_at"] < post["second_scrape_at"]
+ assert post["target_count"] == 1
+ assert post["target_health"] == "up"
+ assert post["probe_success"] == 1
+ assert post["legacy_target_count"] == 0
+ assert independent["mode"] == "independent_readback"
+ assert independent["active_sha_matches_runtime"] is True
+ assert independent["target_count"] == 1
+ assert independent["target_health"] == "up"
+ assert independent["probe_success"] == 1
+ assert independent["legacy_target_count"] == 0
+ assert independent["terminal"] == "pass"
+ assert route["rollback"] == {"required": False, "available": True}
+ assert route["terminal"] == "completed_target_first"
+
+
+def test_alert_rules_receipt_hashes_and_runtime_rule_match() -> None:
+ receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+ alert = receipt["controlled_applies"]["alert_rules"]
+ source_sha = alert["check_receipt"]["source_sha"]
+ execution = alert["execution_receipt"]
+ verifier = alert["post_verifier"]
+
+ assert alert["run_id"] == "P0-OBS-002-alert-rules-20260714T215328Z-apply1"
+ assert alert["check_receipt"]["promtool"] == "pass"
+ assert source_sha == execution["remote_sha"]
+ assert source_sha == execution["canonical_sha"]
+ assert source_sha == execution["runtime_sha"]
+ assert execution["hashes_match"] is True
+ assert execution["prometheus_ready"] is True
+ assert verifier["window_seconds"] == 130
+ assert verifier["terminal"] == "pass"
+ assert verifier["signoz_rule_count"] == 1
+ assert verifier["signoz_rule_health"] == "ok"
+ assert verifier["signoz_rule_state"] == "inactive"
+ assert verifier["exact_query"] == (
+ '(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
+ 'job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
+ 'job="blackbox-http"})'
+ )
+ assert alert["terminal"] == "completed_verified"
+
+
+def test_failed_backup_canary_preserves_failure_and_verified_rollback() -> None:
+ receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+ backup = receipt["controlled_applies"]["backup_canary"]
+ assert backup["guard_source"]["deployment_status"] == "deployed_verified"
+ assert len(backup["attempts"]) == 2
+
+ attempt = backup["attempts"][0]
+ assert attempt["run_id"] == ("P0-OBS-002-backup-canary-20260714T213210Z-retry1")
+ assert attempt["terminal"] == "failed_rolled_back"
+ assert attempt["classification"] == (
+ "failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention"
+ )
+ assert attempt["failed_stage"] == "clickhouse_archive_create"
+ assert attempt["archive_container_exit_code"] == 137
+ assert attempt["root_cause"] == (
+ "docker_health_monitor_restarted_collector_during_intentional_backup_stop"
+ )
+ for false_field in (
+ "consistent_archive_valid",
+ "clickhouse_archive_committed",
+ "sqlite_stage_reached",
+ "restic_stage_reached",
+ "retention_stage_reached",
+ "success_stage_reached",
+ ):
+ assert attempt[false_field] is False
+
+ rollback = attempt["rollback"]
+ assert rollback["collector_running"] is True
+ assert rollback["collector_restart_count"] == 0
+ assert rollback["grpc_4317_listening"] is True
+ assert rollback["http_4318_listening"] is True
+ assert rollback["canary_processes_absent"] is True
+ assert rollback["canary_container_absent"] is True
+ assert rollback["current_temp_dir_absent"] is True
+ assert rollback["partial_artifacts_absent"] is True
+ assert not attempt["terminal"].startswith("completed")
+ assert "learning_recorded" not in attempt
+
+ retry2 = backup["attempts"][1]
+ assert retry2["run_id"] == ("P0-OBS-002-backup-canary-20260714T221230Z-retry2")
+ assert retry2["terminal"] == "failed_rolled_back"
+ assert retry2["backup_script_exit_code"] == 1
+ assert retry2["archive_container_exit_code"] == "unknown_not_durably_captured"
+ assert retry2["archive_stderr_receipt"] == "suppressed_by_legacy_pipeline"
+ assert retry2["root_cause"] == (
+ "raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract"
+ )
+ assert retry2["exact_tar_error_proven"] is False
+ assert retry2["monitor_window"]["cron_intervals_crossed"] == 2
+ assert retry2["monitor_window"]["auto_repair_count"] == 0
+ for false_field in (
+ "consistent_archive_valid",
+ "clickhouse_archive_committed",
+ "sqlite_stage_reached",
+ "restic_stage_reached",
+ "retention_stage_reached",
+ "success_stage_reached",
+ ):
+ assert retry2[false_field] is False
+ retry2_rollback = retry2["rollback"]
+ assert retry2_rollback["cooldown_lease_restored"] is True
+ assert retry2_rollback["collector_running"] is True
+ assert retry2_rollback["collector_restart_count"] == 0
+ assert retry2_rollback["grpc_4317_listening"] is True
+ assert retry2_rollback["http_4318_listening"] is True
+ assert retry2_rollback["current_temp_dir_absent"] is True
+ assert retry2_rollback["partial_artifacts_absent"] is True
+ assert retry2["residual_cleanup_debt"] == []
+
+
+def test_clickhouse_native_backup_disk_preserves_failed_attempt_and_closes_retry() -> (
+ None
+):
+ receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+ native_disk = receipt["controlled_applies"]["clickhouse_native_backup_disk"]
+
+ assert native_disk["source"]["override_sha256"] == (
+ "cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488"
+ )
+ assert native_disk["source"]["config_sha256"] == (
+ "c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9"
+ )
+ assert native_disk["check_receipt"]["terminal"] == "check_pass_no_write"
+ assert len(native_disk["attempts"]) == 2
+
+ apply1, retry2 = native_disk["attempts"]
+ assert apply1["terminal"] == "failed_rolled_back"
+ assert apply1["exit_code"] == 2
+ assert apply1["rollback"]["terminal"] == "pass"
+ assert apply1["rollback"]["original_data_volume_preserved"] is True
+ assert apply1["rollback"]["residue_count"] == 0
+
+ assert retry2["run_id"] == (
+ "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
+ )
+ assert retry2["terminal"] == "pass"
+ assert retry2["execution"]["image_digest_unchanged"] is True
+ assert retry2["execution"]["data_volume_unchanged"] is True
+ assert retry2["post_verifier"]["independent"] is True
+ assert retry2["post_verifier"]["server_write_access"] is True
+ assert retry2["post_verifier"]["dependent_restart_count_delta"] == 0
+ assert retry2["post_verifier"]["api_http_status"] == 200
+ assert retry2["post_verifier"]["active_backup_or_restore_count"] == 0
+ assert retry2["post_verifier"]["stage_candidate_process_residue_count"] == 0
+ assert native_disk["terminal"] == "deployed_verified"
+
+
+def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> None:
+ receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+ canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"]
+ assert canary["completion_scope"] == "clickhouse_native_only"
+ assert canary["source"]["post_canary_focused_tests_passed"] == 94
+ assert canary["source"]["post_format_focused_tests_passed"] == 94
+ assert canary["source"]["ruff_format"] == "pass"
+ assert canary["toolchain_controlled_apply"]["apply_terminal"] == "pass"
+ assert canary["toolchain_controlled_apply"]["postcheck_diff"] == (
+ "matching_7_drift_0_absent_0_active_operations_0"
+ )
+ assert len(canary["attempts"]) == 4
+ _, apply2, apply3, apply4 = canary["attempts"]
+ assert apply2["artifact_retention"] == "preserved_failed_run_evidence"
+ assert apply2["cleanup_verified"] is True
+ assert apply3["artifact_retention"] == "preserved_failed_run_evidence"
+ assert apply3["cleanup_verified"] is True
+
+ assert apply4["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
+ assert apply4["check_terminal"] == "check_pass_no_write"
+ assert apply4["terminal"] == "pass"
+ assert apply4["backup"]["terminal"] == "BACKUP_CREATED"
+ assert apply4["backup"]["error"] == ""
+ assert apply4["backup"]["artifact_sha256"] == (
+ "4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc"
+ )
+
+ restore = apply4["isolated_restore"]
+ assert restore["terminal"] == "verified"
+ assert restore["clickhouse_restore_terminal"] == "RESTORED"
+ assert restore["production_network_attached"] is False
+ assert restore["production_restore_performed"] is False
+ assert restore["source_staged_clickhouse_artifact_sha_match"] is True
+ assert restore["database_count"] == restore["restored_database_count"] == 6
+ assert restore["table_count"] == restore["restored_table_count"] == 100
+ assert restore["table_engine_schema_manifest_parity"] is True
+ assert restore["check_table_count"] == restore["check_table_pass_count"] == 44
+ assert restore["critical_nonzero_count"] == 4
+ assert restore["aggregate_counter_exact_parity_gate"] is False
+
+ restic = apply4["restic"]
+ assert restic["snapshot_id"] == "574e2a1a"
+ assert restic["snapshot_readback"] == "pass"
+ assert restic["repository_scope"] == "local_same_failure_domain"
+ assert restic["offsite_verified"] is False
+
+ post = apply4["independent_post_verifier"]
+ assert post["production_restart_count_delta"] == 0
+ assert post["api_http_status"] == 200
+ assert post["new_server_artifact_removed"] is True
+ assert post["apply2_apply3_preserved_artifact_hashes_unchanged"] is True
+ assert (
+ post["exact_ephemeral_container_volume_network_process_tmp_residue_count"] == 0
+ )
+ assert post["cleanup_verified"] is True
+ assert post["secret_values_collected"] is False
+ assert canary["terminal"] == "production_closed_clickhouse_native_scope"
+
+
+def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None:
+ receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
+ verifier = receipt["post_release_runtime_verifier"]
+ assert verifier["run_id"] == "P0-OBS-002-post-release-20260714T215500Z"
+ assert verifier["window_seconds"] == 600
+ assert verifier["terminal"] == "pass"
+ assert verifier["restart_delta"] == 0
+ assert verifier["exporter_trace_errors"] == 0
+ assert verifier["exporter_metric_errors"] == 0
+ assert verifier["trace_count"] == 4026
+ assert verifier["metric_sample_count"] == 76369
+ assert verifier["api_runtime_cooldown_closed"] is True
+ assert (
+ receipt["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is True
+ )
+
+ closed = receipt["completed_verifiers"]["clickhouse_native_backup_migration"]
+ assert closed["prior_run_id"] == (
+ "P0-OBS-002-backup-canary-20260714T221230Z-retry2"
+ )
+ assert closed["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
+ assert closed["status"] == "production_closed_clickhouse_native_scope"
+ assert closed["native_backup_terminal"] == "BACKUP_CREATED"
+ assert closed["isolated_restore_terminal"] == "verified"
+ assert closed["restic_snapshot_id"] == "574e2a1a"
+ assert closed["independent_verifier"] == "pass"
+ assert closed["exact_cleanup"] == "pass"
+ sqlite_pending = receipt["pending_verifiers"][
+ "signoz_sqlite_application_consistent_backup"
+ ]
+ assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False
+ assert sqlite_pending["sqlite_cli_present_in_runtime"] is False
+ offsite = receipt["pending_verifiers"]["signoz_backup_offsite_dr"]
+ assert offsite["current_repository_scope"] == "local_same_failure_domain"
+ assert offsite["offsite_verified"] is False
+ artifacts = receipt["pending_verifiers"][
+ "clickhouse_native_failed_artifact_retention"
+ ]
+ assert artifacts["destructive_cleanup_authorized"] is False
+ assert artifacts["active_backup_or_restore_count"] == 0
+ assert len(artifacts["preserved_artifacts"]) == 2
+ resume = receipt["pending_verifiers"][
+ "clickhouse_native_resume_submitted_inventory"
+ ]
+ assert resume["same_run_resume_safe"] is False
+ assert len(resume["required_preconditions"]) == 4
+ assets = receipt["asset_reconciliation"]
+ assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001"
+ assert assets["source_service_count"] == 27
+ assert assets["runtime_mirror_service_count"] == 24
+ assert assets["exact_shared_service_count"] == 24
+ assert assets["shared_service_drift_count"] == 0
+ assert assets["source_only_services"] == ["bitan-app", "sentry", "signoz"]
+ assert assets["signoz_clickhouse_exact_match"] is True
+ assert assets["live_signoz_query_host"] == "192.168.0.110"
+ assert assets["source_signoz_host"] == "192.168.0.188"
+ github_drift = assets["github_freeze_legacy_asset_drift"]
+ assert github_drift["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-002"
+ assert github_drift["superseded_by"] == "global_product_governance_v2"
+ assert github_drift["active_github_use_in_this_run"] is False
+ assert len(github_drift["source_references"]) == 3
+ assert github_drift["p0_p1_github_recovery_scheduled"] is False
+ duplicate_identity = assets["monitoring_generator_duplicate_identity_drift"]
+ assert duplicate_identity["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-003"
+ assert duplicate_identity["status"] == "pending_source_deduplication"
+ assert duplicate_identity["awoooi_api_registry_entry_count"] == 2
+ assert duplicate_identity["generated_scrape_config_count"] == 24
+ assert duplicate_identity["generated_awoooi_api_job_count"] == 2
+ assert duplicate_identity["generated_blackbox_target_count"] == 20
+ assert duplicate_identity["runtime_apply_performed"] is False
+ assert receipt["terminal"]["status"] == "partial_degraded"
+ assert receipt["terminal"]["blockers"] == [
+ "signoz_sqlite_application_consistent_backup_pending",
+ "signoz_backup_offsite_dr_pending",
+ "clickhouse_native_failed_artifact_retention_pending",
+ "clickhouse_native_resume_submitted_inventory_pending",
+ "service_registry_runtime_mirror_reconciliation_pending",
+ "github_freeze_legacy_asset_retirement_pending",
+ "monitoring_generator_duplicate_awoooi_api_identity_pending",
+ ]
+
+
+def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> None:
+ contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
+ post_closure = contract["runtime_observations"]["post_closure_regression"]
+ assert post_closure["snapshot_schema"] == (
+ "awoooi_signoz_post_closure_regression_v2"
+ )
+ assert post_closure["gitea_cd_run_id"] == 5140
+ assert post_closure["api_runtime_cooldown_closed"] is True
+ assert post_closure["cd_probe_safety_fix_deployed"] is True
+ assert post_closure["config_route_patch_deployed"] is True
+ assert post_closure["signoz_control_plane_health_route_deployed"] is True
+ assert post_closure["backup_collector_restore_guard_deployed"] is True
+ assert post_closure["backup_canary_scope"] == "clickhouse_native_only"
+ assert post_closure["backup_canary_verified"] is True
+ assert post_closure["backup_canary_last_terminal"] == "pass"
+ assert post_closure["backup_canary_last_run_id"] == (
+ "P0-OBS-002-native-canary-20260715T001329Z-apply4"
+ )
+ assert post_closure["backup_canary_cooldown_retry_status"] == (
+ "superseded_by_online_native_no_stop_contract"
+ )
+ assert post_closure["clickhouse_native_backup_migration_status"] == (
+ "production_closed"
+ )
+ assert post_closure["clickhouse_native_backup_disk_run_id"] == (
+ "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
+ )
+ assert post_closure["clickhouse_native_backup_disk_status"] == "deployed_verified"
+ assert post_closure["clickhouse_native_backup_toolchain_run_id"] == (
+ "P0-OBS-002-native-toolchain-20260715T001252Z-apply4"
+ )
+ assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == (
+ "P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4"
+ )
+ assert post_closure["clickhouse_native_restore_terminal"] == "verified"
+ assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a"
+ assert post_closure["clickhouse_native_offsite_status"] == "pending"
+ assert post_closure["clickhouse_native_failed_artifact_retention_status"] == (
+ "pending_bounded_retention_decision"
+ )
+ assert post_closure["clickhouse_native_resume_inventory_status"] == (
+ "pending_durable_submitted_inventory_contract"
+ )
+ assert (
+ post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending"
+ )
+ assert post_closure["service_registry_runtime_mirror_status"] == (
+ "partial_degraded"
+ )
+ assert post_closure["service_registry_runtime_mirror_work_item_id"] == (
+ "P0-OBS-002-ASSET-DRIFT-001"
+ )
+ assert post_closure["github_freeze_legacy_asset_status"] == (
+ "pending_controlled_retirement"
+ )
+ assert post_closure["github_freeze_legacy_asset_work_item_id"] == (
+ "P0-OBS-002-ASSET-DRIFT-002"
+ )
+ assert post_closure["monitoring_generator_duplicate_identity_status"] == (
+ "pending_source_deduplication"
+ )
+ assert post_closure["monitoring_generator_duplicate_identity_work_item_id"] == (
+ "P0-OBS-002-ASSET-DRIFT-003"
+ )
+ assert post_closure["post_release_otlp_grpc_600s_verifier_status"] == "pass"
+ assert contract["terminal"]["status"] == "partial_degraded"
+ assert contract["terminal"]["blockers"] == [
+ "organization_not_initialized",
+ "direct_ingress_policy_not_closed",
+ "http_bridge_not_supervised",
+ "filelog_normalization_degraded",
+ "wave_b_not_authorized",
+ ]
+
+
def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
env = os.environ.copy()
env.pop("MANAGED_PROMOTION_RECEIPT", None)