Merge remote-tracking branch 'origin/main' into codex/playwright-mcp-supply-20260715
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m33s
MCP External Artifact Mirror / mirror-and-verify (push) Successful in 36s
MCP External Protocol Replay / replay-and-verify (push) Successful in 49s
CD Pipeline / build-and-deploy (push) Successful in 14m48s
CD Pipeline / post-deploy-checks (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m33s
MCP External Artifact Mirror / mirror-and-verify (push) Successful in 36s
MCP External Protocol Replay / replay-and-verify (push) Successful in 49s
CD Pipeline / build-and-deploy (push) Successful in 14m48s
CD Pipeline / post-deploy-checks (push) Has been cancelled
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
|
||||
> **管轄範圍**: Docker, K3s, Nginx, NetworkPolicy, CI/CD
|
||||
> **觸發條件**: 修改 `k8s/`, `Dockerfile`, `.github/workflows/`, `nginx`
|
||||
> **治理覆寫**: `superseded_by=global_product_governance_v2`;本文 GitHub/GitHub Actions 段落只保留歷史背景,不得執行。現行 source truth 與 workflow 一律使用 Gitea;舊 SigNoz `192.168.0.188:3301` UI/query 路由同樣失效。
|
||||
|
||||
---
|
||||
|
||||
@@ -190,12 +191,12 @@ echo "${{ secrets.SUDO_PASSWORD }}" | sudo -S kubectl ...
|
||||
|
||||
| 主機 | IP | 角色 | 部署內容 |
|
||||
|------|-----|------|----------|
|
||||
| DevOps | 192.168.0.110 | 金庫 | Harbor:5000, GitHub Runner, Sentry:9000, Langfuse:3100 |
|
||||
| DevOps | 192.168.0.110 | 金庫 | Harbor:5000, Gitea Actions Runner, Sentry:9000, Langfuse:3100, SigNoz health:8080 |
|
||||
| Security | 192.168.0.112 | 安全 | Kali Scanner:8080 |
|
||||
| K3s Server #1 | 192.168.0.120 | 叢集 | Control-Plane MASTER (keepalived priority=101) |
|
||||
| K3s Server #2 | 192.168.0.121 | 叢集 | Control-Plane BACKUP (keepalived priority=100) |
|
||||
| **VIP** | **192.168.0.125** | **HA 入口** | **K3s API:6443 + NodePort (32334/32335)** |
|
||||
| AI/Web | 192.168.0.188 | 大腦 | Nginx, **PostgreSQL:5432 (K3s Datastore)**, Redis:6380, Ollama:11434, OpenClaw:8089, SigNoz:3301 |
|
||||
| AI/Web | 192.168.0.188 | 大腦 | Nginx, **PostgreSQL:5432 (K3s Datastore)**, Redis:6380, Ollama:11434, OpenClaw:8089, OTLP bridge:24317/24318 |
|
||||
|
||||
### 絕對禁令
|
||||
|
||||
@@ -421,7 +422,8 @@ kubectl --server=https://192.168.0.125:6443 get nodes
|
||||
|
||||
| 服務 | 端點 | 用途 |
|
||||
|------|------|------|
|
||||
| **SignOz Web UI** | `http://192.168.0.188:3301` | APM Dashboard |
|
||||
| **SigNoz machine health** | `http://192.168.0.110:8080/api/v1/health` | service-to-service health/readback |
|
||||
| **SigNoz Web UI** | `https://signoz.wooo.work` | human APM dashboard / deep links |
|
||||
| **OTEL gRPC** | `192.168.0.188:24317` | K8s/應用程式 Traces |
|
||||
| **OTEL HTTP** | `http://192.168.0.188:24318` | CI/CD workflows |
|
||||
| **Sentry** | `http://192.168.0.110:9000` | Error Tracking |
|
||||
@@ -451,8 +453,8 @@ OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318
|
||||
### 驗證指令
|
||||
|
||||
```bash
|
||||
# 檢查 SignOz 容器運行狀態
|
||||
ssh ollama@192.168.0.188 "docker ps | grep signoz"
|
||||
# 檢查 SigNoz machine health
|
||||
curl -fsS http://192.168.0.110:8080/api/v1/health | jq .
|
||||
|
||||
# 測試 OTEL 端點
|
||||
curl -s http://192.168.0.188:24318/v1/traces -X POST | head -c 100
|
||||
@@ -1165,7 +1167,7 @@ def verify_github_signature(payload: bytes, signature: str, secret: str) -> bool
|
||||
┌─────────────┐ trace_id ┌─────────────┐ trace_id ┌─────────────┐
|
||||
│ Sentry │ ◄────────────────► │ SignOz │ ◄────────────────► │ Langfuse │
|
||||
│ Errors │ │ Traces │ │ LLMOps │
|
||||
│ :9000 (.110)│ │ :3301 (.188)│ │ :3100 (.110)│
|
||||
│ :9000 (.110)│ │ signoz.wooo │ │ :3100 (.110)│
|
||||
└─────────────┘ └─────────────┘ └─────────────┘
|
||||
```
|
||||
|
||||
@@ -1173,7 +1175,7 @@ def verify_github_signature(payload: bytes, signature: str, secret: str) -> bool
|
||||
|
||||
| 系統 | URL 格式 |
|
||||
|------|----------|
|
||||
| SignOz Trace | `http://192.168.0.188:3301/trace/{trace_id}` |
|
||||
| SigNoz Trace | `https://signoz.wooo.work/trace/{trace_id}` |
|
||||
| Langfuse Trace | `http://192.168.0.110:3100/project/awoooi-openclaw/traces/{id}` |
|
||||
| Sentry Issue | `http://192.168.0.110:9000/organizations/sentry/issues/{id}/` |
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
> **管轄範圍**: Playwright, API Testing, Health Monitoring
|
||||
> **觸發條件**: 懷疑 UI 異常、API 錯誤、部署驗證
|
||||
> **治理覆寫**: `superseded_by=global_product_governance_v2`;本文 `.github/workflows` 與舊 SigNoz `192.168.0.188:3301` 測試僅屬歷史背景。現行 CI/CD 使用 Gitea,SigNoz health/UI/OTLP 依 canonical route 分流。
|
||||
|
||||
---
|
||||
|
||||
@@ -494,8 +495,8 @@ kubectl rollout undo deployment/awoooi-api -n awoooi-prod
|
||||
### 觀測系統健康檢查
|
||||
|
||||
```bash
|
||||
# SignOz (Traces)
|
||||
curl -f http://192.168.0.188:3301/api/v1/services | jq '.data[] | .serviceName'
|
||||
# SigNoz control-plane health (machine readback)
|
||||
curl -fsS http://192.168.0.110:8080/api/v1/health | jq .
|
||||
|
||||
# Langfuse (LLMOps)
|
||||
curl -f http://192.168.0.110:3100/api/health
|
||||
@@ -511,7 +512,7 @@ curl -f http://192.168.0.110:9000/api/0/internal/health/
|
||||
from src.core.deep_linking import DeepLinking
|
||||
|
||||
# 應返回有效 URL
|
||||
assert DeepLinking.signoz_trace_url("abc123") == "http://192.168.0.188:3301/trace/abc123"
|
||||
assert DeepLinking.signoz_trace_url("abc123") == "https://signoz.wooo.work/trace/abc123"
|
||||
assert DeepLinking.langfuse_trace_url("lf-123") != ""
|
||||
assert DeepLinking.sentry_issue_url("456") != ""
|
||||
```
|
||||
@@ -539,7 +540,7 @@ with restore_trace_context({"trace_id": "", "span_id": ""}) as span:
|
||||
|
||||
| 系統 | URL | 狀態 |
|
||||
|------|-----|------|
|
||||
| SignOz | http://192.168.0.188:3301 | ✅ |
|
||||
| SigNoz | https://signoz.wooo.work (machine health: `192.168.0.110:8080/api/v1/health`) | ✅ |
|
||||
| Langfuse | http://192.168.0.110:3100 | ✅ |
|
||||
| Sentry | http://192.168.0.110:9000 | ✅ |
|
||||
| Deep Linking | N/A (內建) | ✅ |
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
> **管轄範圍**: MCP Bridge, 外部系統連接, RAG 向量化
|
||||
> **觸發條件**: 修改 `plugins/mcp/`, `services/*_tool.py`, 向量資料庫
|
||||
> **治理覆寫**: `superseded_by=global_product_governance_v2`;舊 SigNoz `192.168.0.188:3301` query/UI route 已失效,machine health、public query/UI 與 OTLP ingestion 必須使用下列 canonical endpoints。
|
||||
|
||||
---
|
||||
|
||||
@@ -100,7 +101,11 @@ Phase 13.2 Tool 實作 (P0 最優先):
|
||||
|
||||
```
|
||||
ClickHouse HTTP API: http://192.168.0.188:8123
|
||||
SignOz Query API: http://192.168.0.188:3301/api/v3
|
||||
SigNoz machine health: http://192.168.0.110:8080/api/v1/health
|
||||
SigNoz Query API (machine/MCP): http://192.168.0.110:8080/api/v3
|
||||
SigNoz UI / deep links: https://signoz.wooo.work
|
||||
OTLP gRPC: http://192.168.0.188:24317
|
||||
OTLP HTTP: http://192.168.0.188:24318
|
||||
```
|
||||
|
||||
### 查詢範例
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
> **生效日期**: 2026-03-21
|
||||
> **適用對象**: Claude Code / Cursor / GitHub Copilot
|
||||
> **強制等級**: 絕對遵守 (違反等同事故)
|
||||
> **治理覆寫**: `superseded_by=global_product_governance_v2`;本文內以 GitHub 為 source truth、以 `192.168.0.188:3301` 存取 SigNoz UI/query,或以未受控人工 gate 作為終局的舊指令均已失效。現行 source truth/CI/CD 僅使用 Gitea;SigNoz machine health、human deep link 與 OTLP ingestion 必須分流。
|
||||
|
||||
---
|
||||
|
||||
@@ -310,9 +311,9 @@ import { Eye, Sparkles } from 'lucide-react'
|
||||
|
||||
| IP | 名稱 | 職責 |
|
||||
|-----|------|------|
|
||||
| **192.168.0.110** | DevOps 金庫 | Harbor:5000 (`awoooi/` Project), GH Runner (`awoooi-runner`) |
|
||||
| **192.168.0.110** | DevOps 金庫 | Harbor:5000 (`awoooi/` Project), Gitea:3001, **SigNoz machine health:8080 (`/api/v1/health`)** |
|
||||
| **192.168.0.112** | Kali Security | Scanner API:8080 (Header 區分: `X-Source: awoooi`) |
|
||||
| **192.168.0.188** | AI+Web 中心 | Nginx SSL, Ollama:11434, **OpenClaw:8089**, **SigNoz:3301 (OTEL 收集器)** |
|
||||
| **192.168.0.188** | AI+Web 中心 | Nginx SSL, Ollama:11434, **OpenClaw:8089**, **OTLP bridge:24317/24318** |
|
||||
| **192.168.0.120** | K3s Master | **awoooi-prod (API:32334, Frontend:32335)** ⚠️ 唯一部署目標 |
|
||||
| **192.168.0.121** | K3s Worker | HA 備援 |
|
||||
|
||||
@@ -344,10 +345,10 @@ import { Eye, Sparkles } from 'lucide-react'
|
||||
| 1 | **kustomization 禁寫 newTag** | 導致 Ghost Rollback,Tag 由 CI 動態注入 |
|
||||
| 2 | **CI/CD 禁止 git push 回 main** | GitOps Lite 已永久停用 |
|
||||
| 3 | **映像標籤必須唯一不可變** | 格式: `{sha}-{run_id}`,禁用 latest/main/dev |
|
||||
| 4 | **禁止 Gitea ↔ GitHub 雙向同步** | 唯一來源: GitHub |
|
||||
| 4 | **禁止 Gitea ↔ GitHub 雙向同步** | 唯一來源: Gitea;GitHub 全面停用 |
|
||||
| 5 | **LLM 調用必須走快取** | 使用 `_call_with_cache`,禁止直接調用 |
|
||||
| 6 | **Markdown 產出必須遵守格式** | 頭部元數據 + 更新 LOGBOOK |
|
||||
| 7 | **可觀測性必須打向 SigNoz** | OTEL Traces/Logs → 192.168.0.188:3301 |
|
||||
| 7 | **SigNoz 流量必須依用途分流** | machine health → `192.168.0.110:8080/api/v1/health`;human UI → `https://signoz.wooo.work`;OTLP → `192.168.0.188:24317/24318` |
|
||||
| 8 | **UI 文字禁止硬編碼** | 100% next-intl 字典檔 (zh-TW/en) |
|
||||
|
||||
---
|
||||
@@ -366,12 +367,14 @@ from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
|
||||
# SigNoz 端點 (絕對禁止寫死其他地址)
|
||||
SIGNOZ_ENDPOINT = "http://192.168.0.188:4317"
|
||||
# SigNoz canonical endpoints:健康、UI、遙測攝取不可混用
|
||||
SIGNOZ_HEALTH_URL = "http://192.168.0.110:8080/api/v1/health"
|
||||
SIGNOZ_PUBLIC_URL = "https://signoz.wooo.work"
|
||||
OTEL_GRPC_ENDPOINT = "http://192.168.0.188:24317"
|
||||
|
||||
def setup_telemetry(app):
|
||||
provider = TracerProvider()
|
||||
processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=SIGNOZ_ENDPOINT))
|
||||
processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=OTEL_GRPC_ENDPOINT))
|
||||
provider.add_span_processor(processor)
|
||||
trace.set_tracer_provider(provider)
|
||||
FastAPIInstrumentor.instrument_app(app)
|
||||
@@ -440,9 +443,9 @@ AI 可跨資料夾讀取以下舊專案文檔 (唯讀):
|
||||
|
||||
| IP | 職責 | 允許操作 |
|
||||
|-----|------|----------|
|
||||
| **192.168.0.110** | DevOps 金庫 (Harbor) | Docker Registry 操作 |
|
||||
| **192.168.0.110** | DevOps 金庫 (Harbor + SigNoz control plane) | Docker Registry、SigNoz machine health (`:8080/api/v1/health`) |
|
||||
| **192.168.0.112** | Kali Security | 安全掃描 API 呼叫 |
|
||||
| **192.168.0.188** | AI+Web 中心 | **Nginx 部署**、Ollama、SigNoz |
|
||||
| **192.168.0.188** | AI+Web 中心 | **Nginx 部署**、Ollama、OTLP bridge (`:24317/:24318`) |
|
||||
| **192.168.0.120** | K3s Master | kubectl 操作、NodePort 存取 |
|
||||
| **192.168.0.121** | K3s Worker | HA 備援 (禁止直接操作) |
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import time
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
@@ -29,9 +30,12 @@ from src.services.ai_automation_asset_capability_matrix import (
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
_FIRST_DELAY_SECONDS = 30
|
||||
_FIRST_DELAY_SECONDS = 7 * 60
|
||||
_LOOP_BACKOFF_SECONDS = 5 * 60
|
||||
_RECONCILIATION_POLL_SECONDS = 5 * 60
|
||||
_SOURCE_SETTLE_SECONDS = 5 * 60
|
||||
_SOURCE_STABILITY_DELAY_SECONDS = 5
|
||||
_SOURCE_STABILITY_RETRY_SECONDS = 60
|
||||
_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS = 30.0
|
||||
_WORK_ITEM_INSERT_BATCH_SIZE = 250
|
||||
_MAX_RECONCILIATION_CANDIDATES = 20_000
|
||||
@@ -39,6 +43,8 @@ _RETRYABLE_RESULT_STATUSES = {
|
||||
"blocked_safe_no_write",
|
||||
"degraded_cache_refresh",
|
||||
"degraded_no_write",
|
||||
"source_snapshot_settling",
|
||||
"source_snapshot_unstable",
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +56,24 @@ def _canonical_json(value: Any) -> str:
|
||||
return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":"))
|
||||
|
||||
|
||||
def _latest_discovery_age_seconds(
|
||||
matrix: Mapping[str, Any],
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> float | None:
|
||||
ended_at = str(_dict(matrix.get("latest_discovery_run")).get("ended_at") or "")
|
||||
if not ended_at:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(ended_at.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
current = (now or datetime.now(UTC)).astimezone(UTC)
|
||||
return max(0.0, (current - parsed.astimezone(UTC)).total_seconds())
|
||||
|
||||
|
||||
def build_reconciliation_summary_payload(
|
||||
matrix: Mapping[str, Any],
|
||||
candidates: Sequence[Mapping[str, Any]],
|
||||
@@ -177,11 +201,13 @@ def build_reconciliation_work_item_input(
|
||||
|
||||
|
||||
async def run_asset_capability_reconciliation_loop() -> None:
|
||||
"""Reconcile shortly after startup and retry each discovery fingerprint."""
|
||||
"""Reconcile settled source snapshots and retry each fingerprint."""
|
||||
|
||||
logger.info(
|
||||
"asset_capability_reconciliation_loop_started",
|
||||
first_delay_seconds=_FIRST_DELAY_SECONDS,
|
||||
poll_interval_seconds=_RECONCILIATION_POLL_SECONDS,
|
||||
source_settle_seconds=_SOURCE_SETTLE_SECONDS,
|
||||
)
|
||||
await asyncio.sleep(_FIRST_DELAY_SECONDS)
|
||||
triggered_by = "startup"
|
||||
@@ -197,13 +223,20 @@ async def run_asset_capability_reconciliation_loop() -> None:
|
||||
triggered_by = "retry"
|
||||
continue
|
||||
if str(result.get("status") or "") in _RETRYABLE_RESULT_STATUSES:
|
||||
retry_seconds = max(
|
||||
30,
|
||||
min(
|
||||
_LOOP_BACKOFF_SECONDS,
|
||||
int(result.get("retry_seconds") or _LOOP_BACKOFF_SECONDS),
|
||||
),
|
||||
)
|
||||
logger.warning(
|
||||
"asset_capability_reconciliation_retry_scheduled",
|
||||
status=result.get("status"),
|
||||
reason=result.get("reason"),
|
||||
retry_seconds=_LOOP_BACKOFF_SECONDS,
|
||||
retry_seconds=retry_seconds,
|
||||
)
|
||||
await asyncio.sleep(_LOOP_BACKOFF_SECONDS)
|
||||
await asyncio.sleep(retry_seconds)
|
||||
triggered_by = "retry"
|
||||
continue
|
||||
await asyncio.sleep(_RECONCILIATION_POLL_SECONDS)
|
||||
@@ -235,6 +268,70 @@ async def reconcile_once(
|
||||
"candidate_count": 0,
|
||||
}
|
||||
|
||||
discovery_age_seconds = _latest_discovery_age_seconds(matrix)
|
||||
if discovery_age_seconds is None or discovery_age_seconds < _SOURCE_SETTLE_SECONDS:
|
||||
remaining_seconds = (
|
||||
_SOURCE_STABILITY_RETRY_SECONDS
|
||||
if discovery_age_seconds is None
|
||||
else max(
|
||||
_SOURCE_STABILITY_RETRY_SECONDS,
|
||||
int(_SOURCE_SETTLE_SECONDS - discovery_age_seconds) + 1,
|
||||
)
|
||||
)
|
||||
logger.info(
|
||||
"asset_capability_reconciliation_source_snapshot_settling",
|
||||
discovery_age_seconds=discovery_age_seconds,
|
||||
required_age_seconds=_SOURCE_SETTLE_SECONDS,
|
||||
retry_seconds=remaining_seconds,
|
||||
)
|
||||
return {
|
||||
"status": "source_snapshot_settling",
|
||||
"reason": "latest_discovery_run_not_settled",
|
||||
"discovery_age_seconds": discovery_age_seconds,
|
||||
"required_age_seconds": _SOURCE_SETTLE_SECONDS,
|
||||
"retry_seconds": remaining_seconds,
|
||||
"external_runtime_write_performed": False,
|
||||
}
|
||||
|
||||
await asyncio.sleep(_SOURCE_STABILITY_DELAY_SECONDS)
|
||||
stable_matrix = await build_asset_capability_matrix_with_live_readback(
|
||||
project_id=project_id,
|
||||
timeout_seconds=_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS,
|
||||
include_live_only_assets=True,
|
||||
)
|
||||
if stable_matrix.get("live_readback_status") != "ready":
|
||||
return {
|
||||
"status": "degraded_no_write",
|
||||
"reason": "stability_verifier_live_readback_degraded",
|
||||
"live_readback_status": stable_matrix.get("live_readback_status"),
|
||||
"candidate_count": 0,
|
||||
"retry_seconds": _SOURCE_STABILITY_RETRY_SECONDS,
|
||||
"external_runtime_write_performed": False,
|
||||
}
|
||||
|
||||
first_identity = (
|
||||
str(matrix.get("matrix_fingerprint") or ""),
|
||||
str(_dict(matrix.get("latest_discovery_run")).get("run_id") or ""),
|
||||
)
|
||||
stable_identity = (
|
||||
str(stable_matrix.get("matrix_fingerprint") or ""),
|
||||
str(_dict(stable_matrix.get("latest_discovery_run")).get("run_id") or ""),
|
||||
)
|
||||
if not all(stable_identity) or stable_identity != first_identity:
|
||||
logger.warning(
|
||||
"asset_capability_reconciliation_source_snapshot_unstable",
|
||||
discovery_run_changed=first_identity[1] != stable_identity[1],
|
||||
matrix_fingerprint_changed=first_identity[0] != stable_identity[0],
|
||||
retry_seconds=_SOURCE_STABILITY_RETRY_SECONDS,
|
||||
)
|
||||
return {
|
||||
"status": "source_snapshot_unstable",
|
||||
"reason": "matrix_changed_during_stability_verifier",
|
||||
"retry_seconds": _SOURCE_STABILITY_RETRY_SECONDS,
|
||||
"external_runtime_write_performed": False,
|
||||
}
|
||||
matrix = stable_matrix
|
||||
|
||||
candidates = build_reconciliation_candidates(matrix)
|
||||
if len(candidates) > _MAX_RECONCILIATION_CANDIDATES:
|
||||
logger.error(
|
||||
|
||||
@@ -29,7 +29,6 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json as _json
|
||||
import time as _time
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
|
||||
@@ -38,6 +37,7 @@ logger = structlog.get_logger(__name__)
|
||||
_TRACK_INTERVAL_SEC = 3600
|
||||
_FIRST_DELAY_SEC = 360
|
||||
_LOOP_BACKOFF_SEC = 600
|
||||
_PROJECT_ID = "awoooi"
|
||||
|
||||
|
||||
async def run_asset_change_tracker_loop() -> None:
|
||||
@@ -92,9 +92,10 @@ async def track_once() -> dict[str, int]:
|
||||
async def _get_recent_runs(limit: int = 2) -> list[str]:
|
||||
"""取最近 N 個 success 的 run_id (降序)."""
|
||||
from sqlalchemy import text as _sql
|
||||
|
||||
from src.db.base import get_db_context
|
||||
|
||||
async with get_db_context() as db:
|
||||
async with get_db_context(_PROJECT_ID) as db:
|
||||
rows = await db.execute(
|
||||
_sql("SELECT run_id FROM asset_discovery_run WHERE status='success' ORDER BY ended_at DESC LIMIT :lim"),
|
||||
{"lim": limit},
|
||||
@@ -113,11 +114,12 @@ async def _diff_runs(newer_run: str, older_run: str) -> dict[str, int]:
|
||||
- newer ∩ older 且 metadata 變 = modified
|
||||
"""
|
||||
from sqlalchemy import text as _sql
|
||||
|
||||
from src.db.base import get_db_context
|
||||
|
||||
stats = {"added": 0, "removed": 0, "modified": 0}
|
||||
|
||||
async with get_db_context() as db:
|
||||
async with get_db_context(_PROJECT_ID) as db:
|
||||
# 1. Added: newer run 有但 older run 沒有
|
||||
result = await db.execute(
|
||||
_sql("""
|
||||
@@ -216,10 +218,11 @@ async def _diff_runs(newer_run: str, older_run: str) -> dict[str, int]:
|
||||
async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) -> None:
|
||||
try:
|
||||
from sqlalchemy import text as _sql
|
||||
|
||||
from src.db.base import get_db_context
|
||||
|
||||
aol_status = "failed" if error else "success"
|
||||
async with get_db_context() as db:
|
||||
async with get_db_context(_PROJECT_ID) as db:
|
||||
await db.execute(
|
||||
_sql("""
|
||||
INSERT INTO automation_operation_log (
|
||||
|
||||
@@ -206,7 +206,7 @@ async def run_asset_scanner_loop() -> None:
|
||||
|
||||
while True:
|
||||
try:
|
||||
await scan_once(triggered_by="cron")
|
||||
await scan_once(triggered_by="cron", raise_on_failure=True)
|
||||
except Exception as e:
|
||||
logger.exception("asset_scanner_loop_error", error=str(e))
|
||||
await asyncio.sleep(_LOOP_BACKOFF_SEC)
|
||||
@@ -227,6 +227,7 @@ async def scan_once(
|
||||
"domain_tls_inventory",
|
||||
),
|
||||
scan_depth: str = "shallow",
|
||||
raise_on_failure: bool = False,
|
||||
) -> str:
|
||||
"""
|
||||
執行一次資產盤點。
|
||||
@@ -235,6 +236,7 @@ async def scan_once(
|
||||
triggered_by: cron / ai / human / incident
|
||||
scope: 本次掃描範圍標籤 (寫入 asset_discovery_run.scope)
|
||||
scan_depth: shallow (僅 list) / deep (含 describe) / full
|
||||
raise_on_failure: durable terminal 寫入後拋錯,供排程進入 bounded retry
|
||||
|
||||
Returns:
|
||||
run_id (UUID 字串)。無法建立 durable header 時會拋出例外。
|
||||
@@ -324,6 +326,8 @@ async def scan_once(
|
||||
disappeared=disappeared_count,
|
||||
duration_ms=duration_ms,
|
||||
)
|
||||
if error_msg and raise_on_failure:
|
||||
raise RuntimeError(f"asset_scan_failed:{run_id}:{error_msg}")
|
||||
return run_id
|
||||
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ _SYNC_INTERVAL_SEC = 3600 # 每 1 小時
|
||||
_FIRST_DELAY_SEC = 90 # 啟動後等 90s (等 Prometheus 就緒)
|
||||
_HTTP_TIMEOUT_SEC = 10
|
||||
_LOOP_BACKOFF_SEC = 300
|
||||
_PROJECT_ID = "awoooi"
|
||||
|
||||
_PROM_RULES_ENDPOINT = "/api/v1/rules"
|
||||
|
||||
@@ -65,11 +66,19 @@ async def run_rule_catalog_sync_loop() -> None:
|
||||
|
||||
while True:
|
||||
try:
|
||||
await sync_once()
|
||||
stats = await sync_once()
|
||||
except Exception as e:
|
||||
logger.exception("rule_catalog_sync_loop_error", error=str(e))
|
||||
await asyncio.sleep(_LOOP_BACKOFF_SEC)
|
||||
continue
|
||||
if stats.get("failed", 0) > 0:
|
||||
logger.warning(
|
||||
"rule_catalog_sync_retry_scheduled",
|
||||
failed=stats["failed"],
|
||||
retry_seconds=_LOOP_BACKOFF_SEC,
|
||||
)
|
||||
await asyncio.sleep(_LOOP_BACKOFF_SEC)
|
||||
continue
|
||||
await asyncio.sleep(_SYNC_INTERVAL_SEC)
|
||||
|
||||
|
||||
@@ -78,11 +87,11 @@ async def sync_once() -> dict[str, int]:
|
||||
執行一次 Prometheus → alert_rule_catalog 同步.
|
||||
|
||||
Returns:
|
||||
{"total": N, "new": M, "updated": K, "unchanged": L}
|
||||
失敗時所有值為 0,error 會寫 aol.
|
||||
{"total": N, "new": M, "updated": K, "unchanged": L, "failed": F}
|
||||
失敗時 failed > 0,error 會寫 AOL 並由排程 bounded retry.
|
||||
"""
|
||||
started_ms = _time.time()
|
||||
stats = {"total": 0, "new": 0, "updated": 0, "unchanged": 0}
|
||||
stats = {"total": 0, "new": 0, "updated": 0, "unchanged": 0, "failed": 0}
|
||||
error_msg: str | None = None
|
||||
|
||||
try:
|
||||
@@ -98,6 +107,8 @@ async def sync_once() -> dict[str, int]:
|
||||
stats["unchanged"] += 1
|
||||
except Exception as e:
|
||||
error_msg = f"{type(e).__name__}: {e}"[:1000]
|
||||
processed = stats["new"] + stats["updated"] + stats["unchanged"]
|
||||
stats["failed"] = max(1, stats["total"] - processed)
|
||||
logger.exception("rule_catalog_sync_once_failed", error=error_msg)
|
||||
|
||||
duration_ms = int((_time.time() - started_ms) * 1000)
|
||||
@@ -159,7 +170,7 @@ async def _fetch_prometheus_rules() -> list[dict[str, Any]]:
|
||||
|
||||
def _parse_duration(d: Any) -> int:
|
||||
"""Prometheus 的 duration 可能是 秒 (int/float) 或 '5m' 字串."""
|
||||
if isinstance(d, (int, float)):
|
||||
if isinstance(d, int | float):
|
||||
return int(d)
|
||||
if isinstance(d, str) and d:
|
||||
try:
|
||||
@@ -189,10 +200,11 @@ async def _upsert_rule(rule: dict[str, Any]) -> str:
|
||||
比較 expr/severity/labels/annotations,有變化 → updated,否則 unchanged.
|
||||
"""
|
||||
from sqlalchemy import text as _sql
|
||||
|
||||
from src.db.base import get_db_context
|
||||
|
||||
try:
|
||||
async with get_db_context() as db:
|
||||
async with get_db_context(_PROJECT_ID) as db:
|
||||
row = await db.execute(
|
||||
_sql("""
|
||||
INSERT INTO alert_rule_catalog (
|
||||
@@ -237,7 +249,7 @@ async def _upsert_rule(rule: dict[str, Any]) -> str:
|
||||
return "new" if inserted else "updated"
|
||||
except Exception as e:
|
||||
logger.warning("rule_upsert_failed", rule_name=rule["rule_name"], error=str(e))
|
||||
return "unchanged"
|
||||
raise
|
||||
|
||||
|
||||
async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) -> None:
|
||||
@@ -256,9 +268,10 @@ async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) -
|
||||
|
||||
try:
|
||||
from sqlalchemy import text as _sql
|
||||
|
||||
from src.db.base import get_db_context
|
||||
|
||||
async with get_db_context() as db:
|
||||
async with get_db_context(_PROJECT_ID) as db:
|
||||
await db.execute(
|
||||
_sql("""
|
||||
INSERT INTO automation_operation_log (
|
||||
|
||||
@@ -454,13 +454,24 @@ def test_reconcile_once_publishes_closed_public_matrix_cache(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
live_readback_calls: list[dict[str, object]] = []
|
||||
settled_at = (datetime.now(UTC) - timedelta(minutes=10)).isoformat()
|
||||
|
||||
async def _fake_live_readback(**kwargs: object) -> dict[str, object]:
|
||||
live_readback_calls.append(dict(kwargs))
|
||||
if kwargs["include_live_only_assets"] is True:
|
||||
return {"live_readback_status": "ready"}
|
||||
return {
|
||||
"live_readback_status": "ready",
|
||||
"matrix_fingerprint": "stable-fingerprint",
|
||||
"latest_discovery_run": {
|
||||
"run_id": "discovery-run-1",
|
||||
"ended_at": settled_at,
|
||||
},
|
||||
}
|
||||
return {"live_readback_status": "ready", "closed": True}
|
||||
|
||||
async def _no_wait(_seconds: int) -> None:
|
||||
return None
|
||||
|
||||
async def _fake_persist(
|
||||
_matrix: object,
|
||||
_candidates: object,
|
||||
@@ -481,6 +492,7 @@ def test_reconcile_once_publishes_closed_public_matrix_cache(
|
||||
lambda _matrix: [],
|
||||
)
|
||||
monkeypatch.setattr(reconciliation_job, "_persist_reconciliation", _fake_persist)
|
||||
monkeypatch.setattr(reconciliation_job.asyncio, "sleep", _no_wait)
|
||||
|
||||
result = asyncio.run(
|
||||
reconciliation_job.reconcile_once(
|
||||
@@ -495,6 +507,11 @@ def test_reconcile_once_publishes_closed_public_matrix_cache(
|
||||
"timeout_seconds": 30.0,
|
||||
"include_live_only_assets": True,
|
||||
},
|
||||
{
|
||||
"project_id": "awoooi",
|
||||
"timeout_seconds": 30.0,
|
||||
"include_live_only_assets": True,
|
||||
},
|
||||
{
|
||||
"project_id": "awoooi",
|
||||
"timeout_seconds": 30.0,
|
||||
@@ -505,6 +522,95 @@ def test_reconcile_once_publishes_closed_public_matrix_cache(
|
||||
assert result["public_cache_refreshed"] is True
|
||||
|
||||
|
||||
def test_reconcile_once_waits_for_settled_discovery_before_write(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
async def _fake_live_readback(**_kwargs: object) -> dict[str, object]:
|
||||
return {
|
||||
"live_readback_status": "ready",
|
||||
"matrix_fingerprint": "too-fresh",
|
||||
"latest_discovery_run": {
|
||||
"run_id": "discovery-run-1",
|
||||
"ended_at": datetime.now(UTC).isoformat(),
|
||||
},
|
||||
}
|
||||
|
||||
async def _unexpected_persist(*_args: object, **_kwargs: object) -> None:
|
||||
raise AssertionError("unsettled source must not persist reconciliation")
|
||||
|
||||
monkeypatch.setattr(
|
||||
reconciliation_job,
|
||||
"build_asset_capability_matrix_with_live_readback",
|
||||
_fake_live_readback,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
reconciliation_job,
|
||||
"_persist_reconciliation",
|
||||
_unexpected_persist,
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
reconciliation_job.reconcile_once(
|
||||
triggered_by="controlled_replay",
|
||||
project_id="awoooi",
|
||||
)
|
||||
)
|
||||
|
||||
assert result["status"] == "source_snapshot_settling"
|
||||
assert result["reason"] == "latest_discovery_run_not_settled"
|
||||
assert result["retry_seconds"] >= 60
|
||||
assert result["external_runtime_write_performed"] is False
|
||||
|
||||
|
||||
def test_reconcile_once_retries_when_source_changes_during_stability_check(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
fingerprints = iter(("first-fingerprint", "second-fingerprint"))
|
||||
settled_at = (datetime.now(UTC) - timedelta(minutes=10)).isoformat()
|
||||
|
||||
async def _fake_live_readback(**_kwargs: object) -> dict[str, object]:
|
||||
return {
|
||||
"live_readback_status": "ready",
|
||||
"matrix_fingerprint": next(fingerprints),
|
||||
"latest_discovery_run": {
|
||||
"run_id": "discovery-run-1",
|
||||
"ended_at": settled_at,
|
||||
},
|
||||
}
|
||||
|
||||
async def _no_wait(_seconds: int) -> None:
|
||||
return None
|
||||
|
||||
async def _unexpected_persist(*_args: object, **_kwargs: object) -> None:
|
||||
raise AssertionError("unstable source must not persist reconciliation")
|
||||
|
||||
monkeypatch.setattr(
|
||||
reconciliation_job,
|
||||
"build_asset_capability_matrix_with_live_readback",
|
||||
_fake_live_readback,
|
||||
)
|
||||
monkeypatch.setattr(reconciliation_job.asyncio, "sleep", _no_wait)
|
||||
monkeypatch.setattr(
|
||||
reconciliation_job,
|
||||
"_persist_reconciliation",
|
||||
_unexpected_persist,
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
reconciliation_job.reconcile_once(
|
||||
triggered_by="controlled_replay",
|
||||
project_id="awoooi",
|
||||
)
|
||||
)
|
||||
|
||||
assert result == {
|
||||
"status": "source_snapshot_unstable",
|
||||
"reason": "matrix_changed_during_stability_verifier",
|
||||
"retry_seconds": 60,
|
||||
"external_runtime_write_performed": False,
|
||||
}
|
||||
|
||||
|
||||
def test_reconciliation_loop_retries_degraded_result_before_poll(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
@@ -524,9 +630,9 @@ def test_reconciliation_loop_retries_degraded_result_before_poll(
|
||||
|
||||
monkeypatch.setattr(reconciliation_job, "reconcile_once", _fake_reconcile_once)
|
||||
monkeypatch.setattr(reconciliation_job.asyncio, "sleep", _fake_sleep)
|
||||
monkeypatch.setattr(reconciliation_job, "_FIRST_DELAY_SECONDS", 90)
|
||||
monkeypatch.setattr(reconciliation_job, "_FIRST_DELAY_SECONDS", 420)
|
||||
monkeypatch.setattr(reconciliation_job, "_LOOP_BACKOFF_SECONDS", 1_800)
|
||||
monkeypatch.setattr(reconciliation_job, "_RECONCILIATION_POLL_SECONDS", 1)
|
||||
monkeypatch.setattr(reconciliation_job, "_RECONCILIATION_POLL_SECONDS", 3_600)
|
||||
|
||||
try:
|
||||
asyncio.run(reconciliation_job.run_asset_capability_reconciliation_loop())
|
||||
@@ -536,7 +642,7 @@ def test_reconciliation_loop_retries_degraded_result_before_poll(
|
||||
raise AssertionError("loop should stop at the test cancellation point")
|
||||
|
||||
assert triggered_by_values == ["startup", "retry"]
|
||||
assert sleep_values == [90, 1_800, 1]
|
||||
assert sleep_values == [420, 1_800, 3_600]
|
||||
|
||||
|
||||
def test_priority_overlay_projects_aia_p0_006_runtime_controls() -> None:
|
||||
|
||||
54
apps/api/tests/test_asset_change_tracker_job.py
Normal file
54
apps/api/tests/test_asset_change_tracker_job.py
Normal file
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import src.jobs.asset_change_tracker_job as tracker
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
def __init__(self, rows: list[tuple[str]] | None = None) -> None:
|
||||
self._rows = rows or []
|
||||
self.rowcount = 0
|
||||
|
||||
def fetchall(self) -> list[tuple[str]]:
|
||||
return self._rows
|
||||
|
||||
|
||||
class _FakeDb:
|
||||
async def execute(
|
||||
self,
|
||||
statement: Any,
|
||||
_params: dict[str, Any] | None = None,
|
||||
) -> _FakeResult:
|
||||
if "SELECT run_id FROM asset_discovery_run" in str(statement):
|
||||
return _FakeResult([("newer-run",), ("older-run",)])
|
||||
return _FakeResult()
|
||||
|
||||
|
||||
class _FakeDbContext:
|
||||
async def __aenter__(self) -> _FakeDb:
|
||||
return _FakeDb()
|
||||
|
||||
async def __aexit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_change_tracker_scopes_every_database_path_to_awoooi(monkeypatch: Any) -> None:
|
||||
project_ids: list[str] = []
|
||||
|
||||
def _get_db_context(project_id: str) -> _FakeDbContext:
|
||||
project_ids.append(project_id)
|
||||
return _FakeDbContext()
|
||||
|
||||
monkeypatch.setattr("src.db.base.get_db_context", _get_db_context)
|
||||
|
||||
assert asyncio.run(tracker._get_recent_runs()) == ["newer-run", "older-run"]
|
||||
assert asyncio.run(tracker._diff_runs("newer-run", "older-run")) == {
|
||||
"added": 0,
|
||||
"removed": 0,
|
||||
"modified": 0,
|
||||
}
|
||||
asyncio.run(tracker._log_aol({"added": 0, "removed": 0, "modified": 0}, 1, None))
|
||||
|
||||
assert project_ids == ["awoooi", "awoooi", "awoooi"]
|
||||
@@ -4,6 +4,8 @@ import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from src.jobs import asset_scanner_job
|
||||
|
||||
|
||||
@@ -1547,6 +1549,10 @@ def test_scan_terminal_is_failed_when_any_collector_is_incomplete(monkeypatch) -
|
||||
"RuntimeError: asset_collector_failures=prometheus_targets"
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="asset_scan_failed"):
|
||||
asyncio.run(asset_scanner_job.scan_once(raise_on_failure=True))
|
||||
assert terminal["status"] == "failed"
|
||||
|
||||
|
||||
def test_database_catalog_collector_reads_metadata_only(monkeypatch) -> None:
|
||||
requested_projects: list[str | None] = []
|
||||
|
||||
133
apps/api/tests/test_rule_catalog_sync_job.py
Normal file
133
apps/api/tests/test_rule_catalog_sync_job.py
Normal file
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
import src.jobs.rule_catalog_sync_job as rule_sync
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
def one_or_none(self) -> tuple[str, bool]:
|
||||
return "rule-id", True
|
||||
|
||||
|
||||
class _FakeDb:
|
||||
async def execute(
|
||||
self,
|
||||
_statement: Any,
|
||||
_parameters: dict[str, Any] | None = None,
|
||||
) -> _FakeResult:
|
||||
return _FakeResult()
|
||||
|
||||
|
||||
class _FakeDbContext:
|
||||
async def __aenter__(self) -> _FakeDb:
|
||||
return _FakeDb()
|
||||
|
||||
async def __aexit__(self, *_args: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _rule(name: str = "ExampleRule") -> dict[str, Any]:
|
||||
return {
|
||||
"rule_name": name,
|
||||
"expr": "vector(1)",
|
||||
"duration_seconds": 60,
|
||||
"severity": "warning",
|
||||
"labels": {},
|
||||
"annotations": {},
|
||||
"group_name": "example",
|
||||
}
|
||||
|
||||
|
||||
def test_rule_catalog_database_paths_are_tenant_scoped(monkeypatch: Any) -> None:
|
||||
project_ids: list[str] = []
|
||||
|
||||
def _get_db_context(project_id: str) -> _FakeDbContext:
|
||||
project_ids.append(project_id)
|
||||
return _FakeDbContext()
|
||||
|
||||
monkeypatch.setattr("src.db.base.get_db_context", _get_db_context)
|
||||
|
||||
assert asyncio.run(rule_sync._upsert_rule(_rule())) == "new"
|
||||
asyncio.run(
|
||||
rule_sync._log_aol(
|
||||
{"total": 1, "new": 1, "updated": 0, "unchanged": 0, "failed": 0},
|
||||
duration_ms=1,
|
||||
error=None,
|
||||
)
|
||||
)
|
||||
|
||||
assert project_ids == ["awoooi", "awoooi"]
|
||||
|
||||
|
||||
def test_rule_catalog_failure_is_visible_and_durably_logged(monkeypatch: Any) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
async def _fetch() -> list[dict[str, Any]]:
|
||||
return [_rule("RuleA"), _rule("RuleB")]
|
||||
|
||||
async def _fail_upsert(_item: dict[str, Any]) -> str:
|
||||
raise RuntimeError("database unavailable")
|
||||
|
||||
async def _capture_log(
|
||||
stats: dict[str, int],
|
||||
duration_ms: int,
|
||||
error: str | None,
|
||||
) -> None:
|
||||
captured.update(stats=stats.copy(), duration_ms=duration_ms, error=error)
|
||||
|
||||
monkeypatch.setattr(rule_sync, "_fetch_prometheus_rules", _fetch)
|
||||
monkeypatch.setattr(rule_sync, "_upsert_rule", _fail_upsert)
|
||||
monkeypatch.setattr(rule_sync, "_log_aol", _capture_log)
|
||||
|
||||
result = asyncio.run(rule_sync.sync_once())
|
||||
|
||||
assert result == {
|
||||
"total": 2,
|
||||
"new": 0,
|
||||
"updated": 0,
|
||||
"unchanged": 0,
|
||||
"failed": 2,
|
||||
}
|
||||
assert str(captured["error"]).startswith("RuntimeError: database unavailable")
|
||||
assert captured["stats"] == result
|
||||
|
||||
|
||||
def test_rule_catalog_loop_retries_failed_sync_before_hourly_wait(
|
||||
monkeypatch: Any,
|
||||
) -> None:
|
||||
sync_count = 0
|
||||
sleep_values: list[int] = []
|
||||
|
||||
async def _sync_once() -> dict[str, int]:
|
||||
nonlocal sync_count
|
||||
sync_count += 1
|
||||
return {
|
||||
"total": 1,
|
||||
"new": 0,
|
||||
"updated": 0,
|
||||
"unchanged": 0 if sync_count == 1 else 1,
|
||||
"failed": 1 if sync_count == 1 else 0,
|
||||
}
|
||||
|
||||
async def _sleep(seconds: int) -> None:
|
||||
sleep_values.append(seconds)
|
||||
if len(sleep_values) == 3:
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr(rule_sync, "sync_once", _sync_once)
|
||||
monkeypatch.setattr(rule_sync.asyncio, "sleep", _sleep)
|
||||
monkeypatch.setattr(rule_sync, "_FIRST_DELAY_SEC", 1)
|
||||
monkeypatch.setattr(rule_sync, "_LOOP_BACKOFF_SEC", 2)
|
||||
monkeypatch.setattr(rule_sync, "_SYNC_INTERVAL_SEC", 3)
|
||||
|
||||
try:
|
||||
asyncio.run(rule_sync.run_rule_catalog_sync_loop())
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("loop should stop at the test cancellation point")
|
||||
|
||||
assert sync_count == 2
|
||||
assert sleep_values == [1, 2, 3]
|
||||
@@ -93,6 +93,7 @@ def test_worker_manifest_has_no_execution_transport_or_write_loop() -> None:
|
||||
assert worker_env["AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WINDOW_HOURS"] == "168"
|
||||
assert worker_env["DATABASE_NULL_POOL"] == "true"
|
||||
assert worker_env["DATABASE_NULL_POOL_CONCURRENCY_LIMIT"] == "1"
|
||||
assert worker_env["DATABASE_POOL_TIMEOUT_SECONDS"] == "30"
|
||||
|
||||
|
||||
def test_execution_broker_is_only_workload_with_ssh_transport() -> None:
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -15,9 +17,15 @@ from src.services import signoz_client as signoz_client_module
|
||||
from src.services.host_aggregator import HOST_CONFIGS
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
ALERT_CHAIN_SMOKE = REPO_ROOT / "scripts/alert_chain_smoke_test.py"
|
||||
PROMETHEUS_ROUTE_DEPLOY = (
|
||||
REPO_ROOT / "scripts/ops/deploy-signoz-prometheus-route.sh"
|
||||
)
|
||||
LEGACY_PROMETHEUS_DEPLOY = REPO_ROOT / "k8s/monitoring/deploy-prometheus-config.sh"
|
||||
INTERNAL_URL = "http://192.168.0.110:8080"
|
||||
PUBLIC_URL = "https://signoz.wooo.work"
|
||||
LEGACY_URL = "192.168.0.188:3301"
|
||||
INTERNAL_HEALTH_URL = f"{INTERNAL_URL}/api/v1/health"
|
||||
|
||||
|
||||
def _configmap(relative_path: str) -> dict:
|
||||
@@ -81,7 +89,9 @@ def test_user_facing_deep_links_use_public_route(
|
||||
) -> None:
|
||||
monkeypatch.setattr(deep_linking_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL)
|
||||
monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL)
|
||||
monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL)
|
||||
monkeypatch.setattr(
|
||||
signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL
|
||||
)
|
||||
|
||||
trace_id = "0af7651916cd43dd8448eb211c80319c"
|
||||
assert deep_linking_module.DeepLinking.signoz_trace_url(trace_id) == (
|
||||
@@ -165,9 +175,176 @@ def test_active_api_source_has_no_legacy_signoz_ui_route() -> None:
|
||||
assert offenders == []
|
||||
|
||||
|
||||
def test_cd_alert_chain_smoke_uses_canonical_public_signoz_route() -> None:
|
||||
text = ALERT_CHAIN_SMOKE.read_text(encoding="utf-8")
|
||||
|
||||
assert '"SIGNOZ_PUBLIC_URL"' in text
|
||||
assert PUBLIC_URL in text
|
||||
assert LEGACY_URL not in text
|
||||
|
||||
|
||||
def test_host_asset_map_places_signoz_on_the_canonical_query_host() -> None:
|
||||
canonical_services = HOST_CONFIGS["192.168.0.110"]["services"]
|
||||
legacy_services = HOST_CONFIGS["192.168.0.188"]["services"]
|
||||
|
||||
assert ("SigNoz", 8080, "http", "/api/v1/health") in canonical_services
|
||||
assert not any(service[0] == "SigNoz" for service in legacy_services)
|
||||
|
||||
|
||||
def test_monitoring_registry_splits_query_health_from_otlp_transport() -> None:
|
||||
registry = _configmap("ops/monitoring/service-registry.yaml")
|
||||
services = {service["name"]: service for service in registry["services"]}
|
||||
|
||||
query = services["signoz-ui"]
|
||||
assert query["host"] == "192.168.0.110"
|
||||
assert query["port"] == 8080
|
||||
assert query["health_endpoint"] == "/api/v1/health"
|
||||
assert query["health_type"] == "http"
|
||||
assert query["monitoring"]["prometheus"] is True
|
||||
assert query["monitoring"]["prometheus_scrape"] is False
|
||||
|
||||
collector = services["signoz-collector"]
|
||||
assert collector["host"] == "192.168.0.188"
|
||||
assert collector["port"] == 24317
|
||||
assert collector["health_type"] == "grpc"
|
||||
assert collector["monitoring"]["prometheus_scrape"] is False
|
||||
assert collector["monitoring"]["external_verifier"] == (
|
||||
"scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
|
||||
)
|
||||
|
||||
|
||||
def test_stateful_registry_places_clickhouse_on_canonical_host() -> None:
|
||||
source_registry = _configmap("ops/config/service-registry.yaml")
|
||||
configmap = _configmap("k8s/awoooi-prod/15-service-registry-configmap.yaml")
|
||||
runtime_registry = yaml.safe_load(configmap["data"]["service-registry.yaml"])
|
||||
|
||||
for registry in (source_registry, runtime_registry):
|
||||
services = {service["name"]: service for service in registry["services"]}
|
||||
clickhouse = services["signoz-clickhouse"]
|
||||
assert clickhouse["host"] == "192.168.0.110"
|
||||
assert clickhouse["stateful_level"] == "BLOCK"
|
||||
assert clickhouse["alert_only"] is True
|
||||
|
||||
|
||||
def test_active_prometheus_targets_canonical_http_health_not_legacy_tcp() -> None:
|
||||
config = _configmap("k8s/monitoring/prometheus.yml")
|
||||
jobs = {job["job_name"]: job for job in config["scrape_configs"]}
|
||||
http_job = jobs["blackbox-http"]
|
||||
http_targets = http_job["static_configs"][0]["targets"]
|
||||
tcp_targets = jobs["blackbox-tcp"]["static_configs"][0]["targets"]
|
||||
|
||||
assert INTERNAL_HEALTH_URL in http_targets
|
||||
assert LEGACY_URL not in tcp_targets
|
||||
assert http_job["relabel_configs"] == [
|
||||
{
|
||||
"source_labels": ["__address__"],
|
||||
"target_label": "__param_target",
|
||||
},
|
||||
{
|
||||
"source_labels": ["__param_target"],
|
||||
"target_label": "instance",
|
||||
},
|
||||
{
|
||||
"target_label": "__address__",
|
||||
"replacement": "192.168.0.110:9115",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_signoz_prometheus_route_deploy_is_target_first_and_rollback_safe() -> None:
|
||||
text = PROMETHEUS_ROUTE_DEPLOY.read_text(encoding="utf-8")
|
||||
|
||||
assert 'MODE="${1:-apply}"' in text
|
||||
assert "promtool check config" in text
|
||||
assert "cp -p \"${REMOTE_CONFIG}\" \"${BACKUP_CONFIG}\"" in text
|
||||
assert 'action=target_first_config_apply' in text
|
||||
assert text.index("promtool check config") < text.index(
|
||||
'action=target_first_config_apply'
|
||||
)
|
||||
assert text.index('action=target_first_config_apply') < text.index(
|
||||
"terminal=target_up_two_scrapes"
|
||||
)
|
||||
assert "rollback_deploy" in text
|
||||
assert INTERNAL_HEALTH_URL in text
|
||||
assert LEGACY_URL in text
|
||||
|
||||
|
||||
def test_legacy_prometheus_deployer_routes_to_bounded_host110_replacement() -> None:
|
||||
text = LEGACY_PROMETHEUS_DEPLOY.read_text(encoding="utf-8")
|
||||
|
||||
assert "superseded_by=global_product_governance_v2" in text
|
||||
assert "expiry=2026-07-15" in text
|
||||
assert "scripts/ops/deploy-signoz-prometheus-route.sh" in text
|
||||
assert "192.168.0.188" not in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"relative_path",
|
||||
[
|
||||
"ops/monitoring/alerts-unified.yml",
|
||||
"ops/monitoring/alerts.yml",
|
||||
],
|
||||
)
|
||||
def test_signoz_down_alert_is_exact_and_fails_closed_on_missing_series(
|
||||
relative_path: str,
|
||||
) -> None:
|
||||
payload = _configmap(relative_path)
|
||||
rules = [
|
||||
rule
|
||||
for group in payload["groups"]
|
||||
for rule in group.get("rules", [])
|
||||
if rule.get("alert") == "SignOzDown"
|
||||
]
|
||||
|
||||
assert len(rules) == 1
|
||||
rule = rules[0]
|
||||
assert 'job="blackbox-http"' in rule["expr"]
|
||||
assert INTERNAL_HEALTH_URL in rule["expr"]
|
||||
assert "absent(" in rule["expr"]
|
||||
assert LEGACY_URL not in rule["expr"]
|
||||
assert rule["labels"]["host"] == "110"
|
||||
assert rule["labels"]["target_host"] == "192.168.0.110"
|
||||
|
||||
|
||||
def test_legacy_188_rule_plane_does_not_duplicate_canonical_signoz_alert() -> None:
|
||||
payload = _configmap("k8s/monitoring/k3s-alerts.yaml")
|
||||
signoz_rules = [
|
||||
rule
|
||||
for group in payload["groups"]
|
||||
for rule in group.get("rules", [])
|
||||
if rule.get("alert") == "SignOzDown"
|
||||
]
|
||||
text = (REPO_ROOT / "k8s/monitoring/k3s-alerts.yaml").read_text(encoding="utf-8")
|
||||
|
||||
assert signoz_rules == []
|
||||
assert LEGACY_URL not in text
|
||||
|
||||
|
||||
def test_monitoring_generator_uses_blackbox_without_scraping_signoz_spa(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "ops/monitoring/generate_monitoring.py"),
|
||||
"--output-dir",
|
||||
str(tmp_path),
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
scrape = _configmap(str(tmp_path / "prometheus-scrape-generated.yaml"))
|
||||
blackbox = yaml.safe_load(
|
||||
(tmp_path / "blackbox-targets-generated.yaml").read_text(encoding="utf-8")
|
||||
)
|
||||
generated_jobs = {job["job_name"] for job in scrape["scrape_configs"]}
|
||||
assert "signoz-ui" not in generated_jobs
|
||||
assert "signoz-collector" not in generated_jobs
|
||||
assert any(
|
||||
target["labels"].get("service") == "signoz-ui"
|
||||
and target["url"] == INTERNAL_HEALTH_URL
|
||||
for target in blackbox
|
||||
)
|
||||
|
||||
@@ -66,7 +66,6 @@ HOST_CONFIGS: dict[str, dict] = {
|
||||
{"name": "Redis", "host": "127.0.0.1", "port": 6380},
|
||||
{"name": "Ollama", "host": "127.0.0.1", "port": 11434},
|
||||
{"name": "Nginx", "host": "127.0.0.1", "port": 80},
|
||||
{"name": "SigNoz", "host": "127.0.0.1", "port": 3301},
|
||||
# OpenClaw 跑在 K3s (awoooi-prod),非 188 本機服務,不在此探測
|
||||
],
|
||||
},
|
||||
@@ -77,7 +76,9 @@ HOST_CONFIGS: dict[str, dict] = {
|
||||
"services": [
|
||||
{"name": "Harbor", "host": "127.0.0.1", "port": 5000},
|
||||
{"name": "Gitea", "host": "127.0.0.1", "port": 3001},
|
||||
{"name": "GH-Runner", "host": "127.0.0.1", "port": 8080},
|
||||
# TCP availability for canonical machine health endpoint:
|
||||
# http://192.168.0.110:8080/api/v1/health
|
||||
{"name": "SigNoz", "host": "127.0.0.1", "port": 8080},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -31,6 +31,7 @@ import { HostGrid, type HostInfo, type HostService } from '@/components/infra/ho
|
||||
import { AppLayout } from '@/components/layout'
|
||||
|
||||
const API_BASE = process.env.NEXT_PUBLIC_API_URL ?? ''
|
||||
const SIGNOZ_PUBLIC_URL = 'https://signoz.wooo.work'
|
||||
|
||||
// =============================================================================
|
||||
// Types
|
||||
@@ -152,7 +153,7 @@ function MonitoringTools() {
|
||||
const statusText = isUp ? (hasFiring ? `${tool.firing_count} ${tDash('monitoringStatus.firing')}` : tDash('monitoringStatus.up')) : tDash('monitoringStatus.down')
|
||||
const accentColor = TOOL_ACCENT_COLOR[tool.name] ?? '#b0ad9f'
|
||||
const icon = TOOL_ICON[tool.name] ?? <Activity size={16} />
|
||||
const link = safeExternalMonitoringUrl(tool.url)
|
||||
const link = tool.name === 'SigNoz' ? SIGNOZ_PUBLIC_URL : safeExternalMonitoringUrl(tool.url)
|
||||
const timeStr = (() => {
|
||||
try { return new Date(tool.checked_at).toLocaleTimeString('zh-TW', { timeZone: 'Asia/Taipei', hour: '2-digit', minute: '2-digit' }) }
|
||||
catch { return '--' }
|
||||
@@ -294,6 +295,7 @@ const HOST_CATALOG: Record<string, { services: HostService[]; isK3s?: boolean; r
|
||||
{ name: 'Langfuse', healthy: false, port: 3100, description: 'LLM Tracing' },
|
||||
{ name: 'Grafana', healthy: false, port: 3002, description: '監控面板' },
|
||||
{ name: 'Prometheus', healthy: false, port: 9090, description: '告警規則' },
|
||||
{ name: 'SigNoz', healthy: false, port: 8080, description: 'APM health · /api/v1/health' },
|
||||
],
|
||||
},
|
||||
'host:kali-readonly': {
|
||||
@@ -328,7 +330,6 @@ const HOST_CATALOG: Record<string, { services: HostService[]; isK3s?: boolean; r
|
||||
{ name: 'Redis', healthy: false, port: 6380, description: 'Cache' },
|
||||
{ name: 'Ollama', healthy: false, port: 11434, description: 'LLM' },
|
||||
{ name: 'OpenClaw', healthy: false, port: 8088, description: 'AI Agent' },
|
||||
{ name: 'SigNoz', healthy: false, port: 3301, description: 'APM · OTEL' },
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -150,12 +150,14 @@ const PUBLIC_TEXT_ENDPOINT_ALIASES: Record<string, string> = {
|
||||
'110:3002': 'host:public-gateway/grafana',
|
||||
'110:3100': 'host:public-gateway/langfuse',
|
||||
'110:5000': 'host:public-gateway/registry',
|
||||
'110:8080': 'host:public-gateway/signoz-health',
|
||||
'110:9000': 'host:public-gateway/sentry',
|
||||
'112:8080': 'host:kali-readonly/scanner',
|
||||
'188:11434': 'host:observability-a/ollama',
|
||||
'188:8088': 'host:observability-a/openclaw-legacy',
|
||||
'188:8089': 'host:observability-a/openclaw',
|
||||
'188:3301': 'host:observability-a/signoz',
|
||||
// Historical scrub-only alias; never use this legacy route for live access.
|
||||
'188:3301': 'host:observability-a/signoz-legacy',
|
||||
'188:5432': 'host:observability-a/postgres',
|
||||
'188:6380': 'host:observability-a/redis',
|
||||
}
|
||||
|
||||
@@ -81,6 +81,7 @@ export function LiveDashboard({ locale: _locale }: LiveDashboardProps) {
|
||||
services: [
|
||||
{ name: 'Harbor', status: 'idle', port: 5000 },
|
||||
{ name: 'GH Runner', status: 'idle' },
|
||||
{ name: 'SigNoz', status: 'idle', port: 8080 },
|
||||
{ name: 'Docker', status: 'idle', port: 2375 },
|
||||
],
|
||||
},
|
||||
@@ -111,7 +112,6 @@ export function LiveDashboard({ locale: _locale }: LiveDashboardProps) {
|
||||
{ name: 'Redis', status: 'idle', port: 6380 },
|
||||
{ name: 'Ollama', status: 'idle', port: 11434 },
|
||||
{ name: 'OpenClaw', status: 'idle', port: 8089 },
|
||||
{ name: 'SigNoz', status: 'idle', port: 3301 },
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -242,6 +242,10 @@
|
||||
- backup-awoooi.sh
|
||||
- backup-awoooi-frequent.sh
|
||||
- backup-signoz.sh
|
||||
- clickhouse-native-backup.sh
|
||||
- clickhouse-native-restore-drill.sh
|
||||
- clickhouse-restore-inventory.py
|
||||
- run-signoz-backup-canary.sh
|
||||
- backup-configs.sh
|
||||
- check-backup-integrity.sh
|
||||
- verify-gitea-backup.sh
|
||||
@@ -250,6 +254,36 @@
|
||||
- verify-offsite-full-sync.sh
|
||||
tags: backup_jobs
|
||||
|
||||
- name: "Backup | 建立 SigNoz native backup/restore config 目錄"
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: wooo
|
||||
group: wooo
|
||||
mode: "0755"
|
||||
loop:
|
||||
- /backup/config/signoz/clickhouse/config.d
|
||||
- /backup/config/signoz/clickhouse/restore-drill/config.d
|
||||
tags: backup_jobs
|
||||
|
||||
- name: "Backup | 部署 SigNoz native backup disk config"
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../../../ops/signoz/clickhouse/config.d/backup_disk.xml"
|
||||
dest: /backup/config/signoz/clickhouse/config.d/backup_disk.xml
|
||||
owner: wooo
|
||||
group: wooo
|
||||
mode: "0644"
|
||||
tags: backup_jobs
|
||||
|
||||
- name: "Backup | 部署 SigNoz isolated restore cluster config"
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../../../ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
dest: /backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml
|
||||
owner: wooo
|
||||
group: wooo
|
||||
mode: "0644"
|
||||
tags: backup_jobs
|
||||
|
||||
- name: "Backup | 安裝 AWOOOI daily backup Telegram heartbeat"
|
||||
ansible.builtin.cron:
|
||||
name: "AWOOOI daily backup Telegram heartbeat"
|
||||
|
||||
@@ -160,12 +160,12 @@ spec:
|
||||
- name: AWOOOI_BUILD_COMMIT_SHA
|
||||
# 2026-06-29 Codex: CD rewrites this to the deployed image tag so
|
||||
# production deploy readback does not rely on a stale static snapshot.
|
||||
value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7"
|
||||
value: "6505e2cede2d9e25114052882a054a1c84eab882"
|
||||
- name: AWOOOI_DESIRED_API_IMAGE_TAG
|
||||
# 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA.
|
||||
# Production readback compares runtime image truth against this
|
||||
# GitOps desired tag instead of doing a slow Gitea raw fetch.
|
||||
value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7"
|
||||
value: "6505e2cede2d9e25114052882a054a1c84eab882"
|
||||
- name: DATABASE_POOL_SIZE
|
||||
# 2026-07-01 Codex: production role `awoooi` currently has a low
|
||||
# connection limit. Keep API pool conservative until DB role
|
||||
|
||||
@@ -66,7 +66,7 @@ spec:
|
||||
- name: DATABASE_NULL_POOL
|
||||
value: "true"
|
||||
- name: AWOOOI_BUILD_COMMIT_SHA
|
||||
value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7"
|
||||
value: "6505e2cede2d9e25114052882a054a1c84eab882"
|
||||
- name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER
|
||||
value: "false"
|
||||
- name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER
|
||||
|
||||
@@ -174,10 +174,14 @@ spec:
|
||||
# NullPool releases each connection after use, while this cap
|
||||
# keeps concurrent background loops inside the worker role budget.
|
||||
value: "1"
|
||||
- name: DATABASE_POOL_TIMEOUT_SECONDS
|
||||
# Preserve the one-connection budget while allowing serialized
|
||||
# background jobs to wait through a bounded long query.
|
||||
value: "30"
|
||||
- name: DATABASE_BOOTSTRAP_DDL_ENABLED
|
||||
value: "false"
|
||||
- name: AWOOOI_BUILD_COMMIT_SHA
|
||||
value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7"
|
||||
value: "6505e2cede2d9e25114052882a054a1c84eab882"
|
||||
- name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER
|
||||
value: "true"
|
||||
- name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER
|
||||
|
||||
@@ -70,9 +70,9 @@ data:
|
||||
|
||||
- name: signoz-clickhouse
|
||||
display_name: "ClickHouse (SignOz)"
|
||||
host: "192.168.0.188"
|
||||
host: "192.168.0.110"
|
||||
stateful_level: BLOCK
|
||||
reason: "列欄式 OLAP 資料庫,寫入中重啟可能損壞列欄檔案"
|
||||
reason: "列欄式 OLAP 資料庫;只允許具備 native backup、bounded apply、rollback 與 post-verifier 的受控重建"
|
||||
alert_only: true
|
||||
containers: ["signoz-clickhouse"]
|
||||
|
||||
|
||||
@@ -40,9 +40,9 @@ resources:
|
||||
# ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag)
|
||||
# newName + newTag 由 CI 透過 kustomize edit set image 注入
|
||||
images:
|
||||
- digest: sha256:a06af69531087ef123b91bcae5f4170f322035156a64e5d09ef8ebdc23ea1c7a
|
||||
- digest: sha256:d8304318081a6c4267d78cc1bc6b43a88902a17bfccc7e1146989609282453ad
|
||||
name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER
|
||||
newName: 192.168.0.110:5000/awoooi/api
|
||||
- digest: sha256:9e6ada8cef17243c83230d7890acc1da154add49569b407a590e633d5a5ea70a
|
||||
- digest: sha256:7891f1fb9c2e6473d3ae102efc6c80ae1be48e1fda96a4de0a0c4d8f6cad4238
|
||||
name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER
|
||||
newName: 192.168.0.110:5000/awoooi/web
|
||||
|
||||
@@ -1,118 +1,18 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# Prometheus Config GitOps 部署腳本
|
||||
# =============================================================================
|
||||
# 用途: 將 Git 管理的 Prometheus 配置同步到 192.168.0.188
|
||||
# 建立者: Claude Code (首席架構師)
|
||||
# 日期: 2026-03-29 (台北時間)
|
||||
#!/usr/bin/env bash
|
||||
# Legacy compatibility entrypoint.
|
||||
# superseded_by=global_product_governance_v2
|
||||
# owner=observability-platform
|
||||
# expiry=2026-07-15T00:00:00+08:00
|
||||
# replacement=scripts/ops/deploy-signoz-prometheus-route.sh
|
||||
# verifier=promtool_then_two_successful_blackbox_scrapes
|
||||
#
|
||||
# 使用方式:
|
||||
# ./deploy-prometheus-config.sh [--dry-run]
|
||||
#
|
||||
# 依賴:
|
||||
# - SSH 免密碼登入 ollama@192.168.0.188
|
||||
# - 遠端主機已安裝 Prometheus
|
||||
# =============================================================================
|
||||
# The historical implementation appended partial config to the retired host188
|
||||
# Prometheus plane. Full-file replacement is also unsafe while independently
|
||||
# governed production targets differ from the repository snapshot. Route this
|
||||
# entrypoint to the bounded, target-only host110 deployer instead.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# 配置
|
||||
REMOTE_HOST="ollama@192.168.0.188"
|
||||
REMOTE_CONFIG="/home/ollama/momo-pro/monitoring/prometheus.yml"
|
||||
LOCAL_ADDITIONS="$(dirname "$0")/prometheus-config-additions.yaml"
|
||||
DRY_RUN="${1:-}"
|
||||
|
||||
# 顏色輸出
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
|
||||
|
||||
# 檢查本地配置檔案
|
||||
if [[ ! -f "$LOCAL_ADDITIONS" ]]; then
|
||||
log_error "找不到配置檔案: $LOCAL_ADDITIONS"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Prometheus GitOps 部署開始"
|
||||
log_info "目標主機: $REMOTE_HOST"
|
||||
log_info "遠端配置: $REMOTE_CONFIG"
|
||||
|
||||
# 備份遠端配置
|
||||
BACKUP_FILE="${REMOTE_CONFIG}.bak.$(date +%Y%m%d_%H%M%S)"
|
||||
log_info "備份遠端配置至: $BACKUP_FILE"
|
||||
|
||||
if [[ "$DRY_RUN" == "--dry-run" ]]; then
|
||||
log_warn "DRY RUN 模式 - 不會實際執行"
|
||||
echo ""
|
||||
echo "將執行以下操作:"
|
||||
echo " 1. ssh $REMOTE_HOST \"cp $REMOTE_CONFIG $BACKUP_FILE\""
|
||||
echo " 2. 檢查配置中是否已包含 argocd job"
|
||||
echo " 3. 如需更新,附加新配置"
|
||||
echo " 4. ssh $REMOTE_HOST \"docker exec prometheus kill -SIGHUP 1\""
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 執行備份
|
||||
ssh "$REMOTE_HOST" "cp $REMOTE_CONFIG $BACKUP_FILE"
|
||||
log_info "備份完成"
|
||||
|
||||
# 檢查是否已有 argocd job
|
||||
ARGOCD_EXISTS=$(ssh "$REMOTE_HOST" "grep -c 'job_name.*argocd' $REMOTE_CONFIG || true")
|
||||
|
||||
if [[ "$ARGOCD_EXISTS" -gt 0 ]]; then
|
||||
log_info "ArgoCD job 已存在於配置中,跳過新增"
|
||||
else
|
||||
log_info "新增 ArgoCD 和 Blackbox scrape configs..."
|
||||
|
||||
# 提取配置內容 (去除註解標題)
|
||||
CONFIG_CONTENT=$(grep -A 100 "^- job_name: argocd" "$LOCAL_ADDITIONS" | head -35)
|
||||
|
||||
# 附加到遠端配置
|
||||
ssh "$REMOTE_HOST" "cat >> $REMOTE_CONFIG << 'EOF'
|
||||
|
||||
# === GitOps 新增 ($(date +%Y-%m-%d)) ===
|
||||
$CONFIG_CONTENT
|
||||
EOF"
|
||||
|
||||
log_info "配置已更新"
|
||||
fi
|
||||
|
||||
# 驗證配置語法
|
||||
log_info "驗證 Prometheus 配置語法..."
|
||||
SYNTAX_CHECK=$(ssh "$REMOTE_HOST" "docker exec prometheus promtool check config /etc/prometheus/prometheus.yml 2>&1 || true")
|
||||
|
||||
if echo "$SYNTAX_CHECK" | grep -q "SUCCESS"; then
|
||||
log_info "配置語法正確"
|
||||
else
|
||||
log_error "配置語法錯誤!"
|
||||
echo "$SYNTAX_CHECK"
|
||||
log_warn "正在還原備份..."
|
||||
ssh "$REMOTE_HOST" "cp $BACKUP_FILE $REMOTE_CONFIG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 重載 Prometheus
|
||||
log_info "重載 Prometheus 配置..."
|
||||
ssh "$REMOTE_HOST" "docker exec prometheus kill -SIGHUP 1 || sudo systemctl reload prometheus"
|
||||
|
||||
# 等待並驗證
|
||||
sleep 3
|
||||
log_info "驗證 targets..."
|
||||
|
||||
TARGETS=$(curl -s "http://192.168.0.188:9090/api/v1/targets" 2>/dev/null | \
|
||||
jq -r '.data.activeTargets[] | "\(.labels.job): \(.health)"' 2>/dev/null || echo "驗證失敗")
|
||||
|
||||
echo ""
|
||||
echo "═══════════════════════════════════════════════════════"
|
||||
echo " Prometheus Targets 狀態"
|
||||
echo "═══════════════════════════════════════════════════════"
|
||||
echo "$TARGETS" | grep -E "argocd|blackbox" || echo " (無 argocd/blackbox targets)"
|
||||
echo "═══════════════════════════════════════════════════════"
|
||||
echo ""
|
||||
|
||||
log_info "部署完成!"
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
printf 'NOTICE: deploy-prometheus-config.sh is superseded; using the bounded host110 route deployer\n' >&2
|
||||
exec "${REPO_ROOT}/scripts/ops/deploy-signoz-prometheus-route.sh" "${1:-apply}"
|
||||
|
||||
@@ -92,17 +92,9 @@ groups:
|
||||
# ===== 可觀測性服務告警 =====
|
||||
- name: observability_alerts
|
||||
rules:
|
||||
# SignOz 離線
|
||||
- alert: SignOzDown
|
||||
expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:3301"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
team: ops
|
||||
component: signoz
|
||||
annotations:
|
||||
summary: "⚠️ SignOz 服務離線"
|
||||
description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘"
|
||||
# SigNoz query/UI health belongs to the canonical 110 Prometheus plane in
|
||||
# ops/monitoring/alerts-unified.yml. This legacy 188 rule mirror has no
|
||||
# matching canonical blackbox-http series and must not duplicate it.
|
||||
|
||||
# Sentry 離線
|
||||
- alert: SentryDown
|
||||
|
||||
@@ -56,6 +56,7 @@ scrape_configs:
|
||||
- https://aiops.wooo.work
|
||||
- https://mo.wooo.work
|
||||
- http://192.168.0.110:3001
|
||||
- http://192.168.0.110:8080/api/v1/health
|
||||
- http://192.168.0.125:32334/api/v1/health
|
||||
- http://192.168.0.125:32335
|
||||
- https://www.tsenyang.com
|
||||
@@ -87,7 +88,6 @@ scrape_configs:
|
||||
- 192.168.0.110:5000
|
||||
- 192.168.0.110:3100
|
||||
# 188 服務(Ollama 已於 2026-05-06 自 AWOOOI provider pool 退役)
|
||||
- 192.168.0.188:3301
|
||||
- 192.168.0.188:5432
|
||||
- 192.168.0.188:6380
|
||||
- 192.168.0.188:8089
|
||||
|
||||
@@ -54,9 +54,9 @@ services:
|
||||
|
||||
- name: signoz-clickhouse
|
||||
display_name: "ClickHouse (SignOz)"
|
||||
host: "192.168.0.188"
|
||||
host: "192.168.0.110"
|
||||
stateful_level: BLOCK
|
||||
reason: "列欄式 OLAP 資料庫,寫入中重啟可能損壞列欄檔案"
|
||||
reason: "列欄式 OLAP 資料庫;只允許具備 native backup、bounded apply、rollback 與 post-verifier 的受控重建"
|
||||
alert_only: true
|
||||
containers: ["signoz-clickhouse"]
|
||||
|
||||
|
||||
@@ -719,21 +719,23 @@ groups:
|
||||
description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘"
|
||||
|
||||
- alert: SignOzDown
|
||||
expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0
|
||||
expr: >-
|
||||
(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"} == 0)
|
||||
or absent(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"})
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
layer: docker-188
|
||||
layer: docker-110
|
||||
component: signoz
|
||||
host: "188"
|
||||
host: "110"
|
||||
team: ops
|
||||
auto_repair: "true"
|
||||
mcp_provider: "ssh_host"
|
||||
target_host: "192.168.0.188"
|
||||
target_host: "192.168.0.110"
|
||||
alert_category: "devops_tool"
|
||||
annotations:
|
||||
summary: "SignOz 服務離線"
|
||||
description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘"
|
||||
description: "SignOz (http://192.168.0.110:8080/api/v1/health) 已離線或監控序列缺失超過 2 分鐘"
|
||||
|
||||
# ---- 110 Docker 層 ----
|
||||
- alert: SentryDown
|
||||
|
||||
@@ -400,21 +400,23 @@ groups:
|
||||
description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘"
|
||||
|
||||
- alert: SignOzDown
|
||||
expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0
|
||||
expr: >-
|
||||
(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"} == 0)
|
||||
or absent(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"})
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
layer: docker-188
|
||||
layer: docker-110
|
||||
component: signoz
|
||||
host: "188"
|
||||
host: "110"
|
||||
team: ops
|
||||
auto_repair: "true"
|
||||
mcp_provider: "ssh_host"
|
||||
target_host: "192.168.0.188"
|
||||
target_host: "192.168.0.110"
|
||||
alert_category: "devops_tool"
|
||||
annotations:
|
||||
summary: "SignOz 服務離線"
|
||||
description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘"
|
||||
description: "SignOz (http://192.168.0.110:8080/api/v1/health) 已離線或監控序列缺失超過 2 分鐘"
|
||||
|
||||
# ---- 110 Docker 層 ----
|
||||
- alert: SentryDown
|
||||
|
||||
@@ -54,7 +54,10 @@ def generate_prometheus_scrape_configs(registry: dict) -> list[dict]:
|
||||
|
||||
# K8s 服務
|
||||
for svc in registry.get("services", []):
|
||||
if svc.get("monitoring", {}).get("prometheus"):
|
||||
monitoring = svc.get("monitoring", {})
|
||||
if monitoring.get("prometheus") and monitoring.get(
|
||||
"prometheus_scrape", True
|
||||
):
|
||||
if svc.get("type") == "k8s-deployment":
|
||||
# K8s ServiceMonitor 風格
|
||||
config = {
|
||||
|
||||
@@ -335,6 +335,8 @@ services:
|
||||
health_type: grpc
|
||||
monitoring:
|
||||
prometheus: true
|
||||
prometheus_scrape: false
|
||||
external_verifier: scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh
|
||||
sentry: false
|
||||
alerts:
|
||||
- service_down
|
||||
@@ -345,11 +347,13 @@ services:
|
||||
# --- SignOz UI ---
|
||||
- name: signoz-ui
|
||||
type: docker
|
||||
host: 192.168.0.188
|
||||
port: 3301
|
||||
health_endpoint: /
|
||||
host: 192.168.0.110
|
||||
port: 8080
|
||||
health_endpoint: /api/v1/health
|
||||
health_type: http
|
||||
monitoring:
|
||||
prometheus: true
|
||||
prometheus_scrape: false
|
||||
sentry: false
|
||||
alerts:
|
||||
- service_down
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
# SigNoz Log-Based Alert Rules
|
||||
# 2026-04-05 Claude Code: Sprint 2 — 日誌告警
|
||||
# 設定位置: http://192.168.0.188:3301/alerts (SigNoz UI)
|
||||
# superseded_by=global_product_governance_v2: legacy 192.168.0.188:3301 UI/query route
|
||||
# 設定位置: https://signoz.wooo.work/alerts (SigNoz human UI)
|
||||
# Machine health: http://192.168.0.110:8080/api/v1/health
|
||||
# Webhook: http://192.168.0.121:32334/api/v1/webhooks/signoz
|
||||
|
||||
## 設定步驟
|
||||
|
||||
1. 開啟 http://192.168.0.188:3301/alerts
|
||||
1. 開啟 https://signoz.wooo.work/alerts
|
||||
2. 點擊 "New Alert Rule" → "Logs Based Alert"
|
||||
3. 依照下表填入各欄位
|
||||
4. Notification Channel: 選擇 awoooi-api (指向 /api/v1/webhooks/signoz)
|
||||
@@ -13,6 +15,8 @@
|
||||
|
||||
驗證 Webhook 鏈路:
|
||||
```bash
|
||||
curl -fsS http://192.168.0.110:8080/api/v1/health | jq .
|
||||
|
||||
curl -X POST http://192.168.0.121:32334/api/v1/webhooks/signoz \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"alerts":[{"labels":{"alertname":"SignozWebhookTest","severity":"info","layer":"k8s"},"annotations":{"summary":"Sprint 2 webhook 驗證,請忽略"},"status":"firing"}],"version":"4"}'
|
||||
|
||||
24
ops/signoz/clickhouse/config.d/backup_disk.xml
Normal file
24
ops/signoz/clickhouse/config.d/backup_disk.xml
Normal file
@@ -0,0 +1,24 @@
|
||||
<?xml version="1.0"?>
|
||||
<!--
|
||||
Repo-owned ClickHouse native BACKUP destination for host110.
|
||||
|
||||
The bind mount is supplied by docker-compose.clickhouse-backup.override.yaml.
|
||||
BACKUP data is deliberately kept outside the production ClickHouse volume so
|
||||
it can be ingested by the existing host /backup retention/offsite pipeline.
|
||||
-->
|
||||
<clickhouse>
|
||||
<storage_configuration>
|
||||
<disks>
|
||||
<backups>
|
||||
<type>local</type>
|
||||
<path>/backups/</path>
|
||||
</backups>
|
||||
</disks>
|
||||
</storage_configuration>
|
||||
<backups>
|
||||
<allowed_disk>backups</allowed_disk>
|
||||
<allowed_path>/backups/</allowed_path>
|
||||
<allow_concurrent_backups>false</allow_concurrent_backups>
|
||||
<allow_concurrent_restores>false</allow_concurrent_restores>
|
||||
</backups>
|
||||
</clickhouse>
|
||||
@@ -0,0 +1,35 @@
|
||||
<?xml version="1.0"?>
|
||||
<!--
|
||||
P0-OBS-002 isolated native RESTORE drill topology.
|
||||
|
||||
This file is mounted read-only only into an ephemeral ClickHouse container.
|
||||
The driver creates an internal-only Docker network and a fresh ZooKeeper
|
||||
volume for every run. It never points at the production Keeper namespace.
|
||||
CLICKHOUSE_RESTORE_REPLICA is a non-secret, run-derived value supplied
|
||||
directly by the driver (never from an env file).
|
||||
-->
|
||||
<clickhouse>
|
||||
<zookeeper>
|
||||
<node index="1">
|
||||
<host>restore-zookeeper</host>
|
||||
<port>2181</port>
|
||||
</node>
|
||||
</zookeeper>
|
||||
|
||||
<remote_servers>
|
||||
<cluster>
|
||||
<shard>
|
||||
<internal_replication>true</internal_replication>
|
||||
<replica>
|
||||
<host>restore-clickhouse</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</cluster>
|
||||
</remote_servers>
|
||||
|
||||
<macros>
|
||||
<shard>01</shard>
|
||||
<replica from_env="CLICKHOUSE_RESTORE_REPLICA"/>
|
||||
</macros>
|
||||
</clickhouse>
|
||||
13
ops/signoz/docker-compose.clickhouse-backup.override.yaml
Normal file
13
ops/signoz/docker-compose.clickhouse-backup.override.yaml
Normal file
@@ -0,0 +1,13 @@
|
||||
# P0-OBS-002: additive override for the host110 SigNoz ClickHouse service.
|
||||
# Always pass this file explicitly after the upstream compose file with -f.
|
||||
services:
|
||||
clickhouse:
|
||||
volumes:
|
||||
- type: bind
|
||||
source: /backup/staging/signoz-clickhouse
|
||||
target: /backups
|
||||
read_only: false
|
||||
- type: bind
|
||||
source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml
|
||||
target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml
|
||||
read_only: true
|
||||
@@ -2,7 +2,7 @@ schema: awoooi_signoz_organization_canonical_route_migration_v1
|
||||
work_item_id: P0-OBS-002
|
||||
owner_lane: platform-observability
|
||||
phase: wave_a_dual_allow
|
||||
generated_at: "2026-07-15T03:51:00+08:00"
|
||||
generated_at: "2026-07-15T06:06:46+08:00"
|
||||
policy:
|
||||
superseded_by: global_product_governance_v2
|
||||
controlled_apply_allowed: true
|
||||
@@ -154,11 +154,62 @@ runtime_observations:
|
||||
promotion_authorized: false
|
||||
post_closure_regression:
|
||||
snapshot: ops/signoz/p0-obs-002-post-closure-regression.yaml
|
||||
snapshot_schema: awoooi_signoz_post_closure_regression_v2
|
||||
gitea_cd_run_id: 5140
|
||||
release_source_sha: e4da6570a645e504cc93647d396f45f167b3172a
|
||||
release_terminal: success
|
||||
grpc_bridge_rollback_indicated: false
|
||||
canonical_collector_recovered: true
|
||||
api_runtime_cooldown_closed: false
|
||||
cd_probe_safety_fix_deployed: false
|
||||
backup_collector_restore_guard_deployed: false
|
||||
api_runtime_cooldown_closed: true
|
||||
cd_probe_safety_fix_deployed: true
|
||||
config_route_patch_run_id: P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1
|
||||
config_route_patch_deployed: true
|
||||
signoz_control_plane_health_route_deployed: true
|
||||
prometheus_target_route_run_id: P0-OBS-002-prometheus-route-20260714T215106Z-apply1
|
||||
prometheus_target_route_terminal: completed_target_first
|
||||
alert_rules_run_id: P0-OBS-002-alert-rules-20260714T215328Z-apply1
|
||||
alert_rules_terminal: completed_verified
|
||||
backup_collector_restore_guard_deployed: true
|
||||
backup_canary_scope: clickhouse_native_only
|
||||
backup_canary_verified: true
|
||||
backup_canary_last_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
backup_canary_last_terminal: partial_degraded
|
||||
backup_canary_runtime_operation_terminal: pass
|
||||
backup_canary_machine_receipt_verifier_terminal: pass
|
||||
backup_canary_prior_root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract
|
||||
backup_canary_cooldown_retry_status: superseded_by_online_native_no_stop_contract
|
||||
clickhouse_native_backup_migration_status: production_closed
|
||||
clickhouse_native_backup_disk_run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2
|
||||
clickhouse_native_backup_disk_status: deployed_verified
|
||||
clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4
|
||||
clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4
|
||||
clickhouse_native_current_source_status: deployed_verified
|
||||
clickhouse_native_current_source_deployed: true
|
||||
clickhouse_native_current_source_runtime_verifier_terminal: pass
|
||||
clickhouse_native_current_source_toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
|
||||
clickhouse_native_current_source_toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
|
||||
clickhouse_native_current_source_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
|
||||
clickhouse_native_current_source_toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
|
||||
clickhouse_native_current_source_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
|
||||
clickhouse_native_current_source_canary_apply_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
clickhouse_native_current_source_wrapper_terminal: partial_degraded
|
||||
clickhouse_native_current_source_runtime_operation_terminal: pass
|
||||
clickhouse_native_current_source_machine_receipt_verifier_terminal: pass
|
||||
clickhouse_native_current_source_artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
|
||||
clickhouse_native_restore_terminal: verified
|
||||
clickhouse_native_restic_snapshot_id: 135485ab
|
||||
clickhouse_native_offsite_status: pending
|
||||
clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision
|
||||
clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract
|
||||
signoz_sqlite_application_consistent_backup_status: pending
|
||||
service_registry_runtime_mirror_status: partial_degraded
|
||||
service_registry_runtime_mirror_work_item_id: P0-OBS-002-ASSET-DRIFT-001
|
||||
github_freeze_legacy_asset_status: pending_controlled_retirement
|
||||
github_freeze_legacy_asset_work_item_id: P0-OBS-002-ASSET-DRIFT-002
|
||||
monitoring_generator_duplicate_identity_status: pending_source_deduplication
|
||||
monitoring_generator_duplicate_identity_work_item_id: P0-OBS-002-ASSET-DRIFT-003
|
||||
post_release_otlp_grpc_run_id: P0-OBS-002-post-release-20260714T215500Z
|
||||
post_release_otlp_grpc_600s_verifier_status: pass
|
||||
|
||||
terminal:
|
||||
status: partial_degraded
|
||||
@@ -167,8 +218,4 @@ terminal:
|
||||
- direct_ingress_policy_not_closed
|
||||
- http_bridge_not_supervised
|
||||
- filelog_normalization_degraded
|
||||
- api_runtime_cooldown_pending
|
||||
- cd_probe_safety_fix_not_deployed
|
||||
- backup_collector_restore_guard_not_deployed
|
||||
- signoz_control_plane_health_route_not_deployed
|
||||
- wave_b_not_authorized
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
schema: awoooi_signoz_post_closure_regression_v1
|
||||
schema: awoooi_signoz_post_closure_regression_v2
|
||||
work_item_id: P0-OBS-002
|
||||
trace_id: P0-OBS-002
|
||||
observed_at: "2026-07-15T02:20:00+08:00"
|
||||
observed_at: "2026-07-15T08:16:07+08:00"
|
||||
|
||||
baseline:
|
||||
grpc_runtime_run_id: P0-OBS-002-20260715T013800+0800
|
||||
@@ -11,6 +11,91 @@ baseline:
|
||||
trace_count: 1098
|
||||
metric_sample_count: 60180
|
||||
|
||||
delivery_validation:
|
||||
backup_restore_deployer_governance_focused_tests:
|
||||
passed: 94
|
||||
failed: 0
|
||||
post_format_rerun: pass
|
||||
api_monitoring_alert_adjacent_tests:
|
||||
passed: 115
|
||||
failed: 0
|
||||
environment: explicit_dummy_local_database_and_redis_urls_no_env_file_read
|
||||
web:
|
||||
dependency_install: offline_frozen_lockfile
|
||||
packages_reused: 851
|
||||
packages_downloaded: 0
|
||||
typecheck: pass
|
||||
prettier: legacy_file_style_not_enforced
|
||||
bulk_format_churn_removed: true
|
||||
eslint_errors: 0
|
||||
eslint_existing_warnings: 6
|
||||
production_build: pass
|
||||
build_environment: explicit_local_public_api_and_signoz_urls_no_env_file_read
|
||||
monitoring:
|
||||
generator_validate_only: pass
|
||||
registry_service_count: 26
|
||||
registry_monitored_count: 25
|
||||
registry_coverage_percent: 96.2
|
||||
generated_scrape_config_count: 24
|
||||
generated_blackbox_target_count: 20
|
||||
promtool_rules:
|
||||
alerts: 81
|
||||
alerts_unified: 147
|
||||
k3s_alerts: 14
|
||||
prometheus_config_syntax: pass
|
||||
infrastructure:
|
||||
configmap_client_dry_run: pass
|
||||
ansible_validation_script: pass_with_local_ansible_syntax_check_unavailable
|
||||
ansible_110_playbook_isolated_syntax_check: pass
|
||||
shell_syntax: pass
|
||||
python_compile: pass
|
||||
documentation_secret_sanity_files_scanned: 1090
|
||||
github_used: false
|
||||
secret_read: false
|
||||
|
||||
asset_reconciliation:
|
||||
drift_work_item_id: P0-OBS-002-ASSET-DRIFT-001
|
||||
source: ops/config/service-registry.yaml
|
||||
runtime_mirror: k8s/awoooi-prod/15-service-registry-configmap.yaml
|
||||
source_service_count: 27
|
||||
runtime_mirror_service_count: 24
|
||||
exact_shared_service_count: 24
|
||||
shared_service_drift_count: 0
|
||||
source_only_services:
|
||||
- bitan-app
|
||||
- sentry
|
||||
- signoz
|
||||
signoz_clickhouse_exact_match: true
|
||||
live_signoz_query_host: 192.168.0.110
|
||||
source_signoz_host: 192.168.0.188
|
||||
terminal: partial_degraded
|
||||
safe_next_action: >-
|
||||
Reconcile the three source-only services and the stale SignOz query host
|
||||
against production runtime before atomically regenerating the ConfigMap;
|
||||
do not copy the stale 188 SignOz row into the runtime mirror.
|
||||
github_freeze_legacy_asset_drift:
|
||||
drift_work_item_id: P0-OBS-002-ASSET-DRIFT-002
|
||||
superseded_by: global_product_governance_v2
|
||||
status: pending_controlled_retirement
|
||||
active_github_use_in_this_run: false
|
||||
source_references:
|
||||
- ops/monitoring/service-registry.yaml:github-runner
|
||||
- infra/ansible/playbooks/110-devops.yml:github_runner_management
|
||||
- apps/web/src/components/dashboard/live-dashboard.tsx:gh_runner_fallback
|
||||
replacement: gitea_runner_readiness_and_queue_receipts
|
||||
exit_condition: legacy_runner_assets_absent_and_gitea_replacement_verified
|
||||
p0_p1_github_recovery_scheduled: false
|
||||
monitoring_generator_duplicate_identity_drift:
|
||||
drift_work_item_id: P0-OBS-002-ASSET-DRIFT-003
|
||||
status: pending_source_deduplication
|
||||
registry_service_count: 26
|
||||
awoooi_api_registry_entry_count: 2
|
||||
generated_scrape_config_count: 24
|
||||
generated_awoooi_api_job_count: 2
|
||||
generated_blackbox_target_count: 20
|
||||
runtime_apply_performed: false
|
||||
exit_condition: one_canonical_awoooi_api_asset_and_one_generated_scrape_job
|
||||
|
||||
regressions:
|
||||
api_runtime:
|
||||
first_probe_timeout_at: "2026-07-15T02:11:22+08:00"
|
||||
@@ -21,7 +106,7 @@ regressions:
|
||||
expected_replicas: 2
|
||||
restart_count_per_pod_at_recovery: 4
|
||||
last_observed_ready_replicas: 2
|
||||
last_observed_cooldown_closed: false
|
||||
last_observed_cooldown_closed: true
|
||||
primary_classification: db_connection_budget_and_request_pressure
|
||||
structural_risk_confidence: 0.98
|
||||
direct_application_commit_regression_confidence: 0.12
|
||||
@@ -54,50 +139,695 @@ controlled_recovery:
|
||||
- 192.168.0.120
|
||||
- 192.168.0.121
|
||||
|
||||
source_candidates:
|
||||
cd_probe:
|
||||
path: .gitea/workflows/cd.yaml
|
||||
connection_budget_precheck_required: true
|
||||
max_workers: 4
|
||||
max_attempts: 1
|
||||
request_timeout_seconds: 5
|
||||
safety_reserve_connections: 4
|
||||
receipt_fields_added: true
|
||||
deployment_status: pending
|
||||
backup_restore_guard:
|
||||
path: scripts/backup/backup-signoz.sh
|
||||
deployment_playbook: infra/ansible/playbooks/110-devops.yml
|
||||
deployment_tag: backup_jobs
|
||||
deployment_contract_present: true
|
||||
trap_exit_hup_int_term: true
|
||||
restore_armed_before_stop: true
|
||||
bounded_exit_retry: true
|
||||
deployment_status: pending
|
||||
canonical_health_route:
|
||||
internal_url: http://192.168.0.110:8080
|
||||
public_url: https://signoz.wooo.work
|
||||
production_networkpolicy_tcp_port: 8080
|
||||
legacy_query_route_removed: http://192.168.0.188:3301
|
||||
otlp_producer_route_changed: false
|
||||
deployment_status: pending
|
||||
release_receipts:
|
||||
gitea_cd_5140:
|
||||
gitea_run_id: 5140
|
||||
source_sha: e4da6570a645e504cc93647d396f45f167b3172a
|
||||
workflow_terminal: success
|
||||
focused_tests_passed: 6
|
||||
unit_tests:
|
||||
passed: 4817
|
||||
skipped: 23
|
||||
warnings: 69
|
||||
integration_tests_passed: 5
|
||||
production_readback:
|
||||
build_tag_matched: true
|
||||
desired_tag_matched: true
|
||||
attempt: 1
|
||||
rollout_terminal: success
|
||||
transient_rollout_risk_observed: true
|
||||
postdeploy:
|
||||
alert_chain_terminal: pass
|
||||
alert_chain_checks_passed: 8
|
||||
alert_chain_checks_total: 9
|
||||
playwright_passed: 5
|
||||
notification_mirrored: true
|
||||
cd_probe:
|
||||
receipt_provenance: gitea_run_5140_job_log
|
||||
passed: true
|
||||
workers: 4
|
||||
safety_reserve_connections: 4
|
||||
max_attempts: 1
|
||||
request_timeout_seconds: 5
|
||||
skip_reason: ""
|
||||
restart_free: true
|
||||
restart_count_before: 0
|
||||
restart_count_after: 0
|
||||
ready_replicas: 2
|
||||
desired_replicas: 2
|
||||
|
||||
controlled_applies:
|
||||
config_route_patch:
|
||||
trace_id: P0-OBS-002
|
||||
run_id: P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1
|
||||
work_item_id: P0-OBS-002
|
||||
source_commit: 9a9d1464a586ba172d1613db5bc285def43eb3e8
|
||||
included_in_release_sha: e4da6570a645e504cc93647d396f45f167b3172a
|
||||
normalized_asset:
|
||||
canonical_id: signoz-110-query-health-route
|
||||
internal_url: http://192.168.0.110:8080
|
||||
public_url: https://signoz.wooo.work
|
||||
health_path: /api/v1/health
|
||||
production_networkpolicy_tcp_port: 8080
|
||||
source_truth_diff:
|
||||
legacy_query_route_removed: http://192.168.0.188:3301
|
||||
otlp_producer_route_changed: false
|
||||
check_receipt:
|
||||
source_tests_passed: true
|
||||
execution_receipt:
|
||||
gitea_run_id: 5140
|
||||
deployment_status: deployed_verified
|
||||
post_verifier:
|
||||
release_readback_matched: true
|
||||
ready_replicas: 2
|
||||
desired_replicas: 2
|
||||
rollback: not_required
|
||||
terminal: applied_post_verifier_pass
|
||||
|
||||
prometheus_target_route:
|
||||
trace_id: P0-OBS-002
|
||||
run_id: P0-OBS-002-prometheus-route-20260714T215106Z-apply1
|
||||
work_item_id: P0-OBS-002
|
||||
risk: medium
|
||||
check_receipt:
|
||||
promtool: pass
|
||||
source_sha: 462cb5c2397d1a6ec0542f2fde83e06e0baa05d82df299ac9ae9464d08d9947d
|
||||
candidate_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c
|
||||
execution_receipt:
|
||||
action: target_first_config_apply
|
||||
active_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c
|
||||
runtime_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c
|
||||
ready: true
|
||||
backup_path: /home/wooo/monitoring/prometheus.yml.bak.P0-OBS-002-prometheus-route-20260714T215106Z-apply1
|
||||
backup_sha: 462cb5c2397d1a6ec0542f2fde83e06e0baa05d82df299ac9ae9464d08d9947d
|
||||
post_verifier:
|
||||
terminal: target_up_two_scrapes
|
||||
target_count: 1
|
||||
target_health: up
|
||||
first_scrape_at: "2026-07-15T05:52:38.146935853+08:00"
|
||||
second_scrape_at: "2026-07-15T05:52:53.146935853+08:00"
|
||||
probe_success: 1
|
||||
legacy_target_count: 0
|
||||
independent_verifier:
|
||||
mode: independent_readback
|
||||
observed_at: "2026-07-15T05:58:09+08:00"
|
||||
active_sha_matches_runtime: true
|
||||
target_count: 1
|
||||
target_health: up
|
||||
probe_success: 1
|
||||
legacy_target_count: 0
|
||||
terminal: pass
|
||||
rollback:
|
||||
required: false
|
||||
available: true
|
||||
terminal: completed_target_first
|
||||
|
||||
alert_rules:
|
||||
trace_id: P0-OBS-002
|
||||
run_id: P0-OBS-002-alert-rules-20260714T215328Z-apply1
|
||||
work_item_id: P0-OBS-002
|
||||
check_receipt:
|
||||
promtool: pass
|
||||
source_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58
|
||||
execution_receipt:
|
||||
remote_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58
|
||||
canonical_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58
|
||||
runtime_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58
|
||||
hashes_match: true
|
||||
prometheus_ready: true
|
||||
post_verifier:
|
||||
window_seconds: 130
|
||||
terminal: pass
|
||||
signoz_rule_count: 1
|
||||
signoz_rule_health: ok
|
||||
signoz_rule_state: inactive
|
||||
exact_query: '(probe_success{instance="http://192.168.0.110:8080/api/v1/health",job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",job="blackbox-http"})'
|
||||
rollback:
|
||||
required: false
|
||||
available: true
|
||||
terminal: completed_verified
|
||||
|
||||
clickhouse_native_backup_disk:
|
||||
trace_id: P0-OBS-002
|
||||
work_item_id: P0-OBS-002
|
||||
risk: high
|
||||
normalized_asset:
|
||||
canonical_id: host110-signoz-clickhouse-backup-disk
|
||||
compose_project: signoz
|
||||
compose_service: clickhouse
|
||||
clickhouse_disk: backups
|
||||
container_path: /backups/
|
||||
host_path: /backup/staging/signoz-clickhouse
|
||||
source:
|
||||
override_sha256: cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488
|
||||
config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
|
||||
focused_tests_passed: 12
|
||||
bash_syntax: pass
|
||||
check_receipt:
|
||||
run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064800P0800-check2
|
||||
terminal: check_pass_no_write
|
||||
prior_disk_count: 0
|
||||
prior_backup_mount_count: 0
|
||||
collector_running: true
|
||||
collector_restart_count: 0
|
||||
attempts:
|
||||
- run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064300P0800-apply1
|
||||
terminal: failed_rolled_back
|
||||
failed_stage: post_recreate_server_identity_verifier
|
||||
exit_code: 2
|
||||
root_cause: remote_heredoc_expanded_awk_positional_parameter
|
||||
source_fix: direct_awk_arguments_without_remote_shell_expansion
|
||||
rollback:
|
||||
terminal: pass
|
||||
prior_compose_state_restored: true
|
||||
managed_files_absent: true
|
||||
host_backup_directory_absent: true
|
||||
clickhouse_healthy: true
|
||||
clickhouse_restart_count: 0
|
||||
original_data_volume_preserved: true
|
||||
dependent_container_identity_preserved: true
|
||||
active_backup_or_restore_count: 0
|
||||
residue_count: 0
|
||||
- run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2
|
||||
terminal: pass
|
||||
action: clickhouse_only_force_recreate_no_pull
|
||||
execution:
|
||||
image_digest_unchanged: true
|
||||
data_volume_name: signoz-clickhouse
|
||||
data_volume_unchanged: true
|
||||
clickhouse_restart_count: 0
|
||||
post_verifier:
|
||||
independent: true
|
||||
disk_type: Local
|
||||
disk_read_only: false
|
||||
disk_remote: false
|
||||
disk_broken: false
|
||||
disk_free_space_positive: true
|
||||
server_uid: 101
|
||||
server_gid: 101
|
||||
server_write_access: true
|
||||
concurrent_backups_allowed: false
|
||||
concurrent_restores_allowed: false
|
||||
collector_identity_unchanged: true
|
||||
signoz_identity_unchanged: true
|
||||
zookeeper_identity_unchanged: true
|
||||
dependent_restart_count_delta: 0
|
||||
grpc_4317_listening: true
|
||||
http_4318_listening: true
|
||||
health_8080_listening: true
|
||||
api_http_status: 200
|
||||
active_backup_or_restore_count: 0
|
||||
backup_directory_entry_count: 0
|
||||
stage_candidate_process_residue_count: 0
|
||||
terminal: deployed_verified
|
||||
|
||||
clickhouse_native_backup_restore_canary:
|
||||
trace_id: P0-OBS-002
|
||||
work_item_id: P0-OBS-002
|
||||
risk: high
|
||||
completion_scope: clickhouse_native_only
|
||||
source:
|
||||
evidence_scope: historical_production_receipt
|
||||
deployment_status: deployed_verified
|
||||
backup_orchestrator_sha256: 2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520
|
||||
native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a
|
||||
isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7
|
||||
inventory_verifier_sha256: 79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c
|
||||
canary_wrapper_sha256: 64d23d9f7a64d1e4e1347b98d45e50e26eadea1741d0e28a5e2e5d74b22322d8
|
||||
backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
|
||||
isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
|
||||
latest_restore_focused_tests_passed: 12
|
||||
post_canary_focused_tests_passed: 94
|
||||
post_format_focused_tests_passed: 94
|
||||
bash_syntax: pass
|
||||
ruff: pass
|
||||
ruff_format: pass
|
||||
yaml_xml_parse: pass
|
||||
current_source:
|
||||
observed_at: "2026-07-15T09:53:46+08:00"
|
||||
evidence_scope: committed_current_source
|
||||
status: deployed_verified
|
||||
deployment_status: deployed
|
||||
current_source_deployed: true
|
||||
runtime_verifier_terminal: pass
|
||||
historical_production_receipt_preserved: true
|
||||
expected_file_count: 7
|
||||
hashes:
|
||||
backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2
|
||||
native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a
|
||||
isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7
|
||||
inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863
|
||||
canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886
|
||||
backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
|
||||
isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
|
||||
deployed_source_hashes:
|
||||
backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2
|
||||
native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a
|
||||
isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7
|
||||
inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863
|
||||
canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886
|
||||
backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9
|
||||
isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b
|
||||
drift_from_historical_production_receipt:
|
||||
- backup_orchestrator_sha256
|
||||
- canary_wrapper_sha256
|
||||
- inventory_verifier_sha256
|
||||
toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
|
||||
toolchain_check_terminal: check_pass_no_write
|
||||
toolchain_check_diff: matching_4_drift_3_absent_0_active_operations_0
|
||||
toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
|
||||
toolchain_apply_terminal: pass
|
||||
toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
|
||||
toolchain_postcheck_terminal: check_pass_no_write
|
||||
toolchain_postcheck_diff: matching_7_drift_0_absent_0_active_operations_0
|
||||
toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
|
||||
toolchain_postcanary_terminal: check_pass_no_write
|
||||
toolchain_postcanary_diff: matching_7_drift_0_absent_0_active_operations_0
|
||||
default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
|
||||
default_canary_check_terminal: check_pass_no_write
|
||||
runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
wrapper_terminal: partial_degraded
|
||||
runtime_operation_terminal: pass
|
||||
nested_machine_receipt_verifier_terminal: pass
|
||||
source_validation:
|
||||
backup_restore_deployer_tests_passed: 92
|
||||
canonical_preflight_tests_passed: 24
|
||||
bash_syntax: pass
|
||||
ruff: pass
|
||||
ruff_format: pass
|
||||
yaml_parse: pass
|
||||
diff_check: pass
|
||||
independent_reaudit: no_commit_blocker
|
||||
toolchain_controlled_apply:
|
||||
check_run_id: P0-OBS-002-native-toolchain-20260715T001124Z-check5
|
||||
check_terminal: pass
|
||||
check_diff: matching_6_drift_1_absent_0_active_operations_0
|
||||
apply_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4
|
||||
apply_terminal: pass
|
||||
action: seven_candidates_atomically_replaced
|
||||
rollback_available: true
|
||||
runtime_backup_restore_restart_or_pull: false
|
||||
postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4
|
||||
postcheck_terminal: check_pass_no_write
|
||||
postcheck_diff: matching_7_drift_0_absent_0_active_operations_0
|
||||
attempts:
|
||||
- run_id: P0-OBS-002-native-canary-20260714T233524Z-apply1
|
||||
terminal: failed_cleanup_verified
|
||||
failed_stage: backup_submit_parser
|
||||
exit_code: 62
|
||||
root_cause: clickhouse_25_5_async_settings_id_parameter_syntax_invalid
|
||||
backup_submitted: false
|
||||
restore_submitted: false
|
||||
server_artifact_created: false
|
||||
ephemeral_residue_count: 0
|
||||
- run_id: P0-OBS-002-native-canary-20260714T234436Z-apply2
|
||||
terminal: failed_cleanup_verified_artifact_preserved
|
||||
failed_stage: isolated_restore_startup
|
||||
root_cause:
|
||||
- read_only_source_artifact_was_chowned_by_clickhouse_entrypoint
|
||||
- isolated_zookeeper_anonymous_login_contract_missing
|
||||
backup_terminal: BACKUP_CREATED
|
||||
backup_file_count: 3690
|
||||
backup_total_size_bytes: 88261384
|
||||
artifact_size_bytes: 81474109
|
||||
artifact_sha256: 139230bc6aa9f53b7b351b5d8f37eed87a68cfbba31a0b4ff4caa31906dfdaac
|
||||
restore_submitted: false
|
||||
cleanup_verified: true
|
||||
artifact_retention: preserved_failed_run_evidence
|
||||
- run_id: P0-OBS-002-native-canary-20260715T000638Z-apply3
|
||||
terminal: failed_cleanup_verified_artifact_preserved
|
||||
failed_stage: macro_contract
|
||||
exit_code: 47
|
||||
root_cause: clickhouse_25_5_system_macros_uses_macro_and_substitution_columns
|
||||
backup_terminal: BACKUP_CREATED
|
||||
backup_file_count: 4414
|
||||
backup_total_size_bytes: 92901923
|
||||
artifact_size_bytes: 85561114
|
||||
artifact_sha256: e4da648d36462e3362ff6532b8bee8f5c6745c2de4bf188fe95fe68946287993
|
||||
staging_artifact_verified: true
|
||||
restore_submitted: false
|
||||
cleanup_verified: true
|
||||
artifact_retention: preserved_failed_run_evidence
|
||||
- run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4
|
||||
check_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4-check
|
||||
check_terminal: check_pass_no_write
|
||||
terminal: pass
|
||||
exit_code: 0
|
||||
backup:
|
||||
operation_id: awoooi-6d64f408f85ad25cd78a351c0c851db8a29bea958afaad5ba6e07f03ec7a55dc
|
||||
terminal: BACKUP_CREATED
|
||||
error: ""
|
||||
file_count: 4228
|
||||
total_size_bytes: 93405832
|
||||
compressed_size_bytes: 86033128
|
||||
artifact_sha256: 4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc
|
||||
source_inventory_sha256: 1c290facb734d59443eaeeec0d778772f5fd8e06e832a096c0d4f82a4a7210ef
|
||||
isolated_restore:
|
||||
terminal: verified
|
||||
clickhouse_restore_terminal: RESTORED
|
||||
exact_clickhouse_image: true
|
||||
exact_zookeeper_image: true
|
||||
staging_network_mode: none
|
||||
restore_network_mode: docker_internal_only
|
||||
production_network_attached: false
|
||||
production_restore_performed: false
|
||||
artifact_source_mount: read_only_staging_only
|
||||
backup_volume_mount: writable_ephemeral
|
||||
source_staged_clickhouse_artifact_sha_match: true
|
||||
database_count: 6
|
||||
restored_database_count: 6
|
||||
table_count: 100
|
||||
restored_table_count: 100
|
||||
table_engine_schema_manifest_parity: true
|
||||
check_table_count: 44
|
||||
check_table_pass_count: 44
|
||||
critical_nonzero_count: 4
|
||||
source_total_rows: 4772292
|
||||
restored_total_rows: 4752914
|
||||
row_delta: -19378
|
||||
source_total_bytes: 93042212
|
||||
restored_total_bytes: 93243468
|
||||
byte_delta: 201256
|
||||
aggregate_counter_exact_parity_gate: false
|
||||
aggregate_counter_reason: online_ingestion_can_advance_between_source_inventory_and_backup_snapshot
|
||||
restic:
|
||||
snapshot_id: 574e2a1a
|
||||
snapshot_readback: pass
|
||||
repository_check_read_data_subset: 1%
|
||||
repository_scope: local_same_failure_domain
|
||||
offsite_verified: false
|
||||
independent_post_verifier:
|
||||
independent_monitor: pass
|
||||
collector_identity_started_at_restart_health_listeners_unchanged: true
|
||||
production_container_identity_count_unchanged: 4
|
||||
production_restart_count_delta: 0
|
||||
api_http_status: 200
|
||||
new_server_artifact_removed: true
|
||||
apply2_apply3_preserved_artifact_hashes_unchanged: true
|
||||
exact_ephemeral_container_volume_network_process_tmp_residue_count: 0
|
||||
cleanup_verified: true
|
||||
secret_values_collected: false
|
||||
closure_writeback: durable_local_receipts_acknowledged
|
||||
- run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
|
||||
check_terminal: check_pass_no_write
|
||||
wrapper_terminal: partial_degraded
|
||||
terminal: partial_degraded
|
||||
runtime_operation:
|
||||
terminal: pass
|
||||
scope: clickhouse_native_backup_and_isolated_restore
|
||||
nested_machine_receipt_verifier:
|
||||
stage: native_restore_receipt_verifier
|
||||
terminal: pass
|
||||
backup:
|
||||
terminal: BACKUP_CREATED
|
||||
terminal_sequence:
|
||||
- check_pass_no_write
|
||||
- pass
|
||||
independent_verifier_terminal: pass
|
||||
operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1
|
||||
operation_status: "BACKUP_CREATED|4885|108006575|99461897|"
|
||||
file_count: 4885
|
||||
total_size_bytes: 108006575
|
||||
compressed_size_bytes: 99461897
|
||||
artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
|
||||
active_operation_count_after: 0
|
||||
server_artifact_absent_after: true
|
||||
isolated_restore:
|
||||
terminal: verified
|
||||
terminal_sequence:
|
||||
- check_pass_no_runtime_write
|
||||
- verified
|
||||
clickhouse_restore_terminal: RESTORED
|
||||
independent_post_verifier_terminal: pass
|
||||
cleanup_terminal: pass
|
||||
network_scope: internal_only
|
||||
internal_network_attached: true
|
||||
staging_network_attached: false
|
||||
production_network_attached: false
|
||||
production_restore_performed: false
|
||||
cleanup_verified: true
|
||||
database_count: 6
|
||||
restored_database_count: 6
|
||||
table_count: 100
|
||||
restored_table_count: 100
|
||||
table_engine_schema_manifest_parity: true
|
||||
check_table_count: 44
|
||||
check_table_pass_count: 44
|
||||
critical_nonzero_count: 4
|
||||
restic:
|
||||
snapshot_id: 135485ab
|
||||
snapshot_readback: pass
|
||||
repository_scope: local_same_failure_domain
|
||||
offsite_verified: false
|
||||
sqlite:
|
||||
terminal: pending_application_consistent_export
|
||||
independent_post_verifier:
|
||||
terminal: pass
|
||||
independent_monitor: pass
|
||||
deployed_source_hash_drift_count: 0
|
||||
runtime_drift_count: 0
|
||||
production_container_count: 4
|
||||
production_container_identity_unchanged: true
|
||||
production_container_restart_count_delta: 0
|
||||
production_containers:
|
||||
clickhouse:
|
||||
started_at: "2026-07-14T22:49:13.665350281Z"
|
||||
restart_count: 0
|
||||
health: healthy
|
||||
collector:
|
||||
started_at: "2026-07-14T22:20:53.725583861Z"
|
||||
restart_count: 0
|
||||
health: not_configured
|
||||
signoz:
|
||||
started_at: "2026-07-11T08:45:17.179074963Z"
|
||||
restart_count: 0
|
||||
health: healthy
|
||||
zookeeper:
|
||||
started_at: "2026-07-11T08:45:17.055893064Z"
|
||||
restart_count: 0
|
||||
health: healthy
|
||||
api_http_status: 200
|
||||
listener_counts:
|
||||
"4317": 2
|
||||
"4318": 2
|
||||
"8080": 2
|
||||
active_backup_operation_count: 0
|
||||
restore_container_count: 0
|
||||
restore_volume_count: 0
|
||||
restore_network_count: 0
|
||||
toolchain_stage_residue_count: 0
|
||||
toolchain_candidate_residue_count: 0
|
||||
toolchain_rollback_residue_count: 0
|
||||
backup_process_count: 0
|
||||
lock_holders:
|
||||
canary: none
|
||||
operation: none
|
||||
toolchain: none
|
||||
server_artifact_absent: true
|
||||
exact_ephemeral_container_volume_network_process_tmp_residue_count: 0
|
||||
cleanup_verified: true
|
||||
closure_writeback:
|
||||
terminal: partial_degraded
|
||||
current_source_runtime_closed: true
|
||||
overall_program_partial_degraded: true
|
||||
terminal: production_closed_clickhouse_native_scope
|
||||
|
||||
backup_canary:
|
||||
guard_source:
|
||||
path: scripts/backup/run-signoz-backup-canary.sh
|
||||
monitor_cooldown_contract_present: true
|
||||
deployment_status: deployed_verified
|
||||
attempts:
|
||||
- trace_id: P0-OBS-002
|
||||
run_id: P0-OBS-002-backup-canary-20260714T213210Z-retry1
|
||||
work_item_id: P0-OBS-002
|
||||
terminal: failed_rolled_back
|
||||
classification: failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention
|
||||
failed_stage: clickhouse_archive_create
|
||||
archive_container_exit_code: 137
|
||||
root_cause: docker_health_monitor_restarted_collector_during_intentional_backup_stop
|
||||
consistent_archive_valid: false
|
||||
clickhouse_archive_committed: false
|
||||
sqlite_stage_reached: false
|
||||
restic_stage_reached: false
|
||||
retention_stage_reached: false
|
||||
success_stage_reached: false
|
||||
rollback:
|
||||
collector_running: true
|
||||
collector_pid_at_verifier: 1518279
|
||||
collector_started_at: "2026-07-14T21:35:01Z"
|
||||
collector_restart_count: 0
|
||||
grpc_4317_listening: true
|
||||
http_4318_listening: true
|
||||
canary_processes_absent: true
|
||||
canary_container_absent: true
|
||||
current_temp_dir_absent: true
|
||||
partial_artifacts_absent: true
|
||||
residual_cleanup_debt:
|
||||
- /tmp/signoz-backup-3723877
|
||||
- trace_id: P0-OBS-002
|
||||
run_id: P0-OBS-002-backup-canary-20260714T221230Z-retry2
|
||||
work_item_id: P0-OBS-002
|
||||
terminal: failed_rolled_back
|
||||
classification: failed_live_rw_volume_archive_native_backup_required_no_restic_no_retention
|
||||
failed_stage: clickhouse_archive_create
|
||||
backup_script_exit_code: 1
|
||||
archive_container_exit_code: unknown_not_durably_captured
|
||||
archive_stderr_receipt: suppressed_by_legacy_pipeline
|
||||
root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract
|
||||
exact_tar_error_proven: false
|
||||
likely_tar_file_changed_during_read: true
|
||||
consistent_archive_valid: false
|
||||
clickhouse_archive_committed: false
|
||||
sqlite_stage_reached: false
|
||||
restic_stage_reached: false
|
||||
retention_stage_reached: false
|
||||
success_stage_reached: false
|
||||
monitor_window:
|
||||
cron_intervals_crossed: 2
|
||||
collector_detected_during_intentional_stop: true
|
||||
cooldown_receipt_observed: true
|
||||
auto_repair_count: 0
|
||||
rollback:
|
||||
cooldown_lease_restored: true
|
||||
collector_running: true
|
||||
collector_started_at: "2026-07-14T22:20:53Z"
|
||||
collector_restart_count: 0
|
||||
grpc_4317_listening: true
|
||||
http_4318_listening: true
|
||||
health_8080_listening: true
|
||||
canary_processes_absent: true
|
||||
canary_container_absent: true
|
||||
current_temp_dir_absent: true
|
||||
partial_artifacts_absent: true
|
||||
residual_cleanup_debt: []
|
||||
|
||||
post_release_runtime_verifier:
|
||||
trace_id: P0-OBS-002
|
||||
run_id: P0-OBS-002-post-release-20260714T215500Z
|
||||
work_item_id: P0-OBS-002
|
||||
window_seconds: 600
|
||||
terminal: pass
|
||||
restart_delta: 0
|
||||
exporter_trace_errors: 0
|
||||
exporter_metric_errors: 0
|
||||
trace_count: 4026
|
||||
metric_sample_count: 76369
|
||||
api_runtime_cooldown_closed: true
|
||||
rollback: not_required
|
||||
|
||||
completed_verifiers:
|
||||
clickhouse_native_backup_migration:
|
||||
prior_run_id: P0-OBS-002-backup-canary-20260714T221230Z-retry2
|
||||
run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4
|
||||
status: production_closed_clickhouse_native_scope
|
||||
native_backup_terminal: BACKUP_CREATED
|
||||
isolated_restore_terminal: verified
|
||||
restic_snapshot_id: 574e2a1a
|
||||
independent_verifier: pass
|
||||
exact_cleanup: pass
|
||||
clickhouse_native_current_source_toolchain:
|
||||
status: deployed_verified
|
||||
current_source_deployed: true
|
||||
runtime_verifier_terminal: pass
|
||||
historical_production_receipt_preserved: true
|
||||
toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6
|
||||
toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5
|
||||
toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5
|
||||
toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5
|
||||
default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5
|
||||
runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5
|
||||
wrapper_terminal: partial_degraded
|
||||
runtime_operation_terminal: pass
|
||||
nested_machine_receipt_verifier_terminal: pass
|
||||
operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1
|
||||
operation_status: "BACKUP_CREATED|4885|108006575|99461897|"
|
||||
restic_snapshot_id: 135485ab
|
||||
artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83
|
||||
offsite_verified: false
|
||||
sqlite_terminal: pending_application_consistent_export
|
||||
independent_monitor: pass
|
||||
api_http_status: 200
|
||||
production_container_identity_unchanged: true
|
||||
production_container_restart_count_delta: 0
|
||||
active_backup_operation_count: 0
|
||||
runtime_drift_count: 0
|
||||
residue_count: 0
|
||||
exact_cleanup: pass
|
||||
|
||||
pending_verifiers:
|
||||
signoz_sqlite_application_consistent_backup:
|
||||
status: pending_supported_non_raw_export_or_coordinated_snapshot
|
||||
raw_sqlite_read_or_sync_allowed: false
|
||||
sqlite_cli_present_in_runtime: false
|
||||
required_preconditions:
|
||||
- supported_metadata_export_or_application_consistent_snapshot_design
|
||||
- independent_restore_verifier_without_raw_session_or_secret_read
|
||||
signoz_backup_offsite_dr:
|
||||
status: pending_offsite_snapshot_and_restore_verifier
|
||||
current_repository_scope: local_same_failure_domain
|
||||
latest_local_snapshot_id: 135485ab
|
||||
offsite_verified: false
|
||||
required_preconditions:
|
||||
- immutable_or_versioned_offsite_target
|
||||
- bounded_snapshot_copy_receipt
|
||||
- independent_offsite_restore_drill
|
||||
clickhouse_native_failed_artifact_retention:
|
||||
status: pending_bounded_retention_decision
|
||||
destructive_cleanup_authorized: false
|
||||
active_backup_or_restore_count: 0
|
||||
preserved_artifacts:
|
||||
- run_id: P0-OBS-002-native-canary-20260714T234436Z-apply2
|
||||
path: /backups/signoz-35003fe7ee97b8f74547b5fb2c4b1df01b9f472137e3b8299fd8e95ded6d220e.zip
|
||||
sha256: 139230bc6aa9f53b7b351b5d8f37eed87a68cfbba31a0b4ff4caa31906dfdaac
|
||||
size_bytes: 81474109
|
||||
mode: "0640"
|
||||
- run_id: P0-OBS-002-native-canary-20260715T000638Z-apply3
|
||||
path: /backups/signoz-a011c242647d8a58cf47c38e90605b5325fa0800871256a04e0fbcdeb9b9a289.zip
|
||||
sha256: e4da648d36462e3362ff6532b8bee8f5c6745c2de4bf188fe95fe68946287993
|
||||
size_bytes: 85561114
|
||||
mode: "0640"
|
||||
clickhouse_native_resume_submitted_inventory:
|
||||
status: pending_durable_submitted_inventory_contract
|
||||
same_run_resume_safe: false
|
||||
reason: live_inventory_drift_must_not_rebind_an_existing_backup_artifact
|
||||
required_preconditions:
|
||||
- atomically_persist_inventory_before_first_backup_submit
|
||||
- bind_inventory_hash_operation_id_artifact_hash_and_run_identity
|
||||
- resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered
|
||||
- regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits
|
||||
scope_boundaries:
|
||||
grpc_bridge_rollback_indicated: false
|
||||
api_manual_rollback_performed: false
|
||||
firewall_changed: false
|
||||
database_changed: false
|
||||
stateful_volume_read: false
|
||||
stateful_volume_read: true
|
||||
stateful_volume_write: true
|
||||
clickhouse_container_recreated: true
|
||||
clickhouse_data_volume_identity_changed: false
|
||||
secret_read: false
|
||||
github_used: false
|
||||
|
||||
terminal:
|
||||
status: partial_degraded
|
||||
blockers:
|
||||
- api_runtime_cooldown_pending
|
||||
- cd_probe_safety_fix_not_deployed
|
||||
- backup_collector_restore_guard_not_deployed
|
||||
- newer_main_cd_nonterminal
|
||||
- signoz_sqlite_application_consistent_backup_pending
|
||||
- signoz_backup_offsite_dr_pending
|
||||
- clickhouse_native_failed_artifact_retention_pending
|
||||
- clickhouse_native_resume_submitted_inventory_pending
|
||||
- service_registry_runtime_mirror_reconciliation_pending
|
||||
- github_freeze_legacy_asset_retirement_pending
|
||||
- monitoring_generator_duplicate_awoooi_api_identity_pending
|
||||
safe_next_action: >-
|
||||
Merge current Gitea main, validate and deploy the bounded CD probe and backup
|
||||
restore guard, then require a fresh CD terminal plus ten-minute API and OTLP
|
||||
runtime verifier with zero restart and exporter-error deltas.
|
||||
Preserve the verified native ClickHouse backup and isolated restore lane.
|
||||
Add a supported application-consistent SigNoz metadata export and restore
|
||||
verifier, copy a bounded snapshot to an immutable or versioned offsite
|
||||
target and restore it independently, then apply an explicit failed-artifact
|
||||
retention decision. Persist the submitted source inventory before BACKUP so
|
||||
same-run resume cannot bind an old artifact to newer live counters. Keep raw
|
||||
SQLite reads and syncs disabled. Reconcile the source-only service registry
|
||||
rows against live runtime before regenerating its K8s ConfigMap mirror, and
|
||||
retire legacy GitHub runner assets only after the Gitea replacement verifier
|
||||
is ready. Deduplicate the monitoring registry's awoooi-api identity before
|
||||
applying regenerated scrape configuration.
|
||||
|
||||
@@ -9,7 +9,7 @@ Wave A.6 (ADR-037): 驗證告警鏈路 E2E 完整性
|
||||
2. Alert Chain Metric — awoooi_alert_chain_last_success_timestamp 不超過 2h
|
||||
3. Webhook 可達性 — /api/v1/webhooks/alertmanager, /signoz, /sentry health
|
||||
4. Telegram Secret — K8s Secret 存在且非空
|
||||
5. SigNoz 可達 — 192.168.0.188:3301
|
||||
5. SigNoz 可達 — canonical public route (https://signoz.wooo.work)
|
||||
6. Prometheus Alertmanager — 192.168.0.188:9093 (可選)
|
||||
|
||||
使用方式:
|
||||
@@ -56,7 +56,11 @@ def _env_float(name: str, default: float) -> float:
|
||||
|
||||
|
||||
DEFAULT_API_URL = "https://awoooi.wooo.work"
|
||||
SIGNOZ_URL = "http://192.168.0.188:3301"
|
||||
DEFAULT_SIGNOZ_PUBLIC_URL = "https://signoz.wooo.work"
|
||||
SIGNOZ_URL = (
|
||||
os.environ.get("SIGNOZ_PUBLIC_URL", DEFAULT_SIGNOZ_PUBLIC_URL).strip().rstrip("/")
|
||||
or DEFAULT_SIGNOZ_PUBLIC_URL
|
||||
)
|
||||
ALERTMANAGER_URL = "http://192.168.0.188:9093"
|
||||
PROMETHEUS_URL = "http://192.168.0.110:9090"
|
||||
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite)
|
||||
# 版本: 1.3.0
|
||||
# 建立日期: 2026-04-05
|
||||
# 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計
|
||||
# 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true
|
||||
# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector
|
||||
# 2026-07-15 Codex: v1.3 加入初始狀態、全 run 有界還原與 success 前獨立 verifier
|
||||
# WOOO AIOps - SigNoz ClickHouse native backup orchestrator
|
||||
# Version: 2.0.0
|
||||
#
|
||||
# ClickHouse is backed up online through its native BACKUP engine. This script
|
||||
# never reads or archives the live ClickHouse/SQLite Docker volumes. SigNoz
|
||||
# metadata/SQLite remains an explicit application-consistent export gap and is
|
||||
# recorded in the Restic payload without being misreported as completed.
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
@@ -15,18 +15,38 @@ source "$(dirname "$0")/common.sh"
|
||||
|
||||
SERVICE="signoz"
|
||||
LOCAL_REPO="${BACKUP_BASE}/signoz"
|
||||
DUMP_DIR="/tmp/signoz-backup-$$"
|
||||
COLLECTOR_NAME="signoz-otel-collector"
|
||||
TMP_ROOT="${SIGNOZ_BACKUP_TMP_ROOT:-/tmp}"
|
||||
DUMP_DIR=""
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}"
|
||||
COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}"
|
||||
COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"
|
||||
COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"
|
||||
COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}"
|
||||
ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"
|
||||
ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"
|
||||
COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}"
|
||||
TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"
|
||||
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"
|
||||
COLLECTOR_WAS_RUNNING=0
|
||||
COLLECTOR_RESTORE_PENDING=0
|
||||
COLLECTOR_RESTORE_ATTEMPTS_USED=0
|
||||
# Destructive Restic forget/prune is a separate critical break-glass workflow.
|
||||
# The regular and canary backup paths are backup-only and fail closed if a
|
||||
# caller attempts to re-enable inline retention cleanup.
|
||||
BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-1}"
|
||||
OPERATION_LOCK_FILE="${SIGNOZ_BACKUP_OPERATION_LOCK_FILE:-/tmp/awoooi-signoz-backup-operation.lock}"
|
||||
RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}"
|
||||
RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}"
|
||||
RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}"
|
||||
RESTIC_READBACK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_READBACK_TIMEOUT_SECONDS:-120}"
|
||||
RESTIC_CHECK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_CHECK_TIMEOUT_SECONDS:-1800}"
|
||||
|
||||
BACKUP_ID_SEED="$(date -u '+%Y%m%dT%H%M%SZ')-$$"
|
||||
TRACE_ID="${TRACE_ID:-signoz-backup-${BACKUP_ID_SEED}}"
|
||||
RUN_ID="${RUN_ID:-signoz-${BACKUP_ID_SEED}}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:-SIGNOZ-BACKUP}"
|
||||
export TRACE_ID RUN_ID WORK_ITEM_ID
|
||||
|
||||
COLLECTOR_ID_BEFORE=""
|
||||
COLLECTOR_STARTED_AT_BEFORE=""
|
||||
COLLECTOR_RESTARTS_BEFORE=""
|
||||
COLLECTOR_HEALTH_BEFORE=""
|
||||
COLLECTOR_OBSERVER_PID=""
|
||||
COLLECTOR_OBSERVATIONS=""
|
||||
WORKSPACE_CREATED=0
|
||||
CLEANUP_COMPLETE=0
|
||||
FAILURE_NOTIFIED=0
|
||||
|
||||
@@ -35,27 +55,9 @@ run_collector_docker() {
|
||||
"${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@"
|
||||
}
|
||||
|
||||
collector_running_state() {
|
||||
local running
|
||||
if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
case "${running}" in
|
||||
true)
|
||||
printf '%s\n' "running"
|
||||
;;
|
||||
false)
|
||||
printf '%s\n' "stopped"
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
collector_is_running() {
|
||||
[ "$(collector_running_state)" = "running" ]
|
||||
collector_identity_state() {
|
||||
run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
|
||||
"${COLLECTOR_NAME}" 2>/dev/null
|
||||
}
|
||||
|
||||
notify_failure_once() {
|
||||
@@ -67,67 +69,87 @@ notify_failure_once() {
|
||||
notify_clawbot "failed" "${SERVICE}" "${detail}" || true
|
||||
}
|
||||
|
||||
restore_collector() {
|
||||
if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
observe_collector_continuity() {
|
||||
local epoch observation
|
||||
|
||||
while [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; do
|
||||
COLLECTOR_RESTORE_ATTEMPTS_USED=$((COLLECTOR_RESTORE_ATTEMPTS_USED + 1))
|
||||
log_info "重啟 ${COLLECTOR_NAME} (${COLLECTOR_RESTORE_ATTEMPTS_USED}/${COLLECTOR_RESTORE_MAX_ATTEMPTS})..."
|
||||
|
||||
if run_collector_docker start "${COLLECTOR_NAME}" >/dev/null 2>&1 \
|
||||
&& collector_is_running; then
|
||||
COLLECTOR_RESTORE_PENDING=0
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_warn "${COLLECTOR_NAME} 重啟或 running readback 失敗"
|
||||
if [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; then
|
||||
sleep "${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS}"
|
||||
while true; do
|
||||
epoch="$(date +%s)"
|
||||
if observation="$(collector_identity_state)"; then
|
||||
printf '%s\t%s\n' "${epoch}" "${observation}" >> "${COLLECTOR_OBSERVATIONS}"
|
||||
else
|
||||
printf '%s\tunknown\tunknown\tunknown\tunknown\tunknown\n' "${epoch}" >> "${COLLECTOR_OBSERVATIONS}"
|
||||
fi
|
||||
sleep "${COLLECTOR_CONTINUITY_POLL_SECONDS}"
|
||||
done
|
||||
|
||||
log_error "${COLLECTOR_NAME} 在有界重試後仍無法恢復"
|
||||
return 1
|
||||
}
|
||||
|
||||
verify_collector_final_state() {
|
||||
if [ "${COLLECTOR_WAS_RUNNING}" -ne 1 ]; then
|
||||
log_info "${COLLECTOR_NAME} 原本即為 stopped,不執行自動啟動"
|
||||
return 0
|
||||
fi
|
||||
start_collector_observer() {
|
||||
: > "${COLLECTOR_OBSERVATIONS}"
|
||||
observe_collector_continuity &
|
||||
COLLECTOR_OBSERVER_PID=$!
|
||||
}
|
||||
|
||||
if collector_is_running; then
|
||||
log_success "${COLLECTOR_NAME} final running verifier 通過"
|
||||
return 0
|
||||
stop_collector_observer() {
|
||||
if [ -n "${COLLECTOR_OBSERVER_PID}" ] \
|
||||
&& kill -0 "${COLLECTOR_OBSERVER_PID}" 2>/dev/null; then
|
||||
kill -TERM "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
|
||||
wait "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true
|
||||
fi
|
||||
COLLECTOR_OBSERVER_PID=""
|
||||
}
|
||||
|
||||
log_warn "${COLLECTOR_NAME} final running verifier 首次失敗,使用剩餘有界還原額度"
|
||||
COLLECTOR_RESTORE_PENDING=1
|
||||
if ! restore_collector; then
|
||||
verify_collector_continuity() {
|
||||
local observation final_id final_running final_started_at final_restarts final_health
|
||||
|
||||
stop_collector_observer
|
||||
[ -s "${COLLECTOR_OBSERVATIONS}" ] || {
|
||||
log_error "${COLLECTOR_NAME} continuity receipt 為空"
|
||||
return 1
|
||||
}
|
||||
if awk -F '\t' -v expected_id="${COLLECTOR_ID_BEFORE}" \
|
||||
-v expected_started="${COLLECTOR_STARTED_AT_BEFORE}" \
|
||||
-v expected_restarts="${COLLECTOR_RESTARTS_BEFORE}" \
|
||||
-v expected_health="${COLLECTOR_HEALTH_BEFORE}" \
|
||||
'$2 == "unknown" || $3 != "true" || $2 != expected_id || \
|
||||
$4 != expected_started || $5 != expected_restarts || $6 != expected_health { exit 1 }' \
|
||||
"${COLLECTOR_OBSERVATIONS}"; then
|
||||
:
|
||||
else
|
||||
log_error "${COLLECTOR_NAME} 在 native backup window 發生停止或 identity drift"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 與 docker start 的回傳值與 restore 內部 readback 分離,作為 success 前獨立 verifier。
|
||||
collector_is_running
|
||||
observation="$(collector_identity_state)" || return 1
|
||||
IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${observation}"
|
||||
[ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
|
||||
&& [ "${final_running}" = "true" ] \
|
||||
&& [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||||
&& [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
|
||||
&& [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ]
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
|
||||
if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then
|
||||
return "${exit_code}"
|
||||
exit "${exit_code}"
|
||||
fi
|
||||
CLEANUP_COMPLETE=1
|
||||
|
||||
# cleanup 期間忽略重複 signal,避免還原與刪除流程重入。
|
||||
trap '' HUP INT TERM
|
||||
if ! restore_collector; then
|
||||
notify_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}"
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
stop_collector_observer
|
||||
if [ "${WORKSPACE_CREATED}" -eq 1 ]; then
|
||||
if [ -n "${DUMP_DIR}" ] && [ -d "${DUMP_DIR}" ] \
|
||||
&& [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
|
||||
&& [[ "${DUMP_DIR}" == "${TMP_ROOT%/}"/signoz-backup.* ]]; then
|
||||
rm -rf -- "${DUMP_DIR}" || exit_code=90
|
||||
[ ! -e "${DUMP_DIR}" ] || exit_code=90
|
||||
else
|
||||
log_error "拒絕清理不符合本 run identity 的 SigNoz workspace"
|
||||
exit_code=90
|
||||
fi
|
||||
fi
|
||||
rm -rf "${DUMP_DIR}"
|
||||
return "${exit_code}"
|
||||
exit "${exit_code}"
|
||||
}
|
||||
|
||||
on_signal() {
|
||||
@@ -135,166 +157,253 @@ on_signal() {
|
||||
exit "$((128 + signal_number))"
|
||||
}
|
||||
|
||||
create_volume_archive() {
|
||||
local volume_name="$1"
|
||||
local output_file="$2"
|
||||
local extra_exclude="${3:-}"
|
||||
local partial_file="${output_file}.partial"
|
||||
|
||||
log_info "備份 volume: ${volume_name}"
|
||||
rm -f "${partial_file}" "${output_file}"
|
||||
|
||||
if [ -n "${extra_exclude}" ]; then
|
||||
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
|
||||
docker run --rm -v "${volume_name}:/data" alpine \
|
||||
tar czf - "${extra_exclude}" /data \
|
||||
> "${partial_file}" 2>/dev/null; then
|
||||
log_error " Volume ${volume_name} archive 建立失敗"
|
||||
rm -f "${partial_file}" "${output_file}"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \
|
||||
docker run --rm -v "${volume_name}:/data" alpine \
|
||||
tar czf - /data \
|
||||
> "${partial_file}" 2>/dev/null; then
|
||||
log_error " Volume ${volume_name} archive 建立失敗"
|
||||
rm -f "${partial_file}" "${output_file}"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -s "${partial_file}" ]; then
|
||||
log_error " Volume ${volume_name} 備份失敗 (空檔案)"
|
||||
rm -f "${partial_file}" "${output_file}"
|
||||
create_private_workspace() {
|
||||
[[ "${TMP_ROOT}" == /* ]] \
|
||||
&& [[ ! "${TMP_ROOT}" =~ (^|/)\.\.?(/|$) ]] \
|
||||
&& [[ "${TMP_ROOT}" != *$'\n'* ]] \
|
||||
&& [ -d "${TMP_ROOT}" ] && [ -w "${TMP_ROOT}" ] \
|
||||
&& [ ! -L "${TMP_ROOT}" ] || {
|
||||
log_error "SigNoz workspace root 必須是安全、可寫、非 symlink 的絕對目錄"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
mv "${partial_file}" "${output_file}"
|
||||
DUMP_DIR="$(mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX")" || return 1
|
||||
WORKSPACE_CREATED=1
|
||||
[ -d "${DUMP_DIR}" ] && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \
|
||||
&& [ -r "${DUMP_DIR}" ] && [ -w "${DUMP_DIR}" ] && [ -x "${DUMP_DIR}" ] || {
|
||||
log_error "mktemp 未建立本 run 擁有的 private SigNoz workspace"
|
||||
return 1
|
||||
}
|
||||
chmod 700 "${DUMP_DIR}" || return 1
|
||||
COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv"
|
||||
log_info "建立 SigNoz private workspace: ${DUMP_DIR}"
|
||||
}
|
||||
|
||||
verify_volume_archive() {
|
||||
local volume_name="$1"
|
||||
local output_file="$2"
|
||||
run_native_clickhouse_backup() {
|
||||
local mode="$1"
|
||||
|
||||
if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${ARCHIVE_VERIFY_TIMEOUT_SECONDS}" \
|
||||
tar -tzf "${output_file}" >/dev/null 2>&1; then
|
||||
log_error " Volume ${volume_name} archive integrity verifier 失敗"
|
||||
rm -f "${output_file}" "${output_file}.partial"
|
||||
[ -f "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||||
&& [ -x "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||||
&& [ ! -L "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \
|
||||
|| {
|
||||
log_error "ClickHouse native backup adapter 缺失、不可執行或為 symlink"
|
||||
return 1
|
||||
}
|
||||
|
||||
CLICKHOUSE_NATIVE_OUTPUT_DIR="${DUMP_DIR}" \
|
||||
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
||||
"${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" "${mode}"
|
||||
}
|
||||
|
||||
write_metadata_gap_receipt() {
|
||||
cat > "${DUMP_DIR}/signoz-metadata-gap.json" <<EOF
|
||||
{"trace_id":"${TRACE_ID}","run_id":"${RUN_ID}","work_item_id":"${WORK_ITEM_ID}","asset":"signoz-metadata-sqlite","terminal":"pending_application_consistent_export","raw_volume_read":false,"raw_volume_archive":false,"completion_claim":false}
|
||||
EOF
|
||||
log_warn "SigNoz metadata/SQLite 尚缺 application-consistent export;本 run 未讀取 live volume"
|
||||
}
|
||||
|
||||
write_restic_receipt() {
|
||||
local snapshot_id="$1"
|
||||
local artifact_hash="$2"
|
||||
local manifest_hash="$3"
|
||||
local inventory_hash="$4"
|
||||
local receipt_dir receipt_tmp receipt_file
|
||||
|
||||
[[ "${RESTIC_RECEIPT_ROOT}" == /* ]] && [ ! -L "${RESTIC_RECEIPT_ROOT}" ] || return 1
|
||||
mkdir -p "${RESTIC_RECEIPT_ROOT}"
|
||||
receipt_dir="${RESTIC_RECEIPT_ROOT}/${RUN_ID}"
|
||||
[ ! -L "${receipt_dir}" ] || return 1
|
||||
mkdir -m 700 -p "${receipt_dir}"
|
||||
receipt_file="${receipt_dir}/restic-snapshot.json"
|
||||
receipt_tmp="${receipt_file}.partial.$$"
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","service":"signoz","snapshot_id":"%s","artifact_sha256":"%s","manifest_sha256":"%s","source_inventory_sha256":"%s","repository_scope":"local_same_failure_domain","restic_check_read_data_subset":"1%%","offsite_verified":false,"metadata_sqlite_terminal":"pending_application_consistent_export","completion_claim":"clickhouse_native_only"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${snapshot_id}" \
|
||||
"${artifact_hash}" "${manifest_hash}" "${inventory_hash}" > "${receipt_tmp}"
|
||||
chmod 600 "${receipt_tmp}"
|
||||
mv "${receipt_tmp}" "${receipt_file}"
|
||||
}
|
||||
|
||||
acquire_operation_lock() {
|
||||
case "${OPERATION_LOCK_FILE}" in
|
||||
/*) ;;
|
||||
*)
|
||||
log_error "SigNoz operation lock path 必須是絕對路徑"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
[ ! -L "${OPERATION_LOCK_FILE}" ] || {
|
||||
log_error "SigNoz operation lock 不得是 symlink"
|
||||
return 1
|
||||
}
|
||||
if [ -e "${OPERATION_LOCK_FILE}" ]; then
|
||||
[ -f "${OPERATION_LOCK_FILE}" ] || {
|
||||
log_error "SigNoz operation lock 必須是 regular file"
|
||||
return 1
|
||||
}
|
||||
fi
|
||||
|
||||
local size
|
||||
size="$(du -h "${output_file}" | cut -f1)"
|
||||
log_success " Volume ${volume_name} 備份與 integrity verifier 完成 (${size})"
|
||||
exec 8>>"${OPERATION_LOCK_FILE}"
|
||||
flock -n 8 || {
|
||||
log_error "另一個 SigNoz backup/deploy operation 正在執行"
|
||||
return 1
|
||||
}
|
||||
log_info "取得 SigNoz backup/deploy 共用 operation lock"
|
||||
}
|
||||
|
||||
main() {
|
||||
local start_time=$(date +%s)
|
||||
local start_time end_time duration snapshot_id snapshot_json tags
|
||||
local initial_observation initial_running final_id final_running
|
||||
local final_started_at final_restarts final_health
|
||||
local artifact_file manifest_file inventory_file artifact_hash manifest_hash inventory_hash
|
||||
|
||||
start_time="$(date +%s)"
|
||||
trap cleanup EXIT
|
||||
trap 'on_signal 1' HUP
|
||||
trap 'on_signal 2' INT
|
||||
trap 'on_signal 15' TERM
|
||||
|
||||
log_info "========== 開始 SignOz 備份 =========="
|
||||
mkdir -p "${DUMP_DIR}"
|
||||
|
||||
local timestamp=$(date "+%Y%m%d_%H%M%S")
|
||||
|
||||
# Step 1: 只在 Collector 原本 running 時暫停,並在 stop 前 arm 還原。
|
||||
local collector_initial_state
|
||||
if ! collector_initial_state="$(collector_running_state)"; then
|
||||
log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態"
|
||||
notify_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態"
|
||||
log_info "========== 開始 SigNoz ClickHouse native 備份 =========="
|
||||
[[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]] \
|
||||
&& [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0" ]] \
|
||||
&& [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0.0" ]] || {
|
||||
notify_failure_once "COLLECTOR_CONTINUITY_POLL_SECONDS 無效"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${collector_initial_state}" = "running" ]; then
|
||||
COLLECTOR_WAS_RUNNING=1
|
||||
COLLECTOR_RESTORE_PENDING=1
|
||||
log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..."
|
||||
if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then
|
||||
log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止"
|
||||
notify_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}"
|
||||
}
|
||||
for timeout_value in "${RESTIC_INIT_TIMEOUT_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
|
||||
"${RESTIC_READBACK_TIMEOUT_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}"; do
|
||||
[[ "${timeout_value}" =~ ^[0-9]+$ ]] && [ "${timeout_value}" -gt 0 ] || {
|
||||
notify_failure_once "Restic timeout contract 無效"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
log_error "${COLLECTOR_NAME} 初始狀態為 stopped,停止備份且不自動啟動"
|
||||
notify_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止"
|
||||
}
|
||||
done
|
||||
[ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ] || {
|
||||
notify_failure_once "inline Restic forget/prune 已停用;僅允許獨立 critical break-glass workflow"
|
||||
exit 1
|
||||
fi
|
||||
docker stop signoz-telemetrystore-migrator 2>/dev/null || true
|
||||
|
||||
# Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積)
|
||||
local clickhouse_archive="${DUMP_DIR}/clickhouse_${timestamp}.tar.gz"
|
||||
local sqlite_archive="${DUMP_DIR}/sqlite_${timestamp}.tar.gz"
|
||||
create_volume_archive "signoz-clickhouse" "${clickhouse_archive}" "--exclude=/data/tmp" || {
|
||||
log_error "ClickHouse volume 備份失敗"
|
||||
notify_failure_once "SignOz ClickHouse 備份失敗"
|
||||
}
|
||||
acquire_operation_lock || {
|
||||
notify_failure_once "SigNoz backup/deploy operation lock 取得失敗"
|
||||
exit 1
|
||||
}
|
||||
create_private_workspace || {
|
||||
notify_failure_once "建立 private SigNoz backup workspace 失敗"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Step 3: 備份 SQLite volume (SignOz metadata)
|
||||
create_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
|
||||
log_error "SQLite volume 備份失敗"
|
||||
notify_failure_once "SignOz SQLite 備份失敗"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。
|
||||
if ! restore_collector; then
|
||||
notify_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}"
|
||||
# Check-mode must pass before the backup window. It writes only durable
|
||||
# receipts and does not submit BACKUP or create a server artifact.
|
||||
if ! run_native_clickhouse_backup --check; then
|
||||
log_error "ClickHouse native BACKUP check-mode 失敗;沒有 volume tar fallback"
|
||||
notify_failure_once "SigNoz ClickHouse native BACKUP runtime contract 未就緒"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Step 5: Collector 恢復後才在 host 做有界 archive integrity verifier。
|
||||
verify_volume_archive "signoz-clickhouse" "${clickhouse_archive}" || {
|
||||
notify_failure_once "SignOz ClickHouse archive integrity verifier 失敗"
|
||||
initial_observation="$(collector_identity_state)" || {
|
||||
notify_failure_once "無法讀取 ${COLLECTOR_NAME} 初始 identity/state"
|
||||
exit 1
|
||||
}
|
||||
verify_volume_archive "signoz-sqlite" "${sqlite_archive}" || {
|
||||
notify_failure_once "SignOz SQLite archive integrity verifier 失敗"
|
||||
IFS=$'\t' read -r COLLECTOR_ID_BEFORE initial_running COLLECTOR_STARTED_AT_BEFORE \
|
||||
COLLECTOR_RESTARTS_BEFORE COLLECTOR_HEALTH_BEFORE <<< "${initial_observation}"
|
||||
[ -n "${COLLECTOR_ID_BEFORE}" ] && [ "${initial_running}" = "true" ] || {
|
||||
notify_failure_once "${COLLECTOR_NAME} 初始狀態不是 running"
|
||||
exit 1
|
||||
}
|
||||
[ -n "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||||
&& [[ "${COLLECTOR_RESTARTS_BEFORE}" =~ ^[0-9]+$ ]] \
|
||||
&& [[ "${COLLECTOR_HEALTH_BEFORE}" =~ ^(healthy|none)$ ]] || {
|
||||
notify_failure_once "${COLLECTOR_NAME} 初始 runtime metadata 不完整"
|
||||
exit 1
|
||||
}
|
||||
start_collector_observer
|
||||
|
||||
# Online native backup: no collector stop and no live-volume read.
|
||||
if ! run_native_clickhouse_backup --apply; then
|
||||
notify_failure_once "SigNoz ClickHouse native BACKUP 失敗"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! verify_collector_continuity; then
|
||||
notify_failure_once "${COLLECTOR_NAME} no-downtime continuity verifier 失敗"
|
||||
exit 1
|
||||
fi
|
||||
log_success "${COLLECTOR_NAME} no-downtime continuity verifier 通過"
|
||||
|
||||
write_metadata_gap_receipt
|
||||
|
||||
artifact_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip' -print)"
|
||||
manifest_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip.manifest.json' -print)"
|
||||
inventory_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.source-inventory.tsv' -print)"
|
||||
[ "$(printf '%s\n' "${artifact_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
|
||||
&& [ "$(printf '%s\n' "${manifest_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \
|
||||
&& [ "$(printf '%s\n' "${inventory_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] || {
|
||||
log_error "ClickHouse native artifact set 不唯一或不完整"
|
||||
exit 1
|
||||
}
|
||||
artifact_hash="$(sha256sum "${artifact_file}" | awk '{print $1}')"
|
||||
manifest_hash="$(sha256sum "${manifest_file}" | awk '{print $1}')"
|
||||
inventory_hash="$(sha256sum "${inventory_file}" | awk '{print $1}')"
|
||||
|
||||
# Step 6: 初始化 Restic 倉庫
|
||||
if [ ! -d "${LOCAL_REPO}/data" ]; then
|
||||
log_info "初始化 Restic 倉庫: ${LOCAL_REPO}"
|
||||
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_INIT_TIMEOUT_SECONDS}" \
|
||||
restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||||
log_error "Restic 倉庫初始化失敗"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
|
||||
# Step 7: Restic 備份
|
||||
log_info "建立 Restic 備份..."
|
||||
local tags=$(build_tags "${SERVICE}")
|
||||
restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" --password-file "${RESTIC_PASSWORD_FILE}" ${tags} 2>&1
|
||||
log_info "建立 ClickHouse native artifact 的 Restic 備份..."
|
||||
tags="$(build_tags "${SERVICE}")"
|
||||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \
|
||||
restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \
|
||||
--password-file "${RESTIC_PASSWORD_FILE}" ${tags} \
|
||||
--tag "trace:${TRACE_ID}" --tag "run:${RUN_ID}" \
|
||||
--tag "work-item:${WORK_ITEM_ID}" 2>&1
|
||||
|
||||
local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1)
|
||||
snapshot_json="$(timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${RESTIC_READBACK_TIMEOUT_SECONDS}" restic -r "${LOCAL_REPO}" snapshots \
|
||||
--tag "run:${RUN_ID}" --latest 1 --json \
|
||||
--password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)"
|
||||
snapshot_id="$(printf '%s\n' "${snapshot_json}" \
|
||||
| sed -n 's/.*"short_id":"\([^"]*\)".*/\1/p' | head -1)"
|
||||
[[ "${snapshot_id}" =~ ^[A-Za-z0-9]+$ ]] || {
|
||||
log_error "Restic same-run snapshot durable readback 缺失"
|
||||
exit 1
|
||||
}
|
||||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}" \
|
||||
restic -r "${LOCAL_REPO}" check --read-data-subset=1% \
|
||||
--password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || {
|
||||
log_error "Restic repository/data-subset verifier 失敗"
|
||||
exit 1
|
||||
}
|
||||
write_restic_receipt "${snapshot_id}" "${artifact_hash}" \
|
||||
"${manifest_hash}" "${inventory_hash}" || {
|
||||
log_error "Restic durable receipt writeback 失敗"
|
||||
exit 1
|
||||
}
|
||||
log_success "Restic 備份完成: ${snapshot_id}"
|
||||
|
||||
# Step 8: GFS 清理;real canary 可明確跳過 destructive retention。
|
||||
if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then
|
||||
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
|
||||
else
|
||||
cleanup_old_backups "${LOCAL_REPO}"
|
||||
fi
|
||||
log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)"
|
||||
|
||||
# Step 9: success 前獨立檢查 runtime state;失敗時不得發送 success。
|
||||
if ! verify_collector_final_state; then
|
||||
notify_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗"
|
||||
# Final readback is separate from the continuity observer receipt.
|
||||
initial_observation="$(collector_identity_state)" || {
|
||||
notify_failure_once "${COLLECTOR_NAME} final readback 失敗"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${initial_observation}"
|
||||
[ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \
|
||||
&& [ "${final_running}" = "true" ] \
|
||||
&& [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \
|
||||
&& [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \
|
||||
&& [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ] || {
|
||||
notify_failure_once "${COLLECTOR_NAME} final identity/start/restart/health verifier 失敗"
|
||||
exit 1
|
||||
}
|
||||
|
||||
local end_time=$(date +%s)
|
||||
local duration=$((end_time - start_time))
|
||||
log_success "========== SignOz 備份完成 (${duration}s) =========="
|
||||
notify_clawbot "success" "${SERVICE}" "SignOz 備份完成 (ClickHouse+SQLite)" "${duration}"
|
||||
end_time="$(date +%s)"
|
||||
duration=$((end_time - start_time))
|
||||
log_success "========== SigNoz ClickHouse native 備份完成 (${duration}s) =========="
|
||||
notify_clawbot "warning" "${SERVICE}" \
|
||||
"ClickHouse native backup 完成;metadata/SQLite application-consistent export 仍 pending" \
|
||||
"${duration}"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
628
scripts/backup/clickhouse-native-backup.sh
Executable file
628
scripts/backup/clickhouse-native-backup.sh
Executable file
@@ -0,0 +1,628 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# P0-OBS-002 - ClickHouse native BACKUP adapter for SigNoz
|
||||
#
|
||||
# This adapter deliberately has no live-volume tar fallback. It requires a
|
||||
# ClickHouse Disk configured and allowed for BACKUP, records every bounded
|
||||
# command's stdout/stderr/exit status, and copies only the completed native
|
||||
# backup artifact into the caller's staging directory.
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CONTAINER_NAME="${CLICKHOUSE_NATIVE_CONTAINER_NAME:-signoz-clickhouse}"
|
||||
BACKUP_DISK="${CLICKHOUSE_NATIVE_BACKUP_DISK:-backups}"
|
||||
EXPECTED_DISK_PATH="${CLICKHOUSE_NATIVE_EXPECTED_DISK_PATH:-/backups/}"
|
||||
OUTPUT_DIR="${CLICKHOUSE_NATIVE_OUTPUT_DIR:-}"
|
||||
RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}"
|
||||
COMMAND_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS:-30}"
|
||||
BACKUP_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS:-1800}"
|
||||
POLL_INTERVAL_SECONDS="${CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS:-5}"
|
||||
KILL_AFTER_SECONDS="${CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS:-30}"
|
||||
POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}"
|
||||
EXPECTED_DATABASES_CSV="signoz_analytics,signoz_logs,signoz_metadata,signoz_meter,signoz_metrics,signoz_traces"
|
||||
EXPECTED_DATABASES_JSON='["signoz_analytics","signoz_logs","signoz_metadata","signoz_meter","signoz_metrics","signoz_traces"]'
|
||||
BACKUP_SCOPE_SQL="DATABASE signoz_analytics,DATABASE signoz_logs,DATABASE signoz_metadata,DATABASE signoz_meter,DATABASE signoz_metrics,DATABASE signoz_traces"
|
||||
|
||||
TRACE_ID="${TRACE_ID:-}"
|
||||
RUN_ID="${RUN_ID:-}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
||||
|
||||
MODE=""
|
||||
IDENTITY_HASH=""
|
||||
OPERATION_ID=""
|
||||
ARTIFACT_NAME=""
|
||||
SERVER_ARTIFACT_PATH=""
|
||||
OUTPUT_FILE=""
|
||||
MANIFEST_FILE=""
|
||||
HASH_FILE=""
|
||||
INVENTORY_FILE=""
|
||||
OUTPUT_PARTIAL=""
|
||||
MANIFEST_PARTIAL=""
|
||||
INVENTORY_PARTIAL=""
|
||||
RECEIPT_DIR=""
|
||||
RECEIPT_LOG=""
|
||||
IDENTITY_FILE=""
|
||||
INVOCATION_ID=""
|
||||
|
||||
LAST_STDOUT=""
|
||||
LAST_STDERR=""
|
||||
LAST_STATUS_FILE=""
|
||||
LAST_EXIT_CODE=0
|
||||
COMMAND_SEQUENCE=0
|
||||
|
||||
CONTAINER_ID=""
|
||||
CLICKHOUSE_VERSION=""
|
||||
DATABASES_HASH=""
|
||||
SOURCE_INVENTORY_RECEIPT=""
|
||||
SOURCE_INVENTORY_HASH=""
|
||||
STATUS_PRESENT=0
|
||||
BACKUP_NAME=""
|
||||
BACKUP_STATUS=""
|
||||
BACKUP_NUM_FILES="0"
|
||||
BACKUP_TOTAL_SIZE="0"
|
||||
BACKUP_UNCOMPRESSED_SIZE="0"
|
||||
BACKUP_COMPRESSED_SIZE="0"
|
||||
BACKUP_ERROR_HEX="none"
|
||||
BACKUP_ERROR_HASH=""
|
||||
|
||||
RECEIPTS_READY=0
|
||||
CHECK_PASS=0
|
||||
APPLY_PASS=0
|
||||
OUTPUT_CREATED=0
|
||||
SERVER_ARTIFACT_ACTIVE=0
|
||||
|
||||
log_info() { printf '[INFO] %s\n' "$*"; }
|
||||
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||
|
||||
valid_identity() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
||||
}
|
||||
|
||||
valid_name() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]
|
||||
}
|
||||
|
||||
valid_positive_integer() {
|
||||
[[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
|
||||
}
|
||||
|
||||
emit_receipt() {
|
||||
local stage="$1"
|
||||
local terminal="$2"
|
||||
local detail="$3"
|
||||
local observed_at
|
||||
|
||||
[ "${RECEIPTS_READY}" -eq 1 ] || return 0
|
||||
observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${INVOCATION_ID}" \
|
||||
"${observed_at}" "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}"
|
||||
}
|
||||
|
||||
fail_closed() {
|
||||
local detail="$1"
|
||||
log_error "${detail}"
|
||||
emit_receipt "failure" "failed" "${detail}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
local terminal="failed"
|
||||
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
[ -z "${OUTPUT_PARTIAL}" ] || rm -f -- "${OUTPUT_PARTIAL}"
|
||||
[ -z "${MANIFEST_PARTIAL}" ] || rm -f -- "${MANIFEST_PARTIAL}"
|
||||
[ -z "${INVENTORY_PARTIAL}" ] || rm -f -- "${INVENTORY_PARTIAL}"
|
||||
|
||||
if [ "${OUTPUT_CREATED}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then
|
||||
rm -f -- "${OUTPUT_FILE}" "${MANIFEST_FILE}" "${HASH_FILE}" "${INVENTORY_FILE}"
|
||||
emit_receipt "cleanup" "removed" "unverified_local_artifact_removed"
|
||||
fi
|
||||
|
||||
if [ "${SERVER_ARTIFACT_ACTIVE}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then
|
||||
emit_receipt "cleanup" "preserved" \
|
||||
"active_or_unverified_server_artifact_preserved_for_safe_followup"
|
||||
fi
|
||||
|
||||
if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_PASS}" -eq 1 ]; then
|
||||
terminal="check_pass_no_write"
|
||||
elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "apply" ] && [ "${APPLY_PASS}" -eq 1 ]; then
|
||||
terminal="pass"
|
||||
fi
|
||||
|
||||
emit_receipt "terminal" "${terminal}" "clickhouse_native_backup_${terminal}"
|
||||
printf 'CLICKHOUSE_NATIVE_BACKUP_TERMINAL trace_id=%s run_id=%s work_item_id=%s mode=%s terminal=%s exit_code=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" "${terminal}" "${exit_code}"
|
||||
exit "${exit_code}"
|
||||
}
|
||||
|
||||
on_signal() {
|
||||
local signal_number="$1"
|
||||
emit_receipt "signal" "failed" "signal_${signal_number}"
|
||||
exit "$((128 + signal_number))"
|
||||
}
|
||||
|
||||
setup() {
|
||||
local identity_content identity_tmp
|
||||
|
||||
[ "$#" -eq 1 ] || { log_error "exactly one mode is required: --check or --apply"; exit 2; }
|
||||
case "$1" in
|
||||
--check) MODE="check" ;;
|
||||
--apply) MODE="apply" ;;
|
||||
*) log_error "unsupported mode: use --check or --apply"; exit 2 ;;
|
||||
esac
|
||||
|
||||
valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and path-safe"; exit 2; }
|
||||
valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and path-safe"; exit 2; }
|
||||
valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and path-safe"; exit 2; }
|
||||
valid_name "${CONTAINER_NAME}" || { log_error "invalid container name"; exit 2; }
|
||||
valid_name "${BACKUP_DISK}" || { log_error "invalid backup disk name"; exit 2; }
|
||||
valid_positive_integer "${COMMAND_TIMEOUT_SECONDS}" || { log_error "invalid command timeout"; exit 2; }
|
||||
valid_positive_integer "${BACKUP_TIMEOUT_SECONDS}" || { log_error "invalid backup timeout"; exit 2; }
|
||||
valid_positive_integer "${POLL_INTERVAL_SECONDS}" || { log_error "invalid poll interval"; exit 2; }
|
||||
valid_positive_integer "${KILL_AFTER_SECONDS}" || { log_error "invalid kill-after timeout"; exit 2; }
|
||||
[[ "${EXPECTED_DISK_PATH}" == /*/ ]] && [[ "${EXPECTED_DISK_PATH}" != *".."* ]] \
|
||||
|| { log_error "expected disk path must be an absolute trailing-slash path without dot-dot"; exit 2; }
|
||||
[ -n "${OUTPUT_DIR}" ] && [[ "${OUTPUT_DIR}" == /* ]] \
|
||||
|| { log_error "CLICKHOUSE_NATIVE_OUTPUT_DIR must be absolute"; exit 2; }
|
||||
[ -d "${OUTPUT_DIR}" ] && [ -w "${OUTPUT_DIR}" ] && [ ! -L "${OUTPUT_DIR}" ] \
|
||||
|| { log_error "output directory missing, unwritable, or symlink"; exit 2; }
|
||||
if [ -n "${POST_VERIFY_HOOK}" ]; then
|
||||
[[ "${POST_VERIFY_HOOK}" == /* ]] && [ -f "${POST_VERIFY_HOOK}" ] \
|
||||
&& [ -x "${POST_VERIFY_HOOK}" ] && [ ! -L "${POST_VERIFY_HOOK}" ] \
|
||||
|| { log_error "post verifier hook must be an absolute executable regular file"; exit 2; }
|
||||
fi
|
||||
|
||||
for command_name in awk basename cat chmod cmp cp date docker env flock grep mkdir mktemp mv rm sha256sum sleep timeout touch tr unzip wc; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 \
|
||||
|| { log_error "required command missing: ${command_name}"; exit 2; }
|
||||
done
|
||||
|
||||
IDENTITY_HASH="$(printf '%s\000%s\000%s' "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" | sha256sum | awk '{print $1}')"
|
||||
OPERATION_ID="awoooi-${IDENTITY_HASH}"
|
||||
ARTIFACT_NAME="signoz-${IDENTITY_HASH}.zip"
|
||||
SERVER_ARTIFACT_PATH="${EXPECTED_DISK_PATH%/}/${ARTIFACT_NAME}"
|
||||
OUTPUT_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.zip"
|
||||
MANIFEST_FILE="${OUTPUT_FILE}.manifest.json"
|
||||
HASH_FILE="${OUTPUT_FILE}.sha256"
|
||||
INVENTORY_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.source-inventory.tsv"
|
||||
OUTPUT_PARTIAL="${OUTPUT_FILE}.partial"
|
||||
MANIFEST_PARTIAL="${MANIFEST_FILE}.partial"
|
||||
INVENTORY_PARTIAL="${INVENTORY_FILE}.partial"
|
||||
|
||||
[ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root symlink is unsafe"; exit 2; }
|
||||
mkdir -p "${RECEIPT_ROOT}"
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
[ ! -L "${RECEIPT_DIR}" ] || { log_error "receipt directory symlink is unsafe"; exit 2; }
|
||||
mkdir -m 700 -p "${RECEIPT_DIR}"
|
||||
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
||||
IDENTITY_FILE="${RECEIPT_DIR}/identity.json"
|
||||
INVOCATION_ID="${MODE}-$(date -u '+%Y%m%dT%H%M%SZ')-$$"
|
||||
|
||||
identity_content="{\"trace_id\":\"${TRACE_ID}\",\"run_id\":\"${RUN_ID}\",\"work_item_id\":\"${WORK_ITEM_ID}\",\"identity_hash\":\"${IDENTITY_HASH}\",\"operation_id\":\"${OPERATION_ID}\",\"artifact_name\":\"${ARTIFACT_NAME}\"}"
|
||||
if [ -e "${IDENTITY_FILE}" ]; then
|
||||
[ -f "${IDENTITY_FILE}" ] && [ ! -L "${IDENTITY_FILE}" ] \
|
||||
|| { log_error "existing identity receipt is unsafe"; exit 2; }
|
||||
[ "$(cat "${IDENTITY_FILE}")" = "${identity_content}" ] \
|
||||
|| { log_error "run identity conflicts with durable receipt"; exit 2; }
|
||||
else
|
||||
identity_tmp="$(mktemp "${RECEIPT_DIR}/.identity.XXXXXX")"
|
||||
printf '%s\n' "${identity_content}" > "${identity_tmp}"
|
||||
chmod 600 "${identity_tmp}"
|
||||
mv "${identity_tmp}" "${IDENTITY_FILE}"
|
||||
fi
|
||||
touch "${RECEIPT_LOG}"
|
||||
RECEIPTS_READY=1
|
||||
|
||||
[ ! -L "${RECEIPT_DIR}/run.lock" ] || fail_closed "run_lock_symlink_unsafe"
|
||||
exec 9>>"${RECEIPT_DIR}/run.lock"
|
||||
flock -n 9 || fail_closed "same_run_native_backup_is_active"
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'on_signal 1' HUP
|
||||
trap 'on_signal 2' INT
|
||||
trap 'on_signal 15' TERM
|
||||
}
|
||||
|
||||
run_captured() {
|
||||
local step="$1"
|
||||
local stdout_hash stderr_hash terminal
|
||||
shift
|
||||
|
||||
valid_name "${step}" || fail_closed "invalid_command_step"
|
||||
COMMAND_SEQUENCE=$((COMMAND_SEQUENCE + 1))
|
||||
LAST_STDOUT="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stdout"
|
||||
LAST_STDERR="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stderr"
|
||||
LAST_STATUS_FILE="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.status"
|
||||
|
||||
set +e
|
||||
"$@" >"${LAST_STDOUT}" 2>"${LAST_STDERR}"
|
||||
LAST_EXIT_CODE=$?
|
||||
set -e
|
||||
printf '%s\n' "${LAST_EXIT_CODE}" > "${LAST_STATUS_FILE}"
|
||||
stdout_hash="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')"
|
||||
stderr_hash="$(sha256sum "${LAST_STDERR}" | awk '{print $1}')"
|
||||
terminal="pass"
|
||||
[ "${LAST_EXIT_CODE}" -eq 0 ] || terminal="failed"
|
||||
emit_receipt "command" "${terminal}" \
|
||||
"step_${step}_exit_${LAST_EXIT_CODE}_stdout_${stdout_hash}_stderr_${stderr_hash}"
|
||||
return "${LAST_EXIT_CODE}"
|
||||
}
|
||||
|
||||
run_sql() {
|
||||
local step="$1"
|
||||
local query="$2"
|
||||
shift 2
|
||||
run_captured "${step}" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
||||
clickhouse-client --format TSVRaw "$@" --query "${query}"
|
||||
}
|
||||
|
||||
perform_check() {
|
||||
local inspect_line running disk_path system_backups_count database_count backup_grant
|
||||
local expected_databases_file
|
||||
|
||||
run_captured "container" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" docker inspect \
|
||||
--format $'{{.Id}}\t{{.State.Running}}' "${CONTAINER_NAME}" \
|
||||
|| fail_closed "clickhouse_container_inspect_failed"
|
||||
inspect_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
||||
IFS=$'\t' read -r CONTAINER_ID running <<< "${inspect_line}"
|
||||
[ -n "${CONTAINER_ID}" ] && [ "${running}" = "true" ] \
|
||||
|| fail_closed "clickhouse_container_not_running_or_identity_ambiguous"
|
||||
|
||||
run_sql "version" 'SELECT version()' || fail_closed "clickhouse_version_query_failed"
|
||||
CLICKHOUSE_VERSION="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
||||
[ -n "${CLICKHOUSE_VERSION}" ] || fail_closed "clickhouse_version_empty"
|
||||
|
||||
run_sql "system_backups" \
|
||||
"SELECT count() FROM system.tables WHERE database='system' AND name='backups'" \
|
||||
|| fail_closed "system_backups_capability_query_failed"
|
||||
system_backups_count="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
||||
[ "${system_backups_count}" = "1" ] || fail_closed "native_backup_capability_absent"
|
||||
|
||||
run_sql "backup_grant" 'CHECK GRANT BACKUP ON *.*' \
|
||||
|| fail_closed "native_backup_grant_check_failed"
|
||||
backup_grant="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
||||
[ "${backup_grant}" = "1" ] || fail_closed "native_backup_grant_absent"
|
||||
|
||||
run_sql "backup_disk" \
|
||||
'SELECT path FROM system.disks WHERE name={disk:String}' \
|
||||
--param_disk "${BACKUP_DISK}" || fail_closed "backup_disk_query_failed"
|
||||
disk_path="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
||||
[ "${disk_path}" = "${EXPECTED_DISK_PATH}" ] \
|
||||
|| fail_closed "backup_disk_missing_or_path_mismatch"
|
||||
|
||||
run_captured "disk_readable" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
||||
test -d "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_absent"
|
||||
run_captured "disk_writable" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
||||
test -w "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_unwritable"
|
||||
|
||||
run_sql "databases" \
|
||||
"SELECT name FROM system.databases WHERE name NOT IN ('system','information_schema','INFORMATION_SCHEMA','default') ORDER BY name" \
|
||||
|| fail_closed "user_database_inventory_query_failed"
|
||||
database_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
|
||||
expected_databases_file="${RECEIPT_DIR}/${INVOCATION_ID}-expected-databases.tsv"
|
||||
printf '%s\n' signoz_analytics signoz_logs signoz_metadata signoz_meter signoz_metrics signoz_traces \
|
||||
> "${expected_databases_file}"
|
||||
cmp -s "${expected_databases_file}" "${LAST_STDOUT}" \
|
||||
|| fail_closed "signoz_database_allowlist_drift"
|
||||
[ "${database_count}" -eq 6 ] || fail_closed "signoz_database_count_drift"
|
||||
DATABASES_HASH="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')"
|
||||
|
||||
run_sql "table_inventory" \
|
||||
"SELECT database,name,engine,toString(ifNull(total_rows,0)),toString(ifNull(total_bytes,0)),hex(SHA256(create_table_query)) FROM system.tables WHERE database IN ('signoz_analytics','signoz_logs','signoz_metadata','signoz_meter','signoz_metrics','signoz_traces') ORDER BY database,name" \
|
||||
|| fail_closed "source_table_inventory_query_failed"
|
||||
[ "$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")" -gt 0 ] \
|
||||
|| fail_closed "source_table_inventory_empty"
|
||||
SOURCE_INVENTORY_RECEIPT="${LAST_STDOUT}"
|
||||
SOURCE_INVENTORY_HASH="$(sha256sum "${SOURCE_INVENTORY_RECEIPT}" | awk '{print $1}')"
|
||||
|
||||
emit_receipt "sensor_source" "pass" \
|
||||
"container_${CONTAINER_ID}_version_${CLICKHOUSE_VERSION}_disk_${BACKUP_DISK}"
|
||||
emit_receipt "normalized_asset_identity" "pass" \
|
||||
"identity_${IDENTITY_HASH}_operation_${OPERATION_ID}_artifact_${ARTIFACT_NAME}"
|
||||
emit_receipt "source_of_truth_diff" "pass" \
|
||||
"configured_disk_path_exact_six_databases_hash_${DATABASES_HASH}_table_inventory_${SOURCE_INVENTORY_HASH}"
|
||||
emit_receipt "risk_policy" "pass" \
|
||||
"risk_medium_native_backup_async_bounded_poll_exact_allowlist_no_volume_tar_fallback"
|
||||
}
|
||||
|
||||
query_backup_status() {
|
||||
local line_count status_line
|
||||
|
||||
run_sql "backup_status" \
|
||||
"SELECT name,toString(status),toString(num_files),toString(total_size),toString(uncompressed_size),toString(compressed_size),if(empty(error),'none',hex(substring(error,1,2048))),hex(SHA256(error)) FROM system.backups WHERE id={operation_id:String}" \
|
||||
--param_operation_id "${OPERATION_ID}" || fail_closed "system_backups_status_query_failed"
|
||||
line_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")"
|
||||
if [ "${line_count}" -eq 0 ]; then
|
||||
STATUS_PRESENT=0
|
||||
return 0
|
||||
fi
|
||||
[ "${line_count}" -eq 1 ] || fail_closed "system_backups_operation_id_not_unique"
|
||||
STATUS_PRESENT=1
|
||||
status_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
||||
IFS=$'\t' read -r BACKUP_NAME BACKUP_STATUS BACKUP_NUM_FILES BACKUP_TOTAL_SIZE \
|
||||
BACKUP_UNCOMPRESSED_SIZE BACKUP_COMPRESSED_SIZE BACKUP_ERROR_HEX \
|
||||
BACKUP_ERROR_HASH <<< "${status_line}"
|
||||
[[ "${BACKUP_NUM_FILES}" =~ ^[0-9]+$ ]] \
|
||||
&& [[ "${BACKUP_TOTAL_SIZE}" =~ ^[0-9]+$ ]] \
|
||||
&& [[ "${BACKUP_UNCOMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \
|
||||
&& [[ "${BACKUP_COMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \
|
||||
|| fail_closed "system_backups_metrics_ambiguous"
|
||||
}
|
||||
|
||||
server_artifact_absent() {
|
||||
run_captured "artifact_absent" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
||||
test ! -e "${SERVER_ARTIFACT_PATH}"
|
||||
}
|
||||
|
||||
cleanup_server_artifact() {
|
||||
run_captured "artifact_cleanup" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \
|
||||
rm -f -- "${SERVER_ARTIFACT_PATH}" || return 1
|
||||
server_artifact_absent || return 1
|
||||
SERVER_ARTIFACT_ACTIVE=0
|
||||
emit_receipt "cleanup" "pass" "exact_server_artifact_removed_and_absent"
|
||||
}
|
||||
|
||||
poll_backup_terminal() {
|
||||
local deadline now expected_name
|
||||
expected_name="Disk('${BACKUP_DISK}', '${ARTIFACT_NAME}')"
|
||||
deadline=$(( $(date +%s) + BACKUP_TIMEOUT_SECONDS ))
|
||||
|
||||
while true; do
|
||||
query_backup_status
|
||||
[ "${STATUS_PRESENT}" -eq 1 ] || fail_closed "async_backup_status_disappeared"
|
||||
[ "${BACKUP_NAME}" = "${expected_name}" ] || fail_closed "async_backup_name_mismatch"
|
||||
emit_receipt "backup_status" "observed" \
|
||||
"status_${BACKUP_STATUS}_files_${BACKUP_NUM_FILES}_total_${BACKUP_TOTAL_SIZE}_error_hash_${BACKUP_ERROR_HASH}"
|
||||
|
||||
case "${BACKUP_STATUS}" in
|
||||
BACKUP_CREATED)
|
||||
[ "${BACKUP_NUM_FILES}" -gt 0 ] \
|
||||
&& [ "${BACKUP_TOTAL_SIZE}" -gt 0 ] \
|
||||
&& [ "${BACKUP_UNCOMPRESSED_SIZE}" -gt 0 ] \
|
||||
&& [ "${BACKUP_COMPRESSED_SIZE}" -gt 0 ] \
|
||||
|| fail_closed "native_backup_terminal_metrics_empty"
|
||||
SERVER_ARTIFACT_ACTIVE=1
|
||||
return 0
|
||||
;;
|
||||
CREATING_BACKUP)
|
||||
SERVER_ARTIFACT_ACTIVE=1
|
||||
;;
|
||||
BACKUP_FAILED|BACKUP_CANCELLED)
|
||||
printf '%s\n' "${BACKUP_ERROR_HEX}" \
|
||||
> "${RECEIPT_DIR}/${INVOCATION_ID}-backup-error-bounded.hex"
|
||||
emit_receipt "backup_error" "retained" \
|
||||
"bounded_hex_2048_bytes_hash_${BACKUP_ERROR_HASH}"
|
||||
SERVER_ARTIFACT_ACTIVE=1
|
||||
if ! cleanup_server_artifact; then
|
||||
emit_receipt "cleanup" "failed" "failed_terminal_exact_artifact_cleanup_failed"
|
||||
fi
|
||||
fail_closed "native_backup_terminal_${BACKUP_STATUS}"
|
||||
;;
|
||||
*) fail_closed "unexpected_native_backup_status_${BACKUP_STATUS}" ;;
|
||||
esac
|
||||
|
||||
now="$(date +%s)"
|
||||
if [ "${now}" -ge "${deadline}" ]; then
|
||||
fail_closed "native_backup_poll_timeout_artifact_preserved"
|
||||
fi
|
||||
sleep "${POLL_INTERVAL_SECONDS}"
|
||||
done
|
||||
}
|
||||
|
||||
verify_existing_local_artifact() {
|
||||
local recorded_hash actual_hash inventory_hash artifact_size
|
||||
local manifest_hash hook_manifest_hash hook_inventory_hash
|
||||
|
||||
if [ ! -e "${OUTPUT_FILE}" ] && [ ! -e "${MANIFEST_FILE}" ] \
|
||||
&& [ ! -e "${HASH_FILE}" ] && [ ! -e "${INVENTORY_FILE}" ]; then
|
||||
return 1
|
||||
fi
|
||||
[ -f "${OUTPUT_FILE}" ] && [ ! -L "${OUTPUT_FILE}" ] \
|
||||
&& [ -f "${MANIFEST_FILE}" ] && [ ! -L "${MANIFEST_FILE}" ] \
|
||||
&& [ -f "${HASH_FILE}" ] && [ ! -L "${HASH_FILE}" ] \
|
||||
&& [ -f "${INVENTORY_FILE}" ] && [ ! -L "${INVENTORY_FILE}" ] \
|
||||
|| fail_closed "local_artifact_identity_incomplete_or_unsafe"
|
||||
grep -F -q "\"identity_hash\":\"${IDENTITY_HASH}\"" "${MANIFEST_FILE}" \
|
||||
|| fail_closed "local_artifact_manifest_identity_mismatch"
|
||||
recorded_hash="$(awk 'NR == 1 { print $1 }' "${HASH_FILE}")"
|
||||
[[ "${recorded_hash}" =~ ^[0-9a-f]{64}$ ]] || fail_closed "local_artifact_hash_receipt_invalid"
|
||||
actual_hash="$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')"
|
||||
[ "${actual_hash}" = "${recorded_hash}" ] || fail_closed "local_artifact_hash_mismatch"
|
||||
artifact_size="$(wc -c < "${OUTPUT_FILE}" | tr -d ' ')"
|
||||
inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')"
|
||||
manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')"
|
||||
grep -F -q "\"operation_id\":\"${OPERATION_ID}\"" "${MANIFEST_FILE}" \
|
||||
|| fail_closed "local_artifact_manifest_operation_mismatch"
|
||||
grep -F -q "\"artifact_name\":\"${ARTIFACT_NAME}\"" "${MANIFEST_FILE}" \
|
||||
|| fail_closed "local_artifact_manifest_name_mismatch"
|
||||
grep -F -q "\"databases\":${EXPECTED_DATABASES_JSON}" "${MANIFEST_FILE}" \
|
||||
|| fail_closed "local_artifact_manifest_database_allowlist_mismatch"
|
||||
grep -F -q "\"source_inventory_sha256\":\"${inventory_hash}\"" "${MANIFEST_FILE}" \
|
||||
|| fail_closed "local_source_inventory_hash_mismatch"
|
||||
grep -F -q "\"sha256\":\"${actual_hash}\"" "${MANIFEST_FILE}" \
|
||||
|| fail_closed "local_artifact_manifest_hash_mismatch"
|
||||
grep -F -q "\"size_bytes\":${artifact_size}" "${MANIFEST_FILE}" \
|
||||
|| fail_closed "local_artifact_manifest_size_mismatch"
|
||||
grep -F -q '"status":"BACKUP_CREATED"' "${MANIFEST_FILE}" \
|
||||
|| fail_closed "local_artifact_manifest_terminal_mismatch"
|
||||
run_captured "local_zip_reuse" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_FILE}" \
|
||||
|| fail_closed "local_native_archive_reuse_verifier_failed"
|
||||
if [ -n "${POST_VERIFY_HOOK}" ]; then
|
||||
run_captured "reuse_verify_hook_check" env \
|
||||
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
||||
CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \
|
||||
CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \
|
||||
CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \
|
||||
CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
|
||||
CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
|
||||
CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \
|
||||
"${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_reuse_check_failed"
|
||||
run_captured "reuse_verify_hook" env \
|
||||
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
||||
CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \
|
||||
CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \
|
||||
CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \
|
||||
CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
|
||||
CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
|
||||
CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \
|
||||
"${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_reuse_failed"
|
||||
[ "$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')" = "${actual_hash}" ] \
|
||||
|| fail_closed "post_verify_hook_modified_reused_artifact"
|
||||
hook_inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')"
|
||||
[ "${hook_inventory_hash}" = "${inventory_hash}" ] \
|
||||
|| fail_closed "post_verify_hook_modified_reused_source_inventory"
|
||||
hook_manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')"
|
||||
[ "${hook_manifest_hash}" = "${manifest_hash}" ] \
|
||||
|| fail_closed "post_verify_hook_modified_reused_manifest"
|
||||
fi
|
||||
emit_receipt "idempotency" "reused" "verified_local_artifact_hash_${actual_hash}"
|
||||
APPLY_PASS=1
|
||||
return 0
|
||||
}
|
||||
|
||||
copy_and_verify_artifact() {
|
||||
local artifact_hash artifact_size hook_hash inventory_hash hook_inventory_hash
|
||||
local manifest_hash hook_manifest_hash
|
||||
|
||||
rm -f -- "${OUTPUT_PARTIAL}" "${MANIFEST_PARTIAL}" "${INVENTORY_PARTIAL}"
|
||||
run_captured "artifact_copy" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${BACKUP_TIMEOUT_SECONDS}" docker cp \
|
||||
"${CONTAINER_NAME}:${SERVER_ARTIFACT_PATH}" "${OUTPUT_PARTIAL}" \
|
||||
|| fail_closed "native_backup_artifact_copy_failed"
|
||||
[ -s "${OUTPUT_PARTIAL}" ] || fail_closed "native_backup_artifact_copy_empty"
|
||||
|
||||
run_captured "local_zip_verify" timeout --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_PARTIAL}" \
|
||||
|| fail_closed "native_backup_archive_integrity_failed"
|
||||
artifact_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')"
|
||||
artifact_size="$(wc -c < "${OUTPUT_PARTIAL}" | tr -d ' ')"
|
||||
[[ "${artifact_size}" =~ ^[0-9]+$ ]] && [ "${artifact_size}" -gt 0 ] \
|
||||
|| fail_closed "native_backup_local_size_invalid"
|
||||
|
||||
cp "${SOURCE_INVENTORY_RECEIPT}" "${INVENTORY_PARTIAL}"
|
||||
inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')"
|
||||
[ "${inventory_hash}" = "${SOURCE_INVENTORY_HASH}" ] \
|
||||
|| fail_closed "copied_source_inventory_hash_mismatch"
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","identity_hash":"%s","operation_id":"%s","artifact_name":"%s","databases":%s,"source_inventory_sha256":"%s","sha256":"%s","size_bytes":%s,"status":"BACKUP_CREATED","num_files":%s,"total_size":%s,"uncompressed_size":%s,"compressed_size":%s}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${IDENTITY_HASH}" \
|
||||
"${OPERATION_ID}" "${ARTIFACT_NAME}" "${EXPECTED_DATABASES_JSON}" \
|
||||
"${SOURCE_INVENTORY_HASH}" "${artifact_hash}" "${artifact_size}" \
|
||||
"${BACKUP_NUM_FILES}" "${BACKUP_TOTAL_SIZE}" "${BACKUP_UNCOMPRESSED_SIZE}" \
|
||||
"${BACKUP_COMPRESSED_SIZE}" > "${MANIFEST_PARTIAL}"
|
||||
|
||||
if [ -n "${POST_VERIFY_HOOK}" ]; then
|
||||
manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')"
|
||||
run_captured "post_verify_hook_check" env \
|
||||
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
||||
CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \
|
||||
CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \
|
||||
CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \
|
||||
CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
|
||||
CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
|
||||
CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \
|
||||
"${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_check_failed"
|
||||
run_captured "post_verify_hook" env \
|
||||
TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
||||
CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \
|
||||
CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \
|
||||
CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \
|
||||
CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \
|
||||
CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \
|
||||
CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \
|
||||
"${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_failed"
|
||||
hook_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')"
|
||||
[ "${hook_hash}" = "${artifact_hash}" ] || fail_closed "post_verify_hook_modified_artifact"
|
||||
hook_inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')"
|
||||
[ "${hook_inventory_hash}" = "${inventory_hash}" ] \
|
||||
|| fail_closed "post_verify_hook_modified_source_inventory"
|
||||
hook_manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')"
|
||||
[ "${hook_manifest_hash}" = "${manifest_hash}" ] \
|
||||
|| fail_closed "post_verify_hook_modified_manifest"
|
||||
fi
|
||||
|
||||
mv "${OUTPUT_PARTIAL}" "${OUTPUT_FILE}"
|
||||
OUTPUT_CREATED=1
|
||||
mv "${INVENTORY_PARTIAL}" "${INVENTORY_FILE}"
|
||||
printf '%s %s\n' "${artifact_hash}" "$(basename "${OUTPUT_FILE}")" > "${HASH_FILE}"
|
||||
mv "${MANIFEST_PARTIAL}" "${MANIFEST_FILE}"
|
||||
|
||||
cleanup_server_artifact || fail_closed "completed_server_artifact_cleanup_failed"
|
||||
emit_receipt "independent_verifier" "pass" \
|
||||
"system_status_archive_integrity_hash_${artifact_hash}_size_${artifact_size}"
|
||||
APPLY_PASS=1
|
||||
}
|
||||
|
||||
apply_backup() {
|
||||
local submission_line submitted_id submitted_status
|
||||
|
||||
grep -F -q '"stage":"check","terminal":"check_pass_no_write"' "${RECEIPT_LOG}" \
|
||||
|| fail_closed "same_run_check_pass_receipt_required_before_apply"
|
||||
perform_check
|
||||
emit_receipt "check" "pass" "apply_revalidation_pass"
|
||||
|
||||
if verify_existing_local_artifact; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
query_backup_status
|
||||
if [ "${STATUS_PRESENT}" -eq 0 ]; then
|
||||
server_artifact_absent || fail_closed "orphan_server_artifact_without_system_status"
|
||||
emit_receipt "ai_decision" "candidate" \
|
||||
"submit_native_async_backup_exact_six_database_allowlist_to_exact_artifact"
|
||||
SERVER_ARTIFACT_ACTIVE=1
|
||||
run_sql "backup_submit" \
|
||||
"BACKUP ${BACKUP_SCOPE_SQL} TO Disk({disk:String},{artifact:String}) SETTINGS id='${OPERATION_ID}' ASYNC" \
|
||||
--param_disk "${BACKUP_DISK}" --param_artifact "${ARTIFACT_NAME}" \
|
||||
|| fail_closed "native_backup_submit_failed"
|
||||
submission_line="$(tr -d '\r\n' < "${LAST_STDOUT}")"
|
||||
IFS=$'\t' read -r submitted_id submitted_status <<< "${submission_line}"
|
||||
[ "${submitted_id}" = "${OPERATION_ID}" ] \
|
||||
|| fail_closed "native_backup_submission_id_mismatch"
|
||||
case "${submitted_status}" in
|
||||
CREATING_BACKUP|BACKUP_CREATED) ;;
|
||||
*) fail_closed "native_backup_submission_status_${submitted_status:-empty}" ;;
|
||||
esac
|
||||
SERVER_ARTIFACT_ACTIVE=1
|
||||
emit_receipt "execution" "submitted" "operation_${OPERATION_ID}_status_${submitted_status}"
|
||||
else
|
||||
emit_receipt "idempotency" "resumed" "existing_operation_status_${BACKUP_STATUS}"
|
||||
fi
|
||||
|
||||
poll_backup_terminal
|
||||
copy_and_verify_artifact
|
||||
}
|
||||
|
||||
main() {
|
||||
setup "$@"
|
||||
|
||||
if [ "${MODE}" = "check" ]; then
|
||||
perform_check
|
||||
emit_receipt "ai_decision" "candidate" \
|
||||
"native_backup_apply_requires_same_run_check_receipt"
|
||||
emit_receipt "check" "check_pass_no_write" \
|
||||
"runtime_contract_valid_no_backup_or_artifact_write"
|
||||
CHECK_PASS=1
|
||||
log_info "ClickHouse native backup check passed without runtime write"
|
||||
return 0
|
||||
fi
|
||||
|
||||
apply_backup
|
||||
log_info "ClickHouse native backup verified: ${OUTPUT_FILE}"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
1152
scripts/backup/clickhouse-native-restore-drill.sh
Executable file
1152
scripts/backup/clickhouse-native-restore-drill.sh
Executable file
File diff suppressed because it is too large
Load Diff
284
scripts/backup/clickhouse-restore-inventory.py
Executable file
284
scripts/backup/clickhouse-restore-inventory.py
Executable file
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate SigNoz ClickHouse native-backup inventories without data access.
|
||||
|
||||
The shell restore-drill driver uses this helper for two narrow jobs:
|
||||
|
||||
* prove that a local artifact is a completed, hashed native BACKUP with the
|
||||
exact six-database source inventory expected by P0-OBS-002; and
|
||||
* compare the restored database/table/engine set with that source inventory.
|
||||
|
||||
Row and byte values are recorded as bounded aggregate evidence, not exact
|
||||
parity gates, because the source inventory is captured while online ingestion
|
||||
can still advance before the native BACKUP snapshot is committed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
EXPECTED_DATABASES = (
|
||||
"signoz_analytics",
|
||||
"signoz_logs",
|
||||
"signoz_metadata",
|
||||
"signoz_meter",
|
||||
"signoz_metrics",
|
||||
"signoz_traces",
|
||||
)
|
||||
CRITICAL_TABLES = (
|
||||
("signoz_logs", "logs_v2"),
|
||||
("signoz_metrics", "time_series_v4"),
|
||||
("signoz_metrics", "samples_v4"),
|
||||
("signoz_traces", "signoz_index_v3"),
|
||||
)
|
||||
SAFE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
SAFE_OPERATION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
|
||||
SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
SCHEMA_SHA256 = re.compile(r"^[0-9A-F]{64}$")
|
||||
|
||||
|
||||
class ContractError(ValueError):
|
||||
"""A bounded, non-secret backup or inventory contract failure."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TableReceipt:
|
||||
database: str
|
||||
table: str
|
||||
engine: str
|
||||
rows: int
|
||||
bytes: int
|
||||
schema_hash: str
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def parse_inventory(path: Path) -> dict[tuple[str, str], TableReceipt]:
|
||||
receipts: dict[tuple[str, str], TableReceipt] = {}
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
except (OSError, UnicodeError) as exc:
|
||||
raise ContractError("inventory_read_failed") from exc
|
||||
|
||||
if not lines:
|
||||
raise ContractError("inventory_empty")
|
||||
|
||||
for line_number, line in enumerate(lines, start=1):
|
||||
fields = line.split("\t")
|
||||
if len(fields) != 6:
|
||||
raise ContractError(f"inventory_field_count_invalid_line_{line_number}")
|
||||
database, table, engine, rows_text, bytes_text, schema_hash = fields
|
||||
if not SAFE_IDENTIFIER.fullmatch(database):
|
||||
raise ContractError(
|
||||
f"inventory_database_identifier_unsafe_line_{line_number}"
|
||||
)
|
||||
if not SAFE_IDENTIFIER.fullmatch(table):
|
||||
raise ContractError(f"inventory_table_identifier_unsafe_line_{line_number}")
|
||||
if not SAFE_IDENTIFIER.fullmatch(engine):
|
||||
raise ContractError(
|
||||
f"inventory_engine_identifier_unsafe_line_{line_number}"
|
||||
)
|
||||
if not rows_text.isdecimal() or not bytes_text.isdecimal():
|
||||
raise ContractError(f"inventory_counter_invalid_line_{line_number}")
|
||||
if not SCHEMA_SHA256.fullmatch(schema_hash):
|
||||
raise ContractError(f"inventory_schema_hash_invalid_line_{line_number}")
|
||||
key = (database, table)
|
||||
if key in receipts:
|
||||
raise ContractError(f"inventory_duplicate_table_line_{line_number}")
|
||||
receipts[key] = TableReceipt(
|
||||
database=database,
|
||||
table=table,
|
||||
engine=engine,
|
||||
rows=int(rows_text),
|
||||
bytes=int(bytes_text),
|
||||
schema_hash=schema_hash,
|
||||
)
|
||||
return receipts
|
||||
|
||||
|
||||
def require_database_set(receipts: dict[tuple[str, str], TableReceipt]) -> None:
|
||||
observed = tuple(sorted({receipt.database for receipt in receipts.values()}))
|
||||
if observed != EXPECTED_DATABASES:
|
||||
raise ContractError("signoz_database_set_mismatch")
|
||||
|
||||
|
||||
def require_critical_tables(
|
||||
receipts: dict[tuple[str, str], TableReceipt], *, require_nonzero: bool
|
||||
) -> None:
|
||||
for key in CRITICAL_TABLES:
|
||||
receipt = receipts.get(key)
|
||||
if receipt is None:
|
||||
raise ContractError(f"critical_table_missing_{key[0]}_{key[1]}")
|
||||
if require_nonzero and receipt.rows <= 0:
|
||||
raise ContractError(f"critical_table_empty_{key[0]}_{key[1]}")
|
||||
|
||||
|
||||
def emit(values: list[tuple[str, object]]) -> None:
|
||||
for key, value in values:
|
||||
print(f"{key}={value}")
|
||||
|
||||
|
||||
def preflight(args: argparse.Namespace) -> None:
|
||||
manifest_path = Path(args.manifest)
|
||||
artifact_path = Path(args.artifact)
|
||||
inventory_path = Path(args.source_inventory)
|
||||
try:
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise ContractError("backup_manifest_read_failed") from exc
|
||||
if not isinstance(manifest, dict):
|
||||
raise ContractError("backup_manifest_not_object")
|
||||
if manifest.get("status") != "BACKUP_CREATED":
|
||||
raise ContractError("backup_terminal_not_BACKUP_CREATED")
|
||||
|
||||
artifact_sha256 = sha256_file(artifact_path)
|
||||
inventory_sha256 = sha256_file(inventory_path)
|
||||
artifact_size = artifact_path.stat().st_size
|
||||
manifest_sha256 = manifest.get("sha256")
|
||||
manifest_inventory_sha256 = manifest.get("source_inventory_sha256")
|
||||
manifest_size = manifest.get("size_bytes")
|
||||
if not isinstance(manifest_sha256, str) or not SHA256.fullmatch(manifest_sha256):
|
||||
raise ContractError("backup_manifest_sha256_invalid")
|
||||
if manifest_sha256 != artifact_sha256:
|
||||
raise ContractError("backup_artifact_sha256_mismatch")
|
||||
if manifest_inventory_sha256 != inventory_sha256:
|
||||
raise ContractError("source_inventory_sha256_mismatch")
|
||||
if not isinstance(manifest_size, int) or manifest_size <= 0:
|
||||
raise ContractError("backup_manifest_size_invalid")
|
||||
if manifest_size != artifact_size:
|
||||
raise ContractError("backup_artifact_size_mismatch")
|
||||
if tuple(manifest.get("databases", ())) != EXPECTED_DATABASES:
|
||||
raise ContractError("backup_manifest_database_allowlist_mismatch")
|
||||
|
||||
operation_id = manifest.get("operation_id")
|
||||
if not isinstance(operation_id, str) or not SAFE_OPERATION_ID.fullmatch(
|
||||
operation_id
|
||||
):
|
||||
raise ContractError("backup_operation_id_invalid")
|
||||
|
||||
receipts = parse_inventory(inventory_path)
|
||||
require_database_set(receipts)
|
||||
require_critical_tables(receipts, require_nonzero=True)
|
||||
replicated_count = sum(
|
||||
receipt.engine.startswith("Replicated") for receipt in receipts.values()
|
||||
)
|
||||
distributed_count = sum(
|
||||
receipt.engine == "Distributed" for receipt in receipts.values()
|
||||
)
|
||||
physical_count = sum("MergeTree" in receipt.engine for receipt in receipts.values())
|
||||
if replicated_count <= 0:
|
||||
raise ContractError("replicated_table_precondition_missing")
|
||||
if distributed_count <= 0:
|
||||
raise ContractError("distributed_table_precondition_missing")
|
||||
if physical_count <= 0:
|
||||
raise ContractError("physical_table_precondition_missing")
|
||||
|
||||
emit(
|
||||
[
|
||||
("artifact_sha256", artifact_sha256),
|
||||
("artifact_size_bytes", artifact_size),
|
||||
("source_inventory_sha256", inventory_sha256),
|
||||
("source_database_count", len(EXPECTED_DATABASES)),
|
||||
("source_table_count", len(receipts)),
|
||||
("replicated_table_count", replicated_count),
|
||||
("distributed_table_count", distributed_count),
|
||||
("physical_table_count", physical_count),
|
||||
("backup_operation_id", operation_id),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def compare(args: argparse.Namespace) -> None:
|
||||
source = parse_inventory(Path(args.source_inventory))
|
||||
restored = parse_inventory(Path(args.restored_inventory))
|
||||
require_database_set(source)
|
||||
require_database_set(restored)
|
||||
require_critical_tables(source, require_nonzero=True)
|
||||
require_critical_tables(restored, require_nonzero=True)
|
||||
|
||||
source_keys = set(source)
|
||||
restored_keys = set(restored)
|
||||
if source_keys != restored_keys:
|
||||
raise ContractError("database_table_manifest_set_mismatch")
|
||||
for key in sorted(source_keys):
|
||||
if source[key].engine != restored[key].engine:
|
||||
raise ContractError(f"table_engine_mismatch_{key[0]}_{key[1]}")
|
||||
if source[key].schema_hash != restored[key].schema_hash:
|
||||
raise ContractError(f"table_schema_mismatch_{key[0]}_{key[1]}")
|
||||
|
||||
source_rows = sum(receipt.rows for receipt in source.values())
|
||||
restored_rows = sum(receipt.rows for receipt in restored.values())
|
||||
source_bytes = sum(receipt.bytes for receipt in source.values())
|
||||
restored_bytes = sum(receipt.bytes for receipt in restored.values())
|
||||
physical_count = sum("MergeTree" in receipt.engine for receipt in restored.values())
|
||||
emit(
|
||||
[
|
||||
("manifest_parity", 1),
|
||||
("restored_database_count", len(EXPECTED_DATABASES)),
|
||||
("restored_table_count", len(restored)),
|
||||
("physical_table_count", physical_count),
|
||||
("critical_nonzero_count", len(CRITICAL_TABLES)),
|
||||
("source_total_rows", source_rows),
|
||||
("restored_total_rows", restored_rows),
|
||||
("row_delta", restored_rows - source_rows),
|
||||
("source_total_bytes", source_bytes),
|
||||
("restored_total_bytes", restored_bytes),
|
||||
("byte_delta", restored_bytes - source_bytes),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def check_list(args: argparse.Namespace) -> None:
|
||||
receipts = parse_inventory(Path(args.inventory))
|
||||
require_database_set(receipts)
|
||||
for key in sorted(receipts):
|
||||
receipt = receipts[key]
|
||||
if "MergeTree" in receipt.engine:
|
||||
print(f"{receipt.database}\t{receipt.table}")
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
root = argparse.ArgumentParser()
|
||||
commands = root.add_subparsers(dest="command", required=True)
|
||||
|
||||
preflight_parser = commands.add_parser("preflight")
|
||||
preflight_parser.add_argument("--manifest", required=True)
|
||||
preflight_parser.add_argument("--artifact", required=True)
|
||||
preflight_parser.add_argument("--source-inventory", required=True)
|
||||
preflight_parser.set_defaults(handler=preflight)
|
||||
|
||||
compare_parser = commands.add_parser("compare")
|
||||
compare_parser.add_argument("--source-inventory", required=True)
|
||||
compare_parser.add_argument("--restored-inventory", required=True)
|
||||
compare_parser.set_defaults(handler=compare)
|
||||
|
||||
check_parser = commands.add_parser("check-list")
|
||||
check_parser.add_argument("--inventory", required=True)
|
||||
check_parser.set_defaults(handler=check_list)
|
||||
return root
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parser().parse_args()
|
||||
try:
|
||||
args.handler(args)
|
||||
except (ContractError, OSError) as exc:
|
||||
print(f"ERROR={exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
987
scripts/backup/run-signoz-backup-canary.sh
Executable file
987
scripts/backup/run-signoz-backup-canary.sh
Executable file
@@ -0,0 +1,987 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# P0-OBS-002 - SigNoz online backup canary and legacy maintenance lease verifier
|
||||
#
|
||||
# The default native-backup path never stops the collector and therefore does
|
||||
# not depend on, read, or mutate the legacy monitor/cooldown contract. The
|
||||
# bounded lease path remains available only when an explicit legacy stop-mode
|
||||
# canary is requested. Neither mode reads or sources a secret/.env file.
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}"
|
||||
BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}"
|
||||
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-/backup/scripts/clickhouse-native-restore-drill.sh}"
|
||||
CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-/backup/scripts/clickhouse-restore-inventory.py}"
|
||||
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-/backup/config/signoz/clickhouse/config.d/backup_disk.xml}"
|
||||
CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml}"
|
||||
CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}"
|
||||
CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-/backup/logs/clickhouse-native-restore-drill}"
|
||||
MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}"
|
||||
MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}"
|
||||
MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}"
|
||||
COOLDOWN_FILE="${MONITOR_COOLDOWN_DIR}/${COLLECTOR_NAME}.cooldown"
|
||||
RECEIPT_ROOT="${SIGNOZ_CANARY_RECEIPT_ROOT:-/backup/logs/signoz-canary}"
|
||||
LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}"
|
||||
|
||||
CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}"
|
||||
TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}"
|
||||
DOCKER_TIMEOUT_SECONDS="${SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS:-15}"
|
||||
LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}"
|
||||
MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}"
|
||||
STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}"
|
||||
EXPECT_COLLECTOR_STOP="${SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP:-0}"
|
||||
|
||||
TRACE_ID="${TRACE_ID:-}"
|
||||
RUN_ID="${RUN_ID:-}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
||||
|
||||
RECEIPT_DIR=""
|
||||
RECEIPT_LOG=""
|
||||
BACKUP_OUTPUT=""
|
||||
MONITOR_WINDOW=""
|
||||
STATE_OBSERVATIONS=""
|
||||
ROLLBACK_FILE=""
|
||||
ROLLBACK_METADATA=""
|
||||
|
||||
RECEIPTS_READY=0
|
||||
LEASE_ROLLBACK_READY=0
|
||||
LEASE_MUTATION_STARTED=0
|
||||
LEASE_ARMED=0
|
||||
PRIOR_LEASE_PRESENT=0
|
||||
PRIOR_LEASE_MODE=""
|
||||
PRIOR_LEASE_UID=""
|
||||
PRIOR_LEASE_GID=""
|
||||
PRIOR_LEASE_HASH=""
|
||||
PRIOR_LEASE_VALUE=""
|
||||
LEASE_TMP=""
|
||||
BACKUP_PID=""
|
||||
OBSERVER_PID=""
|
||||
CANARY_VERIFIED=0
|
||||
CHECK_VERIFIED=0
|
||||
MODE=""
|
||||
|
||||
log_info() { printf '[INFO] %s\n' "$*"; }
|
||||
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||
|
||||
valid_identity() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
||||
}
|
||||
|
||||
valid_positive_integer() {
|
||||
[[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ]
|
||||
}
|
||||
|
||||
valid_poll_interval() {
|
||||
[[ "$1" =~ ^[0-9]+([.][0-9]+)?$ ]] && [[ "$1" != "0" ]] && [[ "$1" != "0.0" ]]
|
||||
}
|
||||
|
||||
emit_receipt() {
|
||||
local stage="$1"
|
||||
local terminal="$2"
|
||||
local detail="$3"
|
||||
local observed_at
|
||||
|
||||
[ "${RECEIPTS_READY}" -eq 1 ] || return 0
|
||||
observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${observed_at}" \
|
||||
"${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}"
|
||||
}
|
||||
|
||||
fail_closed() {
|
||||
local detail="$1"
|
||||
log_error "${detail}"
|
||||
emit_receipt "failure" "failed" "${detail}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}"
|
||||
}
|
||||
|
||||
run_collector_docker() {
|
||||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${DOCKER_TIMEOUT_SECONDS}" docker "$@"
|
||||
}
|
||||
|
||||
require_native_restore_contract() {
|
||||
local label path
|
||||
|
||||
for label in \
|
||||
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK \
|
||||
CLICKHOUSE_RESTORE_INVENTORY_HELPER \
|
||||
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG \
|
||||
CLICKHOUSE_RESTORE_CLUSTER_CONFIG; do
|
||||
path="${!label}"
|
||||
case "${path}" in
|
||||
/*) ;;
|
||||
*) fail_closed "native_restore_contract_${label}_path_not_absolute" ;;
|
||||
esac
|
||||
[ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \
|
||||
|| fail_closed "native_restore_contract_${label}_missing_unreadable_or_symlink"
|
||||
done
|
||||
|
||||
[ -x "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" ] \
|
||||
|| fail_closed "native_restore_contract_post_verify_hook_not_executable"
|
||||
[ -x "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" ] \
|
||||
|| fail_closed "native_restore_contract_inventory_helper_not_executable"
|
||||
}
|
||||
|
||||
acquire_canary_lock() {
|
||||
[ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe"
|
||||
if [ -e "${LOCK_FILE}" ]; then
|
||||
[ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file"
|
||||
fi
|
||||
exec 9>>"${LOCK_FILE}"
|
||||
flock -n 9 || fail_closed "another_signoz_canary_holds_lock"
|
||||
}
|
||||
|
||||
require_native_receipt_contract() {
|
||||
local label path run_dir
|
||||
|
||||
for label in CLICKHOUSE_NATIVE_RECEIPT_ROOT CLICKHOUSE_RESTORE_RECEIPT_ROOT; do
|
||||
path="${!label}"
|
||||
case "${path}" in
|
||||
/*) ;;
|
||||
*) fail_closed "native_receipt_contract_${label}_path_not_absolute" ;;
|
||||
esac
|
||||
[[ "${path}" != *$'\n'* ]] \
|
||||
|| fail_closed "native_receipt_contract_${label}_path_contains_newline"
|
||||
[ ! -L "${path}" ] \
|
||||
|| fail_closed "native_receipt_contract_${label}_root_symlink_unsafe"
|
||||
if [ -e "${path}" ]; then
|
||||
[ -d "${path}" ] && [ -r "${path}" ] && [ -w "${path}" ] \
|
||||
|| fail_closed "native_receipt_contract_${label}_root_not_readable_writable_directory"
|
||||
fi
|
||||
run_dir="${path}/${RUN_ID}"
|
||||
[ ! -e "${run_dir}" ] && [ ! -L "${run_dir}" ] \
|
||||
|| fail_closed "native_receipt_contract_${label}_stale_same_run_path_present"
|
||||
done
|
||||
|
||||
[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" != "${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" ] \
|
||||
|| fail_closed "native_and_restore_receipt_roots_must_be_distinct"
|
||||
}
|
||||
|
||||
verify_same_run_native_restore_receipts() {
|
||||
local native_receipts restore_receipts restore_status
|
||||
|
||||
native_receipts="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl"
|
||||
restore_receipts="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl"
|
||||
restore_status="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/status.json"
|
||||
|
||||
for path in "${native_receipts}" "${restore_receipts}" "${restore_status}"; do
|
||||
[ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \
|
||||
|| fail_closed "same_run_native_restore_receipt_missing_unreadable_or_symlink"
|
||||
done
|
||||
|
||||
python3 - "${native_receipts}" "${restore_receipts}" "${restore_status}" \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" <<'PY' \
|
||||
|| fail_closed "same_run_native_restore_receipt_validation_failed"
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def reject(message: str) -> None:
|
||||
print(f"receipt_verifier:{message}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
native_path, restore_path, status_path, trace_id, run_id, work_item_id = sys.argv[1:]
|
||||
identity = {
|
||||
"trace_id": trace_id,
|
||||
"run_id": run_id,
|
||||
"work_item_id": work_item_id,
|
||||
}
|
||||
|
||||
|
||||
def read_jsonl(path: str, label: str) -> list[dict[str, object]]:
|
||||
if os.path.islink(path) or not os.path.isfile(path):
|
||||
reject(f"{label}_unsafe")
|
||||
records: list[dict[str, object]] = []
|
||||
try:
|
||||
with open(path, encoding="utf-8") as source:
|
||||
for line_number, raw_line in enumerate(source, start=1):
|
||||
if not raw_line.strip():
|
||||
continue
|
||||
value = json.loads(raw_line)
|
||||
if not isinstance(value, dict):
|
||||
reject(f"{label}_line_{line_number}_not_object")
|
||||
for key, expected in identity.items():
|
||||
if value.get(key) != expected:
|
||||
reject(f"{label}_line_{line_number}_{key}_mismatch")
|
||||
invocation_id = value.get("invocation_id")
|
||||
if not isinstance(invocation_id, str) or not invocation_id:
|
||||
reject(f"{label}_line_{line_number}_invocation_id_invalid")
|
||||
records.append(value)
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
reject(f"{label}_malformed_{type(exc).__name__}")
|
||||
if not records:
|
||||
reject(f"{label}_empty")
|
||||
return records
|
||||
|
||||
|
||||
def mode_records(
|
||||
records: list[dict[str, object]], mode: str, label: str
|
||||
) -> list[dict[str, object]]:
|
||||
selected = [
|
||||
record
|
||||
for record in records
|
||||
if str(record.get("invocation_id", "")).startswith(f"{mode}-")
|
||||
]
|
||||
if not selected:
|
||||
reject(f"{label}_{mode}_invocation_missing")
|
||||
return selected
|
||||
|
||||
|
||||
def require_exact(
|
||||
records: list[dict[str, object]],
|
||||
*,
|
||||
label: str,
|
||||
stage: str,
|
||||
terminal: str,
|
||||
detail_prefix: str | None = None,
|
||||
) -> None:
|
||||
matches = [
|
||||
record
|
||||
for record in records
|
||||
if record.get("stage") == stage
|
||||
and record.get("terminal") == terminal
|
||||
and (
|
||||
detail_prefix is None
|
||||
or str(record.get("detail", "")).startswith(detail_prefix)
|
||||
)
|
||||
]
|
||||
if len(matches) != 1:
|
||||
reject(f"{label}_{stage}_{terminal}_count_{len(matches)}")
|
||||
|
||||
|
||||
native_records = read_jsonl(native_path, "native_receipts")
|
||||
native_check = mode_records(native_records, "check", "native")
|
||||
native_apply = mode_records(native_records, "apply", "native")
|
||||
require_exact(
|
||||
native_check,
|
||||
label="native_check",
|
||||
stage="terminal",
|
||||
terminal="check_pass_no_write",
|
||||
)
|
||||
require_exact(
|
||||
native_apply,
|
||||
label="native_apply",
|
||||
stage="terminal",
|
||||
terminal="pass",
|
||||
)
|
||||
require_exact(
|
||||
native_apply,
|
||||
label="native_apply",
|
||||
stage="independent_verifier",
|
||||
terminal="pass",
|
||||
)
|
||||
require_exact(
|
||||
native_apply,
|
||||
label="native_apply_hook_check",
|
||||
stage="command",
|
||||
terminal="pass",
|
||||
detail_prefix="step_post_verify_hook_check_exit_0_",
|
||||
)
|
||||
require_exact(
|
||||
native_apply,
|
||||
label="native_apply_hook",
|
||||
stage="command",
|
||||
terminal="pass",
|
||||
detail_prefix="step_post_verify_hook_exit_0_",
|
||||
)
|
||||
|
||||
restore_records = read_jsonl(restore_path, "restore_receipts")
|
||||
restore_check = mode_records(restore_records, "check", "restore")
|
||||
restore_apply = mode_records(restore_records, "apply", "restore")
|
||||
require_exact(
|
||||
restore_check,
|
||||
label="restore_check",
|
||||
stage="terminal",
|
||||
terminal="check_pass_no_runtime_write",
|
||||
)
|
||||
require_exact(
|
||||
restore_apply,
|
||||
label="restore_apply",
|
||||
stage="terminal",
|
||||
terminal="verified",
|
||||
)
|
||||
require_exact(
|
||||
restore_apply,
|
||||
label="restore_apply",
|
||||
stage="independent_post_verifier",
|
||||
terminal="pass",
|
||||
)
|
||||
require_exact(
|
||||
restore_apply,
|
||||
label="restore_apply",
|
||||
stage="cleanup",
|
||||
terminal="pass",
|
||||
)
|
||||
|
||||
if os.path.islink(status_path) or not os.path.isfile(status_path):
|
||||
reject("restore_status_unsafe")
|
||||
try:
|
||||
with open(status_path, encoding="utf-8") as source:
|
||||
status = json.load(source)
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
reject(f"restore_status_malformed_{type(exc).__name__}")
|
||||
if not isinstance(status, dict):
|
||||
reject("restore_status_not_object")
|
||||
for key, expected in identity.items():
|
||||
if status.get(key) != expected:
|
||||
reject(f"restore_status_{key}_mismatch")
|
||||
|
||||
expected_strings = {
|
||||
"schema_version": "clickhouse_native_restore_drill_status_v1",
|
||||
"mode": "apply",
|
||||
"terminal": "verified",
|
||||
"restore_terminal": "RESTORED",
|
||||
"network_mode": "docker_internal_only",
|
||||
"staging_network_mode": "none",
|
||||
}
|
||||
for key, expected in expected_strings.items():
|
||||
if not isinstance(status.get(key), str) or status.get(key) != expected:
|
||||
reject(f"restore_status_{key}_invalid")
|
||||
for key, expected in (("exit_code", 0), ("manifest_parity", 1)):
|
||||
if type(status.get(key)) is not int or status.get(key) != expected:
|
||||
reject(f"restore_status_{key}_invalid")
|
||||
for key in (
|
||||
"check_pass",
|
||||
"apply_pass",
|
||||
"cleanup_verified",
|
||||
"artifact_staging_verified",
|
||||
"clickhouse_artifact_readback_verified",
|
||||
"ephemeral_runtime_created",
|
||||
"isolated_keeper",
|
||||
):
|
||||
if status.get(key) is not True:
|
||||
reject(f"restore_status_{key}_not_true")
|
||||
for key in (
|
||||
"production_network_attached",
|
||||
"production_restore_performed",
|
||||
"allow_non_empty_tables",
|
||||
):
|
||||
if status.get(key) is not False:
|
||||
reject(f"restore_status_{key}_not_false")
|
||||
|
||||
|
||||
def positive_integer(key: str) -> int:
|
||||
value = status.get(key)
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
||||
reject(f"restore_status_{key}_not_positive_integer")
|
||||
return value
|
||||
|
||||
|
||||
source_database_count = positive_integer("source_database_count")
|
||||
restored_database_count = positive_integer("restored_database_count")
|
||||
if source_database_count != restored_database_count:
|
||||
reject("restore_status_database_count_mismatch")
|
||||
source_table_count = positive_integer("source_table_count")
|
||||
restored_table_count = positive_integer("restored_table_count")
|
||||
if source_table_count != restored_table_count:
|
||||
reject("restore_status_table_count_mismatch")
|
||||
positive_integer("critical_nonzero_count")
|
||||
check_count = status.get("check_table_count")
|
||||
check_pass_count = status.get("check_table_pass_count")
|
||||
if (
|
||||
isinstance(check_count, bool)
|
||||
or not isinstance(check_count, int)
|
||||
or isinstance(check_pass_count, bool)
|
||||
or not isinstance(check_pass_count, int)
|
||||
or check_count <= 0
|
||||
or check_count != check_pass_count
|
||||
):
|
||||
reject("restore_status_check_table_counts_invalid")
|
||||
PY
|
||||
|
||||
emit_receipt "native_restore_receipt_verifier" "pass" \
|
||||
"same_run_machine_readable_native_check_apply_hook_and_isolated_restore_status_verified"
|
||||
}
|
||||
|
||||
require_monitor_fragment() {
|
||||
local fragment="$1"
|
||||
local count
|
||||
count="$(grep -F -c -- "${fragment}" "${MONITOR_SCRIPT}" || true)"
|
||||
[ "${count}" -eq 1 ] || fail_closed "monitor_contract_ambiguous_fragment_count_${count}"
|
||||
}
|
||||
|
||||
require_monitor_function_fragment() {
|
||||
local function_name="$1"
|
||||
local fragment="$2"
|
||||
local count
|
||||
|
||||
count="$(awk -v header="${function_name}() {" -v needle="${fragment}" '
|
||||
$0 == header {
|
||||
inside = 1
|
||||
depth = 0
|
||||
}
|
||||
inside {
|
||||
original = $0
|
||||
if (index(original, needle)) {
|
||||
count++
|
||||
}
|
||||
open_line = original
|
||||
close_line = original
|
||||
depth += gsub(/\{/, "", open_line)
|
||||
depth -= gsub(/\}/, "", close_line)
|
||||
if (depth == 0) {
|
||||
print count + 0
|
||||
printed = 1
|
||||
exit
|
||||
}
|
||||
}
|
||||
END {
|
||||
if (!printed) {
|
||||
print count + 0
|
||||
}
|
||||
}
|
||||
' "${MONITOR_SCRIPT}")"
|
||||
[ "${count}" -eq 1 ] \
|
||||
|| fail_closed "monitor_${function_name}_contract_ambiguous_fragment_count_${count}"
|
||||
}
|
||||
|
||||
collector_running_state() {
|
||||
local running
|
||||
if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then
|
||||
return 1
|
||||
fi
|
||||
case "${running}" in
|
||||
true) printf '%s\n' "running" ;;
|
||||
false) printf '%s\n' "stopped" ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
collector_runtime_metadata() {
|
||||
run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \
|
||||
"${COLLECTOR_NAME}" 2>/dev/null
|
||||
}
|
||||
|
||||
listener_is_up() {
|
||||
local port="$1"
|
||||
ss -H -lnt | awk -v port="${port}" '
|
||||
$4 ~ (":" port "$") { found = 1 }
|
||||
END { exit(found ? 0 : 1) }
|
||||
'
|
||||
}
|
||||
|
||||
verify_collector_runtime() {
|
||||
local expected_id="$1"
|
||||
local expected_started_at="$2"
|
||||
local expected_restart_count="$3"
|
||||
local expected_health="$4"
|
||||
local metadata actual_id actual_running actual_started_at actual_restart_count actual_health
|
||||
|
||||
metadata="$(collector_runtime_metadata)" || return 1
|
||||
IFS=$'\t' read -r actual_id actual_running actual_started_at actual_restart_count actual_health <<< "${metadata}"
|
||||
[ "${actual_id}" = "${expected_id}" ] || return 1
|
||||
[ "${actual_running}" = "true" ] || return 1
|
||||
[ "${actual_started_at}" = "${expected_started_at}" ] || return 1
|
||||
[ "${actual_restart_count}" = "${expected_restart_count}" ] || return 1
|
||||
[ "${actual_health}" = "${expected_health}" ] || return 1
|
||||
listener_is_up 4317 || return 1
|
||||
listener_is_up 4318 || return 1
|
||||
}
|
||||
|
||||
restore_lease() {
|
||||
local restore_tmp restore_hash current_value
|
||||
|
||||
[ "${LEASE_MUTATION_STARTED}" -eq 1 ] || return 0
|
||||
[ "${LEASE_ROLLBACK_READY}" -eq 1 ] || return 1
|
||||
|
||||
if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then
|
||||
[ -f "${ROLLBACK_FILE}" ] || return 1
|
||||
restore_tmp="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.restore.${RUN_ID}.XXXXXX")" || return 1
|
||||
cp -p "${ROLLBACK_FILE}" "${restore_tmp}" || return 1
|
||||
chmod "${PRIOR_LEASE_MODE}" "${restore_tmp}" || return 1
|
||||
mv -f "${restore_tmp}" "${COOLDOWN_FILE}" || return 1
|
||||
restore_tmp=""
|
||||
|
||||
restore_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" || return 1
|
||||
[ "${restore_hash}" = "${PRIOR_LEASE_HASH}" ] || return 1
|
||||
[ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] || return 1
|
||||
[ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] || return 1
|
||||
[ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] || return 1
|
||||
current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
|
||||
[ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] || return 1
|
||||
else
|
||||
rm -f "${COOLDOWN_FILE}" || return 1
|
||||
[ ! -e "${COOLDOWN_FILE}" ] || return 1
|
||||
fi
|
||||
|
||||
LEASE_ARMED=0
|
||||
LEASE_MUTATION_STARTED=0
|
||||
return 0
|
||||
}
|
||||
|
||||
terminate_children() {
|
||||
if [ -n "${BACKUP_PID}" ] && kill -0 "${BACKUP_PID}" 2>/dev/null; then
|
||||
kill -TERM "${BACKUP_PID}" 2>/dev/null || true
|
||||
wait "${BACKUP_PID}" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${OBSERVER_PID}" ] && kill -0 "${OBSERVER_PID}" 2>/dev/null; then
|
||||
kill -TERM "${OBSERVER_PID}" 2>/dev/null || true
|
||||
wait "${OBSERVER_PID}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
local lease_terminal="not_armed"
|
||||
local overall_terminal="failed"
|
||||
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
terminate_children
|
||||
rm -f "${LEASE_TMP}" 2>/dev/null || true
|
||||
|
||||
if [ "${LEASE_MUTATION_STARTED}" -eq 1 ]; then
|
||||
if restore_lease; then
|
||||
lease_terminal="restored"
|
||||
else
|
||||
lease_terminal="restore_failed"
|
||||
exit_code=91
|
||||
fi
|
||||
elif [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then
|
||||
lease_terminal="no_write"
|
||||
elif [ "${MODE}" = "apply" ] && [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
|
||||
lease_terminal="no_write"
|
||||
fi
|
||||
|
||||
if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then
|
||||
overall_terminal="check_pass_no_write"
|
||||
elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then
|
||||
overall_terminal="partial_degraded"
|
||||
fi
|
||||
|
||||
emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}"
|
||||
if [ "${overall_terminal}" = "partial_degraded" ]; then
|
||||
emit_receipt "runtime_operation" "pass" \
|
||||
"clickhouse_native_backup_and_isolated_restore_runtime_verifier_pass"
|
||||
emit_receipt "closure_writeback" "partial_degraded" \
|
||||
"local_durable_receipt_ack_incident_not_applicable_telegram_not_sent_km_rag_writeback_pending_offsite_false_sqlite_application_consistent_export_pending"
|
||||
elif [ "${overall_terminal}" = "check_pass_no_write" ]; then
|
||||
emit_receipt "closure_writeback" "not_applicable" \
|
||||
"check_mode_no_runtime_change_local_durable_receipt_ack"
|
||||
else
|
||||
emit_receipt "closure_writeback" "deferred" \
|
||||
"failed_terminal_requires_followup_no_completion_claim"
|
||||
fi
|
||||
emit_receipt "terminal" "${overall_terminal}" \
|
||||
"backup_canary_${overall_terminal}_clickhouse_native_scope_only_offsite_false_sqlite_pending"
|
||||
printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \
|
||||
"${lease_terminal}" "${exit_code}"
|
||||
exit "${exit_code}"
|
||||
}
|
||||
|
||||
on_signal() {
|
||||
local signal_number="$1"
|
||||
emit_receipt "signal" "failed" "signal_${signal_number}"
|
||||
exit "$((128 + signal_number))"
|
||||
}
|
||||
|
||||
setup_receipts() {
|
||||
valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and must be path-safe"; exit 2; }
|
||||
valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and must be path-safe"; exit 2; }
|
||||
valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and must be path-safe"; exit 2; }
|
||||
|
||||
[ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root must not be a symlink"; exit 2; }
|
||||
mkdir -p "${RECEIPT_ROOT}"
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
[ ! -e "${RECEIPT_DIR}" ] || { log_error "receipt directory already exists: ${RECEIPT_DIR}"; exit 2; }
|
||||
mkdir -m 700 "${RECEIPT_DIR}"
|
||||
|
||||
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
||||
BACKUP_OUTPUT="${RECEIPT_DIR}/backup-output.log"
|
||||
MONITOR_WINDOW="${RECEIPT_DIR}/monitor-window.log"
|
||||
STATE_OBSERVATIONS="${RECEIPT_DIR}/collector-state.tsv"
|
||||
ROLLBACK_FILE="${RECEIPT_DIR}/cooldown.rollback"
|
||||
ROLLBACK_METADATA="${RECEIPT_DIR}/cooldown.rollback.json"
|
||||
: > "${RECEIPT_LOG}"
|
||||
RECEIPTS_READY=1
|
||||
|
||||
trap cleanup EXIT
|
||||
trap 'on_signal 1' HUP
|
||||
trap 'on_signal 2' INT
|
||||
trap 'on_signal 15' TERM
|
||||
}
|
||||
|
||||
verify_monitor_contract() {
|
||||
local cron_output cron_count cron_line exclude_count exclude_line
|
||||
|
||||
[ -f "${MONITOR_SCRIPT}" ] && [ -r "${MONITOR_SCRIPT}" ] && [ ! -L "${MONITOR_SCRIPT}" ] \
|
||||
|| fail_closed "monitor_script_missing_unreadable_or_symlink"
|
||||
[ -f "${MONITOR_LOG}" ] && [ -r "${MONITOR_LOG}" ] && [ ! -L "${MONITOR_LOG}" ] \
|
||||
|| fail_closed "monitor_log_missing_unreadable_or_symlink"
|
||||
[ -d "${MONITOR_COOLDOWN_DIR}" ] && [ -w "${MONITOR_COOLDOWN_DIR}" ] && [ ! -L "${MONITOR_COOLDOWN_DIR}" ] \
|
||||
|| fail_closed "monitor_cooldown_directory_missing_unwritable_or_symlink"
|
||||
|
||||
require_monitor_fragment "COOLDOWN_DIR:=${MONITOR_COOLDOWN_DIR}"
|
||||
require_monitor_fragment 'local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown"'
|
||||
require_monitor_fragment 'last_sent=$(cat "$cooldown_file")'
|
||||
# The live monitor has another cooldown path with the same elapsed
|
||||
# calculation. Bind this assertion to is_in_cooldown instead of requiring
|
||||
# a globally unique string.
|
||||
require_monitor_function_fragment is_in_cooldown 'elapsed=$(( now - last_sent ))'
|
||||
require_monitor_fragment 'if (( elapsed < ACTION_COOLDOWN_SECONDS )); then'
|
||||
require_monitor_fragment 'COOLDOWN: ${container}'
|
||||
require_monitor_fragment 'is_in_cooldown "$container_name" && continue'
|
||||
require_monitor_fragment 'set_cooldown "$container_name"'
|
||||
require_monitor_fragment 'log "AUTO_REPAIR: docker restart ${container}"'
|
||||
require_monitor_fragment 'if docker restart "$container"'
|
||||
|
||||
exclude_count="$(grep -F -c 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}" || true)"
|
||||
[ "${exclude_count}" -eq 1 ] || fail_closed "monitor_exclude_contract_ambiguous"
|
||||
exclude_line="$(grep -F 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}")"
|
||||
[[ "${exclude_line}" != *"${COLLECTOR_NAME}"* ]] || fail_closed "collector_already_excluded_monitor_contract_changed"
|
||||
[ "$(grep -F -c "${COLLECTOR_NAME}" "${MONITOR_SCRIPT}" || true)" -eq 0 ] \
|
||||
|| fail_closed "collector_literal_present_monitor_contract_changed"
|
||||
|
||||
cron_output="$(crontab -l 2>/dev/null)" || fail_closed "monitor_crontab_unreadable"
|
||||
cron_count="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" '
|
||||
$0 !~ /^[[:space:]]*#/ && index($0, script) { count++ }
|
||||
END { print count + 0 }
|
||||
')"
|
||||
[ "${cron_count}" -eq 1 ] || fail_closed "monitor_cron_contract_ambiguous_count_${cron_count}"
|
||||
cron_line="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" '
|
||||
$0 !~ /^[[:space:]]*#/ && index($0, script) { print }
|
||||
')"
|
||||
[[ "${cron_line}" == "*/5 * * * * "* ]] || fail_closed "monitor_cron_schedule_not_every_five_minutes"
|
||||
[[ "${cron_line}" == *">> ${MONITOR_LOG}"* ]] || fail_closed "monitor_cron_log_route_mismatch"
|
||||
[[ "${cron_line}" != *"COOLDOWN_DIR="* ]] || fail_closed "monitor_cron_cooldown_override_ambiguous"
|
||||
|
||||
}
|
||||
|
||||
inspect_prior_lease() {
|
||||
local current_uid current_gid
|
||||
current_uid="$(id -u)"
|
||||
current_gid="$(id -g)"
|
||||
|
||||
if [ -e "${COOLDOWN_FILE}" ]; then
|
||||
[ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \
|
||||
|| fail_closed "existing_cooldown_receipt_unsafe"
|
||||
PRIOR_LEASE_PRESENT=1
|
||||
PRIOR_LEASE_MODE="$(stat -c '%a' "${COOLDOWN_FILE}")"
|
||||
PRIOR_LEASE_UID="$(stat -c '%u' "${COOLDOWN_FILE}")"
|
||||
PRIOR_LEASE_GID="$(stat -c '%g' "${COOLDOWN_FILE}")"
|
||||
[ "${PRIOR_LEASE_UID}" = "${current_uid}" ] && [ "${PRIOR_LEASE_GID}" = "${current_gid}" ] \
|
||||
|| fail_closed "existing_cooldown_receipt_owner_mismatch"
|
||||
PRIOR_LEASE_VALUE="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
|
||||
[[ "${PRIOR_LEASE_VALUE}" =~ ^[0-9]+$ ]] || fail_closed "existing_cooldown_receipt_not_epoch"
|
||||
PRIOR_LEASE_HASH="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
|
||||
else
|
||||
PRIOR_LEASE_PRESENT=0
|
||||
PRIOR_LEASE_MODE="664"
|
||||
PRIOR_LEASE_UID="${current_uid}"
|
||||
PRIOR_LEASE_GID="${current_gid}"
|
||||
PRIOR_LEASE_HASH="absent"
|
||||
PRIOR_LEASE_VALUE="absent"
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
capture_prior_lease() {
|
||||
local current_hash current_value
|
||||
|
||||
if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then
|
||||
[ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \
|
||||
|| fail_closed "cooldown_changed_between_check_and_rollback_capture"
|
||||
current_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
|
||||
current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")"
|
||||
[ "${current_hash}" = "${PRIOR_LEASE_HASH}" ] \
|
||||
&& [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] \
|
||||
&& [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] \
|
||||
&& [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] \
|
||||
&& [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] \
|
||||
|| fail_closed "cooldown_changed_between_check_and_rollback_capture"
|
||||
cp -p "${COOLDOWN_FILE}" "${ROLLBACK_FILE}"
|
||||
[ "$(sha256sum "${ROLLBACK_FILE}" | awk '{print $1}')" = "${PRIOR_LEASE_HASH}" ] \
|
||||
|| fail_closed "cooldown_changed_between_check_and_rollback_capture"
|
||||
else
|
||||
[ ! -e "${COOLDOWN_FILE}" ] \
|
||||
|| fail_closed "cooldown_appeared_between_check_and_rollback_capture"
|
||||
fi
|
||||
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","prior_present":%s,"prior_mode":"%s","prior_uid":"%s","prior_gid":"%s","prior_hash":"%s","prior_value":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${PRIOR_LEASE_PRESENT}" \
|
||||
"${PRIOR_LEASE_MODE}" "${PRIOR_LEASE_UID}" "${PRIOR_LEASE_GID}" \
|
||||
"${PRIOR_LEASE_HASH}" "${PRIOR_LEASE_VALUE}" > "${ROLLBACK_METADATA}"
|
||||
LEASE_ROLLBACK_READY=1
|
||||
emit_receipt "rollback_capture" "pass" "prior_cooldown_receipt_captured"
|
||||
}
|
||||
|
||||
arm_lease() {
|
||||
local now lease_epoch lease_hash
|
||||
now="$(date +%s)"
|
||||
lease_epoch=$((now + CANARY_TIMEOUT_SECONDS + LEASE_MARGIN_SECONDS))
|
||||
LEASE_TMP="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.lease.${RUN_ID}.XXXXXX")"
|
||||
printf '%s\n' "${lease_epoch}" > "${LEASE_TMP}"
|
||||
chmod "${PRIOR_LEASE_MODE}" "${LEASE_TMP}"
|
||||
|
||||
LEASE_MUTATION_STARTED=1
|
||||
mv -f "${LEASE_TMP}" "${COOLDOWN_FILE}"
|
||||
LEASE_TMP=""
|
||||
LEASE_ARMED=1
|
||||
|
||||
[ "$(tr -d '\r\n' < "${COOLDOWN_FILE}")" = "${lease_epoch}" ] \
|
||||
|| fail_closed "cooldown_lease_epoch_readback_mismatch"
|
||||
[ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "$(id -u)" ] \
|
||||
|| fail_closed "cooldown_lease_owner_readback_mismatch"
|
||||
[ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "$(id -g)" ] \
|
||||
|| fail_closed "cooldown_lease_group_readback_mismatch"
|
||||
lease_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')"
|
||||
emit_receipt "lease" "armed" "future_epoch_${lease_epoch}_hash_${lease_hash}"
|
||||
}
|
||||
|
||||
observe_collector() {
|
||||
local epoch state
|
||||
while kill -0 "${BACKUP_PID}" 2>/dev/null; do
|
||||
epoch="$(date +%s)"
|
||||
if state="$(collector_running_state)"; then
|
||||
printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}"
|
||||
else
|
||||
printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}"
|
||||
fi
|
||||
sleep "${STATE_POLL_SECONDS}"
|
||||
done
|
||||
|
||||
epoch="$(date +%s)"
|
||||
if state="$(collector_running_state)"; then
|
||||
printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}"
|
||||
else
|
||||
printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}"
|
||||
fi
|
||||
}
|
||||
|
||||
verify_monitor_window() {
|
||||
local log_identity_before="$1"
|
||||
local log_size_before="$2"
|
||||
local log_identity_after log_size_after start_byte
|
||||
local auto_repair_count detected_count cooldown_count
|
||||
local stop_epoch restore_epoch next_cron_epoch cron_crossed=0
|
||||
|
||||
log_identity_after="$(stat -c '%d:%i' "${MONITOR_LOG}")"
|
||||
log_size_after="$(stat -c '%s' "${MONITOR_LOG}")"
|
||||
[ "${log_identity_after}" = "${log_identity_before}" ] \
|
||||
|| fail_closed "monitor_log_rotated_during_canary"
|
||||
[ "${log_size_after}" -ge "${log_size_before}" ] \
|
||||
|| fail_closed "monitor_log_shrank_during_canary"
|
||||
|
||||
start_byte=$((log_size_before + 1))
|
||||
tail -c "+${start_byte}" "${MONITOR_LOG}" > "${MONITOR_WINDOW}"
|
||||
auto_repair_count="$(grep -a -F -c "AUTO_REPAIR: docker restart ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)"
|
||||
detected_count="$(grep -a -F -c "DETECTED: ${COLLECTOR_NAME} state=exited" "${MONITOR_WINDOW}" || true)"
|
||||
cooldown_count="$(grep -a -F -c "COOLDOWN: ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)"
|
||||
|
||||
[ "${auto_repair_count}" -eq 0 ] || fail_closed "monitor_auto_repair_contaminated_canary"
|
||||
[ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
|
||||
|| fail_closed "collector_state_observer_ambiguous"
|
||||
|
||||
if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
|
||||
[ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
|
||||
|| fail_closed "online_native_backup_collector_continuity_failed"
|
||||
[ "${detected_count}" -eq 0 ] \
|
||||
|| fail_closed "online_native_backup_monitor_detected_collector"
|
||||
emit_receipt "monitor_verifier" "pass" \
|
||||
"online_native_collector_continuous_auto_repair_0_detected_0"
|
||||
return 0
|
||||
fi
|
||||
|
||||
stop_epoch="$(awk '$2 == "stopped" { print $1; exit }' "${STATE_OBSERVATIONS}")"
|
||||
[ -n "${stop_epoch}" ] || fail_closed "collector_stop_transition_not_observed"
|
||||
restore_epoch="$(awk '
|
||||
$2 == "stopped" { stopped_seen = 1; next }
|
||||
stopped_seen && $2 == "running" { print $1; exit }
|
||||
' "${STATE_OBSERVATIONS}")"
|
||||
[ -n "${restore_epoch}" ] || fail_closed "collector_restore_transition_not_observed"
|
||||
|
||||
next_cron_epoch=$(( ((stop_epoch / MONITOR_CRON_INTERVAL_SECONDS) + 1) * MONITOR_CRON_INTERVAL_SECONDS ))
|
||||
if [ "${next_cron_epoch}" -le "${restore_epoch}" ]; then
|
||||
cron_crossed=1
|
||||
fi
|
||||
|
||||
if [ "${detected_count}" -gt 0 ]; then
|
||||
[ "${cooldown_count}" -ge "${detected_count}" ] \
|
||||
|| fail_closed "monitor_detected_without_matching_cooldown"
|
||||
fi
|
||||
if [ "${cron_crossed}" -eq 1 ]; then
|
||||
[ "${detected_count}" -gt 0 ] && [ "${cooldown_count}" -gt 0 ] \
|
||||
|| fail_closed "cron_crossed_without_collector_cooldown_receipt"
|
||||
fi
|
||||
|
||||
emit_receipt "monitor_verifier" "pass" \
|
||||
"auto_repair_${auto_repair_count}_detected_${detected_count}_cooldown_${cooldown_count}_cron_crossed_${cron_crossed}"
|
||||
}
|
||||
|
||||
verify_online_collector_window() {
|
||||
[ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
|
||||
|| fail_closed "collector_state_observer_ambiguous"
|
||||
[ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \
|
||||
|| fail_closed "online_native_backup_collector_continuity_failed"
|
||||
emit_receipt "monitor_verifier" "pass" \
|
||||
"online_native_collector_continuous_monitor_contract_not_applicable"
|
||||
}
|
||||
|
||||
main() {
|
||||
local container_id_before container_started_at_before container_restart_count_before
|
||||
local container_health_before collector_metadata_before
|
||||
local monitor_log_identity="" monitor_log_size=""
|
||||
local source_diff_detail ai_decision_detail
|
||||
local backup_status native_restore_contract_detail="not_applicable"
|
||||
|
||||
setup_receipts
|
||||
|
||||
[ "$#" -eq 1 ] || fail_closed "exactly_one_mode_required_use_check_or_apply"
|
||||
case "$1" in
|
||||
--check) MODE="check" ;;
|
||||
--apply) MODE="apply" ;;
|
||||
*) fail_closed "unsupported_mode_use_check_or_apply" ;;
|
||||
esac
|
||||
|
||||
# Hold the canary lock before inspecting or hashing any mutable backup or
|
||||
# restore toolchain path. This closes the check/apply substitution window.
|
||||
require_command flock
|
||||
acquire_canary_lock
|
||||
|
||||
for command_name in awk date docker env grep kill pgrep python3 sha256sum sleep ss timeout tr; do
|
||||
require_command "${command_name}"
|
||||
done
|
||||
valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout"
|
||||
valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after"
|
||||
valid_positive_integer "${DOCKER_TIMEOUT_SECONDS}" || fail_closed "invalid_docker_timeout"
|
||||
valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval"
|
||||
case "${EXPECT_COLLECTOR_STOP}" in
|
||||
0|1) ;;
|
||||
*) fail_closed "invalid_expect_collector_stop_mode" ;;
|
||||
esac
|
||||
|
||||
[ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \
|
||||
|| fail_closed "backup_script_missing_unexecutable_or_symlink"
|
||||
if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then
|
||||
require_native_restore_contract
|
||||
require_native_receipt_contract
|
||||
native_restore_contract_detail="hook_$(sha256sum "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" | awk '{print $1}')_inventory_$(sha256sum "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" | awk '{print $1}')_disk_config_$(sha256sum "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" | awk '{print $1}')_cluster_config_$(sha256sum "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" | awk '{print $1}')"
|
||||
else
|
||||
for command_name in cp crontab id mktemp mv rm stat tail; do
|
||||
require_command "${command_name}"
|
||||
done
|
||||
valid_positive_integer "${LEASE_MARGIN_SECONDS}" || fail_closed "invalid_lease_margin"
|
||||
valid_positive_integer "${MONITOR_CRON_INTERVAL_SECONDS}" || fail_closed "invalid_monitor_interval"
|
||||
[ "${MONITOR_CRON_INTERVAL_SECONDS}" -le 300 ] || fail_closed "monitor_interval_exceeds_cron_contract"
|
||||
verify_monitor_contract
|
||||
fi
|
||||
|
||||
if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
||||
fail_closed "another_signoz_backup_is_running"
|
||||
fi
|
||||
|
||||
collector_metadata_before="$(collector_runtime_metadata)" \
|
||||
|| fail_closed "collector_runtime_metadata_unreadable"
|
||||
IFS=$'\t' read -r container_id_before _container_running_before \
|
||||
container_started_at_before container_restart_count_before container_health_before \
|
||||
<<< "${collector_metadata_before}"
|
||||
[ -n "${container_id_before}" ] \
|
||||
&& [ -n "${container_started_at_before}" ] \
|
||||
&& [[ "${container_restart_count_before}" =~ ^[0-9]+$ ]] \
|
||||
&& [[ "${container_health_before}" =~ ^(healthy|none)$ ]] \
|
||||
|| fail_closed "collector_runtime_metadata_invalid"
|
||||
verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \
|
||||
"${container_restart_count_before}" "${container_health_before}" \
|
||||
|| fail_closed "collector_preflight_runtime_failed"
|
||||
|
||||
if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
|
||||
monitor_log_identity="$(stat -c '%d:%i' "${MONITOR_LOG}")"
|
||||
monitor_log_size="$(stat -c '%s' "${MONITOR_LOG}")"
|
||||
inspect_prior_lease
|
||||
emit_receipt "sensor_source" "pass" \
|
||||
"monitor_hash_$(sha256sum "${MONITOR_SCRIPT}" | awk '{print $1}')_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
|
||||
source_diff_detail="legacy_monitor_and_cron_contract_exact_cooldown_prior_present_${PRIOR_LEASE_PRESENT}_hash_${PRIOR_LEASE_HASH}"
|
||||
ai_decision_detail="arm_exact_container_future_epoch_run_legacy_stop_backup_then_restore"
|
||||
else
|
||||
PRIOR_LEASE_PRESENT=0
|
||||
PRIOR_LEASE_HASH="not_applicable"
|
||||
emit_receipt "sensor_source" "pass" \
|
||||
"online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_${native_restore_contract_detail}_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}"
|
||||
source_diff_detail="online_native_backup_isolated_restore_contract_bound_monitor_and_cooldown_contract_not_applicable"
|
||||
ai_decision_detail="run_online_native_backup_with_mandatory_isolated_restore_verifier_without_collector_or_monitor_mutation"
|
||||
fi
|
||||
emit_receipt "normalized_asset_identity" "pass" \
|
||||
"asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker"
|
||||
emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}"
|
||||
emit_receipt "ai_decision" "candidate" "${ai_decision_detail}"
|
||||
emit_receipt "risk_policy" "pass" \
|
||||
"risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_mandatory_isolated_restore_independent_verifier"
|
||||
emit_receipt "check" "pass" \
|
||||
"mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_${native_restore_contract_detail}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent"
|
||||
|
||||
if [ "${MODE}" = "check" ]; then
|
||||
CHECK_VERIFIED=1
|
||||
log_info "SigNoz backup canary check passed without arming lease or launching backup"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
|
||||
capture_prior_lease
|
||||
arm_lease
|
||||
else
|
||||
emit_receipt "lease" "not_required" \
|
||||
"online_native_backup_has_no_collector_stop_or_cooldown_write"
|
||||
fi
|
||||
|
||||
: > "${STATE_OBSERVATIONS}"
|
||||
set +e
|
||||
timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \
|
||||
env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \
|
||||
CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" \
|
||||
CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" \
|
||||
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" \
|
||||
CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" \
|
||||
CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" \
|
||||
CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" \
|
||||
BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \
|
||||
> "${BACKUP_OUTPUT}" 2>&1 &
|
||||
BACKUP_PID=$!
|
||||
observe_collector &
|
||||
OBSERVER_PID=$!
|
||||
wait "${BACKUP_PID}"
|
||||
backup_status=$?
|
||||
BACKUP_PID=""
|
||||
wait "${OBSERVER_PID}"
|
||||
OBSERVER_PID=""
|
||||
set -e
|
||||
|
||||
emit_receipt "execution" "$([ "${backup_status}" -eq 0 ] && printf pass || printf failed)" \
|
||||
"backup_exit_${backup_status}"
|
||||
[ "${backup_status}" -eq 0 ] || fail_closed "backup_terminal_nonzero_${backup_status}"
|
||||
grep -a -F -q '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)' "${BACKUP_OUTPUT}" \
|
||||
|| fail_closed "retention_skip_receipt_missing"
|
||||
if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
|
||||
grep -a -F -q '========== SignOz 備份完成 (' "${BACKUP_OUTPUT}" \
|
||||
|| fail_closed "backup_success_terminal_missing"
|
||||
else
|
||||
grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \
|
||||
|| fail_closed "native_backup_success_terminal_missing"
|
||||
verify_same_run_native_restore_receipts
|
||||
fi
|
||||
|
||||
if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then
|
||||
verify_monitor_window "${monitor_log_identity}" "${monitor_log_size}"
|
||||
else
|
||||
verify_online_collector_window
|
||||
fi
|
||||
verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \
|
||||
"${container_restart_count_before}" "${container_health_before}" \
|
||||
|| fail_closed "collector_post_backup_runtime_failed"
|
||||
emit_receipt "post_verifier" "pass" \
|
||||
"collector_running_same_identity_started_at_restart_count_health_listeners_4317_4318"
|
||||
|
||||
CANARY_VERIFIED=1
|
||||
log_info "SigNoz backup canary verified; cooldown lease will now roll back"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -5,6 +5,7 @@ import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh"
|
||||
BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh"
|
||||
@@ -20,67 +21,98 @@ def _run_backup(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
initial_running: bool = True,
|
||||
docker_run_mode: str = "success",
|
||||
docker_run_target: str = "signoz-clickhouse",
|
||||
archive_timeout_target: str = "",
|
||||
tar_verify_mode: str = "success",
|
||||
tar_verify_target: str = "clickhouse_",
|
||||
tar_timeout_target: str = "",
|
||||
docker_start_failures: int = 0,
|
||||
native_check_status: int = 0,
|
||||
native_apply_status: int = 0,
|
||||
continuity_mode: str = "stable",
|
||||
restic_backup_status: int = 0,
|
||||
restore_max_attempts: int = 3,
|
||||
skip_retention: bool = True,
|
||||
operation_lock_status: int = 0,
|
||||
workspace_root_symlink: bool = False,
|
||||
) -> tuple[subprocess.CompletedProcess[str], list[str], Path]:
|
||||
harness = tmp_path / "backup"
|
||||
fake_bin = tmp_path / "bin"
|
||||
backup_base = tmp_path / "repository"
|
||||
event_log = tmp_path / "events.log"
|
||||
collector_id = tmp_path / "collector.id"
|
||||
collector_state = tmp_path / "collector.state"
|
||||
start_count = tmp_path / "start.count"
|
||||
workspace_root = tmp_path / "workspace-root"
|
||||
harness.mkdir()
|
||||
fake_bin.mkdir()
|
||||
(backup_base / "signoz" / "data").mkdir(parents=True)
|
||||
shutil.copy2(SCRIPT, harness / SCRIPT.name)
|
||||
event_log.touch()
|
||||
collector_id.write_text("sha256:collector-stable\n", encoding="utf-8")
|
||||
collector_state.write_text(
|
||||
"true\n" if initial_running else "false\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
if workspace_root_symlink:
|
||||
workspace_target = tmp_path / "workspace-target"
|
||||
workspace_target.mkdir()
|
||||
workspace_root.symlink_to(workspace_target, target_is_directory=True)
|
||||
else:
|
||||
workspace_root.mkdir()
|
||||
|
||||
(harness / "common.sh").write_text(
|
||||
"""\
|
||||
BACKUP_BASE="${TEST_BACKUP_BASE:?}"
|
||||
RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password"
|
||||
log_info() { :; }
|
||||
log_warn() { :; }
|
||||
log_error() { :; }
|
||||
log_success() { :; }
|
||||
log_info() { printf 'info %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
log_warn() { printf 'warn %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
log_error() { printf 'error %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
log_success() { printf 'success %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
build_tags() { :; }
|
||||
cleanup_old_backups() { :; }
|
||||
build_tags() { printf '%s' '--tag signoz'; }
|
||||
cleanup_old_backups() { printf 'retention %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_write_executable(
|
||||
harness / "clickhouse-native-backup.sh",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'native %s trace=%s run=%s work=%s\\n' \
|
||||
"${1:-}" "${TRACE_ID:-}" "${RUN_ID:-}" "${WORK_ITEM_ID:-}" \
|
||||
>> "${TEST_EVENT_LOG:?}"
|
||||
case "${1:-}" in
|
||||
--check)
|
||||
exit "${FAKE_NATIVE_CHECK_STATUS:-0}"
|
||||
;;
|
||||
--apply)
|
||||
status="${FAKE_NATIVE_APPLY_STATUS:-0}"
|
||||
[ "$status" -eq 0 ] || exit "$status"
|
||||
case "${FAKE_CONTINUITY_MODE:-stable}" in
|
||||
stopped)
|
||||
printf 'false\\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
sleep 0.08
|
||||
printf 'true\\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
;;
|
||||
identity)
|
||||
printf 'sha256:collector-replaced\\n' > "${TEST_COLLECTOR_ID:?}"
|
||||
sleep 0.08
|
||||
;;
|
||||
*) sleep 0.08 ;;
|
||||
esac
|
||||
printf 'native-zip\\n' \
|
||||
> "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip"
|
||||
printf '{}\\n' \
|
||||
> "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip.manifest.json"
|
||||
printf 'db\\ttable\\tengine\\trows\\tbytes\\t%s\\n' \
|
||||
"$(printf 'A%.0s' {1..64})" \
|
||||
> "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.source-inventory.tsv"
|
||||
;;
|
||||
*) exit 64 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do
|
||||
shift
|
||||
done
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do shift; done
|
||||
shift
|
||||
if [ -n "${FAKE_ARCHIVE_TIMEOUT_TARGET:-}" ] \
|
||||
&& [ "${1:-}" = "docker" ] \
|
||||
&& [[ " $* " == *"${FAKE_ARCHIVE_TIMEOUT_TARGET}"* ]]; then
|
||||
exit 124
|
||||
fi
|
||||
if [ -n "${FAKE_TAR_TIMEOUT_TARGET:-}" ] \
|
||||
&& [ "${1:-}" = "tar" ] \
|
||||
&& [[ " $* " == *"${FAKE_TAR_TIMEOUT_TARGET}"* ]]; then
|
||||
exit 124
|
||||
fi
|
||||
exec "$@"
|
||||
""",
|
||||
)
|
||||
@@ -89,61 +121,12 @@ exec "$@"
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'docker %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
case "${1:-}" in
|
||||
inspect)
|
||||
cat "${TEST_COLLECTOR_STATE:?}"
|
||||
;;
|
||||
stop)
|
||||
if [ "${2:-}" = "signoz-otel-collector" ]; then
|
||||
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
fi
|
||||
;;
|
||||
start)
|
||||
count=0
|
||||
if [ -e "${TEST_START_COUNT:?}" ]; then
|
||||
count=$(cat "${TEST_START_COUNT}")
|
||||
fi
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "$count" > "${TEST_START_COUNT}"
|
||||
if [ "$count" -le "${FAKE_DOCKER_START_FAILURES:-0}" ]; then
|
||||
exit 1
|
||||
fi
|
||||
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
;;
|
||||
run)
|
||||
mode=success
|
||||
if [[ " $* " == *"${FAKE_DOCKER_RUN_TARGET:-signoz-clickhouse}"* ]]; then
|
||||
mode="${FAKE_DOCKER_RUN_MODE:-success}"
|
||||
fi
|
||||
if [ "$mode" = "signal" ]; then
|
||||
kill -TERM "${PPID}"
|
||||
sleep 0.1
|
||||
exit 0
|
||||
fi
|
||||
if [ "$mode" = "empty" ]; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$mode" = "partial_nonzero" ]; then
|
||||
printf 'partial-archive-data\n'
|
||||
exit 23
|
||||
fi
|
||||
printf 'archive-data\n'
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "tar",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'tar %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
if [[ " $* " == *"${FAKE_TAR_VERIFY_TARGET:-signoz-clickhouse}"* ]] \
|
||||
&& [ "${FAKE_TAR_VERIFY_MODE:-success}" = "corrupt" ]; then
|
||||
exit 2
|
||||
fi
|
||||
exit 0
|
||||
printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
[ "${1:-}" = "inspect" ] || exit 70
|
||||
printf '%s\\t%s\\t%s\\t%s\\t%s\\n' \
|
||||
"$(tr -d '\\r\\n' < "${TEST_COLLECTOR_ID:?}")" \
|
||||
"$(tr -d '\\r\\n' < "${TEST_COLLECTOR_STATE:?}")" \
|
||||
'2026-07-15T00:00:00Z' '0' 'healthy'
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
@@ -151,18 +134,29 @@ exit 0
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'restic %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
printf 'restic %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
case " $* " in
|
||||
*" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;;
|
||||
*" snapshots "*) printf '[{"short_id":"test123"}]\n' ;;
|
||||
*" backup "*)
|
||||
dump_dir="${4:?}"
|
||||
if [ -f "$dump_dir/signoz-metadata-gap.json" ]; then
|
||||
printf 'gap %s\\n' "$(cat "$dump_dir/signoz-metadata-gap.json")" \
|
||||
>> "${TEST_EVENT_LOG:?}"
|
||||
fi
|
||||
exit "${FAKE_RESTIC_BACKUP_STATUS:-0}"
|
||||
;;
|
||||
*" snapshots "*)
|
||||
printf '[{"short_id":"native123"}]\\n'
|
||||
;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "du",
|
||||
fake_bin / "flock",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
printf '4K\t%s\n' "${2:-unknown}"
|
||||
set -eu
|
||||
printf 'flock %s\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
exit "${FAKE_OPERATION_LOCK_STATUS:-0}"
|
||||
""",
|
||||
)
|
||||
|
||||
@@ -171,22 +165,23 @@ printf '4K\t%s\n' "${2:-unknown}"
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TEST_BACKUP_BASE": str(backup_base),
|
||||
"TEST_EVENT_LOG": str(event_log),
|
||||
"TEST_COLLECTOR_ID": str(collector_id),
|
||||
"TEST_COLLECTOR_STATE": str(collector_state),
|
||||
"TEST_START_COUNT": str(start_count),
|
||||
"FAKE_DOCKER_RUN_MODE": docker_run_mode,
|
||||
"FAKE_DOCKER_RUN_TARGET": docker_run_target,
|
||||
"FAKE_ARCHIVE_TIMEOUT_TARGET": archive_timeout_target,
|
||||
"FAKE_TAR_VERIFY_MODE": tar_verify_mode,
|
||||
"FAKE_TAR_VERIFY_TARGET": tar_verify_target,
|
||||
"FAKE_TAR_TIMEOUT_TARGET": tar_timeout_target,
|
||||
"FAKE_DOCKER_START_FAILURES": str(docker_start_failures),
|
||||
"FAKE_NATIVE_CHECK_STATUS": str(native_check_status),
|
||||
"FAKE_NATIVE_APPLY_STATUS": str(native_apply_status),
|
||||
"FAKE_CONTINUITY_MODE": continuity_mode,
|
||||
"FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status),
|
||||
"COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts),
|
||||
"COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0",
|
||||
"TRACE_ID": "trace-P0-OBS-002",
|
||||
"RUN_ID": f"run-{tmp_path.name}",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"SIGNOZ_BACKUP_RECEIPT_ROOT": str(tmp_path / "receipts"),
|
||||
"COLLECTOR_CONTINUITY_POLL_SECONDS": "0.01",
|
||||
"COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1",
|
||||
"ARCHIVE_CREATE_TIMEOUT_SECONDS": "1",
|
||||
"ARCHIVE_VERIFY_TIMEOUT_SECONDS": "1",
|
||||
"TIMEOUT_KILL_AFTER_SECONDS": "1",
|
||||
"BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0",
|
||||
"SIGNOZ_BACKUP_OPERATION_LOCK_FILE": str(tmp_path / "operation.lock"),
|
||||
"FAKE_OPERATION_LOCK_STATUS": str(operation_lock_status),
|
||||
"SIGNOZ_BACKUP_TMP_ROOT": str(workspace_root),
|
||||
}
|
||||
process = subprocess.Popen(
|
||||
["bash", str(harness / SCRIPT.name)],
|
||||
@@ -203,281 +198,214 @@ printf '4K\t%s\n' "${2:-unknown}"
|
||||
stderr,
|
||||
)
|
||||
events = event_log.read_text(encoding="utf-8").splitlines()
|
||||
return result, events, Path(f"/tmp/signoz-backup-{process.pid}")
|
||||
workspace_events = _events(events, "info 建立 SigNoz private workspace: ")
|
||||
dump_dir = (
|
||||
Path(workspace_events[-1].split(": ", 1)[1])
|
||||
if workspace_events
|
||||
else tmp_path / "workspace-not-created"
|
||||
)
|
||||
return result, events, dump_dir
|
||||
|
||||
|
||||
def _events(events: list[str], prefix: str) -> list[str]:
|
||||
return [event for event in events if event.startswith(prefix)]
|
||||
|
||||
|
||||
def _run_backup_all_with_signoz_failure(
|
||||
def test_source_sunsets_raw_volumes_and_collector_mutation() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "signoz-clickhouse" not in source
|
||||
assert "signoz-sqlite" not in source
|
||||
assert "docker run" not in source
|
||||
assert "docker stop" not in source
|
||||
assert "docker start" not in source
|
||||
assert "create_volume_archive" not in source
|
||||
assert "pending_application_consistent_export" in source
|
||||
assert '"raw_volume_read":false' in source
|
||||
assert "{{.State.StartedAt}}" in source
|
||||
assert "{{.RestartCount}}" in source
|
||||
assert '{{if (index .State "Health")}}' in source
|
||||
assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
|
||||
assert 'notify_clawbot "success"' not in source
|
||||
assert "no-downtime continuity verifier" in source
|
||||
assert "awoooi-signoz-backup-operation.lock" in source
|
||||
assert "flock -n 8" in source
|
||||
assert "cleanup_old_backups" not in source
|
||||
assert 'mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX"' in source
|
||||
assert "/tmp/signoz-backup-$$" not in source
|
||||
assert "WORKSPACE_CREATED=1" in source
|
||||
|
||||
|
||||
def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime(
|
||||
tmp_path: Path,
|
||||
) -> tuple[subprocess.CompletedProcess[str], list[str]]:
|
||||
harness = tmp_path / "backup-all"
|
||||
fake_scripts = tmp_path / "deployed-scripts"
|
||||
event_log = tmp_path / "backup-all-events.log"
|
||||
harness.mkdir()
|
||||
fake_scripts.mkdir()
|
||||
event_log.touch()
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path)
|
||||
|
||||
source = BACKUP_ALL.read_text(encoding="utf-8").replace(
|
||||
"/backup/scripts",
|
||||
str(fake_scripts),
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert len(_events(events, "native --check")) == 1
|
||||
assert len(_events(events, "native --apply")) == 1
|
||||
check_identity = _events(events, "native --check")[0].split(" trace=", 1)[1]
|
||||
apply_identity = _events(events, "native --apply")[0].split(" trace=", 1)[1]
|
||||
assert check_identity == apply_identity
|
||||
assert not _events(events, "docker stop")
|
||||
assert not _events(events, "docker start")
|
||||
assert not _events(events, "docker run")
|
||||
assert any("pending_application_consistent_export" in event for event in events)
|
||||
assert len(_events(events, "notify warning signoz ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert any("SigNoz ClickHouse native 備份完成" in event for event in events)
|
||||
assert any(
|
||||
"--tag run:run-" in event for event in events if event.startswith("restic ")
|
||||
)
|
||||
backup_all = harness / "backup-all.sh"
|
||||
_write_executable(backup_all, source)
|
||||
(harness / "common.sh").write_text(
|
||||
"""\
|
||||
log_info() { :; }
|
||||
log_warn() { :; }
|
||||
log_error() { :; }
|
||||
log_success() { :; }
|
||||
notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; }
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
for name in (
|
||||
"backup-gitea.sh",
|
||||
"backup-momo.sh",
|
||||
"backup-harbor.sh",
|
||||
"backup-awoooi.sh",
|
||||
"backup-langfuse.sh",
|
||||
"backup-monitoring.sh",
|
||||
"backup-signoz.sh",
|
||||
"backup-open-webui.sh",
|
||||
"backup-clawbot.sh",
|
||||
):
|
||||
status = 17 if name == "backup-signoz.sh" else 0
|
||||
_write_executable(
|
||||
fake_scripts / name,
|
||||
f"#!/bin/bash\nprintf 'run {name}\\n' >> \"${{TEST_EVENT_LOG:?}}\"\nexit {status}\n",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(backup_all)],
|
||||
env={**os.environ, "TEST_EVENT_LOG": str(event_log)},
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
)
|
||||
return result, event_log.read_text(encoding="utf-8").splitlines()
|
||||
assert any("restic -r" in event and " check " in event for event in events)
|
||||
receipts = list((tmp_path / "receipts").glob("*/restic-snapshot.json"))
|
||||
assert len(receipts) == 1
|
||||
receipt = receipts[0].read_text(encoding="utf-8")
|
||||
assert '"snapshot_id":"native123"' in receipt
|
||||
assert '"offsite_verified":false' in receipt
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_backup_jobs_deploys_the_guarded_signoz_script() -> None:
|
||||
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
|
||||
|
||||
assert 'dest: "/backup/scripts/{{ item }}"' in playbook
|
||||
assert "- backup-signoz.sh" in playbook
|
||||
assert "tags: backup_jobs" in playbook
|
||||
|
||||
|
||||
def test_restore_policy_is_bounded_timed_and_independently_verified() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source
|
||||
assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source
|
||||
assert 'ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"' in source
|
||||
assert 'ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"' in source
|
||||
assert 'TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"' in source
|
||||
assert 'timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}"' in source
|
||||
assert 'tar -tzf "${output_file}"' in source
|
||||
assert "docker run" in source
|
||||
assert "|| true" not in source.split("create_volume_archive()", 1)[1].split(
|
||||
"verify_volume_archive()",
|
||||
1,
|
||||
)[0]
|
||||
assert "verify_collector_final_state" in source
|
||||
assert source.index("verify_collector_final_state") < source.rindex(
|
||||
'notify_clawbot "success"'
|
||||
)
|
||||
|
||||
|
||||
def test_canary_retention_skip_is_explicit_and_defaults_off() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"' in source
|
||||
assert 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' in source
|
||||
assert source.index('restic -r "${LOCAL_REPO}" backup') < source.index(
|
||||
'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]'
|
||||
)
|
||||
assert source.index('if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]') < source.index(
|
||||
"if ! verify_collector_final_state"
|
||||
)
|
||||
|
||||
|
||||
def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal")
|
||||
def test_operation_lock_contention_fails_before_native_or_restic(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, operation_lock_status=1)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert _events(events, "docker stop signoz-otel-collector")
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert _events(events, "flock -n 8")
|
||||
assert not _events(events, "native ")
|
||||
assert not _events(events, "restic ")
|
||||
assert len(_events(events, "notify failed signoz ")) == 1
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_collector_is_restored_when_clickhouse_backup_fails(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="empty")
|
||||
def test_symlink_workspace_root_fails_before_native_or_restic(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, workspace_root_symlink=True)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert result.returncode != 0
|
||||
assert _events(events, "flock -n 8")
|
||||
assert not _events(events, "native ")
|
||||
assert not _events(events, "restic ")
|
||||
assert any("workspace root" in event for event in events)
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_exit_cleanup_does_not_restart_an_already_restored_collector(
|
||||
def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, native_check_status=19)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert _events(events, "native --check")
|
||||
assert not _events(events, "native --apply")
|
||||
assert not _events(events, "restic ")
|
||||
assert len(_events(events, "notify failed signoz ")) == 1
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_native_apply_failure_never_reports_scoped_success(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, native_apply_status=23)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert _events(events, "native --apply")
|
||||
assert not _events(events, "restic ")
|
||||
assert not _events(events, "notify warning ")
|
||||
assert not _events(events, "notify success ")
|
||||
assert len(_events(events, "notify failed signoz ")) == 1
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_initially_stopped_collector_fails_without_apply(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert _events(events, "native --check")
|
||||
assert not _events(events, "native --apply")
|
||||
assert not _events(events, "restic ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_continuity_observer_rejects_stopped_sample(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, continuity_mode="stopped")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert any("continuity verifier 失敗" in event for event in events)
|
||||
assert not _events(events, "restic ")
|
||||
assert not _events(events, "notify warning ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_continuity_observer_rejects_container_identity_drift(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, continuity_mode="identity")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert any("continuity verifier 失敗" in event for event in events)
|
||||
assert not _events(events, "restic ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_restic_failure_is_nonzero_and_no_completion_notification(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, restic_backup_status=17)
|
||||
|
||||
assert result.returncode == 17
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert _events(events, "restic ")
|
||||
assert not _events(events, "notify warning ")
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_fail_once_restores_within_budget_and_succeeds(tmp_path: Path) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=1)
|
||||
def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None:
|
||||
result, events, _dump_dir = _run_backup(tmp_path, skip_retention=True)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 2
|
||||
assert len(_events(events, "notify success ")) == 1
|
||||
assert not _events(events, "notify failed ")
|
||||
assert not dump_dir.exists()
|
||||
assert any("BACKUP_SKIP_RETENTION_CLEANUP=1" in event for event in events)
|
||||
assert not _events(events, "retention ")
|
||||
|
||||
|
||||
def test_permanent_restore_failure_is_nonzero_and_never_notifies_success(
|
||||
def test_inline_destructive_retention_request_fails_before_backup(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=99)
|
||||
result, events, dump_dir = _run_backup(tmp_path, skip_retention=False)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 3
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not _events(events, "flock ")
|
||||
assert not _events(events, "native ")
|
||||
assert not _events(events, "restic ")
|
||||
assert not _events(events, "retention ")
|
||||
assert any("critical break-glass workflow" in event for event in events)
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_initially_stopped_fails_closed_without_stop_start_or_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(tmp_path, initial_running=False)
|
||||
def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None:
|
||||
playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert not _events(events, "docker stop signoz-otel-collector")
|
||||
assert not _events(events, "docker start signoz-otel-collector")
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_success_notification_follows_independent_final_running_readback(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, _dump_dir = _run_backup(tmp_path)
|
||||
|
||||
assert result.returncode == 0
|
||||
inspect_positions = [
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.startswith("docker inspect --format")
|
||||
]
|
||||
success_position = next(
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.startswith("notify success ")
|
||||
assert 'dest: "/backup/scripts/{{ item }}"' in playbook
|
||||
assert "- backup-signoz.sh" in playbook
|
||||
assert "- clickhouse-native-backup.sh" in playbook
|
||||
assert "- clickhouse-native-restore-drill.sh" in playbook
|
||||
assert "- clickhouse-restore-inventory.py" in playbook
|
||||
assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in playbook
|
||||
assert (
|
||||
"/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
in playbook
|
||||
)
|
||||
assert len(inspect_positions) >= 3
|
||||
assert inspect_positions[-1] < success_position
|
||||
assert "tags: backup_jobs" in playbook
|
||||
|
||||
start_position = next(
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.startswith("docker start signoz-otel-collector")
|
||||
|
||||
def test_scripts_are_valid_bash() -> None:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
"-n",
|
||||
str(SCRIPT),
|
||||
str(ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
tar_positions = [
|
||||
index for index, event in enumerate(events) if event.startswith("tar -tzf")
|
||||
]
|
||||
assert len(tar_positions) == 2
|
||||
assert start_position < tar_positions[0]
|
||||
|
||||
|
||||
def test_archive_create_timeout_is_bounded_restores_and_notifies_once(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
archive_timeout_target="signoz-clickhouse",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_partial_sqlite_archive_nonzero_fails_overall_and_removes_output(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
docker_run_mode="partial_nonzero",
|
||||
docker_run_target="signoz-sqlite",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_corrupt_archive_fails_after_collector_restore_and_notifies_once(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
tar_verify_mode="corrupt",
|
||||
tar_verify_target="clickhouse_",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
start_position = next(
|
||||
index
|
||||
for index, event in enumerate(events)
|
||||
if event.startswith("docker start signoz-otel-collector")
|
||||
)
|
||||
tar_position = next(
|
||||
index for index, event in enumerate(events) if event.startswith("tar -tzf")
|
||||
)
|
||||
assert start_position < tar_position
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_integrity_verifier_timeout_is_bounded_and_never_reports_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events, dump_dir = _run_backup(
|
||||
tmp_path,
|
||||
tar_timeout_target="clickhouse_",
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert len(_events(events, "docker start signoz-otel-collector")) == 1
|
||||
assert len(_events(events, "notify failed ")) == 1
|
||||
assert not _events(events, "notify success ")
|
||||
assert not dump_dir.exists()
|
||||
|
||||
|
||||
def test_backup_all_propagates_signoz_failure_to_nonzero_terminal(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, events = _run_backup_all_with_signoz_failure(tmp_path)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "run backup-signoz.sh" in events
|
||||
assert any(event.startswith("notify warning all ") for event in events)
|
||||
assert not any(event.startswith("notify success all ") for event in events)
|
||||
assert result.returncode == 0, result.stderr
|
||||
|
||||
478
scripts/backup/tests/test_clickhouse_native_backup.py
Normal file
478
scripts/backup/tests/test_clickhouse_native_backup.py
Normal file
@@ -0,0 +1,478 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"
|
||||
PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
|
||||
|
||||
|
||||
def _write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _install_fakes(fake_bin: Path) -> None:
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do shift; done
|
||||
shift
|
||||
exec "$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(fake_bin / "flock", "#!/bin/bash\nexit 0\n")
|
||||
_write_executable(
|
||||
fake_bin / "sha256sum",
|
||||
"""\
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import sys
|
||||
|
||||
if len(sys.argv) == 1:
|
||||
payload = sys.stdin.buffer.read()
|
||||
print(f"{hashlib.sha256(payload).hexdigest()} -")
|
||||
else:
|
||||
with open(sys.argv[1], "rb") as source:
|
||||
payload = source.read()
|
||||
print(f"{hashlib.sha256(payload).hexdigest()} {sys.argv[1]}")
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "unzip",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'unzip %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
exit "${FAKE_UNZIP_STATUS:-0}"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "docker",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"
|
||||
|
||||
find_arg() {
|
||||
local wanted="$1"
|
||||
shift
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = "$wanted" ]; then
|
||||
printf '%s\\n' "${2:-}"
|
||||
return 0
|
||||
fi
|
||||
shift
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
inspect)
|
||||
printf 'sha256:clickhouse-test-id\\ttrue\\n'
|
||||
;;
|
||||
cp)
|
||||
[ -e "${TEST_SERVER_ARTIFACT:?}" ] || exit 44
|
||||
printf 'clickhouse-native-zip-bytes\\n' > "${3:?}"
|
||||
;;
|
||||
exec)
|
||||
shift
|
||||
container="${1:?}"
|
||||
shift
|
||||
case "${1:-}" in
|
||||
test)
|
||||
shift
|
||||
if [ "${1:-}" = "!" ]; then
|
||||
[ ! -e "${TEST_SERVER_ARTIFACT:?}" ]
|
||||
elif [ "${1:-}" = "-d" ] || [ "${1:-}" = "-w" ]; then
|
||||
[ "${FAKE_DISK_READY:-1}" = "1" ]
|
||||
[ "${2:-}" = "${FAKE_DISK_PATH:-/backups/}" ]
|
||||
else
|
||||
exit 45
|
||||
fi
|
||||
;;
|
||||
rm)
|
||||
rm -f "${TEST_SERVER_ARTIFACT:?}"
|
||||
;;
|
||||
clickhouse-client)
|
||||
query="$(find_arg --query "$@")"
|
||||
operation_id="$(find_arg --param_operation_id "$@" || true)"
|
||||
artifact="$(find_arg --param_artifact "$@" || true)"
|
||||
disk="$(find_arg --param_disk "$@" || true)"
|
||||
case "$query" in
|
||||
'SELECT version()')
|
||||
printf '25.5.1.1\\n'
|
||||
;;
|
||||
*"database='system' AND name='backups'"*)
|
||||
printf '%s\\n' "${FAKE_SYSTEM_BACKUPS_COUNT:-1}"
|
||||
;;
|
||||
'CHECK GRANT BACKUP ON *.*')
|
||||
printf '%s\\n' "${FAKE_BACKUP_GRANT:-1}"
|
||||
;;
|
||||
*"FROM system.disks"*)
|
||||
printf '%s\\n' "${FAKE_DISK_PATH:-/backups/}"
|
||||
;;
|
||||
*"FROM system.databases"*)
|
||||
printf '%s\\n' signoz_analytics signoz_logs signoz_metadata \
|
||||
signoz_meter signoz_metrics signoz_traces
|
||||
if [ "${FAKE_EXTRA_DATABASE:-0}" = "1" ]; then
|
||||
printf 'unexpected_db\\n'
|
||||
fi
|
||||
;;
|
||||
*"FROM system.tables WHERE database IN"*)
|
||||
schema_hash="$(printf 'A%.0s' {1..64})"
|
||||
printf 'signoz_analytics\\tanalytics\\tReplicatedMergeTree\\t10\\t1000\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_logs\\tlogs_v2\\tReplicatedMergeTree\\t20\\t2000\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_metadata\\tmetadata\\tMergeTree\\t3\\t300\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_meter\\tmeter\\tDistributed\\t0\\t0\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_metrics\\tsamples_v4\\tReplicatedMergeTree\\t40\\t4000\\t%s\\n' "$schema_hash"
|
||||
printf 'signoz_traces\\tsignoz_index_v3\\tReplicatedMergeTree\\t50\\t5000\\t%s\\n' "$schema_hash"
|
||||
;;
|
||||
BACKUP*)
|
||||
case "$query" in
|
||||
*" SETTINGS id='${EXPECTED_FAKE_OPERATION_ID:?}' ASYNC")
|
||||
operation_id="${EXPECTED_FAKE_OPERATION_ID}"
|
||||
;;
|
||||
*) exit 49 ;;
|
||||
esac
|
||||
printf '%s\\n' "$operation_id" > "${TEST_OPERATION_ID:?}"
|
||||
printf '%s\\n' "$artifact" > "${TEST_ARTIFACT_NAME:?}"
|
||||
printf '%s\\n' "$disk" > "${TEST_DISK_NAME:?}"
|
||||
: > "${TEST_SUBMITTED:?}"
|
||||
: > "${TEST_SERVER_ARTIFACT:?}"
|
||||
printf '%s\\tCREATING_BACKUP\\n' "$operation_id"
|
||||
;;
|
||||
*"FROM system.backups WHERE id="*)
|
||||
mode="${FAKE_STATUS_MODE:-success}"
|
||||
if [ "$mode" = "existing_success" ] || [ -e "${TEST_SUBMITTED:?}" ]; then
|
||||
artifact_name="$(cat "${TEST_ARTIFACT_NAME:?}" 2>/dev/null || true)"
|
||||
[ -n "$artifact_name" ] || artifact_name="${EXPECTED_FAKE_ARTIFACT:?}"
|
||||
backup_name="Disk('backups', '$artifact_name')"
|
||||
case "$mode" in
|
||||
failed)
|
||||
printf '%s\\tBACKUP_FAILED\\t0\\t0\\t0\\t0\\t434F44453A20353938\\t%s\\n' \
|
||||
"$backup_name" "$(printf '1%.0s' {1..64})"
|
||||
;;
|
||||
*)
|
||||
: > "${TEST_SERVER_ARTIFACT:?}"
|
||||
printf '%s\\tBACKUP_CREATED\\t12\\t1200\\t2400\\t1100\\tnone\\t%s\\n' \
|
||||
"$backup_name" "$(printf '0%.0s' {1..64})"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
;;
|
||||
*) exit 46 ;;
|
||||
esac
|
||||
;;
|
||||
*) exit 47 ;;
|
||||
esac
|
||||
;;
|
||||
*) exit 48 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
def _harness(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
status_mode: str = "success",
|
||||
extra_database: bool = False,
|
||||
disk_ready: bool = True,
|
||||
unzip_status: int = 0,
|
||||
with_hook: bool = False,
|
||||
) -> tuple[dict[str, str], dict[str, Path]]:
|
||||
fake_bin = tmp_path / "bin"
|
||||
output = tmp_path / "output"
|
||||
receipts = tmp_path / "receipts"
|
||||
fake_bin.mkdir()
|
||||
output.mkdir()
|
||||
receipts.mkdir()
|
||||
_install_fakes(fake_bin)
|
||||
|
||||
event_log = tmp_path / "events.log"
|
||||
event_log.touch()
|
||||
paths = {
|
||||
"output": output,
|
||||
"receipts": receipts,
|
||||
"events": event_log,
|
||||
"submitted": tmp_path / "submitted",
|
||||
"server_artifact": tmp_path / "server-artifact",
|
||||
"operation_id": tmp_path / "operation-id",
|
||||
"artifact_name": tmp_path / "artifact-name",
|
||||
"disk_name": tmp_path / "disk-name",
|
||||
"hook_events": tmp_path / "hook-events.log",
|
||||
}
|
||||
identity_hash = (
|
||||
subprocess.check_output(
|
||||
[str(fake_bin / "sha256sum")],
|
||||
input=b"trace-native\0run-native\0P0-OBS-002",
|
||||
)
|
||||
.decode()
|
||||
.split()[0]
|
||||
)
|
||||
expected_artifact = f"signoz-{identity_hash}.zip"
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TRACE_ID": "trace-native",
|
||||
"RUN_ID": "run-native",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"CLICKHOUSE_NATIVE_OUTPUT_DIR": str(output),
|
||||
"CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(receipts),
|
||||
"CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS": "2",
|
||||
"CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS": "2",
|
||||
"CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS": "1",
|
||||
"CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS": "1",
|
||||
"TEST_EVENT_LOG": str(event_log),
|
||||
"TEST_SUBMITTED": str(paths["submitted"]),
|
||||
"TEST_SERVER_ARTIFACT": str(paths["server_artifact"]),
|
||||
"TEST_OPERATION_ID": str(paths["operation_id"]),
|
||||
"TEST_ARTIFACT_NAME": str(paths["artifact_name"]),
|
||||
"TEST_DISK_NAME": str(paths["disk_name"]),
|
||||
"EXPECTED_FAKE_ARTIFACT": expected_artifact,
|
||||
"EXPECTED_FAKE_OPERATION_ID": f"awoooi-{identity_hash}",
|
||||
"FAKE_STATUS_MODE": status_mode,
|
||||
"FAKE_EXTRA_DATABASE": "1" if extra_database else "0",
|
||||
"FAKE_DISK_READY": "1" if disk_ready else "0",
|
||||
"FAKE_UNZIP_STATUS": str(unzip_status),
|
||||
}
|
||||
if with_hook:
|
||||
hook = tmp_path / "restore-verifier-hook.sh"
|
||||
_write_executable(
|
||||
hook,
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
if [ "$1" = "--apply" ]; then
|
||||
test -f "$CLICKHOUSE_NATIVE_ARTIFACT_PATH"
|
||||
test -f "$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH"
|
||||
test -f "$CLICKHOUSE_NATIVE_MANIFEST_PATH"
|
||||
grep -q '"status":"BACKUP_CREATED"' "$CLICKHOUSE_NATIVE_MANIFEST_PATH"
|
||||
fi
|
||||
printf '%s trace=%s run=%s work=%s dbs=%s inventory=%s manifest=%s\\n' \
|
||||
"$1" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
|
||||
"$CLICKHOUSE_NATIVE_EXPECTED_DATABASES" \
|
||||
"$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH" \
|
||||
"${CLICKHOUSE_NATIVE_MANIFEST_PATH:-none}" \
|
||||
>> "${TEST_HOOK_EVENTS:?}"
|
||||
""",
|
||||
)
|
||||
env["CLICKHOUSE_NATIVE_POST_VERIFY_HOOK"] = str(hook)
|
||||
env["TEST_HOOK_EVENTS"] = str(paths["hook_events"])
|
||||
return env, paths
|
||||
|
||||
|
||||
def _run(env: dict[str, str], mode: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["bash", str(SCRIPT), mode],
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def _receipts(path: Path) -> list[dict[str, object]]:
|
||||
receipt_file = path / "run-native" / "receipts.jsonl"
|
||||
return [
|
||||
json.loads(line)
|
||||
for line in receipt_file.read_text(encoding="utf-8").splitlines()
|
||||
]
|
||||
|
||||
|
||||
def test_check_validates_exact_scope_without_backup_or_artifact_write(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
result = _run(env, "--check")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
events = paths["events"].read_text(encoding="utf-8")
|
||||
assert "BACKUP DATABASE" not in events
|
||||
assert "docker cp" not in events
|
||||
assert "docker exec signoz-clickhouse rm" not in events
|
||||
assert not list(paths["output"].iterdir())
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert receipts[-1]["terminal"] == "check_pass_no_write"
|
||||
|
||||
|
||||
def test_apply_requires_same_run_check_receipt(tmp_path: Path) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "same_run_check_pass_receipt_required_before_apply" in result.stderr
|
||||
assert paths["events"].read_text(encoding="utf-8") == ""
|
||||
|
||||
|
||||
def test_apply_submits_exact_six_database_backup_and_verifies_zip(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
assert _run(env, "--check").returncode == 0
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
events = paths["events"].read_text(encoding="utf-8")
|
||||
assert "BACKUP DATABASE signoz_analytics,DATABASE signoz_logs" in events
|
||||
assert "DATABASE signoz_metadata,DATABASE signoz_meter" in events
|
||||
assert "DATABASE signoz_metrics,DATABASE signoz_traces" in events
|
||||
assert "BACKUP ALL" not in events
|
||||
backup_line = next(
|
||||
line for line in events.splitlines() if "--query BACKUP " in line
|
||||
)
|
||||
assert " SETTINGS id='awoooi-" in backup_line
|
||||
assert "id={operation_id:String}" not in backup_line
|
||||
assert "--param_operation_id" not in backup_line
|
||||
assert "docker cp signoz-clickhouse:/backups/signoz-" in events
|
||||
assert "unzip -tq" in events
|
||||
assert "docker exec signoz-clickhouse rm -f -- /backups/signoz-" in events
|
||||
assert not paths["server_artifact"].exists()
|
||||
outputs = list(paths["output"].glob("clickhouse-native-*.zip"))
|
||||
assert len(outputs) == 1
|
||||
manifest = json.loads(Path(str(outputs[0]) + ".manifest.json").read_text())
|
||||
assert manifest["databases"] == [
|
||||
"signoz_analytics",
|
||||
"signoz_logs",
|
||||
"signoz_metadata",
|
||||
"signoz_meter",
|
||||
"signoz_metrics",
|
||||
"signoz_traces",
|
||||
]
|
||||
inventory = list(paths["output"].glob("*.source-inventory.tsv"))
|
||||
assert len(inventory) == 1
|
||||
assert "ReplicatedMergeTree" in inventory[0].read_text(encoding="utf-8")
|
||||
assert "Distributed" in inventory[0].read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_database_allowlist_drift_fails_check(tmp_path: Path) -> None:
|
||||
env, _paths = _harness(tmp_path, extra_database=True)
|
||||
result = _run(env, "--check")
|
||||
assert result.returncode != 0
|
||||
assert "signoz_database_allowlist_drift" in result.stderr
|
||||
|
||||
|
||||
def test_missing_backup_grant_fails_check_before_submission(tmp_path: Path) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
env["FAKE_BACKUP_GRANT"] = "0"
|
||||
|
||||
result = _run(env, "--check")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "native_backup_grant_absent" in result.stderr
|
||||
assert not paths["submitted"].exists()
|
||||
|
||||
|
||||
def test_missing_runtime_backup_disk_fails_check(tmp_path: Path) -> None:
|
||||
env, _paths = _harness(tmp_path, disk_ready=False)
|
||||
result = _run(env, "--check")
|
||||
assert result.returncode != 0
|
||||
assert "backup_disk_directory_absent" in result.stderr
|
||||
|
||||
|
||||
def test_failed_async_terminal_has_exact_cleanup_and_no_local_artifact(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, paths = _harness(tmp_path, status_mode="failed")
|
||||
assert _run(env, "--check").returncode == 0
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "native_backup_terminal_BACKUP_FAILED" in result.stderr
|
||||
assert not paths["server_artifact"].exists()
|
||||
assert not list(paths["output"].iterdir())
|
||||
receipt_dir = paths["receipts"] / "run-native"
|
||||
assert list(receipt_dir.glob("*.stderr"))
|
||||
assert list(receipt_dir.glob("*.status"))
|
||||
bounded_errors = list(receipt_dir.glob("*-backup-error-bounded.hex"))
|
||||
assert len(bounded_errors) == 1
|
||||
assert bounded_errors[0].read_text(encoding="utf-8") == ("434F44453A20353938\n")
|
||||
|
||||
|
||||
def test_zip_failure_removes_local_partial_and_preserves_server_evidence(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env, paths = _harness(tmp_path, unzip_status=7)
|
||||
assert _run(env, "--check").returncode == 0
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "native_backup_archive_integrity_failed" in result.stderr
|
||||
assert paths["server_artifact"].exists()
|
||||
assert not list(paths["output"].iterdir())
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert any(
|
||||
item["stage"] == "cleanup" and item["terminal"] == "preserved"
|
||||
for item in receipts
|
||||
)
|
||||
|
||||
|
||||
def test_restore_verifier_hook_uses_check_and_apply_contract(tmp_path: Path) -> None:
|
||||
env, paths = _harness(tmp_path, with_hook=True)
|
||||
assert _run(env, "--check").returncode == 0
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
hook_events = paths["hook_events"].read_text(encoding="utf-8").splitlines()
|
||||
assert sum(event.startswith("--check ") for event in hook_events) == 1
|
||||
assert sum(event.startswith("--apply ") for event in hook_events) == 1
|
||||
assert all("signoz_analytics,signoz_logs" in event for event in hook_events)
|
||||
assert any(
|
||||
"manifest=" in event and "manifest=none" not in event
|
||||
for event in hook_events
|
||||
if event.startswith("--apply ")
|
||||
)
|
||||
|
||||
|
||||
def test_verified_local_artifact_is_idempotently_reused(tmp_path: Path) -> None:
|
||||
env, paths = _harness(tmp_path)
|
||||
assert _run(env, "--check").returncode == 0
|
||||
assert _run(env, "--apply").returncode == 0
|
||||
submissions_before = (
|
||||
paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE")
|
||||
)
|
||||
|
||||
result = _run(env, "--apply")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
submissions_after = (
|
||||
paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE")
|
||||
)
|
||||
assert submissions_after == submissions_before
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert any(
|
||||
item["stage"] == "idempotency" and item["terminal"] == "reused"
|
||||
for item in receipts
|
||||
)
|
||||
|
||||
|
||||
def test_source_contract_has_no_raw_volume_fallback_and_is_deployed() -> None:
|
||||
source = SCRIPT.read_text(encoding="utf-8")
|
||||
playbook = PLAYBOOK.read_text(encoding="utf-8")
|
||||
|
||||
assert "BACKUP ${BACKUP_SCOPE_SQL}" in source
|
||||
assert "SETTINGS id='${OPERATION_ID}' ASYNC" in source
|
||||
assert "SETTINGS id={operation_id:String}" not in source
|
||||
assert "BACKUP ALL" not in source
|
||||
assert ".zip" in source
|
||||
assert "unzip -tq" in source
|
||||
assert "--format $'{{.Id}}\\t{{.State.Running}}'" in source
|
||||
assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source
|
||||
assert "docker run" not in source
|
||||
assert "clickhouse-native-backup.sh" in playbook
|
||||
result = subprocess.run(
|
||||
["bash", "-n", str(SCRIPT)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
848
scripts/backup/tests/test_clickhouse_native_restore_drill.py
Normal file
848
scripts/backup/tests/test_clickhouse_native_restore_drill.py
Normal file
@@ -0,0 +1,848 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "clickhouse-native-restore-drill.sh"
|
||||
INVENTORY_HELPER = ROOT / "clickhouse-restore-inventory.py"
|
||||
REPO_ROOT = ROOT.parents[1]
|
||||
CLICKHOUSE_IMAGE_ID = (
|
||||
"sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6"
|
||||
)
|
||||
ZOOKEEPER_IMAGE_ID = (
|
||||
"sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0"
|
||||
)
|
||||
EMPTY_SHA256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"
|
||||
EXPECTED_DATABASES = [
|
||||
"signoz_analytics",
|
||||
"signoz_logs",
|
||||
"signoz_metadata",
|
||||
"signoz_meter",
|
||||
"signoz_metrics",
|
||||
"signoz_traces",
|
||||
]
|
||||
|
||||
|
||||
def _inventory(
|
||||
*,
|
||||
restored_delta: int = 0,
|
||||
engine_mismatch: bool = False,
|
||||
schema_mismatch: bool = False,
|
||||
) -> str:
|
||||
rows = [
|
||||
("signoz_analytics", "events", "ReplicatedMergeTree", 10, 100),
|
||||
("signoz_analytics", "events_all", "Distributed", 10, 100),
|
||||
("signoz_logs", "logs_v2", "ReplicatedMergeTree", 20, 200),
|
||||
("signoz_metadata", "resource_attrs", "ReplicatedMergeTree", 2, 20),
|
||||
("signoz_meter", "meter", "ReplicatedMergeTree", 3, 30),
|
||||
("signoz_metrics", "samples_v4", "ReplicatedMergeTree", 30, 300),
|
||||
("signoz_metrics", "time_series_v4", "ReplicatedMergeTree", 12, 120),
|
||||
("signoz_traces", "signoz_index_v3", "ReplicatedMergeTree", 40, 400),
|
||||
]
|
||||
rendered: list[str] = []
|
||||
for index, (database, table, engine, row_count, byte_count) in enumerate(rows):
|
||||
if engine_mismatch and table == "meter":
|
||||
engine = "MergeTree"
|
||||
schema_hash = (
|
||||
hashlib.sha256(f"{database}.{table}:{engine}".encode("utf-8"))
|
||||
.hexdigest()
|
||||
.upper()
|
||||
)
|
||||
if schema_mismatch and table == "meter":
|
||||
schema_hash = "F" * 64
|
||||
rendered.append(
|
||||
"\t".join(
|
||||
(
|
||||
database,
|
||||
table,
|
||||
engine,
|
||||
str(row_count + (restored_delta if index == 0 else 0)),
|
||||
str(byte_count + (restored_delta if index == 0 else 0)),
|
||||
schema_hash,
|
||||
)
|
||||
)
|
||||
)
|
||||
return "\n".join(rendered) + "\n"
|
||||
|
||||
|
||||
def _write_backup_fixture(tmp_path: Path) -> tuple[Path, Path, Path]:
|
||||
artifact = tmp_path / "clickhouse-native.zip.partial"
|
||||
with zipfile.ZipFile(artifact, "w") as archive:
|
||||
archive.writestr(".backup", "bounded native backup fixture\n")
|
||||
inventory = tmp_path / "source-inventory.tsv.partial"
|
||||
inventory.write_text(_inventory(), encoding="utf-8")
|
||||
artifact_sha = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
inventory_sha = hashlib.sha256(inventory.read_bytes()).hexdigest()
|
||||
manifest = tmp_path / "manifest.json.partial"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"trace_id": "backup-trace",
|
||||
"run_id": "backup-run",
|
||||
"work_item_id": "P0-OBS-002",
|
||||
"operation_id": "awoooi-backup-operation",
|
||||
"databases": EXPECTED_DATABASES,
|
||||
"source_inventory_sha256": inventory_sha,
|
||||
"sha256": artifact_sha,
|
||||
"size_bytes": artifact.stat().st_size,
|
||||
"status": "BACKUP_CREATED",
|
||||
},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return artifact, manifest, inventory
|
||||
|
||||
|
||||
def _write_fake_docker(tmp_path: Path) -> tuple[Path, Path, Path]:
|
||||
bin_dir = tmp_path / "bin"
|
||||
bin_dir.mkdir()
|
||||
state_dir = tmp_path / "docker-state"
|
||||
state_dir.mkdir()
|
||||
log_path = tmp_path / "docker.log"
|
||||
docker = bin_dir / "docker"
|
||||
docker.write_text(
|
||||
r"""#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
printf '%s\n' "$*" >> "${FAKE_DOCKER_LOG:?}"
|
||||
state="${FAKE_DOCKER_STATE:?}"
|
||||
cmd="${1:?}"
|
||||
shift
|
||||
last_arg() { eval "printf '%s' \"\${$#}\""; }
|
||||
|
||||
case "$cmd" in
|
||||
info)
|
||||
[ "${FAKE_DOCKER_INFO_FAIL:-0}" = 0 ] || exit 78
|
||||
printf '25.0.0\\n'
|
||||
;;
|
||||
image)
|
||||
sub="${1:?}"
|
||||
shift
|
||||
[ "$sub" = inspect ] || exit 90
|
||||
image="$(last_arg "$@")"
|
||||
case "$image" in
|
||||
clickhouse/clickhouse-server:25.5.6) printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" ;;
|
||||
signoz/zookeeper:3.7.1) printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
;;
|
||||
container)
|
||||
sub="${1:?}"
|
||||
shift
|
||||
case "$sub" in
|
||||
inspect)
|
||||
format=""
|
||||
if [ "${1:-}" = --format ]; then
|
||||
format="$2"
|
||||
shift 2
|
||||
fi
|
||||
name="${1:?}"
|
||||
[ -f "$state/container-$name" ] || exit 1
|
||||
if [[ "$format" == *Labels* ]]; then
|
||||
if [ "${FAKE_LABEL_INSPECT_HANG:-0}" = 1 ] && [[ "$name" == *ch-restore* ]]; then
|
||||
sleep 10
|
||||
fi
|
||||
cat "$state/container-label-$name"
|
||||
fi
|
||||
;;
|
||||
ls)
|
||||
for item in "$state"/container-*; do
|
||||
[ -e "$item" ] || continue
|
||||
case "$item" in
|
||||
*-label-*|*-network-*|*-image-*) continue ;;
|
||||
esac
|
||||
basename "$item" | sed 's/^container-//'
|
||||
done
|
||||
;;
|
||||
*) exit 90 ;;
|
||||
esac
|
||||
;;
|
||||
inspect)
|
||||
format=""
|
||||
if [ "${1:-}" = --format ]; then
|
||||
format="$2"
|
||||
shift 2
|
||||
fi
|
||||
name="${1:?}"
|
||||
[ -f "$state/container-$name" ] || exit 1
|
||||
case "$format" in
|
||||
*State.Running*) printf 'true\n' ;;
|
||||
*HostConfig.NetworkMode*) cat "$state/container-network-$name" ;;
|
||||
*.Image*) cat "$state/container-image-$name" ;;
|
||||
*) printf '%s\n' "fake-$name" ;;
|
||||
esac
|
||||
;;
|
||||
network)
|
||||
sub="${1:?}"
|
||||
shift
|
||||
case "$sub" in
|
||||
create)
|
||||
name="$(last_arg "$@")"
|
||||
label=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi
|
||||
done
|
||||
touch "$state/network-$name"
|
||||
printf '%s\n' "$label" > "$state/network-label-$name"
|
||||
printf '%s\n' "$name" > "$state/network-name"
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
inspect)
|
||||
format=""
|
||||
if [ "${1:-}" = --format ]; then
|
||||
format="$2"
|
||||
shift 2
|
||||
fi
|
||||
name="${1:?}"
|
||||
[ -f "$state/network-$name" ] || exit 1
|
||||
if [[ "$format" == *Internal* ]]; then printf 'true\n'; fi
|
||||
if [[ "$format" == *Labels* ]]; then cat "$state/network-label-$name"; fi
|
||||
;;
|
||||
ls)
|
||||
for item in "$state"/network-*; do
|
||||
[ -e "$item" ] || continue
|
||||
case "$item" in *-label-*|*/network-name) continue ;; esac
|
||||
basename "$item" | sed 's/^network-//'
|
||||
done
|
||||
;;
|
||||
rm)
|
||||
name="${1:?}"
|
||||
rm -f "$state/network-$name" "$state/network-label-$name" "$state/network-name"
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
*) exit 90 ;;
|
||||
esac
|
||||
;;
|
||||
volume)
|
||||
sub="${1:?}"
|
||||
shift
|
||||
case "$sub" in
|
||||
create)
|
||||
name="$(last_arg "$@")"
|
||||
label=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi
|
||||
done
|
||||
touch "$state/volume-$name"
|
||||
printf '%s\n' "$label" > "$state/volume-label-$name"
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
inspect)
|
||||
format=""
|
||||
if [ "${1:-}" = --format ]; then format="$2"; shift 2; fi
|
||||
name="${1:?}"
|
||||
[ -f "$state/volume-$name" ] || exit 1
|
||||
if [[ "$format" == *Labels* ]]; then cat "$state/volume-label-$name"; fi
|
||||
;;
|
||||
ls)
|
||||
for item in "$state"/volume-*; do
|
||||
[ -e "$item" ] || continue
|
||||
case "$item" in *-label-*) continue ;; esac
|
||||
basename "$item" | sed 's/^volume-//'
|
||||
done
|
||||
;;
|
||||
rm)
|
||||
name="${1:?}"
|
||||
rm -f "$state/volume-$name" "$state/volume-label-$name"
|
||||
if [[ "$name" == *-backup ]]; then
|
||||
rm -f "$state/staged-artifact-sha256"
|
||||
fi
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
*) exit 90 ;;
|
||||
esac
|
||||
;;
|
||||
run)
|
||||
name=""
|
||||
replica=""
|
||||
label=""
|
||||
network=""
|
||||
image=""
|
||||
expected_artifact_sha256=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--name)
|
||||
name="$2"
|
||||
shift 2
|
||||
;;
|
||||
--env)
|
||||
if [[ "$2" == CLICKHOUSE_RESTORE_REPLICA=* ]]; then
|
||||
replica="${2#*=}"
|
||||
fi
|
||||
if [[ "$2" == EXPECTED_ARTIFACT_SHA256=* ]]; then
|
||||
expected_artifact_sha256="${2#*=}"
|
||||
fi
|
||||
shift 2
|
||||
;;
|
||||
--label)
|
||||
label="${2#*=}"
|
||||
shift 2
|
||||
;;
|
||||
--network)
|
||||
network="$2"
|
||||
shift 2
|
||||
;;
|
||||
--network-alias|--restart|--mount|--user|--entrypoint)
|
||||
shift 2
|
||||
;;
|
||||
--detach|--pull=never)
|
||||
shift
|
||||
;;
|
||||
clickhouse/clickhouse-server:25.5.6|signoz/zookeeper:3.7.1)
|
||||
image="$1"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
[ -n "$name" ] || exit 91
|
||||
touch "$state/container-$name"
|
||||
printf '%s\n' "$label" > "$state/container-label-$name"
|
||||
printf '%s\n' "$network" > "$state/container-network-$name"
|
||||
case "$image" in
|
||||
clickhouse/clickhouse-server:25.5.6)
|
||||
printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" > "$state/container-image-$name"
|
||||
;;
|
||||
signoz/zookeeper:3.7.1)
|
||||
printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" > "$state/container-image-$name"
|
||||
;;
|
||||
*) exit 96 ;;
|
||||
esac
|
||||
if [ -n "$replica" ]; then printf '%s\n' "$replica" > "$state/replica"; fi
|
||||
if [ "$network" = none ]; then
|
||||
if [ "${FAKE_STAGING_HANG:-0}" = 1 ]; then sleep 10; fi
|
||||
[ "${FAKE_STAGING_FAIL:-0}" = 0 ] || exit 94
|
||||
staged_hash="$expected_artifact_sha256"
|
||||
if [ "${FAKE_STAGING_HASH_MISMATCH:-0}" = 1 ]; then
|
||||
staged_hash="$(printf '0%.0s' {1..64})"
|
||||
fi
|
||||
printf '%s\n' "$staged_hash" > "$state/staged-artifact-sha256"
|
||||
printf '%s\n' "$staged_hash"
|
||||
else
|
||||
printf 'fake-container-%s\n' "$name"
|
||||
fi
|
||||
;;
|
||||
exec)
|
||||
container="${1:?}"
|
||||
shift
|
||||
[ -f "$state/container-$container" ] || exit 1
|
||||
if [ "${1:-}" = sha256sum ]; then
|
||||
[ -f "$state/staged-artifact-sha256" ] || exit 97
|
||||
artifact_hash="$(cat "$state/staged-artifact-sha256")"
|
||||
if [ "${FAKE_CLICKHOUSE_HASH_MISMATCH:-0}" = 1 ]; then
|
||||
artifact_hash="$(printf 'f%.0s' {1..64})"
|
||||
fi
|
||||
printf '%s /backups/native-backup.zip\n' "$artifact_hash"
|
||||
exit 0
|
||||
fi
|
||||
query=""
|
||||
operation=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--query)
|
||||
query="$2"
|
||||
shift 2
|
||||
;;
|
||||
--param_operation_id)
|
||||
operation="$2"
|
||||
shift 2
|
||||
;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
case "$query" in
|
||||
CHECK\ TABLE*)
|
||||
if [ "${FAKE_CHECK_TABLE_FAIL:-0}" = 1 ]; then printf '0\n'; else printf '1\n'; fi
|
||||
;;
|
||||
RESTORE\ *)
|
||||
case "$query" in
|
||||
*" SETTINGS id='${EXPECTED_FAKE_RESTORE_OPERATION_ID:?}' ASYNC")
|
||||
operation="${EXPECTED_FAKE_RESTORE_OPERATION_ID}"
|
||||
;;
|
||||
*) exit 93 ;;
|
||||
esac
|
||||
touch "$state/restored"
|
||||
printf '%s\tRESTORING\n' "$operation"
|
||||
;;
|
||||
*FROM\ system.backups*)
|
||||
printf "Disk('backups', 'native-backup.zip')\tRESTORED\t100\t1000\t%s\n" "${EMPTY_SHA256:?}"
|
||||
;;
|
||||
*FROM\ system.disks*) printf '/backups/\n' ;;
|
||||
*FROM\ system.macros*)
|
||||
printf 'replica\t%s\nshard\t01\n' "$(cat "$state/replica")"
|
||||
;;
|
||||
*FROM\ system.clusters*) printf 'cluster\trestore-clickhouse\t9000\n' ;;
|
||||
*FROM\ system.zookeeper*) printf '1\n' ;;
|
||||
*"SELECT count() FROM system.databases"*) printf '0\n' ;;
|
||||
*"SELECT count() FROM system.tables"*) printf '0\n' ;;
|
||||
*"SELECT database,name,engine"*) cat "${FAKE_RESTORED_INVENTORY:?}" ;;
|
||||
"SELECT 1") printf '1\n' ;;
|
||||
*) printf 'UNHANDLED QUERY: %s\n' "$query" >&2; exit 92 ;;
|
||||
esac
|
||||
;;
|
||||
logs)
|
||||
printf 'bounded fake container log\n'
|
||||
;;
|
||||
rm)
|
||||
if [ "${1:-}" = -f ]; then shift; fi
|
||||
name="${1:?}"
|
||||
rm -f "$state/container-$name" "$state/container-label-$name" \
|
||||
"$state/container-network-$name" "$state/container-image-$name"
|
||||
printf '%s\n' "$name"
|
||||
;;
|
||||
*) exit 99 ;;
|
||||
esac
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
docker.chmod(0o755)
|
||||
return bin_dir, state_dir, log_path
|
||||
|
||||
|
||||
def _environment(
|
||||
tmp_path: Path,
|
||||
artifact: Path,
|
||||
manifest: Path,
|
||||
inventory: Path,
|
||||
restored_inventory: Path,
|
||||
) -> dict[str, str]:
|
||||
bin_dir, state_dir, log_path = _write_fake_docker(tmp_path)
|
||||
return os.environ | {
|
||||
"PATH": f"{bin_dir}:{os.environ['PATH']}",
|
||||
"TRACE_ID": "trace-restore-test",
|
||||
"RUN_ID": "run-restore-test",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"CLICKHOUSE_NATIVE_ARTIFACT_PATH": str(artifact),
|
||||
"CLICKHOUSE_NATIVE_MANIFEST_PATH": str(manifest),
|
||||
"CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH": str(inventory),
|
||||
"CLICKHOUSE_NATIVE_ARTIFACT_SHA256": hashlib.sha256(
|
||||
artifact.read_bytes()
|
||||
).hexdigest(),
|
||||
"CLICKHOUSE_NATIVE_OPERATION_ID": "awoooi-backup-operation",
|
||||
"CLICKHOUSE_NATIVE_EXPECTED_DATABASES": ",".join(EXPECTED_DATABASES),
|
||||
"CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(tmp_path / "receipts"),
|
||||
"CLICKHOUSE_RESTORE_POLL_INTERVAL_SECONDS": "1",
|
||||
"CLICKHOUSE_RESTORE_STARTUP_TIMEOUT_SECONDS": "5",
|
||||
"CLICKHOUSE_RESTORE_TIMEOUT_SECONDS": "5",
|
||||
"CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS": "5",
|
||||
"FAKE_DOCKER_LOG": str(log_path),
|
||||
"FAKE_DOCKER_STATE": str(state_dir),
|
||||
"FAKE_CLICKHOUSE_IMAGE_ID": CLICKHOUSE_IMAGE_ID,
|
||||
"FAKE_ZOOKEEPER_IMAGE_ID": ZOOKEEPER_IMAGE_ID,
|
||||
"FAKE_RESTORED_INVENTORY": str(restored_inventory),
|
||||
"EXPECTED_FAKE_RESTORE_OPERATION_ID": "restore-"
|
||||
+ hashlib.sha256(
|
||||
b"trace-restore-test\0run-restore-test\0P0-OBS-002"
|
||||
).hexdigest(),
|
||||
"EMPTY_SHA256": EMPTY_SHA256,
|
||||
}
|
||||
|
||||
|
||||
def _run(mode: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
["bash", str(SCRIPT), mode],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
|
||||
def _status(tmp_path: Path) -> dict[str, object]:
|
||||
return json.loads(
|
||||
(tmp_path / "receipts" / "run-restore-test" / "status.json").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _assert_ephemeral_resources_absent(state_dir: Path) -> None:
|
||||
assert not list(state_dir.glob("container-*"))
|
||||
assert not list(state_dir.glob("volume-*"))
|
||||
assert not list(state_dir.glob("network-*"))
|
||||
assert not (state_dir / "staged-artifact-sha256").exists()
|
||||
|
||||
|
||||
def test_hook_check_uses_env_contract_without_runtime_creation(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(restored_delta=3), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
|
||||
result = _run("--check", env)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "terminal=check_pass_no_runtime_write" in result.stdout
|
||||
docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8")
|
||||
assert "image inspect" in docker_log
|
||||
assert "network create" not in docker_log
|
||||
assert "volume create" not in docker_log
|
||||
assert not any(line.startswith("run ") for line in docker_log.splitlines())
|
||||
status = _status(tmp_path)
|
||||
assert status["check_pass"] is True
|
||||
assert status["ephemeral_runtime_created"] is False
|
||||
assert status["production_network_attached"] is False
|
||||
assert status["production_restore_performed"] is False
|
||||
|
||||
|
||||
def test_apply_restores_on_internal_pair_and_cleans_every_resource(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(restored_delta=3), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
check = _run("--check", env)
|
||||
assert check.returncode == 0, check.stderr
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "terminal=verified" in result.stdout
|
||||
docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines()
|
||||
network_create = next(
|
||||
line for line in docker_lines if line.startswith("network create")
|
||||
)
|
||||
assert "--internal" in network_create
|
||||
run_lines = [line for line in docker_lines if line.startswith("run ")]
|
||||
assert len(run_lines) == 3
|
||||
assert all("--pull=never" in line for line in run_lines)
|
||||
assert all("--publish" not in line and " -p " not in line for line in run_lines)
|
||||
|
||||
staging_run = next(line for line in run_lines if "--network none" in line)
|
||||
zookeeper_run = next(line for line in run_lines if "signoz/zookeeper:3.7.1" in line)
|
||||
clickhouse_run = next(
|
||||
line for line in run_lines if "--network-alias restore-clickhouse" in line
|
||||
)
|
||||
assert "--user 0:0" in staging_run
|
||||
assert "--entrypoint /bin/sh" in staging_run
|
||||
assert (
|
||||
f"type=bind,src={artifact},dst=/tmp/awoooi-source-native-backup.zip,readonly"
|
||||
in staging_run
|
||||
)
|
||||
assert sum(f"src={artifact}" in line for line in run_lines) == 1
|
||||
assert "type=volume,src=awoooi-ch-restore-" in staging_run
|
||||
assert "-backup,dst=/backups" in staging_run
|
||||
assert "ALLOW_ANONYMOUS_LOGIN=yes" in zookeeper_run
|
||||
assert "--network awoooi-ch-restore-" in zookeeper_run
|
||||
assert "--network awoooi-ch-restore-" in clickhouse_run
|
||||
assert str(artifact) not in clickhouse_run
|
||||
assert "type=volume,src=awoooi-ch-restore-" in clickhouse_run
|
||||
assert "-backup,dst=/backups" in clickhouse_run
|
||||
assert "dst=/backups,readonly" not in clickhouse_run
|
||||
assert (
|
||||
"dst=/etc/clickhouse-server/config.d/awoooi-backup-disk.xml,readonly"
|
||||
in clickhouse_run
|
||||
)
|
||||
assert (
|
||||
"dst=/etc/clickhouse-server/config.d/awoooi-restore-isolation.xml,readonly"
|
||||
in clickhouse_run
|
||||
)
|
||||
hash_readback = next(
|
||||
line
|
||||
for line in docker_lines
|
||||
if line.startswith("exec ") and "sha256sum /backups/native-backup.zip" in line
|
||||
)
|
||||
assert "exec awoooi-ch-restore-" in hash_readback
|
||||
restore_line = next(line for line in docker_lines if "RESTORE DATABASE" in line)
|
||||
assert "allow_non_empty_tables" not in restore_line
|
||||
assert " SETTINGS id='restore-" in restore_line
|
||||
assert "id={operation_id:String}" not in restore_line
|
||||
macro_line = next(line for line in docker_lines if "FROM system.macros" in line)
|
||||
assert "SELECT macro,substitution" in macro_line
|
||||
assert "SELECT name,value" not in macro_line
|
||||
state_dir = Path(env["FAKE_DOCKER_STATE"])
|
||||
_assert_ephemeral_resources_absent(state_dir)
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "verified"
|
||||
assert status["check_pass"] is True
|
||||
assert status["restore_terminal"] == "RESTORED"
|
||||
assert status["manifest_parity"] == 1
|
||||
assert status["check_table_count"] == status["check_table_pass_count"] == 7
|
||||
assert status["critical_nonzero_count"] == 4
|
||||
assert status["row_delta"] == 3
|
||||
expected_hash = hashlib.sha256(artifact.read_bytes()).hexdigest()
|
||||
assert status["artifact_sha256"] == expected_hash
|
||||
assert status["staged_artifact_sha256"] == expected_hash
|
||||
assert status["clickhouse_artifact_sha256"] == expected_hash
|
||||
assert status["artifact_staging_verified"] is True
|
||||
assert status["clickhouse_artifact_readback_verified"] is True
|
||||
assert status["staging_network_mode"] == "none"
|
||||
assert status["artifact_source_mount"] == "read_only_staging_only"
|
||||
assert status["backup_volume_mount"] == "writable_ephemeral"
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_apply_failure_still_cleans_pair_network_and_volumes(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_CHECK_TABLE_FAIL"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "terminal=failed" in result.stdout
|
||||
state_dir = Path(env["FAKE_DOCKER_STATE"])
|
||||
_assert_ephemeral_resources_absent(state_dir)
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["cleanup_verified"] is True
|
||||
assert status["apply_pass"] is False
|
||||
|
||||
|
||||
def test_artifact_staging_failure_cleans_exact_resources(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_STAGING_FAIL"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
_assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["reason"] == "isolated_artifact_staging_failed"
|
||||
assert status["artifact_staging_verified"] is False
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_artifact_staging_hang_is_bounded_and_cleans_exact_resources(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1"
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_STAGING_HANG"] = "1"
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(SCRIPT), "--apply"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
timeout=12,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
_assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["reason"] == "isolated_artifact_staging_failed"
|
||||
assert status["artifact_staging_verified"] is False
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_staged_artifact_hash_mismatch_fails_before_isolated_services(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_STAGING_HASH_MISMATCH"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines()
|
||||
assert not any(
|
||||
"signoz/zookeeper:3.7.1" in line
|
||||
for line in docker_lines
|
||||
if line.startswith("run ")
|
||||
)
|
||||
_assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
|
||||
status = _status(tmp_path)
|
||||
assert status["reason"] == "staged_artifact_hash_mismatch"
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_clickhouse_artifact_hash_readback_mismatch_fails_and_cleans(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_CLICKHOUSE_HASH_MISMATCH"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
_assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"]))
|
||||
status = _status(tmp_path)
|
||||
assert status["reason"] == "isolated_clickhouse_artifact_hash_mismatch"
|
||||
assert status["artifact_staging_verified"] is True
|
||||
assert status["clickhouse_artifact_readback_verified"] is False
|
||||
assert status["clickhouse_artifact_sha256"] == "f" * 64
|
||||
assert status["cleanup_verified"] is True
|
||||
|
||||
|
||||
def test_docker_daemon_failure_cannot_claim_cleanup_verified(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_DOCKER_INFO_FAIL"] = "1"
|
||||
|
||||
result = _run("--apply", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["cleanup_verified"] is False
|
||||
assert status["reason"] == "ephemeral_resource_cleanup_failed"
|
||||
|
||||
|
||||
def test_cleanup_label_inspect_hang_is_bounded_and_fails_closed(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1"
|
||||
assert _run("--check", env).returncode == 0
|
||||
env["FAKE_CHECK_TABLE_FAIL"] = "1"
|
||||
env["FAKE_LABEL_INSPECT_HANG"] = "1"
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(SCRIPT), "--apply"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
timeout=12,
|
||||
)
|
||||
|
||||
assert result.returncode == 1
|
||||
status = _status(tmp_path)
|
||||
assert status["terminal"] == "failed"
|
||||
assert status["cleanup_verified"] is False
|
||||
assert status["reason"] == "ephemeral_resource_cleanup_failed"
|
||||
|
||||
|
||||
def test_check_fails_closed_on_local_image_id_drift(tmp_path: Path) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
env["FAKE_CLICKHOUSE_IMAGE_ID"] = "sha256:" + "0" * 64
|
||||
|
||||
result = _run("--check", env)
|
||||
|
||||
assert result.returncode == 1
|
||||
assert "clickhouse_local_image_id_mismatch" in result.stderr
|
||||
docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8")
|
||||
assert "network create" not in docker_log
|
||||
assert "volume create" not in docker_log
|
||||
|
||||
|
||||
def test_flat_host_deploy_accepts_reviewed_absolute_helper_and_config_paths(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
artifact, manifest, inventory = _write_backup_fixture(tmp_path)
|
||||
restored = tmp_path / "restored.tsv"
|
||||
restored.write_text(_inventory(), encoding="utf-8")
|
||||
env = _environment(tmp_path, artifact, manifest, inventory, restored)
|
||||
host_root = tmp_path / "host-backup"
|
||||
host_root.mkdir()
|
||||
helper = host_root / "clickhouse-restore-inventory.py"
|
||||
backup_config = host_root / "backup_disk.xml"
|
||||
cluster_config = host_root / "isolated-cluster.xml"
|
||||
shutil.copyfile(INVENTORY_HELPER, helper)
|
||||
shutil.copyfile(
|
||||
REPO_ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml", backup_config
|
||||
)
|
||||
shutil.copyfile(
|
||||
REPO_ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
cluster_config,
|
||||
)
|
||||
env.update(
|
||||
{
|
||||
"CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(helper),
|
||||
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_config),
|
||||
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config),
|
||||
}
|
||||
)
|
||||
|
||||
result = _run("--check", env)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "terminal=check_pass_no_runtime_write" in result.stdout
|
||||
|
||||
|
||||
def test_inventory_compare_allows_online_row_delta_but_not_engine_drift(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
source = tmp_path / "source.tsv"
|
||||
restored = tmp_path / "restored.tsv"
|
||||
source.write_text(_inventory(), encoding="utf-8")
|
||||
restored.write_text(_inventory(restored_delta=9), encoding="utf-8")
|
||||
|
||||
accepted = subprocess.run(
|
||||
[
|
||||
"python3",
|
||||
str(INVENTORY_HELPER),
|
||||
"compare",
|
||||
"--source-inventory",
|
||||
str(source),
|
||||
"--restored-inventory",
|
||||
str(restored),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
assert accepted.returncode == 0, accepted.stderr
|
||||
assert "manifest_parity=1" in accepted.stdout
|
||||
assert "row_delta=9" in accepted.stdout
|
||||
|
||||
restored.write_text(_inventory(engine_mismatch=True), encoding="utf-8")
|
||||
rejected = subprocess.run(
|
||||
[
|
||||
"python3",
|
||||
str(INVENTORY_HELPER),
|
||||
"compare",
|
||||
"--source-inventory",
|
||||
str(source),
|
||||
"--restored-inventory",
|
||||
str(restored),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
assert rejected.returncode == 1
|
||||
assert "table_engine_mismatch_signoz_meter_meter" in rejected.stderr
|
||||
|
||||
restored.write_text(_inventory(schema_mismatch=True), encoding="utf-8")
|
||||
schema_rejected = subprocess.run(
|
||||
[
|
||||
"python3",
|
||||
str(INVENTORY_HELPER),
|
||||
"compare",
|
||||
"--source-inventory",
|
||||
str(source),
|
||||
"--restored-inventory",
|
||||
str(restored),
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
assert schema_rejected.returncode == 1
|
||||
assert "table_schema_mismatch_signoz_meter_meter" in schema_rejected.stderr
|
||||
823
scripts/backup/tests/test_signoz_backup_canary_wrapper.py
Normal file
823
scripts/backup/tests/test_signoz_backup_canary_wrapper.py
Normal file
@@ -0,0 +1,823 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh"
|
||||
PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml"
|
||||
ANSIBLE_VALIDATE = ROOT / "scripts" / "ops" / "ansible-validate.sh"
|
||||
READINESS_AUDIT = (
|
||||
ROOT / "scripts" / "reboot-recovery" / "reboot-recovery-readiness-audit.sh"
|
||||
)
|
||||
|
||||
|
||||
def _write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _monitor_source(cooldown_dir: Path, *, omit_contract: bool = False) -> str:
|
||||
cooldown_log = (
|
||||
'log "COOLDOWN: ${container} skipped"'
|
||||
if not omit_contract
|
||||
else 'log "maintenance skip for ${container}"'
|
||||
)
|
||||
return f"""\
|
||||
#!/bin/bash
|
||||
: "${{ACTION_COOLDOWN_SECONDS:=${{SEND_COOLDOWN_SECONDS}}}}"
|
||||
: "${{COOLDOWN_DIR:={cooldown_dir}}}"
|
||||
: "${{EXCLUDE_CONTAINERS:=signoz-clickhouse}}"
|
||||
is_in_cooldown() {{
|
||||
local container="$1"
|
||||
local cooldown_file="${{COOLDOWN_DIR}}/${{container}}.cooldown"
|
||||
local last_sent now elapsed
|
||||
last_sent=$(cat "$cooldown_file")
|
||||
now=$(date +%s)
|
||||
elapsed=$(( now - last_sent ))
|
||||
if (( elapsed < ACTION_COOLDOWN_SECONDS )); then
|
||||
{cooldown_log}
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}}
|
||||
set_cooldown() {{ :; }}
|
||||
# A second monitor path may use the same elapsed expression. The wrapper must
|
||||
# validate the expression inside is_in_cooldown, not require global uniqueness.
|
||||
# elapsed=$(( now - last_sent ))
|
||||
while read -r container_name; do
|
||||
is_in_cooldown "$container_name" && continue
|
||||
set_cooldown "$container_name"
|
||||
container="$container_name"
|
||||
log "AUTO_REPAIR: docker restart ${{container}}"
|
||||
if docker restart "$container"; then
|
||||
:
|
||||
fi
|
||||
done
|
||||
"""
|
||||
|
||||
|
||||
def _install_fake_commands(fake_bin: Path) -> None:
|
||||
_write_executable(
|
||||
fake_bin / "crontab",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
[ "${1:-}" = "-l" ]
|
||||
printf '*/5 * * * * %s >> %s 2>&1\n' \
|
||||
"${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
|
||||
if [ "${FAKE_DUPLICATE_CRON:-0}" = "1" ]; then
|
||||
printf '*/5 * * * * %s >> %s 2>&1\n' \
|
||||
"${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}"
|
||||
fi
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "docker",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
[ "${1:-}" = "inspect" ] || exit 70
|
||||
case " $* " in
|
||||
*"{{.State.StartedAt}}"*)
|
||||
printf 'sha256:collector-test-id\\t%s\\t2026-07-15T00:00:00Z\\t0\\thealthy\\n' \
|
||||
"$(cat "${TEST_COLLECTOR_STATE:?}")"
|
||||
;;
|
||||
*"{{.Id}}"*) printf 'sha256:collector-test-id\n' ;;
|
||||
*"{{.State.Running}}"*) cat "${TEST_COLLECTOR_STATE:?}" ;;
|
||||
*) exit 71 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "ss",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
while IFS= read -r port; do
|
||||
[ -n "$port" ] || continue
|
||||
printf 'LISTEN 0 4096 *:%s *:*\n' "$port"
|
||||
done < "${TEST_LISTENERS:?}"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
while [[ "${1:-}" == --kill-after=* ]]; do
|
||||
shift
|
||||
done
|
||||
[ "$#" -gt 1 ]
|
||||
shift
|
||||
if [ "${FAKE_DOCKER_TIMEOUT:-0}" = "1" ] && [ "${1:-}" = "docker" ]; then
|
||||
exit 124
|
||||
fi
|
||||
exec "$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "flock",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
if [ -n "${TEST_FLOCK_MARKER:-}" ]; then
|
||||
printf 'held\n' > "${TEST_FLOCK_MARKER}"
|
||||
fi
|
||||
exit 0
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "pgrep",
|
||||
"""\
|
||||
#!/bin/bash
|
||||
exit 1
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "stat",
|
||||
"""\
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 4 or sys.argv[1] != "-c":
|
||||
raise SystemExit(64)
|
||||
value = os.stat(sys.argv[3])
|
||||
formats = {
|
||||
"%a": format(stat.S_IMODE(value.st_mode), "o"),
|
||||
"%u": str(value.st_uid),
|
||||
"%g": str(value.st_gid),
|
||||
"%s": str(value.st_size),
|
||||
"%d:%i": f"{value.st_dev}:{value.st_ino}",
|
||||
}
|
||||
if sys.argv[2] not in formats:
|
||||
raise SystemExit(65)
|
||||
print(formats[sys.argv[2]])
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "sha256sum",
|
||||
"""\
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import sys
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
raise SystemExit(64)
|
||||
with open(sys.argv[1], "rb") as source:
|
||||
digest = hashlib.sha256(source.read()).hexdigest()
|
||||
print(f"{digest} {sys.argv[1]}")
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
def _restore_hook_source() -> str:
|
||||
return """\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
[ "$#" -eq 1 ]
|
||||
case "$1" in
|
||||
--check) mode="check" ;;
|
||||
--apply) mode="apply" ;;
|
||||
*) exit 64 ;;
|
||||
esac
|
||||
receipt_dir="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:?}/${RUN_ID:?}"
|
||||
mkdir -p "$receipt_dir"
|
||||
receipt_log="$receipt_dir/receipts.jsonl"
|
||||
identity_run="$RUN_ID"
|
||||
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "forged_identity" ]; then
|
||||
identity_run="stale-$RUN_ID"
|
||||
fi
|
||||
invocation_id="${mode}-fake-restore-$$"
|
||||
emit() {
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","stage":"%s","terminal":"%s","detail":"fake"}\n' \
|
||||
"$TRACE_ID" "$identity_run" "$WORK_ITEM_ID" "$invocation_id" "$1" "$2" \
|
||||
>> "$receipt_log"
|
||||
}
|
||||
if [ "$mode" = "check" ]; then
|
||||
emit check check_pass_no_runtime_write
|
||||
emit cleanup pass
|
||||
emit terminal check_pass_no_runtime_write
|
||||
exit 0
|
||||
fi
|
||||
emit check pass
|
||||
emit independent_post_verifier pass
|
||||
emit cleanup pass
|
||||
emit terminal verified
|
||||
if [ "${FAKE_RECEIPT_FAULT:-valid}" != "missing_status" ]; then
|
||||
cat > "$receipt_dir/status.json" <<EOF
|
||||
{
|
||||
"schema_version":"clickhouse_native_restore_drill_status_v1",
|
||||
"trace_id":"$TRACE_ID",
|
||||
"run_id":"$identity_run",
|
||||
"work_item_id":"$WORK_ITEM_ID",
|
||||
"mode":"apply",
|
||||
"terminal":"verified",
|
||||
"exit_code":0,
|
||||
"network_mode":"docker_internal_only",
|
||||
"staging_network_mode":"none",
|
||||
"production_network_attached":false,
|
||||
"production_restore_performed":false,
|
||||
"allow_non_empty_tables":false,
|
||||
"isolated_keeper":true,
|
||||
"artifact_staging_verified":true,
|
||||
"clickhouse_artifact_readback_verified":true,
|
||||
"ephemeral_runtime_created":true,
|
||||
"check_pass":true,
|
||||
"apply_pass":true,
|
||||
"restore_terminal":"RESTORED",
|
||||
"manifest_parity":1,
|
||||
"source_database_count":6,
|
||||
"restored_database_count":6,
|
||||
"source_table_count":100,
|
||||
"restored_table_count":100,
|
||||
"check_table_count":44,
|
||||
"check_table_pass_count":44,
|
||||
"critical_nonzero_count":4,
|
||||
"cleanup_verified":true
|
||||
}
|
||||
EOF
|
||||
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "symlink_status" ]; then
|
||||
mv "$receipt_dir/status.json" "$receipt_dir/status-target.json"
|
||||
ln -s "status-target.json" "$receipt_dir/status.json"
|
||||
fi
|
||||
fi
|
||||
if [ "${FAKE_RECEIPT_FAULT:-valid}" = "malformed_restore_receipt" ]; then
|
||||
printf '{malformed-json\n' >> "$receipt_log"
|
||||
fi
|
||||
"""
|
||||
|
||||
|
||||
def _backup_source() -> str:
|
||||
return """\
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80
|
||||
[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ]
|
||||
[ "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}" = "${TEST_RESTORE_HOOK:?}" ] || exit 82
|
||||
[ "${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-}" = "${TEST_INVENTORY_HELPER:?}" ] || exit 83
|
||||
[ "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-}" = "${TEST_BACKUP_DISK_CONFIG:?}" ] || exit 84
|
||||
[ "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-}" = "${TEST_CLUSTER_CONFIG:?}" ] || exit 85
|
||||
[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-}" = "${TEST_NATIVE_RECEIPT_ROOT:?}" ] || exit 86
|
||||
[ "${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-}" = "${TEST_RESTORE_RECEIPT_ROOT:?}" ] || exit 87
|
||||
printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \
|
||||
> "${TEST_BACKUP_IDENTITIES:?}"
|
||||
printf '%s\t%s\t%s\t%s\n' \
|
||||
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" \
|
||||
"$CLICKHOUSE_RESTORE_INVENTORY_HELPER" \
|
||||
"$CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" \
|
||||
"$CLICKHOUSE_RESTORE_CLUSTER_CONFIG" \
|
||||
> "${TEST_RESTORE_CONTRACT:?}"
|
||||
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
|
||||
cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}"
|
||||
printf 'false\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
case "${FAKE_MONITOR_MODE:-cooldown}" in
|
||||
cooldown)
|
||||
printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
;;
|
||||
auto_repair)
|
||||
printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
printf '[test] AUTO_REPAIR: docker restart signoz-otel-collector\n' \
|
||||
>> "${TEST_MONITOR_LOG:?}"
|
||||
;;
|
||||
none) : ;;
|
||||
*) exit 81 ;;
|
||||
esac
|
||||
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
|
||||
printf 'true\n' > "${TEST_COLLECTOR_STATE:?}"
|
||||
else
|
||||
sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}"
|
||||
native_dir="$CLICKHOUSE_NATIVE_RECEIPT_ROOT/$RUN_ID"
|
||||
mkdir -p "$native_dir"
|
||||
native_receipts="$native_dir/receipts.jsonl"
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"check-fake-native","stage":"terminal","terminal":"check_pass_no_write","detail":"fake"}\n' \
|
||||
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
|
||||
if [ "${FAKE_RECEIPT_FAULT:-valid}" != "no_hook" ]; then
|
||||
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --check
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_check_exit_0_stdout_fake_stderr_fake"}\n' \
|
||||
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
|
||||
"$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --apply
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_exit_0_stdout_fake_stderr_fake"}\n' \
|
||||
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
|
||||
fi
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"independent_verifier","terminal":"pass","detail":"fake"}\n' \
|
||||
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
|
||||
printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"terminal","terminal":"pass","detail":"fake"}\n' \
|
||||
"$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts"
|
||||
fi
|
||||
if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then
|
||||
printf '4317\n' > "${TEST_LISTENERS:?}"
|
||||
fi
|
||||
printf '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)\n'
|
||||
if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then
|
||||
printf '[SUCCESS] ========== SignOz 備份完成 (1s) ==========\n'
|
||||
else
|
||||
printf '[SUCCESS] ========== SigNoz ClickHouse native 備份完成 (1s) ==========\n'
|
||||
fi
|
||||
"""
|
||||
|
||||
|
||||
def _run_canary(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
mode: str = "apply",
|
||||
prior_lease: bool = True,
|
||||
monitor_mode: str = "cooldown",
|
||||
backup_sleep_seconds: float = 0.2,
|
||||
monitor_interval_seconds: int = 300,
|
||||
omit_monitor_contract: bool = False,
|
||||
duplicate_cron: bool = False,
|
||||
drop_listener: bool = False,
|
||||
expect_collector_stop: bool = True,
|
||||
native_restore_contract: str = "valid",
|
||||
native_receipt_state: str = "valid",
|
||||
docker_timeout: bool = False,
|
||||
) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]:
|
||||
if native_receipt_state not in {
|
||||
"valid",
|
||||
"no_hook",
|
||||
"forged_identity",
|
||||
"missing_status",
|
||||
"symlink_status",
|
||||
"malformed_restore_receipt",
|
||||
}:
|
||||
raise ValueError(f"unsupported native receipt state: {native_receipt_state}")
|
||||
|
||||
fake_bin = tmp_path / "bin"
|
||||
cooldown_dir = tmp_path / "cooldown"
|
||||
receipt_root = tmp_path / "receipts"
|
||||
native_receipt_root = tmp_path / "native-receipts"
|
||||
restore_receipt_root = tmp_path / "restore-receipts"
|
||||
fake_bin.mkdir()
|
||||
cooldown_dir.mkdir()
|
||||
receipt_root.mkdir()
|
||||
native_receipt_root.mkdir()
|
||||
restore_receipt_root.mkdir()
|
||||
|
||||
monitor_script = tmp_path / "docker-health-monitor.sh"
|
||||
monitor_log = tmp_path / "monitor.log"
|
||||
backup_script = tmp_path / "backup-signoz.sh"
|
||||
collector_state = tmp_path / "collector.state"
|
||||
listeners = tmp_path / "listeners"
|
||||
identities = tmp_path / "backup-identities.tsv"
|
||||
lease_observed = tmp_path / "lease-observed.txt"
|
||||
restore_contract_observed = tmp_path / "restore-contract.tsv"
|
||||
flock_marker = tmp_path / "canary-lock-held.txt"
|
||||
restore_hook = tmp_path / "clickhouse-native-restore-drill.sh"
|
||||
inventory_helper = tmp_path / "clickhouse-restore-inventory.py"
|
||||
backup_disk_config = tmp_path / "backup_disk.xml"
|
||||
cluster_config = tmp_path / "isolated-cluster.xml"
|
||||
cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown"
|
||||
run_id = f"canary-{tmp_path.name}"
|
||||
|
||||
_write_executable(
|
||||
monitor_script,
|
||||
_monitor_source(cooldown_dir, omit_contract=omit_monitor_contract),
|
||||
)
|
||||
_write_executable(backup_script, _backup_source())
|
||||
if native_restore_contract == "symlink":
|
||||
restore_hook_target = tmp_path / "restore-hook-target.sh"
|
||||
_write_executable(restore_hook_target, _restore_hook_source())
|
||||
restore_hook.symlink_to(restore_hook_target)
|
||||
elif native_restore_contract == "valid":
|
||||
_write_executable(restore_hook, _restore_hook_source())
|
||||
elif native_restore_contract != "missing":
|
||||
raise ValueError(
|
||||
f"unsupported native restore contract: {native_restore_contract}"
|
||||
)
|
||||
_write_executable(inventory_helper, "#!/bin/sh\nexit 0\n")
|
||||
backup_disk_config.write_text("<clickhouse/>\n", encoding="utf-8")
|
||||
cluster_config.write_text("<clickhouse/>\n", encoding="utf-8")
|
||||
_install_fake_commands(fake_bin)
|
||||
monitor_log.write_text("[test] monitor seed\n", encoding="utf-8")
|
||||
collector_state.write_text("true\n", encoding="utf-8")
|
||||
listeners.write_text("4317\n4318\n", encoding="utf-8")
|
||||
if prior_lease:
|
||||
cooldown_file.write_text("123\n", encoding="utf-8")
|
||||
cooldown_file.chmod(0o640)
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"TRACE_ID": "trace-P0-OBS-002",
|
||||
"RUN_ID": run_id,
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"SIGNOZ_CANARY_BACKUP_SCRIPT": str(backup_script),
|
||||
"SIGNOZ_CANARY_MONITOR_SCRIPT": str(monitor_script),
|
||||
"SIGNOZ_CANARY_MONITOR_LOG": str(monitor_log),
|
||||
"SIGNOZ_CANARY_COOLDOWN_DIR": str(cooldown_dir),
|
||||
"SIGNOZ_CANARY_RECEIPT_ROOT": str(receipt_root),
|
||||
"SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"),
|
||||
"SIGNOZ_CANARY_TIMEOUT_SECONDS": "5",
|
||||
"SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2",
|
||||
"SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS": "2",
|
||||
"SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2",
|
||||
"SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds),
|
||||
"SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05",
|
||||
"SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"),
|
||||
"CLICKHOUSE_NATIVE_POST_VERIFY_HOOK": str(restore_hook),
|
||||
"CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(inventory_helper),
|
||||
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_disk_config),
|
||||
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config),
|
||||
"CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(native_receipt_root),
|
||||
"CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(restore_receipt_root),
|
||||
"TEST_MONITOR_SCRIPT": str(monitor_script),
|
||||
"TEST_MONITOR_LOG": str(monitor_log),
|
||||
"TEST_COLLECTOR_STATE": str(collector_state),
|
||||
"TEST_LISTENERS": str(listeners),
|
||||
"TEST_BACKUP_IDENTITIES": str(identities),
|
||||
"TEST_COOLDOWN_FILE": str(cooldown_file),
|
||||
"TEST_LEASE_OBSERVED": str(lease_observed),
|
||||
"TEST_RESTORE_CONTRACT": str(restore_contract_observed),
|
||||
"TEST_RESTORE_HOOK": str(restore_hook),
|
||||
"TEST_INVENTORY_HELPER": str(inventory_helper),
|
||||
"TEST_BACKUP_DISK_CONFIG": str(backup_disk_config),
|
||||
"TEST_CLUSTER_CONFIG": str(cluster_config),
|
||||
"TEST_NATIVE_RECEIPT_ROOT": str(native_receipt_root),
|
||||
"TEST_RESTORE_RECEIPT_ROOT": str(restore_receipt_root),
|
||||
"TEST_FLOCK_MARKER": str(flock_marker),
|
||||
"FAKE_MONITOR_MODE": monitor_mode,
|
||||
"FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds),
|
||||
"FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0",
|
||||
"FAKE_DROP_LISTENER": "1" if drop_listener else "0",
|
||||
"FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0",
|
||||
"FAKE_RECEIPT_FAULT": native_receipt_state,
|
||||
"FAKE_DOCKER_TIMEOUT": "1" if docker_timeout else "0",
|
||||
}
|
||||
started_at = int(time.time())
|
||||
result = subprocess.run(
|
||||
["bash", str(WRAPPER), f"--{mode}"],
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
paths = {
|
||||
"cooldown": cooldown_file,
|
||||
"receipts": receipt_root / run_id / "receipts.jsonl",
|
||||
"rollback_metadata": receipt_root / run_id / "cooldown.rollback.json",
|
||||
"identities": identities,
|
||||
"lease_observed": lease_observed,
|
||||
"collector_state": collector_state,
|
||||
"listeners": listeners,
|
||||
"restore_contract": restore_contract_observed,
|
||||
"restore_hook": restore_hook,
|
||||
"flock_marker": flock_marker,
|
||||
"native_receipts": native_receipt_root / run_id / "receipts.jsonl",
|
||||
"restore_receipts": restore_receipt_root / run_id / "receipts.jsonl",
|
||||
"restore_status": restore_receipt_root / run_id / "status.json",
|
||||
"backup_output": receipt_root / run_id / "backup-output.log",
|
||||
}
|
||||
return result, paths, str(started_at)
|
||||
|
||||
|
||||
def _receipts(path: Path) -> list[dict[str, object]]:
|
||||
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
|
||||
def _terminal(receipts: list[dict[str, object]], stage: str) -> str:
|
||||
return str([item for item in receipts if item["stage"] == stage][-1]["terminal"])
|
||||
|
||||
|
||||
def test_existing_lease_is_atomically_replaced_and_exactly_restored(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, started_at = _run_canary(tmp_path)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
|
||||
assert int(paths["lease_observed"].read_text(encoding="utf-8")) >= (
|
||||
int(started_at) + 7
|
||||
)
|
||||
identities = paths["identities"].read_text(encoding="utf-8").split("\t")
|
||||
assert identities[0] == "trace-P0-OBS-002"
|
||||
assert identities[1].startswith("canary-")
|
||||
assert identities[2] == "P0-OBS-002\n"
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "restored"
|
||||
assert _terminal(receipts, "runtime_operation") == "pass"
|
||||
assert _terminal(receipts, "closure_writeback") == "partial_degraded"
|
||||
assert _terminal(receipts, "terminal") == "partial_degraded"
|
||||
metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8"))
|
||||
assert metadata["trace_id"] == "trace-P0-OBS-002"
|
||||
assert metadata["prior_present"] == 1
|
||||
|
||||
stages = [str(item["stage"]) for item in receipts]
|
||||
ordered = [
|
||||
"sensor_source",
|
||||
"normalized_asset_identity",
|
||||
"source_of_truth_diff",
|
||||
"ai_decision",
|
||||
"risk_policy",
|
||||
"check",
|
||||
"execution",
|
||||
"post_verifier",
|
||||
"rollback",
|
||||
"closure_writeback",
|
||||
"terminal",
|
||||
]
|
||||
assert [stages.index(stage) for stage in ordered] == sorted(
|
||||
stages.index(stage) for stage in ordered
|
||||
)
|
||||
|
||||
|
||||
def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, prior_lease=False)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert not paths["cooldown"].exists()
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "restored"
|
||||
assert _terminal(receipts, "terminal") == "partial_degraded"
|
||||
|
||||
|
||||
def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, expect_collector_stop=False)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert not paths["lease_observed"].exists()
|
||||
assert paths["collector_state"].read_text(encoding="utf-8") == "true\n"
|
||||
assert paths["restore_contract"].read_text(encoding="utf-8").split("\t") == [
|
||||
str(paths["restore_hook"]),
|
||||
str(tmp_path / "clickhouse-restore-inventory.py"),
|
||||
str(tmp_path / "backup_disk.xml"),
|
||||
f"{tmp_path / 'isolated-cluster.xml'}\n",
|
||||
]
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert paths["native_receipts"].is_file()
|
||||
assert paths["restore_receipts"].is_file()
|
||||
restore_status = json.loads(paths["restore_status"].read_text(encoding="utf-8"))
|
||||
assert restore_status["terminal"] == "verified"
|
||||
assert restore_status["check_table_count"] == 44
|
||||
assert restore_status["check_table_pass_count"] == 44
|
||||
assert any(
|
||||
item["stage"] == "native_restore_receipt_verifier"
|
||||
and item["terminal"] == "pass"
|
||||
for item in receipts
|
||||
)
|
||||
assert any(
|
||||
item["stage"] == "lease" and item["terminal"] == "not_required"
|
||||
for item in receipts
|
||||
)
|
||||
assert _terminal(receipts, "rollback") == "no_write"
|
||||
assert _terminal(receipts, "terminal") == "partial_degraded"
|
||||
|
||||
|
||||
def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
expect_collector_stop=False,
|
||||
omit_monitor_contract=True,
|
||||
duplicate_cron=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert not paths["lease_observed"].exists()
|
||||
receipts = _receipts(paths["receipts"])
|
||||
source_diff = [
|
||||
item for item in receipts if item["stage"] == "source_of_truth_diff"
|
||||
][-1]
|
||||
assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"]
|
||||
assert _terminal(receipts, "terminal") == "partial_degraded"
|
||||
|
||||
|
||||
def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, monitor_mode="auto_repair")
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "monitor_auto_repair_contaminated_canary" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "restored"
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
def test_cron_crossing_requires_monitor_cooldown_receipt(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
monitor_mode="none",
|
||||
backup_sleep_seconds=1.2,
|
||||
monitor_interval_seconds=1,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "cron_crossed_without_collector_cooldown_receipt" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
|
||||
|
||||
def test_cron_crossing_with_detected_and_cooldown_receipts_passes(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
monitor_mode="cooldown",
|
||||
backup_sleep_seconds=1.2,
|
||||
monitor_interval_seconds=1,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
receipts = _receipts(paths["receipts"])
|
||||
monitor_receipt = [
|
||||
item for item in receipts if item["stage"] == "monitor_verifier"
|
||||
][-1]
|
||||
assert monitor_receipt["terminal"] == "pass"
|
||||
assert "cron_crossed_1" in str(monitor_receipt["detail"])
|
||||
|
||||
|
||||
def test_missing_monitor_contract_fails_before_lease_or_backup(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, omit_monitor_contract=True)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "monitor_contract_ambiguous_fragment_count_0" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert not paths["lease_observed"].exists()
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "not_armed"
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
def test_ambiguous_cron_contract_fails_before_lease_or_backup(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, duplicate_cron=True)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "monitor_cron_contract_ambiguous_count_2" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert not paths["lease_observed"].exists()
|
||||
|
||||
|
||||
def test_post_backup_listener_failure_fails_and_restores_lease(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, drop_listener=True)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "collector_post_backup_runtime_failed" in result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "rollback") == "restored"
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
def test_wrapper_deployment_and_static_contracts_are_owned() -> None:
|
||||
wrapper = WRAPPER.read_text(encoding="utf-8")
|
||||
assert "BACKUP_SKIP_RETENTION_CLEANUP=1" in wrapper
|
||||
assert 'TRACE_ID="${TRACE_ID:-}"' in wrapper
|
||||
assert 'RUN_ID="${RUN_ID:-}"' in wrapper
|
||||
assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper
|
||||
assert "monitor_auto_repair_contaminated_canary" in wrapper
|
||||
assert "collector_post_backup_runtime_failed" in wrapper
|
||||
assert "/backup/scripts/clickhouse-native-restore-drill.sh" in wrapper
|
||||
assert "/backup/scripts/clickhouse-restore-inventory.py" in wrapper
|
||||
assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in wrapper
|
||||
assert (
|
||||
"/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
in wrapper
|
||||
)
|
||||
assert (
|
||||
'CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}"'
|
||||
in wrapper
|
||||
)
|
||||
assert (
|
||||
'CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}"' in wrapper
|
||||
)
|
||||
assert (
|
||||
'CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}"'
|
||||
in wrapper
|
||||
)
|
||||
main_source = wrapper[wrapper.index("main() {") :]
|
||||
assert main_source.index("acquire_canary_lock") < main_source.index(
|
||||
"require_native_restore_contract"
|
||||
)
|
||||
assert "same_run_native_restore_receipt_validation_failed" in wrapper
|
||||
assert '"check_table_count"' in wrapper
|
||||
assert "mandatory_isolated_restore_verifier" in wrapper
|
||||
assert "SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS" in wrapper
|
||||
assert 'emit_receipt "runtime_operation" "pass"' in wrapper
|
||||
assert 'emit_receipt "closure_writeback" "partial_degraded"' in wrapper
|
||||
assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8")
|
||||
assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
assert str(WRAPPER.relative_to(ROOT)) in READINESS_AUDIT.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_check_mode_proves_contract_without_lease_or_backup_write(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(tmp_path, mode="check")
|
||||
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
assert paths["cooldown"].read_text(encoding="utf-8") == "123\n"
|
||||
assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640
|
||||
assert not paths["identities"].exists()
|
||||
assert not paths["lease_observed"].exists()
|
||||
check_receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(check_receipts, "rollback") == "no_write"
|
||||
assert _terminal(check_receipts, "terminal") == "check_pass_no_write"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("contract_state", ["missing", "symlink"])
|
||||
def test_native_check_fails_closed_for_unsafe_restore_contract(
|
||||
tmp_path: Path,
|
||||
contract_state: str,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
mode="check",
|
||||
expect_collector_stop=False,
|
||||
native_restore_contract=contract_state,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "native_restore_contract_CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" in result.stderr
|
||||
assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n"
|
||||
assert not paths["identities"].exists()
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"receipt_state",
|
||||
[
|
||||
"no_hook",
|
||||
"forged_identity",
|
||||
"missing_status",
|
||||
"symlink_status",
|
||||
"malformed_restore_receipt",
|
||||
],
|
||||
)
|
||||
def test_native_apply_rejects_missing_or_forged_machine_receipts(
|
||||
tmp_path: Path,
|
||||
receipt_state: str,
|
||||
) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
expect_collector_stop=False,
|
||||
native_receipt_state=receipt_state,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "========== SigNoz ClickHouse native 備份完成 (1s) ==========" in paths[
|
||||
"backup_output"
|
||||
].read_text(encoding="utf-8")
|
||||
assert (
|
||||
"same_run_native_restore_receipt_validation_failed" in result.stderr
|
||||
or "same_run_native_restore_receipt_missing_unreadable_or_symlink"
|
||||
in result.stderr
|
||||
)
|
||||
assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n"
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert not any(
|
||||
item["stage"] == "native_restore_receipt_verifier"
|
||||
and item["terminal"] == "pass"
|
||||
for item in receipts
|
||||
)
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
def test_docker_metadata_timeout_fails_before_backup(tmp_path: Path) -> None:
|
||||
result, paths, _ = _run_canary(
|
||||
tmp_path,
|
||||
mode="check",
|
||||
expect_collector_stop=False,
|
||||
docker_timeout=True,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "collector_runtime_metadata_unreadable" in result.stderr
|
||||
assert not paths["identities"].exists()
|
||||
receipts = _receipts(paths["receipts"])
|
||||
assert _terminal(receipts, "terminal") == "failed"
|
||||
|
||||
|
||||
def test_wrapper_is_valid_bash() -> None:
|
||||
result = subprocess.run(
|
||||
["bash", "-n", str(WRAPPER)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
@@ -54,6 +54,9 @@ bash -n \
|
||||
scripts/backup/backup-langfuse.sh \
|
||||
scripts/backup/backup-monitoring.sh \
|
||||
scripts/backup/backup-signoz.sh \
|
||||
scripts/backup/clickhouse-native-backup.sh \
|
||||
scripts/backup/clickhouse-native-restore-drill.sh \
|
||||
scripts/backup/run-signoz-backup-canary.sh \
|
||||
scripts/backup/backup-open-webui.sh \
|
||||
scripts/backup/backup-clawbot.sh \
|
||||
scripts/backup/backup-configs.sh \
|
||||
@@ -81,6 +84,7 @@ python3 -m py_compile \
|
||||
scripts/ops/backup-alert-label-contract-check.py \
|
||||
scripts/ops/backup-alert-live-visibility-check.py \
|
||||
scripts/ops/recovery-scorecard-contract-check.py \
|
||||
scripts/backup/clickhouse-restore-inventory.py \
|
||||
scripts/ops/doc-secrets-sanity-check.py
|
||||
echo "Python 語法 OK"
|
||||
|
||||
|
||||
798
scripts/ops/deploy-signoz-clickhouse-backup-disk.sh
Executable file
798
scripts/ops/deploy-signoz-clickhouse-backup-disk.sh
Executable file
@@ -0,0 +1,798 @@
|
||||
#!/usr/bin/env bash
|
||||
# P0-OBS-002 - controlled deployment of the host110 ClickHouse native BACKUP disk.
|
||||
#
|
||||
# This script owns only an additive Compose override and one ClickHouse config
|
||||
# fragment. It never runs BACKUP/RESTORE, reads a dotenv/secret source, pulls an
|
||||
# image, or recreates another service.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml"
|
||||
LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
|
||||
|
||||
TARGET_HOST="wooo@192.168.0.110"
|
||||
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
||||
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
||||
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
||||
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
||||
REMOTE_PROJECT="signoz"
|
||||
REMOTE_SERVICE="clickhouse"
|
||||
CLICKHOUSE_CONTAINER="signoz-clickhouse"
|
||||
COLLECTOR_CONTAINER="signoz-otel-collector"
|
||||
SIGNOZ_CONTAINER="signoz"
|
||||
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
|
||||
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
||||
CONTAINER_BACKUP_DIR="/backups"
|
||||
CLICKHOUSE_DISK="backups"
|
||||
CLICKHOUSE_UID="101"
|
||||
CLICKHOUSE_GID="101"
|
||||
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
SAFE_HOME="/home/wooo"
|
||||
LOCAL_KILL_AFTER_SECONDS=15
|
||||
REMOTE_CHECK_TIMEOUT_SECONDS=180
|
||||
REMOTE_STAGE_TIMEOUT_SECONDS=60
|
||||
REMOTE_TRANSFER_TIMEOUT_SECONDS=60
|
||||
REMOTE_APPLY_TIMEOUT_SECONDS=3600
|
||||
REMOTE_CLEANUP_TIMEOUT_SECONDS=60
|
||||
|
||||
SSH_OPTS=(
|
||||
-o BatchMode=yes
|
||||
-o ConnectTimeout=8
|
||||
-o ConnectionAttempts=1
|
||||
-o ServerAliveInterval=5
|
||||
-o ServerAliveCountMax=2
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: deploy-signoz-clickhouse-backup-disk.sh [--check|--apply]
|
||||
|
||||
Required environment:
|
||||
TRACE_ID Path-safe controlled-apply trace identifier
|
||||
RUN_ID Unique path-safe execution identifier
|
||||
WORK_ITEM_ID Path-safe work-item identifier
|
||||
|
||||
--check performs local source validation and remote read-only Compose/runtime
|
||||
checks. --apply atomically installs the two repo-owned artifacts, recreates only
|
||||
the ClickHouse service with --pull never, independently verifies runtime state,
|
||||
and restores the exact prior managed Compose state on failure.
|
||||
USAGE
|
||||
}
|
||||
|
||||
MODE="${1:---check}"
|
||||
case "${MODE}" in
|
||||
--check|--apply) ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) usage >&2; exit 64 ;;
|
||||
esac
|
||||
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
||||
|
||||
TRACE_ID="${TRACE_ID:-}"
|
||||
RUN_ID="${RUN_ID:-}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
||||
|
||||
valid_identity() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
||||
}
|
||||
|
||||
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
||||
valid_identity "${value}" || {
|
||||
printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
|
||||
exit 64
|
||||
}
|
||||
done
|
||||
|
||||
for command_name in python3 scp ssh timeout; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 || {
|
||||
printf 'required local command missing: %s\n' "${command_name}" >&2
|
||||
exit 69
|
||||
}
|
||||
done
|
||||
|
||||
hash_file() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 "$1" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
bounded_ssh() {
|
||||
local timeout_seconds="$1"
|
||||
shift
|
||||
timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
|
||||
"${timeout_seconds}" ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" "$@"
|
||||
}
|
||||
|
||||
bounded_scp() {
|
||||
timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \
|
||||
"${REMOTE_TRANSFER_TIMEOUT_SECONDS}" scp -q "${SSH_OPTS[@]}" "$@"
|
||||
}
|
||||
|
||||
validate_local_sources() {
|
||||
[ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ]
|
||||
[ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ]
|
||||
|
||||
python3 - "${LOCAL_CONFIG}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
root = ET.parse(path).getroot()
|
||||
assert root.tag == "clickhouse"
|
||||
disk = root.find("./storage_configuration/disks/backups")
|
||||
assert disk is not None
|
||||
assert disk.findtext("type") == "local"
|
||||
assert disk.findtext("path") == "/backups/"
|
||||
backups = root.find("./backups")
|
||||
assert backups is not None
|
||||
assert backups.findtext("allowed_disk") == "backups"
|
||||
assert backups.findtext("allowed_path") == "/backups/"
|
||||
assert backups.findtext("allow_concurrent_backups") == "false"
|
||||
assert backups.findtext("allow_concurrent_restores") == "false"
|
||||
PY
|
||||
|
||||
[ "$(grep -F -c ' clickhouse:' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
||||
[ "$(grep -F -c 'source: /backup/staging/signoz-clickhouse' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
||||
[ "$(grep -F -c 'target: /backups' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
||||
[ "$(grep -F -c 'target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml' "${LOCAL_OVERRIDE}")" -eq 1 ]
|
||||
! grep -Eq '^[[:space:]]*(image|build|command|environment):' "${LOCAL_OVERRIDE}"
|
||||
}
|
||||
|
||||
validate_local_sources
|
||||
OVERRIDE_SHA="$(hash_file "${LOCAL_OVERRIDE}")"
|
||||
CONFIG_SHA="$(hash_file "${LOCAL_CONFIG}")"
|
||||
|
||||
local_receipt() {
|
||||
local phase="$1" result="$2" detail="$3"
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
|
||||
}
|
||||
|
||||
remote_candidate_compose_check() {
|
||||
bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" \
|
||||
"timeout --signal=TERM --kill-after=15 120 env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \
|
||||
< "${LOCAL_OVERRIDE}"
|
||||
}
|
||||
|
||||
remote_read_only_check() {
|
||||
bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" bash -s -- \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
EXPECTED_OVERRIDE_SHA="$4"
|
||||
EXPECTED_CONFIG_SHA="$5"
|
||||
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
||||
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
||||
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
||||
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
||||
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
||||
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
SAFE_HOME="/home/wooo"
|
||||
COMMAND_TIMEOUT_SECONDS=30
|
||||
COMPOSE_TIMEOUT_SECONDS=120
|
||||
KILL_AFTER_SECONDS=15
|
||||
|
||||
for command_name in docker timeout; do
|
||||
command -v "${command_name}" >/dev/null 2>&1
|
||||
done
|
||||
|
||||
docker_cmd() {
|
||||
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" docker "$@"
|
||||
}
|
||||
|
||||
compose_cmd() {
|
||||
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMPOSE_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p signoz "$@"
|
||||
}
|
||||
|
||||
file_hash_or_absent() {
|
||||
if [ -f "$1" ] && [ ! -L "$1" ]; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
elif [ ! -e "$1" ]; then
|
||||
printf 'absent\n'
|
||||
else
|
||||
printf 'unsafe\n'
|
||||
fi
|
||||
}
|
||||
|
||||
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
|
||||
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
|
||||
docker_cmd inspect "${container}" >/dev/null
|
||||
done
|
||||
compose_cmd -f "${REMOTE_BASE}" config -q >/dev/null
|
||||
compose_cmd -f "${REMOTE_BASE}" config --services | grep -qx clickhouse
|
||||
|
||||
override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")"
|
||||
config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")"
|
||||
override_match=false
|
||||
config_match=false
|
||||
[ "${override_hash}" = "${EXPECTED_OVERRIDE_SHA}" ] && override_match=true
|
||||
[ "${config_hash}" = "${EXPECTED_CONFIG_SHA}" ] && config_match=true
|
||||
|
||||
host_dir=absent
|
||||
if [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]; then
|
||||
host_dir=present
|
||||
elif [ -e "${HOST_BACKUP_DIR}" ]; then
|
||||
host_dir=unsafe
|
||||
fi
|
||||
|
||||
disk_count="$(docker_cmd exec signoz-clickhouse clickhouse-client --query \
|
||||
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
||||
mount_count="$(docker_cmd inspect --format \
|
||||
'{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \
|
||||
signoz-clickhouse | tr -cd '1' | wc -c | xargs)"
|
||||
collector_running="$(docker_cmd inspect --format '{{.State.Running}}' signoz-otel-collector)"
|
||||
collector_restarts="$(docker_cmd inspect --format '{{.RestartCount}}' signoz-otel-collector)"
|
||||
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \
|
||||
"${config_match}" "${host_dir}" "${disk_count}" "${mount_count}" \
|
||||
"${collector_running}" "${collector_restarts}"
|
||||
REMOTE_CHECK
|
||||
}
|
||||
|
||||
local_receipt sensor_source pass "override_sha_${OVERRIDE_SHA}_config_sha_${CONFIG_SHA}"
|
||||
local_receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_disk_backups"
|
||||
remote_candidate_compose_check
|
||||
remote_read_only_check
|
||||
local_receipt check pass "compose_config_q_no_output_no_write"
|
||||
|
||||
if [ "${MODE}" = "--check" ]; then
|
||||
local_receipt terminal check_pass_no_write "source_and_remote_preflight_valid"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}"
|
||||
STAGE_CREATED=0
|
||||
LOCAL_APPLY_VERIFIED=0
|
||||
|
||||
cleanup_stage() {
|
||||
local cleanup_rc
|
||||
[ "${STAGE_CREATED}" -eq 1 ] || return 0
|
||||
bounded_ssh "${REMOTE_CLEANUP_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1
|
||||
set -euo pipefail
|
||||
stage_dir="$1"
|
||||
if [ ! -e "${stage_dir}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
||||
rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml"
|
||||
rmdir "${stage_dir}"
|
||||
[ ! -e "${stage_dir}" ]
|
||||
REMOTE_CLEANUP
|
||||
cleanup_rc=$?
|
||||
[ "${cleanup_rc}" -eq 0 ] || return "${cleanup_rc}"
|
||||
STAGE_CREATED=0
|
||||
}
|
||||
|
||||
on_local_exit() {
|
||||
local rc=$?
|
||||
local cleanup_rc=0 cleanup_terminal=not_created terminal=failed
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
if [ "${STAGE_CREATED}" -eq 1 ]; then
|
||||
cleanup_stage
|
||||
cleanup_rc=$?
|
||||
if [ "${cleanup_rc}" -eq 0 ]; then
|
||||
cleanup_terminal=pass_absence_verified
|
||||
else
|
||||
cleanup_terminal=failed_or_residue_present
|
||||
rc=92
|
||||
fi
|
||||
fi
|
||||
local_receipt cleanup "$([ "${cleanup_rc}" -eq 0 ] && printf pass || printf failed)" \
|
||||
"remote_stage_${cleanup_terminal}"
|
||||
if [ "${rc}" -eq 0 ] && [ "${LOCAL_APPLY_VERIFIED}" -eq 1 ]; then
|
||||
terminal=pass
|
||||
fi
|
||||
local_receipt terminal "${terminal}" \
|
||||
"remote_controlled_apply_${terminal}_stage_${cleanup_terminal}_exit_${rc}"
|
||||
exit "${rc}"
|
||||
}
|
||||
|
||||
trap on_local_exit EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
bounded_ssh "${REMOTE_STAGE_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
|
||||
set -euo pipefail
|
||||
stage_dir="$1"
|
||||
[ ! -e "${stage_dir}" ]
|
||||
umask 077
|
||||
mkdir -m 0700 "${stage_dir}"
|
||||
REMOTE_STAGE
|
||||
STAGE_CREATED=1
|
||||
|
||||
bounded_scp "${LOCAL_OVERRIDE}" \
|
||||
"${TARGET_HOST}:${STAGE_DIR}/override.yaml"
|
||||
bounded_scp "${LOCAL_CONFIG}" \
|
||||
"${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml"
|
||||
|
||||
bounded_ssh "${REMOTE_APPLY_TIMEOUT_SECONDS}" bash -s -- \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
EXPECTED_OVERRIDE_SHA="$4"
|
||||
EXPECTED_CONFIG_SHA="$5"
|
||||
STAGE_DIR="$6"
|
||||
|
||||
REMOTE_DIR="/home/wooo/signoz/deploy/docker"
|
||||
REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml"
|
||||
REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml"
|
||||
REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml"
|
||||
STAGE_OVERRIDE="${STAGE_DIR}/override.yaml"
|
||||
STAGE_CONFIG="${STAGE_DIR}/backup_disk.xml"
|
||||
REMOTE_PROJECT="signoz"
|
||||
REMOTE_SERVICE="clickhouse"
|
||||
CLICKHOUSE_CONTAINER="signoz-clickhouse"
|
||||
COLLECTOR_CONTAINER="signoz-otel-collector"
|
||||
SIGNOZ_CONTAINER="signoz"
|
||||
ZOOKEEPER_CONTAINER="signoz-zookeeper-1"
|
||||
HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse"
|
||||
RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk"
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
SAFE_HOME="/home/wooo"
|
||||
LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock"
|
||||
BACKUP_OPERATION_LOCK_FILE="/tmp/awoooi-signoz-backup-operation.lock"
|
||||
CANARY_LOCK_FILE="/tmp/awoooi-signoz-backup-canary.lock"
|
||||
COMMAND_TIMEOUT_SECONDS=30
|
||||
COMPOSE_CONFIG_TIMEOUT_SECONDS=120
|
||||
COMPOSE_APPLY_TIMEOUT_SECONDS=300
|
||||
KILL_AFTER_SECONDS=15
|
||||
|
||||
APPLY_STARTED=0
|
||||
DEPLOY_VERIFIED=0
|
||||
HOST_DIR_CREATED=0
|
||||
PRIOR_OVERRIDE_PRESENT=0
|
||||
PRIOR_CONFIG_PRESENT=0
|
||||
PRIOR_OVERRIDE_HASH="absent"
|
||||
PRIOR_OVERRIDE_MODE="absent"
|
||||
PRIOR_OVERRIDE_UID="absent"
|
||||
PRIOR_OVERRIDE_GID="absent"
|
||||
PRIOR_CONFIG_HASH="absent"
|
||||
PRIOR_CONFIG_MODE="absent"
|
||||
PRIOR_CONFIG_UID="absent"
|
||||
PRIOR_CONFIG_GID="absent"
|
||||
PRIOR_DISK_COUNT=0
|
||||
BEFORE_IMAGE_ID=""
|
||||
BEFORE_DATA_VOLUME=""
|
||||
BEFORE_COLLECTOR_ID=""
|
||||
BEFORE_COLLECTOR_RESTARTS=""
|
||||
BEFORE_SIGNOZ_ID=""
|
||||
BEFORE_SIGNOZ_RESTARTS=""
|
||||
BEFORE_ZOOKEEPER_ID=""
|
||||
BEFORE_ZOOKEEPER_RESTARTS=""
|
||||
|
||||
receipt() {
|
||||
local phase="$1" result="$2" detail="$3"
|
||||
local line
|
||||
line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
|
||||
printf '%s\n' "${line}"
|
||||
printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
|
||||
}
|
||||
|
||||
docker_cmd() {
|
||||
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" docker "$@"
|
||||
}
|
||||
|
||||
bounded_cmd() {
|
||||
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" "$@"
|
||||
}
|
||||
|
||||
compose_config_cmd() {
|
||||
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMPOSE_CONFIG_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
|
||||
}
|
||||
|
||||
compose_apply_cmd() {
|
||||
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMPOSE_APPLY_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \
|
||||
docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@"
|
||||
}
|
||||
|
||||
pgrep_cmd() {
|
||||
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" pgrep "$@"
|
||||
}
|
||||
|
||||
ss_cmd() {
|
||||
timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \
|
||||
"${COMMAND_TIMEOUT_SECONDS}" ss "$@"
|
||||
}
|
||||
|
||||
compose_base_config() {
|
||||
compose_config_cmd -f "${REMOTE_BASE}" config -q >/dev/null 2>&1
|
||||
}
|
||||
|
||||
compose_managed_config() {
|
||||
compose_config_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
|
||||
config -q >/dev/null 2>&1
|
||||
}
|
||||
|
||||
compose_stage_config() {
|
||||
compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" \
|
||||
config -q >/dev/null 2>&1
|
||||
}
|
||||
|
||||
recreate_managed() {
|
||||
compose_apply_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \
|
||||
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
recreate_base() {
|
||||
compose_apply_cmd -f "${REMOTE_BASE}" \
|
||||
up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
wait_clickhouse() {
|
||||
local attempt running health
|
||||
for attempt in $(seq 1 36); do
|
||||
running="$(docker_cmd inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
|
||||
health="$(docker_cmd inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \
|
||||
"${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)"
|
||||
if [ "${running}" = true ] && [ "${health}" = healthy ] \
|
||||
&& docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
|
||||
clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
listener_up() {
|
||||
local port="$1"
|
||||
ss_cmd -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
||||
}
|
||||
|
||||
wait_api_and_listeners() {
|
||||
local attempt api_code
|
||||
for attempt in $(seq 1 30); do
|
||||
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}" 2>/dev/null || true)"
|
||||
if [ "${api_code}" = 200 ] && listener_up 4317 && listener_up 4318 && listener_up 8080; then
|
||||
return 0
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
file_matches_metadata() {
|
||||
local path="$1" expected_hash="$2" expected_mode="$3" expected_uid="$4" expected_gid="$5"
|
||||
[ -f "${path}" ] && [ ! -L "${path}" ] \
|
||||
&& [ "$(sha256sum "${path}" | awk '{print $1}')" = "${expected_hash}" ] \
|
||||
&& [ "$(stat -c '%a' "${path}")" = "${expected_mode}" ] \
|
||||
&& [ "$(stat -c '%u' "${path}")" = "${expected_uid}" ] \
|
||||
&& [ "$(stat -c '%g' "${path}")" = "${expected_gid}" ]
|
||||
}
|
||||
|
||||
restore_exact_file() {
|
||||
local source="$1" target="$2" expected_hash="$3" expected_mode="$4"
|
||||
local expected_uid="$5" expected_gid="$6" tmp="$7"
|
||||
local current_uid current_gid
|
||||
|
||||
bounded_cmd install -m "${expected_mode}" "${source}" "${tmp}" || return 1
|
||||
current_uid="$(stat -c '%u' "${tmp}")" || return 1
|
||||
current_gid="$(stat -c '%g' "${tmp}")" || return 1
|
||||
if [ "${current_uid}" != "${expected_uid}" ] || [ "${current_gid}" != "${expected_gid}" ]; then
|
||||
bounded_cmd sudo -n chown "${expected_uid}:${expected_gid}" "${tmp}" || return 1
|
||||
fi
|
||||
bounded_cmd mv -f "${tmp}" "${target}" || return 1
|
||||
file_matches_metadata "${target}" "${expected_hash}" "${expected_mode}" \
|
||||
"${expected_uid}" "${expected_gid}"
|
||||
}
|
||||
|
||||
restore_managed_files() {
|
||||
local tmp
|
||||
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
||||
tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}"
|
||||
restore_exact_file "${RECEIPT_DIR}/prior-override.yaml" "${REMOTE_OVERRIDE}" \
|
||||
"${PRIOR_OVERRIDE_HASH}" "${PRIOR_OVERRIDE_MODE}" \
|
||||
"${PRIOR_OVERRIDE_UID}" "${PRIOR_OVERRIDE_GID}" "${tmp}" || return 1
|
||||
else
|
||||
bounded_cmd rm -f "${REMOTE_OVERRIDE}" || return 1
|
||||
[ ! -e "${REMOTE_OVERRIDE}" ] || return 1
|
||||
fi
|
||||
|
||||
if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then
|
||||
tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}"
|
||||
restore_exact_file "${RECEIPT_DIR}/prior-backup-disk.xml" "${REMOTE_CONFIG}" \
|
||||
"${PRIOR_CONFIG_HASH}" "${PRIOR_CONFIG_MODE}" \
|
||||
"${PRIOR_CONFIG_UID}" "${PRIOR_CONFIG_GID}" "${tmp}" || return 1
|
||||
else
|
||||
bounded_cmd rm -f "${REMOTE_CONFIG}" || return 1
|
||||
[ ! -e "${REMOTE_CONFIG}" ] || return 1
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_owned_residue() {
|
||||
local path
|
||||
for path in \
|
||||
"${REMOTE_OVERRIDE}.candidate.${RUN_ID}" \
|
||||
"${REMOTE_CONFIG}.candidate.${RUN_ID}" \
|
||||
"${REMOTE_OVERRIDE}.rollback.${RUN_ID}" \
|
||||
"${REMOTE_CONFIG}.rollback.${RUN_ID}"; do
|
||||
bounded_cmd rm -f "${path}" || return 1
|
||||
[ ! -e "${path}" ] || return 1
|
||||
done
|
||||
if [ -e "${STAGE_DIR}" ]; then
|
||||
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
|
||||
bounded_cmd rm -f "${STAGE_OVERRIDE}" "${STAGE_CONFIG}" || return 1
|
||||
bounded_cmd rmdir "${STAGE_DIR}" || return 1
|
||||
fi
|
||||
[ ! -e "${STAGE_DIR}" ] || return 1
|
||||
}
|
||||
|
||||
rollback_deploy() {
|
||||
local disk_count rollback_dir_terminal=retained
|
||||
restore_managed_files || return 1
|
||||
|
||||
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
||||
compose_managed_config || return 1
|
||||
recreate_managed || return 1
|
||||
else
|
||||
compose_base_config || return 1
|
||||
recreate_base || return 1
|
||||
fi
|
||||
wait_clickhouse || return 1
|
||||
wait_api_and_listeners || return 1
|
||||
|
||||
disk_count="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1
|
||||
[ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1
|
||||
[ "$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1
|
||||
[ "$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1
|
||||
[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1
|
||||
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1
|
||||
|
||||
if [ "${HOST_DIR_CREATED}" -eq 1 ]; then
|
||||
if bounded_cmd sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then
|
||||
rollback_dir_terminal=removed_empty
|
||||
else
|
||||
rollback_dir_terminal=retained_nonempty_no_delete
|
||||
fi
|
||||
fi
|
||||
receipt rollback pass \
|
||||
"prior_compose_state_restored_exact_hash_mode_uid_gid_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}"
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
local rc=$? rollback_rc=0 cleanup_rc=0 terminal=failed
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
||||
rollback_deploy
|
||||
rollback_rc=$?
|
||||
if [ "${rollback_rc}" -ne 0 ]; then
|
||||
receipt rollback failed "prior_compose_state_restore_failed"
|
||||
rc=91
|
||||
fi
|
||||
fi
|
||||
cleanup_owned_residue
|
||||
cleanup_rc=$?
|
||||
if [ "${cleanup_rc}" -ne 0 ]; then
|
||||
receipt cleanup failed "exact_stage_candidate_or_rollback_residue_cleanup_failed"
|
||||
rc=92
|
||||
else
|
||||
receipt cleanup pass "exact_stage_candidate_and_rollback_residue_absence_verified"
|
||||
fi
|
||||
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ] && [ "${cleanup_rc}" -eq 0 ]; then
|
||||
terminal=pass
|
||||
fi
|
||||
receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}"
|
||||
exit "${rc}"
|
||||
}
|
||||
|
||||
mkdir -p "${RECEIPT_ROOT}"
|
||||
[ ! -e "${RECEIPT_DIR}" ]
|
||||
mkdir -m 0700 "${RECEIPT_DIR}"
|
||||
: > "${RECEIPT_LOG}"
|
||||
trap on_exit EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
for lock_path in "${LOCK_FILE}" "${BACKUP_OPERATION_LOCK_FILE}" "${CANARY_LOCK_FILE}"; do
|
||||
[ ! -L "${lock_path}" ] \
|
||||
|| { receipt check failed operation_lock_symlink_unsafe; exit 75; }
|
||||
if [ -e "${lock_path}" ]; then
|
||||
[ -f "${lock_path}" ] \
|
||||
|| { receipt check failed operation_lock_not_regular; exit 75; }
|
||||
fi
|
||||
done
|
||||
exec 9>>"${LOCK_FILE}"
|
||||
flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; }
|
||||
exec 7>>"${CANARY_LOCK_FILE}"
|
||||
flock -n 7 || { receipt check failed backup_canary_lock_held; exit 75; }
|
||||
exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"
|
||||
flock -n 8 || { receipt check failed backup_operation_lock_held; exit 75; }
|
||||
|
||||
for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss stat sudo timeout; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 \
|
||||
|| { receipt check failed "required_command_missing_${command_name}"; exit 69; }
|
||||
done
|
||||
|
||||
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
||||
[ -f "${STAGE_OVERRIDE}" ] && [ ! -L "${STAGE_OVERRIDE}" ]
|
||||
[ -f "${STAGE_CONFIG}" ] && [ ! -L "${STAGE_CONFIG}" ]
|
||||
[ "$(sha256sum "${STAGE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
|
||||
[ "$(sha256sum "${STAGE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
|
||||
[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ]
|
||||
[ -d "${REMOTE_DIR}" ] && [ ! -L "${REMOTE_DIR}" ] && [ -w "${REMOTE_DIR}" ]
|
||||
[ -d /backup ] && [ ! -L /backup ]
|
||||
|
||||
compose_base_config
|
||||
compose_stage_config
|
||||
compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \
|
||||
| grep -qx "${REMOTE_SERVICE}"
|
||||
|
||||
require_backup_idle() {
|
||||
local phase="$1" active_native_backups process_rc
|
||||
set +e
|
||||
pgrep_cmd -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' \
|
||||
>/dev/null 2>&1
|
||||
process_rc=$?
|
||||
set -e
|
||||
case "${process_rc}" in
|
||||
0)
|
||||
receipt check failed "active_signoz_backup_or_restore_process_detected_${phase}"
|
||||
return 75
|
||||
;;
|
||||
1) ;;
|
||||
*)
|
||||
receipt check failed "signoz_backup_process_probe_failed_${phase}_exit_${process_rc}"
|
||||
return 69
|
||||
;;
|
||||
esac
|
||||
active_native_backups="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" \
|
||||
|| return 1
|
||||
[ "${active_native_backups}" = 0 ] \
|
||||
|| { receipt check failed "active_native_backup_or_restore_${phase}"; return 75; }
|
||||
receipt check pass "backup_idle_${phase}_shared_operation_and_canary_locks_held"
|
||||
}
|
||||
|
||||
require_backup_idle preflight
|
||||
|
||||
for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do
|
||||
[ "$(docker_cmd inspect --format '{{.State.Running}}' "${container}")" = true ]
|
||||
done
|
||||
wait_clickhouse
|
||||
wait_api_and_listeners
|
||||
|
||||
if [ -e "${REMOTE_OVERRIDE}" ]; then
|
||||
[ -f "${REMOTE_OVERRIDE}" ] && [ ! -L "${REMOTE_OVERRIDE}" ]
|
||||
PRIOR_OVERRIDE_PRESENT=1
|
||||
fi
|
||||
if [ -e "${REMOTE_CONFIG}" ]; then
|
||||
[ -f "${REMOTE_CONFIG}" ] && [ ! -L "${REMOTE_CONFIG}" ]
|
||||
PRIOR_CONFIG_PRESENT=1
|
||||
fi
|
||||
[ "${PRIOR_OVERRIDE_PRESENT}" -eq "${PRIOR_CONFIG_PRESENT}" ] \
|
||||
|| { receipt check failed partial_managed_file_drift; exit 78; }
|
||||
if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then
|
||||
compose_managed_config
|
||||
PRIOR_OVERRIDE_HASH="$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')"
|
||||
PRIOR_OVERRIDE_MODE="$(stat -c '%a' "${REMOTE_OVERRIDE}")"
|
||||
PRIOR_OVERRIDE_UID="$(stat -c '%u' "${REMOTE_OVERRIDE}")"
|
||||
PRIOR_OVERRIDE_GID="$(stat -c '%g' "${REMOTE_OVERRIDE}")"
|
||||
PRIOR_CONFIG_HASH="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
||||
PRIOR_CONFIG_MODE="$(stat -c '%a' "${REMOTE_CONFIG}")"
|
||||
PRIOR_CONFIG_UID="$(stat -c '%u' "${REMOTE_CONFIG}")"
|
||||
PRIOR_CONFIG_GID="$(stat -c '%g' "${REMOTE_CONFIG}")"
|
||||
bounded_cmd cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml"
|
||||
bounded_cmd cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml"
|
||||
[ "$(sha256sum "${RECEIPT_DIR}/prior-override.yaml" | awk '{print $1}')" = "${PRIOR_OVERRIDE_HASH}" ]
|
||||
[ "$(sha256sum "${RECEIPT_DIR}/prior-backup-disk.xml" | awk '{print $1}')" = "${PRIOR_CONFIG_HASH}" ]
|
||||
fi
|
||||
|
||||
PRIOR_DISK_COUNT="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
||||
BEFORE_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
BEFORE_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
[ -n "${BEFORE_DATA_VOLUME}" ]
|
||||
BEFORE_COLLECTOR_ID="$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")"
|
||||
BEFORE_COLLECTOR_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")"
|
||||
BEFORE_SIGNOZ_ID="$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")"
|
||||
BEFORE_SIGNOZ_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")"
|
||||
BEFORE_ZOOKEEPER_ID="$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")"
|
||||
BEFORE_ZOOKEEPER_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")"
|
||||
|
||||
receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid"
|
||||
receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}"
|
||||
receipt source_of_truth_diff pass \
|
||||
"prior_override_${PRIOR_OVERRIDE_PRESENT}_hash_${PRIOR_OVERRIDE_HASH}_mode_${PRIOR_OVERRIDE_MODE}_uid_${PRIOR_OVERRIDE_UID}_gid_${PRIOR_OVERRIDE_GID}_prior_config_${PRIOR_CONFIG_PRESENT}_hash_${PRIOR_CONFIG_HASH}_mode_${PRIOR_CONFIG_MODE}_uid_${PRIOR_CONFIG_UID}_gid_${PRIOR_CONFIG_GID}_prior_disk_count_${PRIOR_DISK_COUNT}"
|
||||
receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only"
|
||||
receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback"
|
||||
receipt check pass \
|
||||
"compose_config_q_stage_hashes_runtime_preflight_dual_locks_backup_idle"
|
||||
|
||||
require_backup_idle pre_mutation
|
||||
|
||||
APPLY_STARTED=1
|
||||
if [ -e "${HOST_BACKUP_DIR}" ]; then
|
||||
[ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]
|
||||
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
|
||||
else
|
||||
bounded_cmd sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}"
|
||||
HOST_DIR_CREATED=1
|
||||
fi
|
||||
|
||||
bounded_cmd install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}"
|
||||
bounded_cmd install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}"
|
||||
bounded_cmd mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}"
|
||||
bounded_cmd mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}"
|
||||
[ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ]
|
||||
[ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ]
|
||||
compose_managed_config
|
||||
|
||||
recreate_managed
|
||||
wait_clickhouse
|
||||
|
||||
AFTER_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
AFTER_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
[ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ]
|
||||
[ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ]
|
||||
|
||||
backup_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
config_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")"
|
||||
[ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ]
|
||||
[ "${config_mount}" = false ]
|
||||
[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ]
|
||||
server_uid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
|
||||
awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)"
|
||||
server_gid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \
|
||||
awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)"
|
||||
[ "${server_uid}" = 101 ]
|
||||
[ "${server_gid}" = 101 ]
|
||||
docker_cmd exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \
|
||||
'test -d /backups && test -w /backups'
|
||||
|
||||
disk_state="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)"
|
||||
[ "${disk_state}" = "Local|0|0|0|1" ]
|
||||
[ "$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \
|
||||
"EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ]
|
||||
docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \
|
||||
| grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)'
|
||||
|
||||
[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ]
|
||||
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ]
|
||||
[ "$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ]
|
||||
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ]
|
||||
[ "$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ]
|
||||
[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ]
|
||||
wait_api_and_listeners
|
||||
|
||||
receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged"
|
||||
receipt post_verifier pass "disk_backups_local_rw_healthy_server_uid101_write_access_concurrency_disabled_collector_unchanged_listeners_4317_4318_8080_api_200"
|
||||
receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_executed"
|
||||
DEPLOY_VERIFIED=1
|
||||
REMOTE_APPLY
|
||||
|
||||
LOCAL_APPLY_VERIFIED=1
|
||||
908
scripts/ops/deploy-signoz-native-backup-toolchain.sh
Executable file
908
scripts/ops/deploy-signoz-native-backup-toolchain.sh
Executable file
@@ -0,0 +1,908 @@
|
||||
#!/usr/bin/env bash
|
||||
# P0-OBS-002 - bounded controlled deploy of the host110 SigNoz native backup toolchain.
|
||||
#
|
||||
# This deployer owns seven repo-sourced files only. It does not run a backup or
|
||||
# restore, restart a container, pull an image, or inspect a secret-bearing
|
||||
# runtime source.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TARGET_HOST="wooo@192.168.0.110"
|
||||
SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
SAFE_HOME="/home/wooo"
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
CANARY_LOCK="/tmp/awoooi-signoz-backup-canary.lock"
|
||||
OPERATION_LOCK="/tmp/awoooi-signoz-backup-operation.lock"
|
||||
|
||||
SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"
|
||||
SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"
|
||||
DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"
|
||||
TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"
|
||||
|
||||
REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"
|
||||
REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}"
|
||||
RECEIPT_ROOT="${SIGNOZ_TOOLCHAIN_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-native-backup-toolchain}"
|
||||
STAGE_ROOT="${SIGNOZ_TOOLCHAIN_STAGE_ROOT:-/tmp}"
|
||||
|
||||
LABELS=(
|
||||
backup-signoz.sh
|
||||
clickhouse-native-backup.sh
|
||||
clickhouse-native-restore-drill.sh
|
||||
clickhouse-restore-inventory.py
|
||||
run-signoz-backup-canary.sh
|
||||
backup_disk.xml
|
||||
isolated-cluster.xml
|
||||
)
|
||||
LOCAL_SOURCES=(
|
||||
"${ROOT_DIR}/scripts/backup/backup-signoz.sh"
|
||||
"${ROOT_DIR}/scripts/backup/clickhouse-native-backup.sh"
|
||||
"${ROOT_DIR}/scripts/backup/clickhouse-native-restore-drill.sh"
|
||||
"${ROOT_DIR}/scripts/backup/clickhouse-restore-inventory.py"
|
||||
"${ROOT_DIR}/scripts/backup/run-signoz-backup-canary.sh"
|
||||
"${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml"
|
||||
"${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml"
|
||||
)
|
||||
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
||||
|
||||
SSH_OPTS=(
|
||||
-o BatchMode=yes
|
||||
-o ConnectTimeout=8
|
||||
-o ConnectionAttempts=1
|
||||
-o ServerAliveInterval=5
|
||||
-o ServerAliveCountMax=2
|
||||
)
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: deploy-signoz-native-backup-toolchain.sh [--check|--apply]
|
||||
|
||||
Required environment:
|
||||
TRACE_ID Path-safe controlled-apply trace identifier
|
||||
RUN_ID Unique path-safe execution identifier
|
||||
WORK_ITEM_ID Path-safe work-item identifier
|
||||
|
||||
Optional safe absolute path environment:
|
||||
SIGNOZ_TOOLCHAIN_SCRIPT_DIR
|
||||
SIGNOZ_TOOLCHAIN_CONFIG_ROOT
|
||||
SIGNOZ_TOOLCHAIN_RECEIPT_ROOT
|
||||
SIGNOZ_TOOLCHAIN_STAGE_ROOT
|
||||
|
||||
Optional bounded-command timeout environment (positive integer seconds):
|
||||
SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS
|
||||
SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS
|
||||
SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS
|
||||
SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS
|
||||
|
||||
--check validates local sources and performs a remote read-only target diff and
|
||||
runtime readiness check. --apply stages all seven sources, validates hashes and
|
||||
syntax, uses a same-filesystem atomic replacement for each exact target, and
|
||||
rolls every target back to its prior hash/mode/owner if any bounded verifier
|
||||
fails. SSH, SCP, and Docker calls are bounded, and apply holds both the SigNoz
|
||||
canary lock and backup operation lock through post-verification or rollback.
|
||||
|
||||
Flat restore-driver invocation must set:
|
||||
CLICKHOUSE_RESTORE_INVENTORY_HELPER=<script-dir>/clickhouse-restore-inventory.py
|
||||
CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG=<config-root>/config.d/backup_disk.xml
|
||||
CLICKHOUSE_RESTORE_CLUSTER_CONFIG=<config-root>/restore-drill/config.d/isolated-cluster.xml
|
||||
USAGE
|
||||
}
|
||||
|
||||
MODE="${1:---check}"
|
||||
case "${MODE}" in
|
||||
--check|--apply) ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) usage >&2; exit 64 ;;
|
||||
esac
|
||||
[ "$#" -le 1 ] || { usage >&2; exit 64; }
|
||||
|
||||
TRACE_ID="${TRACE_ID:-}"
|
||||
RUN_ID="${RUN_ID:-}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:-}"
|
||||
|
||||
valid_identity() {
|
||||
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]]
|
||||
}
|
||||
|
||||
valid_positive_integer() {
|
||||
[[ "$1" =~ ^[1-9][0-9]*$ ]]
|
||||
}
|
||||
|
||||
safe_absolute_path() {
|
||||
local path="$1"
|
||||
[[ "${path}" == /* ]] || return 1
|
||||
[[ "${path}" != / && "${path}" != */ ]] || return 1
|
||||
[[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1
|
||||
case "/${path#/}/" in
|
||||
*'//'*|*'/../'*|*'/./'*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
paths_overlap() {
|
||||
local left="$1" right="$2"
|
||||
[[ "${left}" = "${right}" || "${left}" == "${right}"/* || "${right}" == "${left}"/* ]]
|
||||
}
|
||||
|
||||
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
||||
valid_identity "${value}" || {
|
||||
printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2
|
||||
exit 64
|
||||
}
|
||||
done
|
||||
|
||||
for value in \
|
||||
"${SSH_COMMAND_TIMEOUT_SECONDS}" \
|
||||
"${SCP_COMMAND_TIMEOUT_SECONDS}" \
|
||||
"${DOCKER_COMMAND_TIMEOUT_SECONDS}" \
|
||||
"${TIMEOUT_KILL_AFTER_SECONDS}"; do
|
||||
valid_positive_integer "${value}" || {
|
||||
printf 'toolchain command timeouts must be positive integer seconds\n' >&2
|
||||
exit 64
|
||||
}
|
||||
done
|
||||
[ "${SSH_COMMAND_TIMEOUT_SECONDS}" -le 1200 ] \
|
||||
&& [ "${SCP_COMMAND_TIMEOUT_SECONDS}" -le 600 ] \
|
||||
&& [ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" -le 300 ] \
|
||||
&& [ "${TIMEOUT_KILL_AFTER_SECONDS}" -le 60 ] || {
|
||||
printf 'toolchain command timeout exceeds bounded maximum\n' >&2
|
||||
exit 64
|
||||
}
|
||||
|
||||
for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do
|
||||
safe_absolute_path "${path}" || {
|
||||
printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2
|
||||
exit 64
|
||||
}
|
||||
done
|
||||
case "${REMOTE_SCRIPT_DIR}" in /backup/*) ;; *) printf 'script dir must stay under /backup\n' >&2; exit 64 ;; esac
|
||||
case "${REMOTE_CONFIG_ROOT}" in /backup/*) ;; *) printf 'config root must stay under /backup\n' >&2; exit 64 ;; esac
|
||||
case "${RECEIPT_ROOT}" in /backup/*) ;; *) printf 'receipt root must stay under /backup\n' >&2; exit 64 ;; esac
|
||||
case "${STAGE_ROOT}" in /tmp|/tmp/*|/backup/deploy-staging|/backup/deploy-staging/*) ;;
|
||||
*) printf 'stage root must stay under /tmp or /backup/deploy-staging\n' >&2; exit 64 ;;
|
||||
esac
|
||||
if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \
|
||||
|| paths_overlap "${REMOTE_SCRIPT_DIR}" "${RECEIPT_ROOT}" \
|
||||
|| paths_overlap "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" \
|
||||
|| paths_overlap "${STAGE_ROOT}" "${REMOTE_SCRIPT_DIR}" \
|
||||
|| paths_overlap "${STAGE_ROOT}" "${REMOTE_CONFIG_ROOT}" \
|
||||
|| paths_overlap "${STAGE_ROOT}" "${RECEIPT_ROOT}"; then
|
||||
printf 'toolchain target, receipt, and stage paths must not overlap\n' >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
for command_name in bash python3 scp ssh timeout; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 || {
|
||||
printf 'required local command missing: %s\n' "${command_name}" >&2
|
||||
exit 69
|
||||
}
|
||||
done
|
||||
|
||||
|
||||
bounded_ssh() {
|
||||
timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${SSH_COMMAND_TIMEOUT_SECONDS}" ssh "$@"
|
||||
}
|
||||
|
||||
|
||||
bounded_scp() {
|
||||
timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${SCP_COMMAND_TIMEOUT_SECONDS}" scp "$@"
|
||||
}
|
||||
|
||||
hash_file() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 "$1" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
validate_xml_pair() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
backup_path = Path(sys.argv[1])
|
||||
cluster_path = Path(sys.argv[2])
|
||||
backup = ET.parse(backup_path).getroot()
|
||||
cluster = ET.parse(cluster_path).getroot()
|
||||
assert backup.tag == "clickhouse"
|
||||
assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
|
||||
assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
|
||||
assert backup.findtext("./backups/allowed_disk") == "backups"
|
||||
assert backup.findtext("./backups/allowed_path") == "/backups/"
|
||||
assert backup.findtext("./backups/allow_concurrent_backups") == "false"
|
||||
assert backup.findtext("./backups/allow_concurrent_restores") == "false"
|
||||
assert cluster.tag == "clickhouse"
|
||||
assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
|
||||
assert cluster.findtext("./zookeeper/node/port") == "2181"
|
||||
assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
|
||||
assert cluster.findtext("./remote_servers/cluster/shard/replica/port") == "9000"
|
||||
assert cluster.findtext("./macros/shard") == "01"
|
||||
replica = cluster.find("./macros/replica")
|
||||
assert replica is not None
|
||||
assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
|
||||
PY
|
||||
}
|
||||
|
||||
validate_local_sources() {
|
||||
local index source
|
||||
for index in "${!LOCAL_SOURCES[@]}"; do
|
||||
source="${LOCAL_SOURCES[index]}"
|
||||
[ -f "${source}" ] && [ ! -L "${source}" ]
|
||||
[ -s "${source}" ]
|
||||
done
|
||||
|
||||
for index in 0 1 2 4; do
|
||||
bash -n "${LOCAL_SOURCES[index]}"
|
||||
done
|
||||
python3 - "${LOCAL_SOURCES[3]}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
compile(path.read_text(encoding="utf-8"), str(path), "exec")
|
||||
PY
|
||||
validate_xml_pair "${LOCAL_SOURCES[5]}" "${LOCAL_SOURCES[6]}"
|
||||
|
||||
python3 - "${LOCAL_SOURCES[2]}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
text = Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||
for token in (
|
||||
"CLICKHOUSE_RESTORE_INVENTORY_HELPER",
|
||||
"CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG",
|
||||
"CLICKHOUSE_RESTORE_CLUSTER_CONFIG",
|
||||
):
|
||||
assert token in text
|
||||
PY
|
||||
}
|
||||
|
||||
validate_local_sources
|
||||
SOURCE_HASHES=()
|
||||
SOURCE_DETAIL=""
|
||||
for index in "${!LOCAL_SOURCES[@]}"; do
|
||||
source_hash="$(hash_file "${LOCAL_SOURCES[index]}")"
|
||||
SOURCE_HASHES+=("${source_hash}")
|
||||
SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_"
|
||||
done
|
||||
SOURCE_DETAIL="${SOURCE_DETAIL%_}"
|
||||
|
||||
CONFIG_DIR="${REMOTE_CONFIG_ROOT}/config.d"
|
||||
RESTORE_CONFIG_DIR="${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
||||
TARGET_PATHS=(
|
||||
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
||||
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
||||
"${CONFIG_DIR}/backup_disk.xml"
|
||||
"${RESTORE_CONFIG_DIR}/isolated-cluster.xml"
|
||||
)
|
||||
STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-native-backup-toolchain.${RUN_ID}"
|
||||
|
||||
local_receipt() {
|
||||
local phase="$1" result="$2" detail="$3"
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}"
|
||||
}
|
||||
|
||||
remote_read_only_check() {
|
||||
bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \
|
||||
"${DOCKER_COMMAND_TIMEOUT_SECONDS}" "${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${SOURCE_HASHES[@]}" <<'REMOTE_CHECK'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
REMOTE_SCRIPT_DIR="$4"
|
||||
REMOTE_CONFIG_ROOT="$5"
|
||||
RECEIPT_ROOT="$6"
|
||||
STAGE_ROOT="$7"
|
||||
DOCKER_COMMAND_TIMEOUT_SECONDS="$8"
|
||||
TIMEOUT_KILL_AFTER_SECONDS="$9"
|
||||
shift 9
|
||||
EXPECTED_HASHES=("$@")
|
||||
LABELS=(
|
||||
backup-signoz.sh
|
||||
clickhouse-native-backup.sh
|
||||
clickhouse-native-restore-drill.sh
|
||||
clickhouse-restore-inventory.py
|
||||
run-signoz-backup-canary.sh
|
||||
backup_disk.xml
|
||||
isolated-cluster.xml
|
||||
)
|
||||
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
||||
TARGETS=(
|
||||
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
||||
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
||||
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
|
||||
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
|
||||
)
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
|
||||
[ -d /backup ] && [ ! -L /backup ]
|
||||
for command_name in awk curl docker pgrep sha256sum ss stat timeout; do
|
||||
command -v "${command_name}" >/dev/null 2>&1
|
||||
done
|
||||
|
||||
[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]]
|
||||
|
||||
bounded_docker() {
|
||||
timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@"
|
||||
}
|
||||
|
||||
listener_up() {
|
||||
local port="$1"
|
||||
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
||||
}
|
||||
|
||||
validate_existing_ancestors() {
|
||||
local requested="$1" current="" component
|
||||
local -a components
|
||||
IFS='/' read -r -a components <<<"${requested#/}"
|
||||
for component in "${components[@]}"; do
|
||||
[ -n "${component}" ] || continue
|
||||
current="${current}/${component}"
|
||||
if [ -e "${current}" ] || [ -L "${current}" ]; then
|
||||
[ -d "${current}" ] && [ ! -L "${current}" ]
|
||||
else
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
runtime_line() {
|
||||
local container="$1" value running health
|
||||
value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
|
||||
IFS='|' read -r _ _ _ running health <<<"${value}"
|
||||
[ "${running}" = true ]
|
||||
case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
||||
printf 'active SigNoz backup toolchain process detected\n' >&2
|
||||
exit 75
|
||||
fi
|
||||
validate_existing_ancestors "${REMOTE_SCRIPT_DIR}"
|
||||
validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/config.d"
|
||||
validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
||||
validate_existing_ancestors "${RECEIPT_ROOT}"
|
||||
validate_existing_ancestors "${STAGE_ROOT}"
|
||||
[ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ]
|
||||
active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \
|
||||
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
|
||||
[ "${active_operations}" = 0 ]
|
||||
|
||||
runtime_summary=""
|
||||
for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do
|
||||
runtime_summary+="${container}_$(runtime_line "${container}")_"
|
||||
done
|
||||
for port in 4317 4318 8080; do listener_up "${port}"; done
|
||||
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
|
||||
[ "${api_code}" = 200 ]
|
||||
|
||||
matching=0
|
||||
drift=0
|
||||
absent=0
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
target="${TARGETS[index]}"
|
||||
if [ -e "${target}" ] || [ -L "${target}" ]; then
|
||||
[ -f "${target}" ] && [ ! -L "${target}" ]
|
||||
current_hash="$(sha256sum "${target}" | awk '{print $1}')"
|
||||
current_mode="$(stat -c '%a' "${target}")"
|
||||
if [ "${current_hash}" = "${EXPECTED_HASHES[index]}" ] && [ "0${current_mode}" = "${MODES[index]}" ]; then
|
||||
matching=$((matching + 1))
|
||||
else
|
||||
drift=$((drift + 1))
|
||||
fi
|
||||
else
|
||||
absent=$((absent + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"source_of_truth_diff","result":"pass","detail":"matching_%s_drift_%s_absent_%s_active_operations_0_api_200_ports_4317_4318_8080"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${matching}" "${drift}" "${absent}"
|
||||
printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"runtime_readiness","result":"pass","detail":"container_identity_started_at_restart_count_health_captured_%s"}\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${runtime_summary%_}"
|
||||
REMOTE_CHECK
|
||||
}
|
||||
|
||||
local_receipt sensor_source pass "${SOURCE_DETAIL}"
|
||||
local_receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
|
||||
remote_read_only_check
|
||||
local_receipt check pass "local_syntax_xml_python_compile_and_remote_read_only_preflight"
|
||||
|
||||
if [ "${MODE}" = --check ]; then
|
||||
local_receipt terminal check_pass_no_write "remote_target_unchanged_no_stage_no_candidate"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
STAGE_CREATED=0
|
||||
cleanup_stage() {
|
||||
if [ "${STAGE_CREATED}" -eq 1 ]; then
|
||||
bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true
|
||||
set -u
|
||||
stage_dir="$1"
|
||||
if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then
|
||||
rm -rf "${stage_dir}/pycache" "${stage_dir}/candidate-pycache"
|
||||
rm -f \
|
||||
"${stage_dir}/backup-signoz.sh" \
|
||||
"${stage_dir}/clickhouse-native-backup.sh" \
|
||||
"${stage_dir}/clickhouse-native-restore-drill.sh" \
|
||||
"${stage_dir}/clickhouse-restore-inventory.py" \
|
||||
"${stage_dir}/run-signoz-backup-canary.sh" \
|
||||
"${stage_dir}/backup_disk.xml" \
|
||||
"${stage_dir}/isolated-cluster.xml"
|
||||
rmdir "${stage_dir}" 2>/dev/null || true
|
||||
fi
|
||||
REMOTE_CLEANUP
|
||||
fi
|
||||
}
|
||||
trap cleanup_stage EXIT
|
||||
|
||||
bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE'
|
||||
set -euo pipefail
|
||||
stage_dir="$1"
|
||||
[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ]
|
||||
stage_root="${stage_dir%/*}"
|
||||
[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ]
|
||||
umask 077
|
||||
mkdir -m 0700 "${stage_dir}"
|
||||
REMOTE_STAGE
|
||||
STAGE_CREATED=1
|
||||
|
||||
for index in "${!LOCAL_SOURCES[@]}"; do
|
||||
bounded_scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \
|
||||
"${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}"
|
||||
done
|
||||
|
||||
bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \
|
||||
"${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \
|
||||
"${CANARY_LOCK}" "${OPERATION_LOCK}" "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \
|
||||
"${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${SOURCE_HASHES[@]}" <<'REMOTE_APPLY'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
REMOTE_SCRIPT_DIR="$4"
|
||||
REMOTE_CONFIG_ROOT="$5"
|
||||
RECEIPT_ROOT="$6"
|
||||
STAGE_DIR="$7"
|
||||
CANARY_LOCK="$8"
|
||||
OPERATION_LOCK="$9"
|
||||
DOCKER_COMMAND_TIMEOUT_SECONDS="${10}"
|
||||
TIMEOUT_KILL_AFTER_SECONDS="${11}"
|
||||
shift 11
|
||||
EXPECTED_HASHES=("$@")
|
||||
LABELS=(
|
||||
backup-signoz.sh
|
||||
clickhouse-native-backup.sh
|
||||
clickhouse-native-restore-drill.sh
|
||||
clickhouse-restore-inventory.py
|
||||
run-signoz-backup-canary.sh
|
||||
backup_disk.xml
|
||||
isolated-cluster.xml
|
||||
)
|
||||
MODES=(0755 0755 0755 0755 0755 0644 0644)
|
||||
STAGED=(
|
||||
"${STAGE_DIR}/backup-signoz.sh"
|
||||
"${STAGE_DIR}/clickhouse-native-backup.sh"
|
||||
"${STAGE_DIR}/clickhouse-native-restore-drill.sh"
|
||||
"${STAGE_DIR}/clickhouse-restore-inventory.py"
|
||||
"${STAGE_DIR}/run-signoz-backup-canary.sh"
|
||||
"${STAGE_DIR}/backup_disk.xml"
|
||||
"${STAGE_DIR}/isolated-cluster.xml"
|
||||
)
|
||||
TARGETS=(
|
||||
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh"
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py"
|
||||
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh"
|
||||
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml"
|
||||
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml"
|
||||
)
|
||||
TARGET_DIRS=(
|
||||
"${REMOTE_SCRIPT_DIR}"
|
||||
"${REMOTE_CONFIG_ROOT}/config.d"
|
||||
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d"
|
||||
)
|
||||
CONTAINERS=(signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1)
|
||||
API_URL="http://127.0.0.1:8080/api/v1/health"
|
||||
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
||||
RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl"
|
||||
PRIOR_DIR="${RECEIPT_DIR}/prior"
|
||||
PRIOR_MANIFEST="${RECEIPT_DIR}/prior-manifest.tsv"
|
||||
DEPLOYED_MANIFEST="${RECEIPT_DIR}/deployed-manifest.tsv"
|
||||
RUNTIME_BEFORE="${RECEIPT_DIR}/runtime-before.tsv"
|
||||
RUNTIME_AFTER="${RECEIPT_DIR}/runtime-after.tsv"
|
||||
LOCK_FILE="/tmp/awoooi-signoz-native-backup-toolchain.lock"
|
||||
|
||||
APPLY_STARTED=0
|
||||
DEPLOY_VERIFIED=0
|
||||
RUNTIME_BASELINE_CAPTURED=0
|
||||
LOCK_HELD=0
|
||||
CANARY_LOCK_HELD=0
|
||||
OPERATION_LOCK_HELD=0
|
||||
REMOTE_UID="$(id -u)"
|
||||
REMOTE_GID="$(id -g)"
|
||||
CREATED_DIRS=()
|
||||
PRIOR_PRESENT=()
|
||||
PRIOR_MODES=()
|
||||
PRIOR_UIDS=()
|
||||
PRIOR_GIDS=()
|
||||
PRIOR_HASHES=()
|
||||
|
||||
for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat timeout; do
|
||||
command -v "${command_name}" >/dev/null 2>&1 || {
|
||||
printf 'required remote command missing: %s\n' "${command_name}" >&2
|
||||
exit 69
|
||||
}
|
||||
done
|
||||
if [ "${REMOTE_UID}" -ne 0 ]; then
|
||||
command -v sudo >/dev/null 2>&1 || { printf 'required remote command missing: sudo\n' >&2; exit 69; }
|
||||
sudo -n true
|
||||
fi
|
||||
[ "${#EXPECTED_HASHES[@]}" -eq 7 ]
|
||||
[ -d /backup ] && [ ! -L /backup ]
|
||||
[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]]
|
||||
[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]]
|
||||
[ "${CANARY_LOCK}" = "/tmp/awoooi-signoz-backup-canary.lock" ]
|
||||
[ "${OPERATION_LOCK}" = "/tmp/awoooi-signoz-backup-operation.lock" ]
|
||||
|
||||
run_root() {
|
||||
if [ "${REMOTE_UID}" -eq 0 ]; then
|
||||
"$@"
|
||||
else
|
||||
sudo -n "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
bounded_docker() {
|
||||
timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \
|
||||
"${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@"
|
||||
}
|
||||
|
||||
root_hash() {
|
||||
if [ "${REMOTE_UID}" -eq 0 ]; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
else
|
||||
sudo -n sha256sum "$1" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_dir_tree() {
|
||||
local requested="$1" record_created="$2" current="" component
|
||||
local -a components
|
||||
IFS='/' read -r -a components <<<"${requested#/}"
|
||||
for component in "${components[@]}"; do
|
||||
[ -n "${component}" ] || continue
|
||||
current="${current}/${component}"
|
||||
if [ -e "${current}" ] || [ -L "${current}" ]; then
|
||||
[ -d "${current}" ] && [ ! -L "${current}" ]
|
||||
else
|
||||
run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0755 "${current}"
|
||||
if [ "${record_created}" = true ]; then
|
||||
CREATED_DIRS+=("${current}")
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
ensure_dir_tree "${RECEIPT_ROOT}" false
|
||||
[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ]
|
||||
run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0700 "${RECEIPT_DIR}"
|
||||
install -d -m 0700 "${PRIOR_DIR}"
|
||||
: > "${RECEIPT_LOG}"
|
||||
|
||||
receipt() {
|
||||
local phase="$1" result="$2" detail="$3" line
|
||||
line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")"
|
||||
printf '%s\n' "${line}"
|
||||
printf '%s\n' "${line}" >> "${RECEIPT_LOG}"
|
||||
}
|
||||
|
||||
listener_up() {
|
||||
local port="$1"
|
||||
ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }'
|
||||
}
|
||||
|
||||
runtime_snapshot() {
|
||||
local container value running health port api_code
|
||||
for container in "${CONTAINERS[@]}"; do
|
||||
value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")"
|
||||
IFS='|' read -r _ _ _ running health <<<"${value}"
|
||||
[ "${running}" = true ]
|
||||
case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac
|
||||
printf 'container\t%s\t%s\n' "${container}" "${value}"
|
||||
done
|
||||
for port in 4317 4318 8080; do
|
||||
listener_up "${port}"
|
||||
printf 'listener\t%s\tup\n' "${port}"
|
||||
done
|
||||
api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")"
|
||||
[ "${api_code}" = 200 ]
|
||||
printf 'api\thealth\t%s\n' "${api_code}"
|
||||
}
|
||||
|
||||
validate_xml_pair() {
|
||||
python3 - "$1" "$2" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
backup_path = Path(sys.argv[1])
|
||||
cluster_path = Path(sys.argv[2])
|
||||
backup = ET.parse(backup_path).getroot()
|
||||
cluster = ET.parse(cluster_path).getroot()
|
||||
assert backup.findtext("./storage_configuration/disks/backups/type") == "local"
|
||||
assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/"
|
||||
assert backup.findtext("./backups/allowed_disk") == "backups"
|
||||
assert backup.findtext("./backups/allowed_path") == "/backups/"
|
||||
assert backup.findtext("./backups/allow_concurrent_backups") == "false"
|
||||
assert backup.findtext("./backups/allow_concurrent_restores") == "false"
|
||||
assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper"
|
||||
assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse"
|
||||
assert cluster.findtext("./macros/shard") == "01"
|
||||
replica = cluster.find("./macros/replica")
|
||||
assert replica is not None
|
||||
assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"}
|
||||
PY
|
||||
}
|
||||
|
||||
cleanup_transients() {
|
||||
local index rc=0
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
run_root rm -f \
|
||||
"${TARGETS[index]}.candidate.${RUN_ID}" \
|
||||
"${TARGETS[index]}.rollback.${RUN_ID}" || rc=1
|
||||
done
|
||||
if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then
|
||||
rm -rf "${STAGE_DIR}/pycache" "${STAGE_DIR}/candidate-pycache" || rc=1
|
||||
rm -f "${STAGED[@]}" || rc=1
|
||||
rmdir "${STAGE_DIR}" 2>/dev/null || rc=1
|
||||
else
|
||||
rc=1
|
||||
fi
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
verify_residue_zero() {
|
||||
local index
|
||||
[ ! -e "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
[ ! -e "${TARGETS[index]}.candidate.${RUN_ID}" ] \
|
||||
&& [ ! -L "${TARGETS[index]}.candidate.${RUN_ID}" ] || return 1
|
||||
[ ! -e "${TARGETS[index]}.rollback.${RUN_ID}" ] \
|
||||
&& [ ! -L "${TARGETS[index]}.rollback.${RUN_ID}" ] || return 1
|
||||
done
|
||||
}
|
||||
|
||||
rollback_targets() (
|
||||
set -euo pipefail
|
||||
local index target rollback_candidate actual_mode actual_hash
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
target="${TARGETS[index]}"
|
||||
if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
|
||||
rollback_candidate="${target}.rollback.${RUN_ID}"
|
||||
run_root install \
|
||||
-o "${PRIOR_UIDS[index]}" -g "${PRIOR_GIDS[index]}" \
|
||||
-m "${PRIOR_MODES[index]}" \
|
||||
"${PRIOR_DIR}/${LABELS[index]}" "${rollback_candidate}"
|
||||
[ "$(root_hash "${rollback_candidate}")" = "${PRIOR_HASHES[index]}" ]
|
||||
run_root mv -f "${rollback_candidate}" "${target}"
|
||||
actual_mode="$(stat -c '%a' "${target}")"
|
||||
actual_hash="$(root_hash "${target}")"
|
||||
[ "${actual_mode}" = "${PRIOR_MODES[index]}" ]
|
||||
[ "${actual_hash}" = "${PRIOR_HASHES[index]}" ]
|
||||
else
|
||||
run_root rm -f "${target}"
|
||||
[ ! -e "${target}" ] && [ ! -L "${target}" ]
|
||||
fi
|
||||
done
|
||||
|
||||
for ((index=${#CREATED_DIRS[@]} - 1; index >= 0; index--)); do
|
||||
run_root rmdir "${CREATED_DIRS[index]}"
|
||||
done
|
||||
|
||||
if [ "${RUNTIME_BASELINE_CAPTURED}" -eq 1 ]; then
|
||||
runtime_snapshot > "${RECEIPT_DIR}/runtime-rollback.tsv"
|
||||
cmp -s "${RUNTIME_BEFORE}" "${RECEIPT_DIR}/runtime-rollback.tsv"
|
||||
fi
|
||||
receipt rollback pass "all_prior_files_modes_hashes_owners_and_runtime_restored"
|
||||
)
|
||||
|
||||
on_exit() {
|
||||
local rc=$? rollback_rc=0 cleanup_rc=0 residue_rc=0 terminal=failed
|
||||
trap - EXIT HUP INT TERM
|
||||
set +e
|
||||
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
||||
rollback_targets
|
||||
rollback_rc=$?
|
||||
if [ "${rollback_rc}" -ne 0 ]; then
|
||||
receipt rollback failed "prior_state_restore_or_runtime_verifier_failed"
|
||||
rc=91
|
||||
fi
|
||||
elif [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 0 ]; then
|
||||
receipt rollback no_write "bounded_execution_not_started_targets_unchanged"
|
||||
fi
|
||||
cleanup_transients
|
||||
cleanup_rc=$?
|
||||
verify_residue_zero
|
||||
residue_rc=$?
|
||||
if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then
|
||||
rc=92
|
||||
fi
|
||||
if [ "${OPERATION_LOCK_HELD}" -eq 1 ]; then
|
||||
flock -u 7 >/dev/null 2>&1
|
||||
exec 7>&-
|
||||
fi
|
||||
if [ "${CANARY_LOCK_HELD}" -eq 1 ]; then
|
||||
flock -u 8 >/dev/null 2>&1
|
||||
exec 8>&-
|
||||
fi
|
||||
if [ "${LOCK_HELD}" -eq 1 ]; then
|
||||
flock -u 9 >/dev/null 2>&1
|
||||
exec 9>&-
|
||||
rm -f "${LOCK_FILE}"
|
||||
fi
|
||||
if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then
|
||||
terminal=pass
|
||||
fi
|
||||
receipt terminal "${terminal}" "exit_${rc}_stage_candidate_residue_0_receipt_ref_${RECEIPT_LOG}"
|
||||
exit "${rc}"
|
||||
}
|
||||
|
||||
trap on_exit EXIT
|
||||
trap 'exit 129' HUP
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
exec 9>>"${LOCK_FILE}"
|
||||
flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; }
|
||||
LOCK_HELD=1
|
||||
|
||||
[ ! -L "${CANARY_LOCK}" ] \
|
||||
|| { receipt check failed canary_lock_symlink_unsafe; exit 75; }
|
||||
if [ -e "${CANARY_LOCK}" ]; then
|
||||
[ -f "${CANARY_LOCK}" ] \
|
||||
|| { receipt check failed canary_lock_not_regular_file; exit 75; }
|
||||
fi
|
||||
exec 8>>"${CANARY_LOCK}"
|
||||
flock -n 8 \
|
||||
|| { receipt check failed another_signoz_canary_holds_canary_lock; exit 75; }
|
||||
CANARY_LOCK_HELD=1
|
||||
|
||||
[ ! -L "${OPERATION_LOCK}" ] \
|
||||
|| { receipt check failed backup_operation_lock_symlink_unsafe; exit 75; }
|
||||
if [ -e "${OPERATION_LOCK}" ]; then
|
||||
[ -f "${OPERATION_LOCK}" ] \
|
||||
|| { receipt check failed backup_operation_lock_not_regular_file; exit 75; }
|
||||
fi
|
||||
exec 7>>"${OPERATION_LOCK}"
|
||||
flock -n 7 \
|
||||
|| { receipt check failed another_signoz_backup_holds_operation_lock; exit 75; }
|
||||
OPERATION_LOCK_HELD=1
|
||||
|
||||
if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then
|
||||
receipt check failed active_signoz_backup_toolchain_process
|
||||
exit 75
|
||||
fi
|
||||
active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \
|
||||
"SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)"
|
||||
[ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; }
|
||||
|
||||
runtime_snapshot > "${RUNTIME_BEFORE}"
|
||||
RUNTIME_BASELINE_CAPTURED=1
|
||||
|
||||
[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]
|
||||
for index in "${!STAGED[@]}"; do
|
||||
[ -f "${STAGED[index]}" ] && [ ! -L "${STAGED[index]}" ]
|
||||
[ "$(sha256sum "${STAGED[index]}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ]
|
||||
done
|
||||
for index in 0 1 2 4; do bash -n "${STAGED[index]}"; done
|
||||
PYTHONPYCACHEPREFIX="${STAGE_DIR}/pycache" python3 -m py_compile "${STAGED[3]}"
|
||||
validate_xml_pair "${STAGED[5]}" "${STAGED[6]}"
|
||||
|
||||
: > "${PRIOR_MANIFEST}"
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
target="${TARGETS[index]}"
|
||||
[ ! -e "${target}.candidate.${RUN_ID}" ] && [ ! -L "${target}.candidate.${RUN_ID}" ]
|
||||
[ ! -e "${target}.rollback.${RUN_ID}" ] && [ ! -L "${target}.rollback.${RUN_ID}" ]
|
||||
if [ -e "${target}" ] || [ -L "${target}" ]; then
|
||||
[ -f "${target}" ] && [ ! -L "${target}" ]
|
||||
PRIOR_PRESENT[index]=1
|
||||
PRIOR_MODES[index]="$(stat -c '%a' "${target}")"
|
||||
PRIOR_UIDS[index]="$(stat -c '%u' "${target}")"
|
||||
PRIOR_GIDS[index]="$(stat -c '%g' "${target}")"
|
||||
PRIOR_HASHES[index]="$(root_hash "${target}")"
|
||||
run_root cp -p "${target}" "${PRIOR_DIR}/${LABELS[index]}"
|
||||
[ "$(root_hash "${PRIOR_DIR}/${LABELS[index]}")" = "${PRIOR_HASHES[index]}" ]
|
||||
else
|
||||
PRIOR_PRESENT[index]=0
|
||||
PRIOR_MODES[index]=absent
|
||||
PRIOR_UIDS[index]=absent
|
||||
PRIOR_GIDS[index]=absent
|
||||
PRIOR_HASHES[index]=absent
|
||||
fi
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"${LABELS[index]}" "${target}" "${PRIOR_PRESENT[index]}" \
|
||||
"${PRIOR_MODES[index]}" "${PRIOR_UIDS[index]}" "${PRIOR_GIDS[index]}" \
|
||||
"${PRIOR_HASHES[index]}" >> "${PRIOR_MANIFEST}"
|
||||
done
|
||||
|
||||
receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources"
|
||||
receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2"
|
||||
receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}"
|
||||
receipt ai_decision candidate "same_filesystem_candidate_replace_native_backup_toolchain_only"
|
||||
receipt risk_policy pass "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_backup_operation_locks_no_runtime_execution_no_container_change_full_rollback"
|
||||
receipt check pass "dual_signoz_canary_and_backup_operation_locks_held_runtime_baseline_active_operations_0_hash_bash_xml_py_compile"
|
||||
|
||||
APPLY_STARTED=1
|
||||
for directory in "${TARGET_DIRS[@]}"; do
|
||||
ensure_dir_tree "${directory}" true
|
||||
done
|
||||
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
target="${TARGETS[index]}"
|
||||
if [ "${PRIOR_PRESENT[index]}" = 1 ]; then
|
||||
target_uid="${PRIOR_UIDS[index]}"
|
||||
target_gid="${PRIOR_GIDS[index]}"
|
||||
else
|
||||
target_uid="${REMOTE_UID}"
|
||||
target_gid="${REMOTE_GID}"
|
||||
fi
|
||||
run_root install -o "${target_uid}" -g "${target_gid}" -m "${MODES[index]}" \
|
||||
"${STAGED[index]}" "${target}.candidate.${RUN_ID}"
|
||||
[ "$(root_hash "${target}.candidate.${RUN_ID}")" = "${EXPECTED_HASHES[index]}" ]
|
||||
[ "0$(stat -c '%a' "${target}.candidate.${RUN_ID}")" = "${MODES[index]}" ]
|
||||
done
|
||||
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"
|
||||
done
|
||||
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
[ -f "${TARGETS[index]}" ] && [ ! -L "${TARGETS[index]}" ]
|
||||
[ "$(root_hash "${TARGETS[index]}")" = "${EXPECTED_HASHES[index]}" ]
|
||||
[ "0$(stat -c '%a' "${TARGETS[index]}")" = "${MODES[index]}" ]
|
||||
done
|
||||
for index in 0 1 2 4; do bash -n "${TARGETS[index]}"; done
|
||||
PYTHONPYCACHEPREFIX="${STAGE_DIR}/candidate-pycache" python3 -m py_compile "${TARGETS[3]}"
|
||||
validate_xml_pair "${TARGETS[5]}" "${TARGETS[6]}"
|
||||
|
||||
: > "${DEPLOYED_MANIFEST}"
|
||||
for index in "${!TARGETS[@]}"; do
|
||||
printf '%s\t%s\t%s\t%s\n' \
|
||||
"${LABELS[index]}" "${TARGETS[index]}" "${MODES[index]}" \
|
||||
"${EXPECTED_HASHES[index]}" >> "${DEPLOYED_MANIFEST}"
|
||||
done
|
||||
|
||||
runtime_snapshot > "${RUNTIME_AFTER}"
|
||||
cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"
|
||||
receipt execution pass "seven_same_filesystem_candidates_replaced_with_full_rollback_expected_hashes_and_modes"
|
||||
receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged"
|
||||
receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull"
|
||||
DEPLOY_VERIFIED=1
|
||||
REMOTE_APPLY
|
||||
|
||||
trap - EXIT
|
||||
cleanup_stage
|
||||
local_receipt terminal pass "remote_controlled_apply_postverify_and_stage_candidate_residue_0"
|
||||
218
scripts/ops/deploy-signoz-prometheus-route.sh
Executable file
218
scripts/ops/deploy-signoz-prometheus-route.sh
Executable file
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deploy only the canonical SigNoz blackbox route to the host110 Prometheus.
|
||||
# The full Prometheus file intentionally remains untouched because production
|
||||
# contains unrelated, independently governed scrape targets.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
|
||||
RUN_ID="${RUN_ID:?RUN_ID is required}"
|
||||
WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
|
||||
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
|
||||
PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
|
||||
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
|
||||
REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}"
|
||||
MODE="${1:-apply}"
|
||||
|
||||
case "${MODE}" in
|
||||
apply|--dry-run) ;;
|
||||
*) printf 'usage: %s [apply|--dry-run]\n' "$0" >&2; exit 64 ;;
|
||||
esac
|
||||
|
||||
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
||||
case "${value}" in
|
||||
*[!A-Za-z0-9._-]*|'')
|
||||
printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
|
||||
bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
||||
"${PROMETHEUS_URL}" "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
TRACE_ID="$1"
|
||||
RUN_ID="$2"
|
||||
WORK_ITEM_ID="$3"
|
||||
MODE="$4"
|
||||
PROMETHEUS_URL="$5"
|
||||
PROMETHEUS_CONTAINER="$6"
|
||||
REMOTE_CONFIG="$7"
|
||||
REMOTE_CANDIDATE="/tmp/awoooi-prometheus-signoz.${RUN_ID}.yml"
|
||||
RUNTIME_CANDIDATE="/etc/prometheus/.awoooi-prometheus-signoz.${RUN_ID}.yml"
|
||||
BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}"
|
||||
APPLY_STARTED=0
|
||||
DEPLOY_VERIFIED=0
|
||||
|
||||
cleanup_candidate() {
|
||||
rm -f "${REMOTE_CANDIDATE}"
|
||||
docker exec -u 0 "${PROMETHEUS_CONTAINER}" \
|
||||
rm -f "${RUNTIME_CANDIDATE}" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
rollback_deploy() {
|
||||
cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}"
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
||||
printf 'rollback_verifier trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}"
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
local rc=$?
|
||||
trap - EXIT
|
||||
set +e
|
||||
cleanup_candidate
|
||||
if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \
|
||||
&& [ "${DEPLOY_VERIFIED}" -eq 0 ]; then
|
||||
rollback_deploy
|
||||
fi
|
||||
exit "${rc}"
|
||||
}
|
||||
trap on_exit EXIT
|
||||
|
||||
test -f "${REMOTE_CONFIG}"
|
||||
test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
||||
|
||||
python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
source = Path(sys.argv[1])
|
||||
candidate = Path(sys.argv[2])
|
||||
text = source.read_text(encoding="utf-8")
|
||||
|
||||
canonical = " - http://192.168.0.110:8080/api/v1/health\n"
|
||||
legacy = " - 192.168.0.188:3301\n"
|
||||
anchor = " - http://192.168.0.110:3001\n"
|
||||
|
||||
|
||||
def block_bounds(payload: str, job: str) -> tuple[int, int]:
|
||||
marker = f' - job_name: "{job}"\n'
|
||||
start = payload.find(marker)
|
||||
if start < 0 or payload.find(marker, start + 1) >= 0:
|
||||
raise SystemExit(f"expected exactly one {job} job")
|
||||
end = payload.find("\n - job_name:", start + len(marker))
|
||||
return start, len(payload) if end < 0 else end + 1
|
||||
|
||||
|
||||
http_start, http_end = block_bounds(text, "blackbox-http")
|
||||
http_block = text[http_start:http_end]
|
||||
if http_block.count(canonical) > 1 or http_block.count(anchor) != 1:
|
||||
raise SystemExit("canonical HTTP target or insertion anchor is ambiguous")
|
||||
if canonical not in http_block:
|
||||
http_block = http_block.replace(anchor, anchor + canonical, 1)
|
||||
text = text[:http_start] + http_block + text[http_end:]
|
||||
|
||||
tcp_start, tcp_end = block_bounds(text, "blackbox-tcp")
|
||||
tcp_block = text[tcp_start:tcp_end]
|
||||
if tcp_block.count(legacy) > 1:
|
||||
raise SystemExit("legacy TCP target is ambiguous")
|
||||
tcp_block = tcp_block.replace(legacy, "", 1)
|
||||
text = text[:tcp_start] + tcp_block + text[tcp_end:]
|
||||
|
||||
http_start, http_end = block_bounds(text, "blackbox-http")
|
||||
tcp_start, tcp_end = block_bounds(text, "blackbox-tcp")
|
||||
if text[http_start:http_end].count(canonical) != 1:
|
||||
raise SystemExit("canonical HTTP target was not normalized exactly once")
|
||||
if legacy in text[tcp_start:tcp_end]:
|
||||
raise SystemExit("legacy TCP target remains after normalization")
|
||||
|
||||
candidate.write_text(text, encoding="utf-8")
|
||||
PY
|
||||
|
||||
docker cp "${REMOTE_CANDIDATE}" \
|
||||
"${PROMETHEUS_CONTAINER}:${RUNTIME_CANDIDATE}" >/dev/null
|
||||
docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${RUNTIME_CANDIDATE}"
|
||||
|
||||
SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
||||
CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')"
|
||||
printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
||||
"${SOURCE_SHA}" "${CANDIDATE_SHA}"
|
||||
|
||||
if [ "${MODE}" = "--dry-run" ]; then
|
||||
printf 'terminal trace_id=%s run_id=%s status=check_pass_no_write\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}"
|
||||
APPLY_STARTED=1
|
||||
cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}"
|
||||
|
||||
ACTIVE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')"
|
||||
RUNTIME_SHA="$(docker exec "${PROMETHEUS_CONTAINER}" \
|
||||
sha256sum /etc/prometheus/prometheus.yml | awk '{print $1}')"
|
||||
test "${ACTIVE_SHA}" = "${CANDIDATE_SHA}"
|
||||
test "${RUNTIME_SHA}" = "${CANDIDATE_SHA}"
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
|
||||
curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null
|
||||
printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=target_first_config_apply active_sha=%s runtime_sha=%s ready=true backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${ACTIVE_SHA}" \
|
||||
"${RUNTIME_SHA}" "${BACKUP_CONFIG}"
|
||||
|
||||
target_state() {
|
||||
curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c '
|
||||
import json, sys
|
||||
payload = json.load(sys.stdin)
|
||||
targets = payload["data"]["activeTargets"]
|
||||
canonical = [t for t in targets if t.get("labels", {}).get("job") == "blackbox-http" and t.get("labels", {}).get("instance") == "http://192.168.0.110:8080/api/v1/health"]
|
||||
legacy = [t for t in targets if t.get("labels", {}).get("instance") == "192.168.0.188:3301"]
|
||||
if len(canonical) == 1:
|
||||
target = canonical[0]
|
||||
print("{}|{}|{}|{}".format(
|
||||
1,
|
||||
target.get("health", "unknown"),
|
||||
target.get("lastScrape", "never"),
|
||||
len(legacy),
|
||||
))
|
||||
else:
|
||||
print(f"{len(canonical)}|missing|never|{len(legacy)}")
|
||||
'
|
||||
}
|
||||
|
||||
FIRST_SCRAPE=""
|
||||
for attempt in $(seq 1 9); do
|
||||
IFS='|' read -r count health last_scrape legacy_count <<<"$(target_state)"
|
||||
printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s legacy_count=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \
|
||||
"${last_scrape}" "${legacy_count}"
|
||||
if [ "${count}" = 1 ] && [ "${health}" = up ] \
|
||||
&& [ "${legacy_count}" = 0 ] && [ "${last_scrape}" != never ]; then
|
||||
FIRST_SCRAPE="${last_scrape}"
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
test -n "${FIRST_SCRAPE}"
|
||||
|
||||
sleep 16
|
||||
IFS='|' read -r count health SECOND_SCRAPE legacy_count <<<"$(target_state)"
|
||||
printf 'target_probe trace_id=%s run_id=%s sample=post_cooldown count=%s health=%s last_scrape=%s legacy_count=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${count}" "${health}" \
|
||||
"${SECOND_SCRAPE}" "${legacy_count}"
|
||||
test "${count}" = 1
|
||||
test "${health}" = up
|
||||
test "${legacy_count}" = 0
|
||||
test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}"
|
||||
|
||||
PROBE_VALUE="$(curl -fsS --get \
|
||||
--data-urlencode 'query=probe_success{job="blackbox-http",instance="http://192.168.0.110:8080/api/v1/health"}' \
|
||||
"${PROMETHEUS_URL}/api/v1/query" | python3 -c '
|
||||
import json, sys
|
||||
result = json.load(sys.stdin)["data"]["result"]
|
||||
print(result[0]["value"][1] if len(result) == 1 else "invalid")
|
||||
')"
|
||||
test "${PROBE_VALUE}" = 1
|
||||
|
||||
DEPLOY_VERIFIED=1
|
||||
printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=target_up_two_scrapes first=%s second=%s probe_success=%s legacy_target_count=0\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \
|
||||
"${SECOND_SCRAPE}" "${PROBE_VALUE}"
|
||||
printf 'terminal trace_id=%s run_id=%s status=completed_target_first backup=%s\n' \
|
||||
"${TRACE_ID}" "${RUN_ID}" "${BACKUP_CONFIG}"
|
||||
REMOTE
|
||||
405
scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
Normal file
405
scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py
Normal file
@@ -0,0 +1,405 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml"
|
||||
OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml"
|
||||
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh"
|
||||
BACKUP_SCRIPT = ROOT / "scripts/backup/backup-signoz.sh"
|
||||
|
||||
|
||||
def write_executable(path: Path, text: str) -> None:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def executable_text() -> str:
|
||||
return "\n".join(
|
||||
line
|
||||
for line in DEPLOYER.read_text(encoding="utf-8").splitlines()
|
||||
if not line.lstrip().startswith("#")
|
||||
)
|
||||
|
||||
|
||||
def test_clickhouse_config_declares_one_allowed_local_backup_disk() -> None:
|
||||
root = ET.parse(CONFIG).getroot()
|
||||
assert root.tag == "clickhouse"
|
||||
|
||||
disks = root.findall("./storage_configuration/disks/*")
|
||||
assert [disk.tag for disk in disks] == ["backups"]
|
||||
assert disks[0].findtext("type") == "local"
|
||||
assert disks[0].findtext("path") == "/backups/"
|
||||
|
||||
backups = root.find("./backups")
|
||||
assert backups is not None
|
||||
assert backups.findtext("allowed_disk") == "backups"
|
||||
assert backups.findtext("allowed_path") == "/backups/"
|
||||
assert backups.findtext("allow_concurrent_backups") == "false"
|
||||
assert backups.findtext("allow_concurrent_restores") == "false"
|
||||
assert CONFIG.read_text(encoding="utf-8").count("<allowed_disk>") == 1
|
||||
|
||||
|
||||
def test_compose_override_is_additive_clickhouse_only_and_uses_dedicated_mounts() -> (
|
||||
None
|
||||
):
|
||||
text = OVERRIDE.read_text(encoding="utf-8")
|
||||
assert text.count(" clickhouse:") == 1
|
||||
assert "otel-collector:" not in text
|
||||
assert "signoz:" not in text
|
||||
assert "zookeeper:" not in text
|
||||
assert "source: /backup/staging/signoz-clickhouse" in text
|
||||
assert "target: /backups" in text
|
||||
assert (
|
||||
"source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml"
|
||||
) in text
|
||||
assert ("target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml") in text
|
||||
assert text.count("read_only: true") == 1
|
||||
assert text.count("read_only: false") == 1
|
||||
for forbidden in ("image:", "build:", "command:", "environment:"):
|
||||
assert forbidden not in text
|
||||
|
||||
|
||||
def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
|
||||
syntax = subprocess.run(
|
||||
["bash", "-n", str(DEPLOYER)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert syntax.returncode == 0, syntax.stderr
|
||||
|
||||
help_result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--help"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
||||
},
|
||||
)
|
||||
assert help_result.returncode == 0
|
||||
assert "--check" in help_result.stdout
|
||||
assert "--apply" in help_result.stdout
|
||||
|
||||
|
||||
def test_deployer_requires_all_three_controlled_apply_identifiers_before_ssh() -> None:
|
||||
env = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
||||
}
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "must be path-safe" in result.stderr
|
||||
assert "ssh:" not in result.stderr
|
||||
|
||||
|
||||
def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
check_terminal = text.index('if [ "${MODE}" = "--check" ]')
|
||||
stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.')
|
||||
first_transfer = text.index('bounded_scp "${LOCAL_OVERRIDE}"')
|
||||
|
||||
assert check_terminal < stage_assignment < first_transfer
|
||||
assert "-f '${REMOTE_BASE}' -f - config -q" in text
|
||||
assert '< "${LOCAL_OVERRIDE}"' in text
|
||||
assert "check_pass_no_write" in text
|
||||
|
||||
|
||||
def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
executable = executable_text()
|
||||
|
||||
assert 'REMOTE_SERVICE="clickhouse"' in text
|
||||
assert "--env-file /dev/null" in text
|
||||
assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text
|
||||
assert "bounded_ssh" in text
|
||||
assert "bounded_scp" in text
|
||||
assert "docker_cmd" in text
|
||||
assert "compose_config_cmd" in text
|
||||
assert "compose_apply_cmd" in text
|
||||
assert "REMOTE_APPLY_TIMEOUT_SECONDS=3600" in text
|
||||
assert "up -d --no-deps --pull never --force-recreate" in text
|
||||
assert '"${REMOTE_SERVICE}"' in text
|
||||
assert "docker compose down" not in executable
|
||||
assert "docker pull" not in executable
|
||||
assert "docker volume" not in executable
|
||||
assert "docker system prune" not in executable
|
||||
assert "github.com" not in text.lower()
|
||||
assert "ghcr.io" not in text.lower()
|
||||
|
||||
|
||||
def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "active_signoz_backup_or_restore_process_detected" in text
|
||||
assert "active_native_backup_or_restore" in text
|
||||
assert "signoz_backup_process_probe_failed" in text
|
||||
assert "partial_managed_file_drift" in text
|
||||
assert "another_deploy_holds_lock" in text
|
||||
for process in (
|
||||
"backup-signoz",
|
||||
"clickhouse-native-backup",
|
||||
"clickhouse-native-restore-drill",
|
||||
"run-signoz-backup-canary",
|
||||
):
|
||||
assert process in text
|
||||
probe_block = text[
|
||||
text.index("require_backup_idle()") : text.index(
|
||||
"require_backup_idle preflight"
|
||||
)
|
||||
]
|
||||
assert "process_rc=$?" in probe_block
|
||||
assert 'case "${process_rc}" in' in probe_block
|
||||
assert "1) ;;" in probe_block
|
||||
assert "return 69" in probe_block
|
||||
assert "CREATING_BACKUP" in text
|
||||
assert "RESTORING" in text
|
||||
|
||||
|
||||
def test_target_host_is_fixed_and_ignores_ambient_override(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
ssh_log = tmp_path / "ssh.log"
|
||||
write_executable(
|
||||
fake_bin / "ssh",
|
||||
"""#!/bin/bash
|
||||
printf '%s\\n' "$*" >> "${FAKE_SSH_LOG:?}"
|
||||
cat >/dev/null
|
||||
""",
|
||||
)
|
||||
write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"PATH": f"{fake_bin}:{env['PATH']}",
|
||||
"TRACE_ID": "P0-OBS-002-test",
|
||||
"RUN_ID": "fixed-host-test",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"TARGET_HOST": "attacker@example.invalid",
|
||||
"FAKE_SSH_LOG": str(ssh_log),
|
||||
}
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
invocations = ssh_log.read_text(encoding="utf-8").splitlines()
|
||||
assert len(invocations) == 2
|
||||
assert all("wooo@192.168.0.110" in line for line in invocations)
|
||||
assert all("attacker@example.invalid" not in line for line in invocations)
|
||||
|
||||
|
||||
def test_all_transport_and_remote_docker_operations_are_bounded() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
executable = executable_text()
|
||||
|
||||
assert executable.count(' ssh "${SSH_OPTS[@]}"') == 1
|
||||
assert executable.count(' scp -q "${SSH_OPTS[@]}"') == 1
|
||||
assert not re.search(r"\bdocker (inspect|exec)\b", executable)
|
||||
assert 'timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}"' in text
|
||||
assert 'docker_cmd exec "${CLICKHOUSE_CONTAINER}"' in text
|
||||
assert "ss_cmd -H -lnt" in text
|
||||
for mutation in (
|
||||
"bounded_cmd cp -p",
|
||||
"bounded_cmd install -m",
|
||||
"bounded_cmd mv -f",
|
||||
"bounded_cmd sudo -n install",
|
||||
"bounded_cmd rm -f",
|
||||
):
|
||||
assert mutation in text
|
||||
|
||||
|
||||
def test_stage_cleanup_failure_changes_apply_terminal_to_failed(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
ssh_count = tmp_path / "ssh-count"
|
||||
write_executable(
|
||||
fake_bin / "ssh",
|
||||
"""#!/bin/bash
|
||||
count=0
|
||||
if [ -f "${FAKE_SSH_COUNT:?}" ]; then
|
||||
count="$(cat "${FAKE_SSH_COUNT}")"
|
||||
fi
|
||||
count=$((count + 1))
|
||||
printf '%s\\n' "${count}" > "${FAKE_SSH_COUNT}"
|
||||
cat >/dev/null
|
||||
if [ "${count}" -eq 5 ]; then
|
||||
exit 1
|
||||
fi
|
||||
""",
|
||||
)
|
||||
write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n")
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"PATH": f"{fake_bin}:{env['PATH']}",
|
||||
"TRACE_ID": "P0-OBS-002-test",
|
||||
"RUN_ID": "cleanup-failure-test",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
"FAKE_SSH_COUNT": str(ssh_count),
|
||||
}
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--apply"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
assert result.returncode == 92
|
||||
assert '"phase":"cleanup","result":"failed"' in result.stdout
|
||||
assert '"phase":"terminal","result":"failed"' in result.stdout
|
||||
assert "failed_or_residue_present" in result.stdout
|
||||
|
||||
|
||||
def test_apply_preserves_image_data_volume_and_non_target_containers() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "BEFORE_IMAGE_ID" in text
|
||||
assert "AFTER_IMAGE_ID" in text
|
||||
assert "BEFORE_DATA_VOLUME" in text
|
||||
assert "AFTER_DATA_VOLUME" in text
|
||||
assert 'eq .Destination "/var/lib/clickhouse"' in text
|
||||
for prefix in ("COLLECTOR", "SIGNOZ", "ZOOKEEPER"):
|
||||
assert f"BEFORE_{prefix}_ID" in text
|
||||
assert f"BEFORE_{prefix}_RESTARTS" in text
|
||||
assert "image_and_data_volume_unchanged" in text
|
||||
|
||||
|
||||
def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "system.disks WHERE name='backups'" in text
|
||||
assert "Local|0|0|0|1" in text
|
||||
assert "EXISTS TABLE system.backups" in text
|
||||
assert "SHOW GRANTS" in text
|
||||
assert "BACKUP([,[:space:]]|$)" in text
|
||||
assert 'eq .Destination "/backups"' in text
|
||||
assert "101:101:750" in text
|
||||
assert "server_uid" in text
|
||||
assert "server_gid" in text
|
||||
assert "docker_cmd exec -u 101:101" in text
|
||||
assert "test -w /backups" in text
|
||||
assert "awk '/^Uid:/{print $2}' /proc/1/status" in text
|
||||
assert "awk '/^Gid:/{print $2}' /proc/1/status" in text
|
||||
identity_start = text.index('server_uid="')
|
||||
identity_block = text[
|
||||
identity_start : text.index("docker_cmd exec -u 101:101", identity_start)
|
||||
]
|
||||
assert "sh -c" not in identity_block
|
||||
assert "concurrency_disabled" in text
|
||||
for port in (4317, 4318, 8080):
|
||||
assert f"listener_up {port}" in text
|
||||
assert "api_code" in text
|
||||
assert "disk_backups_local_rw_healthy" in text
|
||||
|
||||
|
||||
def test_failed_apply_restores_prior_managed_compose_state() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
assert "restore_managed_files" in text
|
||||
assert "rollback_deploy" in text
|
||||
assert "PRIOR_OVERRIDE_PRESENT" in text
|
||||
assert "PRIOR_CONFIG_PRESENT" in text
|
||||
assert "prior-override.yaml" in text
|
||||
assert "prior-backup-disk.xml" in text
|
||||
assert "prior_compose_state_restored" in text
|
||||
assert "retained_nonempty_no_delete" in text
|
||||
assert "rc=91" in text
|
||||
assert "trap on_exit EXIT" in text
|
||||
for token in (
|
||||
"PRIOR_OVERRIDE_HASH",
|
||||
"PRIOR_OVERRIDE_MODE",
|
||||
"PRIOR_OVERRIDE_UID",
|
||||
"PRIOR_OVERRIDE_GID",
|
||||
"PRIOR_CONFIG_HASH",
|
||||
"PRIOR_CONFIG_MODE",
|
||||
"PRIOR_CONFIG_UID",
|
||||
"PRIOR_CONFIG_GID",
|
||||
"file_matches_metadata",
|
||||
"restore_exact_file",
|
||||
"prior_compose_state_restored_exact_hash_mode_uid_gid",
|
||||
):
|
||||
assert token in text
|
||||
assert "exact_stage_candidate_and_rollback_residue_absence_verified" in text
|
||||
|
||||
|
||||
def test_deployer_and_backup_execution_share_operation_lock_contract() -> None:
|
||||
deployer = DEPLOYER.read_text(encoding="utf-8")
|
||||
backup = BACKUP_SCRIPT.read_text(encoding="utf-8")
|
||||
operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
|
||||
canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
|
||||
|
||||
assert operation_lock in deployer
|
||||
assert operation_lock in backup
|
||||
assert canary_lock in deployer
|
||||
assert 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' in deployer
|
||||
assert 'exec 7>>"${CANARY_LOCK_FILE}"' in deployer
|
||||
assert "flock -n 8 || { receipt check failed backup_operation_lock_held" in deployer
|
||||
assert "flock -n 7 || { receipt check failed backup_canary_lock_held" in deployer
|
||||
assert 'exec 8>>"${OPERATION_LOCK_FILE}"' in backup
|
||||
assert "require_backup_idle pre_mutation" in deployer
|
||||
assert deployer.index('exec 7>>"${CANARY_LOCK_FILE}"') < deployer.index(
|
||||
'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"'
|
||||
)
|
||||
assert deployer.index('exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"') < deployer.index(
|
||||
"require_backup_idle pre_mutation"
|
||||
)
|
||||
remote_cleanup_call = deployer.rindex(" cleanup_owned_residue")
|
||||
assert remote_cleanup_call < deployer.index(
|
||||
'receipt terminal "${terminal}"', remote_cleanup_call
|
||||
)
|
||||
|
||||
|
||||
def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None:
|
||||
executable = executable_text()
|
||||
lowered = executable.lower()
|
||||
assert ".config.env" not in lowered
|
||||
assert "config.env" not in lowered
|
||||
assert ".env" not in lowered
|
||||
assert "docker inspect --format '{{json .config.env}}'" not in lowered
|
||||
assert " backup all" not in lowered
|
||||
assert " restore all" not in lowered
|
||||
assert "clickhouse-client --query 'backup" not in lowered
|
||||
assert "clickhouse-client --query 'restore" not in lowered
|
||||
assert "no_backup_or_restore_executed" in executable
|
||||
|
||||
|
||||
def test_receipts_cover_controlled_apply_and_durable_terminal() -> None:
|
||||
text = DEPLOYER.read_text(encoding="utf-8")
|
||||
for phase in (
|
||||
"sensor_source",
|
||||
"normalized_asset_identity",
|
||||
"source_of_truth_diff",
|
||||
"ai_decision",
|
||||
"risk_policy",
|
||||
"check",
|
||||
"execution",
|
||||
"post_verifier",
|
||||
"rollback",
|
||||
"closure_writeback",
|
||||
"terminal",
|
||||
):
|
||||
assert phase in text
|
||||
assert "receipts.jsonl" in text
|
||||
assert "override_sha_" in text
|
||||
assert "config_sha_" in text
|
||||
475
scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
Normal file
475
scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py
Normal file
@@ -0,0 +1,475 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh"
|
||||
BACKUP = ROOT / "scripts/backup/backup-signoz.sh"
|
||||
CANARY = ROOT / "scripts/backup/run-signoz-backup-canary.sh"
|
||||
|
||||
|
||||
def deployer_text() -> str:
|
||||
return DEPLOYER.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def executable_text() -> str:
|
||||
return "\n".join(
|
||||
line
|
||||
for line in deployer_text().splitlines()
|
||||
if not line.lstrip().startswith("#")
|
||||
)
|
||||
|
||||
|
||||
def base_env() -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"TRACE_ID": "P0-OBS-002",
|
||||
"RUN_ID": "toolchain-test-run",
|
||||
"WORK_ITEM_ID": "P0-OBS-002",
|
||||
}
|
||||
)
|
||||
return env
|
||||
|
||||
|
||||
def test_deployer_shell_is_valid_and_help_is_no_write() -> None:
|
||||
syntax = subprocess.run(
|
||||
["bash", "-n", str(DEPLOYER)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert syntax.returncode == 0, syntax.stderr
|
||||
|
||||
help_result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--help"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
||||
},
|
||||
)
|
||||
assert help_result.returncode == 0
|
||||
assert "--check" in help_result.stdout
|
||||
assert "--apply" in help_result.stdout
|
||||
assert "CLICKHOUSE_RESTORE_INVENTORY_HELPER" in help_result.stdout
|
||||
assert "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" in help_result.stdout
|
||||
assert "CLICKHOUSE_RESTORE_CLUSTER_CONFIG" in help_result.stdout
|
||||
|
||||
|
||||
def test_deployer_requires_controlled_apply_ids_before_network_access() -> None:
|
||||
env = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"}
|
||||
}
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "must be path-safe" in result.stderr
|
||||
assert "ssh:" not in result.stderr
|
||||
|
||||
|
||||
def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> None:
|
||||
env = base_env()
|
||||
env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/scripts/../escape"
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "safe absolute paths" in result.stderr
|
||||
assert "ssh:" not in result.stderr
|
||||
|
||||
env = base_env()
|
||||
env["SIGNOZ_TOOLCHAIN_RECEIPT_ROOT"] = "/var/tmp/receipts"
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "receipt root must stay under /backup" in result.stderr
|
||||
|
||||
env = base_env()
|
||||
env["SIGNOZ_TOOLCHAIN_STAGE_ROOT"] = "/backup/deploy-staging"
|
||||
env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/deploy-staging/scripts"
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert "must not overlap" in result.stderr
|
||||
|
||||
|
||||
def test_timeout_overrides_fail_closed_before_network_access() -> None:
|
||||
for key, value, expected in (
|
||||
(
|
||||
"SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS",
|
||||
"0",
|
||||
"positive integer seconds",
|
||||
),
|
||||
(
|
||||
"SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS",
|
||||
"301",
|
||||
"exceeds bounded maximum",
|
||||
),
|
||||
):
|
||||
env = base_env()
|
||||
env[key] = value
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 64
|
||||
assert expected in result.stderr
|
||||
assert "ssh:" not in result.stderr
|
||||
|
||||
|
||||
def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None:
|
||||
text = deployer_text()
|
||||
assert 'TARGET_HOST="wooo@192.168.0.110"' in text
|
||||
assert "TARGET_HOST:-" not in text
|
||||
assert "TARGET_HOST=" not in text.replace('TARGET_HOST="wooo@192.168.0.110"', "", 1)
|
||||
|
||||
|
||||
def test_exact_seven_artifacts_and_target_modes_are_declared() -> None:
|
||||
text = deployer_text()
|
||||
for source in (
|
||||
"scripts/backup/backup-signoz.sh",
|
||||
"scripts/backup/clickhouse-native-backup.sh",
|
||||
"scripts/backup/clickhouse-native-restore-drill.sh",
|
||||
"scripts/backup/clickhouse-restore-inventory.py",
|
||||
"scripts/backup/run-signoz-backup-canary.sh",
|
||||
"ops/signoz/clickhouse/config.d/backup_disk.xml",
|
||||
"ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml",
|
||||
):
|
||||
assert source in text
|
||||
|
||||
for target in (
|
||||
"${REMOTE_SCRIPT_DIR}/backup-signoz.sh",
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh",
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh",
|
||||
"${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py",
|
||||
"${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh",
|
||||
"${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml",
|
||||
"${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml",
|
||||
):
|
||||
assert target in text
|
||||
assert "MODES=(0755 0755 0755 0755 0755 0644 0644)" in text
|
||||
assert 'REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"' in text
|
||||
assert (
|
||||
'REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-'
|
||||
'/backup/config/signoz/clickhouse}"'
|
||||
) in text
|
||||
|
||||
|
||||
def test_local_and_remote_stage_validation_cover_hash_syntax_python_and_xml() -> None:
|
||||
text = deployer_text()
|
||||
assert "validate_local_sources" in text
|
||||
assert "SOURCE_HASHES" in text
|
||||
assert "bash -n" in text
|
||||
assert "python3 -m py_compile" in text
|
||||
assert "validate_xml_pair" in text
|
||||
for expected in (
|
||||
'"/backups/"',
|
||||
'"restore-zookeeper"',
|
||||
'"restore-clickhouse"',
|
||||
'"CLICKHOUSE_RESTORE_REPLICA"',
|
||||
):
|
||||
assert expected in text
|
||||
|
||||
|
||||
def test_check_mode_exits_before_remote_stage_or_scp() -> None:
|
||||
text = deployer_text()
|
||||
check_exit = text.index('if [ "${MODE}" = --check ]')
|
||||
stage_create = text.index("STAGE_CREATED=0")
|
||||
first_scp = text.index("scp -q")
|
||||
assert check_exit < stage_create < first_scp
|
||||
assert "check_pass_no_write" in text
|
||||
assert "remote_target_unchanged_no_stage_no_candidate" in text
|
||||
|
||||
remote_check = text[
|
||||
text.index("remote_read_only_check() {") : text.index(
|
||||
"local_receipt sensor_source", text.index("remote_read_only_check() {")
|
||||
)
|
||||
]
|
||||
for forbidden in ("mkdir ", "install ", "mv ", "rm ", "scp "):
|
||||
assert forbidden not in remote_check
|
||||
|
||||
|
||||
def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
log = tmp_path / "calls.log"
|
||||
ssh = fake_bin / "ssh"
|
||||
ssh.write_text(
|
||||
"#!/bin/sh\n"
|
||||
'printf "ssh %s\\n" "$*" >> "$FAKE_CALL_LOG"\n'
|
||||
"cat >/dev/null\n"
|
||||
'printf \'{"phase":"source_of_truth_diff","result":"pass"}\\n\'\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
scp = fake_bin / "scp"
|
||||
scp.write_text(
|
||||
'#!/bin/sh\nprintf "scp %s\\n" "$*" >> "$FAKE_CALL_LOG"\nexit 99\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
ssh.chmod(0o755)
|
||||
scp.chmod(0o755)
|
||||
|
||||
env = base_env()
|
||||
env["PATH"] = f"{fake_bin}:{env['PATH']}"
|
||||
env["FAKE_CALL_LOG"] = str(log)
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
calls = log.read_text(encoding="utf-8").splitlines()
|
||||
assert len(calls) == 1
|
||||
assert calls[0].startswith("ssh ")
|
||||
assert "wooo@192.168.0.110" in calls[0]
|
||||
assert "check_pass_no_write" in result.stdout
|
||||
|
||||
|
||||
def test_hung_ssh_is_bounded_and_cannot_claim_check_pass(tmp_path: Path) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
ssh = fake_bin / "ssh"
|
||||
ssh.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8")
|
||||
scp = fake_bin / "scp"
|
||||
scp.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8")
|
||||
ssh.chmod(0o755)
|
||||
scp.chmod(0o755)
|
||||
|
||||
env = base_env()
|
||||
env["PATH"] = f"{fake_bin}:{env['PATH']}"
|
||||
env["SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS"] = "1"
|
||||
env["SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS"] = "1"
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(
|
||||
["bash", str(DEPLOYER), "--check"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
timeout=5,
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
assert result.returncode == 124
|
||||
assert elapsed < 4
|
||||
assert "check_pass_no_write" not in result.stdout
|
||||
|
||||
|
||||
def test_all_ssh_scp_and_docker_calls_use_bounded_wrappers() -> None:
|
||||
text = deployer_text()
|
||||
executable = executable_text()
|
||||
|
||||
assert (
|
||||
'SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"'
|
||||
in text
|
||||
)
|
||||
assert (
|
||||
'SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"'
|
||||
in text
|
||||
)
|
||||
assert (
|
||||
'DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"'
|
||||
in text
|
||||
)
|
||||
assert (
|
||||
'TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"'
|
||||
in text
|
||||
)
|
||||
assert "bounded_ssh()" in text
|
||||
assert "bounded_scp()" in text
|
||||
assert text.count('ssh "$@"') == 1
|
||||
assert text.count('scp "$@"') == 1
|
||||
assert executable.count("bounded_docker()") == 2
|
||||
assert "$(docker inspect" not in executable
|
||||
assert "$(docker exec" not in executable
|
||||
assert "\n docker inspect" not in executable
|
||||
assert "\n docker exec" not in executable
|
||||
assert executable.count("bounded_docker inspect") == 2
|
||||
assert executable.count("bounded_docker exec") == 2
|
||||
assert "timeout --signal=TERM" in executable
|
||||
|
||||
|
||||
def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None:
|
||||
text = deployer_text()
|
||||
for token in (
|
||||
"prior-manifest.tsv",
|
||||
"PRIOR_PRESENT",
|
||||
"PRIOR_MODES",
|
||||
"PRIOR_UIDS",
|
||||
"PRIOR_GIDS",
|
||||
"PRIOR_HASHES",
|
||||
"cp -p",
|
||||
"rollback_targets",
|
||||
"all_prior_files_modes_hashes_owners_and_runtime_restored",
|
||||
):
|
||||
assert token in text
|
||||
assert "trap on_exit EXIT" in text
|
||||
assert "rc=91" in text
|
||||
assert 'run_root rm -f "${target}"' in text
|
||||
assert 'run_root rmdir "${CREATED_DIRS[index]}"' in text
|
||||
assert "bounded_execution_not_started_targets_unchanged" in text
|
||||
|
||||
|
||||
def test_apply_uses_same_filesystem_candidates_and_accurate_replace_receipt() -> None:
|
||||
text = deployer_text()
|
||||
assert '"${target}.candidate.${RUN_ID}"' in text
|
||||
assert (
|
||||
'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"'
|
||||
) in text
|
||||
assert "deployed-manifest.tsv" in text
|
||||
assert (
|
||||
"seven_same_filesystem_candidates_replaced_with_full_rollback_"
|
||||
"expected_hashes_and_modes"
|
||||
) in text
|
||||
assert "seven_candidates_atomically_replaced" not in text
|
||||
assert "atomically replaces exact targets" not in text
|
||||
assert "stage_candidate_residue_0" in text
|
||||
assert "verify_residue_zero" in text
|
||||
|
||||
|
||||
def test_apply_holds_dual_backup_locks_through_postverify_or_rollback() -> None:
|
||||
text = deployer_text()
|
||||
backup_text = BACKUP.read_text(encoding="utf-8")
|
||||
canary_text = CANARY.read_text(encoding="utf-8")
|
||||
canary_lock = "/tmp/awoooi-signoz-backup-canary.lock"
|
||||
operation_lock = "/tmp/awoooi-signoz-backup-operation.lock"
|
||||
|
||||
assert f'CANARY_LOCK="{canary_lock}"' in text
|
||||
assert f'OPERATION_LOCK="{operation_lock}"' in text
|
||||
assert canary_lock in canary_text
|
||||
assert operation_lock in backup_text
|
||||
canary_acquire = text.index('exec 8>>"${CANARY_LOCK}"')
|
||||
operation_acquire = text.index('exec 7>>"${OPERATION_LOCK}"')
|
||||
active_check = text.index("if pgrep -af", operation_acquire)
|
||||
apply_started = text.index("APPLY_STARTED=1", operation_acquire)
|
||||
target_replace = text.index(
|
||||
'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}"', apply_started
|
||||
)
|
||||
assert canary_acquire < operation_acquire < active_check < apply_started
|
||||
assert apply_started < target_replace
|
||||
|
||||
on_exit = text[text.index("on_exit() {") : text.index("trap on_exit EXIT")]
|
||||
rollback = on_exit.index("rollback_targets")
|
||||
release_operation = on_exit.index("flock -u 7")
|
||||
release_canary = on_exit.index("flock -u 8")
|
||||
assert rollback < release_operation < release_canary
|
||||
assert 'rm -f "${OPERATION_LOCK}"' not in text
|
||||
assert 'rm -f "${CANARY_LOCK}"' not in text
|
||||
assert "another_signoz_backup_holds_operation_lock" in text
|
||||
assert "another_signoz_canary_holds_canary_lock" in text
|
||||
assert (
|
||||
"risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_"
|
||||
"backup_operation_locks_"
|
||||
"no_runtime_execution_no_container_change_full_rollback"
|
||||
) in text
|
||||
|
||||
|
||||
def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None:
|
||||
text = deployer_text()
|
||||
for container in (
|
||||
"signoz-clickhouse",
|
||||
"signoz-otel-collector",
|
||||
"signoz",
|
||||
"signoz-zookeeper-1",
|
||||
):
|
||||
assert container in text
|
||||
for field in (
|
||||
".Id",
|
||||
".State.StartedAt",
|
||||
".RestartCount",
|
||||
".State.Running",
|
||||
'index .State "Health"',
|
||||
):
|
||||
assert field in text
|
||||
assert "runtime-before.tsv" in text
|
||||
assert "runtime-after.tsv" in text
|
||||
assert 'cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"' in text
|
||||
for port in (4317, 4318, 8080):
|
||||
assert "for port in 4317 4318 8080" in text
|
||||
assert str(port) in text
|
||||
assert "api_200_unchanged" in text
|
||||
|
||||
|
||||
def test_apply_fails_closed_while_backup_or_restore_is_active() -> None:
|
||||
text = deployer_text()
|
||||
assert "active_signoz_backup_toolchain_process" in text
|
||||
assert "active_native_backup_or_restore" in text
|
||||
assert "CREATING_BACKUP" in text
|
||||
assert "RESTORING" in text
|
||||
assert "another_toolchain_deploy_holds_lock" in text
|
||||
|
||||
|
||||
def test_deployer_never_executes_runtime_work_or_uses_forbidden_supply_chain() -> None:
|
||||
executable = executable_text().lower()
|
||||
for forbidden in (
|
||||
"docker restart",
|
||||
"docker compose",
|
||||
"docker pull",
|
||||
"docker run",
|
||||
"backup all",
|
||||
"restore all",
|
||||
"ansible-playbook",
|
||||
"github.com",
|
||||
"api.github.com",
|
||||
"raw.githubusercontent.com",
|
||||
"codeload.github.com",
|
||||
"ghcr.io",
|
||||
" gh ",
|
||||
):
|
||||
assert forbidden not in executable
|
||||
assert "no_backup_restore_restart_or_pull" in executable
|
||||
|
||||
|
||||
def test_receipts_cover_complete_controlled_apply_contract() -> None:
|
||||
text = deployer_text()
|
||||
for phase in (
|
||||
"sensor_source",
|
||||
"normalized_asset_identity",
|
||||
"source_of_truth_diff",
|
||||
"ai_decision",
|
||||
"risk_policy",
|
||||
"check",
|
||||
"execution",
|
||||
"post_verifier",
|
||||
"rollback",
|
||||
"closure_writeback",
|
||||
"terminal",
|
||||
):
|
||||
assert phase in text
|
||||
assert "receipts.jsonl" in text
|
||||
assert '"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"' in text
|
||||
assert "durable_manifests_and_receipts_ack" in text
|
||||
@@ -265,6 +265,12 @@ require_file scripts/backup/backup-awoooi-frequent.sh "AWOOOI high-frequency DB
|
||||
require_file scripts/backup/backup-langfuse.sh "Langfuse backup script"
|
||||
require_file scripts/backup/backup-monitoring.sh "Monitoring backup script"
|
||||
require_file scripts/backup/backup-signoz.sh "SignOz backup script"
|
||||
require_file scripts/backup/clickhouse-native-backup.sh "ClickHouse native backup adapter"
|
||||
require_file scripts/backup/clickhouse-native-restore-drill.sh "ClickHouse isolated native restore drill"
|
||||
require_file scripts/backup/clickhouse-restore-inventory.py "ClickHouse restore inventory verifier"
|
||||
require_file scripts/backup/run-signoz-backup-canary.sh "Guarded SignOz backup canary wrapper"
|
||||
require_file ops/signoz/clickhouse/config.d/backup_disk.xml "ClickHouse backup disk config"
|
||||
require_file ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml "ClickHouse isolated restore config"
|
||||
require_file scripts/backup/backup-open-webui.sh "Open-WebUI backup script"
|
||||
require_file scripts/backup/backup-clawbot.sh "ClawBot backup script"
|
||||
require_file scripts/backup/backup-configs.sh "Host/service config backup"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -25,9 +27,21 @@ GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml"
|
||||
POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml"
|
||||
|
||||
|
||||
def _load_preflight_module():
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"signoz_canonical_route_preflight_under_test", PREFLIGHT
|
||||
)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None:
|
||||
payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
||||
assert payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
|
||||
assert (
|
||||
payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1"
|
||||
)
|
||||
assert payload["work_item_id"] == "P0-OBS-002"
|
||||
assert payload["phase"] == "wave_a_dual_allow"
|
||||
assert payload["routes"]["active_route"] == "incumbent"
|
||||
@@ -43,14 +57,33 @@ def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() ->
|
||||
)
|
||||
|
||||
|
||||
def test_contract_contains_no_credential_values_or_destructive_db_plan() -> None:
|
||||
text = CONTRACT.read_text(encoding="utf-8").lower()
|
||||
assert "signoz_user_root_password:" not in text
|
||||
assert "signoz-api-key:" not in text
|
||||
assert "authorization:" not in text
|
||||
assert "raw_relational_database_update" in text
|
||||
assert "stateful_volume_restore" in text
|
||||
assert "apply_allowed_in_wave_a: false" in text
|
||||
def test_contract_and_receipt_contain_no_secret_values_or_github_dependency() -> None:
|
||||
contract_text = CONTRACT.read_text(encoding="utf-8").lower()
|
||||
regression_text = POST_CLOSURE_REGRESSION.read_text(encoding="utf-8").lower()
|
||||
forbidden = (
|
||||
"signoz_user_root_password:",
|
||||
"signoz-api-key:",
|
||||
"authorization:",
|
||||
"password:",
|
||||
"private_key:",
|
||||
"cookie:",
|
||||
"session:",
|
||||
"github.com",
|
||||
"api.github.com",
|
||||
"raw.githubusercontent.com",
|
||||
"codeload.github.com",
|
||||
"ghcr.io",
|
||||
)
|
||||
for text in (contract_text, regression_text):
|
||||
assert all(value not in text for value in forbidden)
|
||||
|
||||
assert "raw_relational_database_update" in contract_text
|
||||
assert "stateful_volume_restore" in contract_text
|
||||
assert "apply_allowed_in_wave_a: false" in contract_text
|
||||
|
||||
regression = yaml.safe_load(regression_text)
|
||||
assert regression["scope_boundaries"]["secret_read"] is False
|
||||
assert regression["scope_boundaries"]["github_used"] is False
|
||||
|
||||
|
||||
def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
|
||||
@@ -65,13 +98,50 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None:
|
||||
assert payload["writes_performed"] == 0
|
||||
assert payload["terminal"] == "pass"
|
||||
assert payload["runtime_status"] == "partial_degraded"
|
||||
assert payload["backup_toolchain_current_source_status"] == "deployed_verified"
|
||||
assert payload["backup_toolchain_current_source_deployed"] is True
|
||||
assert payload["backup_toolchain_runtime_verifier_terminal"] == "pass"
|
||||
assert payload["backup_toolchain_source_drift_fields"] == [
|
||||
"backup_orchestrator_sha256",
|
||||
"canary_wrapper_sha256",
|
||||
"inventory_verifier_sha256",
|
||||
]
|
||||
assert payload["active_route"] == "incumbent"
|
||||
assert payload["checks"] and all(payload["checks"].values())
|
||||
assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True
|
||||
assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True
|
||||
assert payload["checks"]["post_closure_schema_v2"] is True
|
||||
assert payload["checks"]["native_backup_current_source_hashes_bound"] is True
|
||||
assert (
|
||||
payload["checks"]["native_backup_current_source_deployment_state_explicit"]
|
||||
is True
|
||||
)
|
||||
assert payload["checks"]["native_backup_current_source_runtime_closed"] is True
|
||||
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
|
||||
assert payload["checks"]["post_release_600s_verifier_consistent"] is True
|
||||
assert (
|
||||
payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"]
|
||||
is True
|
||||
)
|
||||
assert payload["checks"]["service_registry_runtime_mirror_drift_explicit"]
|
||||
assert payload["checks"]["github_freeze_legacy_asset_drift_explicit"]
|
||||
assert payload["checks"]["monitoring_generator_duplicate_identity_drift_explicit"]
|
||||
assert "organization_not_initialized" in payload["blockers"]
|
||||
assert "signoz_control_plane_health_route_not_deployed" in payload["blockers"]
|
||||
assert "wave_b_not_authorized" in payload["blockers"]
|
||||
assert payload["blockers"] == [
|
||||
"organization_not_initialized",
|
||||
"direct_ingress_policy_not_closed",
|
||||
"http_bridge_not_supervised",
|
||||
"filelog_normalization_degraded",
|
||||
"wave_b_not_authorized",
|
||||
]
|
||||
for closed_blocker in (
|
||||
"api_runtime_cooldown_pending",
|
||||
"cd_probe_safety_fix_not_deployed",
|
||||
"backup_collector_restore_guard_not_deployed",
|
||||
"signoz_control_plane_health_route_not_deployed",
|
||||
):
|
||||
assert closed_blocker not in payload["blockers"]
|
||||
|
||||
|
||||
def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None:
|
||||
@@ -129,7 +199,10 @@ def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion()
|
||||
assert snapshot["trace_id"] == parity["trace_id"]
|
||||
assert snapshot["route_parity_terminal"] == "pass"
|
||||
assert snapshot["program_terminal"] == "partial_degraded"
|
||||
assert snapshot["controlled_apply_contract"]["learning_writeback"]["status"] == "partial"
|
||||
assert (
|
||||
snapshot["controlled_apply_contract"]["learning_writeback"]["status"]
|
||||
== "partial"
|
||||
)
|
||||
assert (
|
||||
snapshot["controlled_apply_contract"]["learning_writeback"][
|
||||
"durable_gitea_feature_acknowledgement"
|
||||
@@ -151,13 +224,13 @@ def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() ->
|
||||
text = OTEL_DAEMONSET.read_text(encoding="utf-8")
|
||||
assert "id: cri_parser" in text
|
||||
assert "on_error: send_quiet" not in text
|
||||
assert 'if: \'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"\'' in text
|
||||
assert "if: 'body matches \"^[^ ]+ (stdout|stderr) [^ ]* .*$\"'" in text
|
||||
assert "id: mark_cri_parsed" in text
|
||||
assert 'value: parsed_cri' in text
|
||||
assert "value: parsed_cri" in text
|
||||
assert "id: move_cri_body" in text
|
||||
assert "if: attributes.log != nil" in text
|
||||
assert "id: mark_cri_fallback" in text
|
||||
assert 'value: fallback_unparsed' in text
|
||||
assert "value: fallback_unparsed" in text
|
||||
assert 'if: attributes["awoooi.log.parse_status"] == nil' in text
|
||||
|
||||
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
||||
@@ -231,11 +304,10 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier()
|
||||
apply_block = bridge.split(" --apply)", 1)[1].split(" --status)", 1)[0]
|
||||
|
||||
learning = (
|
||||
'receipt learning partial '
|
||||
'"application_trace_metric_10m_post_verifier_pending"'
|
||||
'receipt learning partial "application_trace_metric_10m_post_verifier_pending"'
|
||||
)
|
||||
terminal = (
|
||||
'receipt terminal partial '
|
||||
"receipt terminal partial "
|
||||
'"grpc_transport_restored_independent_600s_post_verifier_pending"'
|
||||
)
|
||||
assert learning in apply_block
|
||||
@@ -243,9 +315,8 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier()
|
||||
assert apply_block.index(learning) < apply_block.index(terminal)
|
||||
assert "receipt terminal pass" not in apply_block
|
||||
assert (
|
||||
'receipt terminal pass '
|
||||
'"real_application_otlp_grpc_trace_metric_freshness_closed"'
|
||||
in verifier
|
||||
"receipt terminal pass "
|
||||
'"real_application_otlp_grpc_trace_metric_freshness_closed"' in verifier
|
||||
)
|
||||
|
||||
|
||||
@@ -266,7 +337,7 @@ def test_contract_grpc_route_state_matches_verified_runtime_evidence() -> None:
|
||||
assert grpc["source_fix"]["runtime_verifier_window_seconds"] >= 600
|
||||
|
||||
|
||||
def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None:
|
||||
def test_grpc_runtime_verifier_uses_real_aggregates_and_ten_minute_gate() -> None:
|
||||
text = GRPC_VERIFIER.read_text(encoding="utf-8")
|
||||
for expected in (
|
||||
'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"',
|
||||
@@ -293,10 +364,7 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
|
||||
assert grpc["recent_awoooi_api_metric_sample_count"] == 60180
|
||||
assert grpc["http_only_verifier_blind_spot"] is True
|
||||
assert grpc["source_fix"]["check_terminal"] == "pass"
|
||||
assert (
|
||||
grpc["source_fix"]["runtime_apply_status"]
|
||||
== "applied_post_verifier_pass"
|
||||
)
|
||||
assert grpc["source_fix"]["runtime_apply_status"] == "applied_post_verifier_pass"
|
||||
assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass"
|
||||
|
||||
receipt = yaml.safe_load(GRPC_RUNTIME_RECEIPT.read_text(encoding="utf-8"))
|
||||
@@ -312,13 +380,657 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate(
|
||||
|
||||
regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
assert regression["baseline"]["verifier_terminal"] == "pass"
|
||||
assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False
|
||||
assert (
|
||||
regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"]
|
||||
is True
|
||||
)
|
||||
assert regression["controlled_recovery"]["stateful_services_touched"] == 0
|
||||
assert regression["controlled_recovery"]["grpc_4317_listening"] is True
|
||||
assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False
|
||||
assert regression["terminal"]["status"] == "partial_degraded"
|
||||
|
||||
|
||||
def test_post_closure_v2_records_cd_5140_and_config_route_patch() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
assert receipt["schema"] == "awoooi_signoz_post_closure_regression_v2"
|
||||
|
||||
cd = receipt["release_receipts"]["gitea_cd_5140"]
|
||||
assert cd["gitea_run_id"] == 5140
|
||||
assert cd["source_sha"] == "e4da6570a645e504cc93647d396f45f167b3172a"
|
||||
assert cd["workflow_terminal"] == "success"
|
||||
assert cd["focused_tests_passed"] == 6
|
||||
assert cd["unit_tests"] == {"passed": 4817, "skipped": 23, "warnings": 69}
|
||||
assert cd["integration_tests_passed"] == 5
|
||||
assert cd["production_readback"]["build_tag_matched"] is True
|
||||
assert cd["production_readback"]["desired_tag_matched"] is True
|
||||
assert cd["production_readback"]["rollout_terminal"] == "success"
|
||||
assert cd["postdeploy"]["alert_chain_terminal"] == "pass"
|
||||
assert cd["postdeploy"]["playwright_passed"] == 5
|
||||
assert cd["cd_probe"]["receipt_provenance"] == "gitea_run_5140_job_log"
|
||||
assert cd["cd_probe"]["restart_free"] is True
|
||||
assert cd["cd_probe"]["workers"] == 4
|
||||
assert cd["cd_probe"]["safety_reserve_connections"] == 4
|
||||
|
||||
route = receipt["controlled_applies"]["config_route_patch"]
|
||||
assert route["run_id"] == ("P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1")
|
||||
assert route["source_commit"] == "9a9d1464a586ba172d1613db5bc285def43eb3e8"
|
||||
assert route["included_in_release_sha"] == cd["source_sha"]
|
||||
assert route["normalized_asset"]["internal_url"] == "http://192.168.0.110:8080"
|
||||
assert route["normalized_asset"]["public_url"] == "https://signoz.wooo.work"
|
||||
assert route["source_truth_diff"]["otlp_producer_route_changed"] is False
|
||||
assert route["execution_receipt"]["deployment_status"] == "deployed_verified"
|
||||
assert route["terminal"] == "applied_post_verifier_pass"
|
||||
|
||||
|
||||
def test_prometheus_target_first_receipt_has_independent_verifier() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
route = receipt["controlled_applies"]["prometheus_target_route"]
|
||||
check = route["check_receipt"]
|
||||
execution = route["execution_receipt"]
|
||||
post = route["post_verifier"]
|
||||
independent = route["independent_verifier"]
|
||||
|
||||
assert route["run_id"] == ("P0-OBS-002-prometheus-route-20260714T215106Z-apply1")
|
||||
assert check["promtool"] == "pass"
|
||||
assert check["candidate_sha"] == execution["active_sha"] == execution["runtime_sha"]
|
||||
assert execution["action"] == "target_first_config_apply"
|
||||
assert execution["backup_sha"] == check["source_sha"]
|
||||
assert post["terminal"] == "target_up_two_scrapes"
|
||||
assert post["first_scrape_at"] == "2026-07-15T05:52:38.146935853+08:00"
|
||||
assert post["second_scrape_at"] == "2026-07-15T05:52:53.146935853+08:00"
|
||||
assert post["first_scrape_at"] < post["second_scrape_at"]
|
||||
assert post["target_count"] == 1
|
||||
assert post["target_health"] == "up"
|
||||
assert post["probe_success"] == 1
|
||||
assert post["legacy_target_count"] == 0
|
||||
assert independent["mode"] == "independent_readback"
|
||||
assert independent["active_sha_matches_runtime"] is True
|
||||
assert independent["target_count"] == 1
|
||||
assert independent["target_health"] == "up"
|
||||
assert independent["probe_success"] == 1
|
||||
assert independent["legacy_target_count"] == 0
|
||||
assert independent["terminal"] == "pass"
|
||||
assert route["rollback"] == {"required": False, "available": True}
|
||||
assert route["terminal"] == "completed_target_first"
|
||||
|
||||
|
||||
def test_alert_rules_receipt_hashes_and_runtime_rule_match() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
alert = receipt["controlled_applies"]["alert_rules"]
|
||||
source_sha = alert["check_receipt"]["source_sha"]
|
||||
execution = alert["execution_receipt"]
|
||||
verifier = alert["post_verifier"]
|
||||
|
||||
assert alert["run_id"] == "P0-OBS-002-alert-rules-20260714T215328Z-apply1"
|
||||
assert alert["check_receipt"]["promtool"] == "pass"
|
||||
assert source_sha == execution["remote_sha"]
|
||||
assert source_sha == execution["canonical_sha"]
|
||||
assert source_sha == execution["runtime_sha"]
|
||||
assert execution["hashes_match"] is True
|
||||
assert execution["prometheus_ready"] is True
|
||||
assert verifier["window_seconds"] == 130
|
||||
assert verifier["terminal"] == "pass"
|
||||
assert verifier["signoz_rule_count"] == 1
|
||||
assert verifier["signoz_rule_health"] == "ok"
|
||||
assert verifier["signoz_rule_state"] == "inactive"
|
||||
assert verifier["exact_query"] == (
|
||||
'(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
|
||||
'job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",'
|
||||
'job="blackbox-http"})'
|
||||
)
|
||||
assert alert["terminal"] == "completed_verified"
|
||||
|
||||
|
||||
def test_failed_backup_canary_preserves_failure_and_verified_rollback() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
backup = receipt["controlled_applies"]["backup_canary"]
|
||||
assert backup["guard_source"]["deployment_status"] == "deployed_verified"
|
||||
assert len(backup["attempts"]) == 2
|
||||
|
||||
attempt = backup["attempts"][0]
|
||||
assert attempt["run_id"] == ("P0-OBS-002-backup-canary-20260714T213210Z-retry1")
|
||||
assert attempt["terminal"] == "failed_rolled_back"
|
||||
assert attempt["classification"] == (
|
||||
"failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention"
|
||||
)
|
||||
assert attempt["failed_stage"] == "clickhouse_archive_create"
|
||||
assert attempt["archive_container_exit_code"] == 137
|
||||
assert attempt["root_cause"] == (
|
||||
"docker_health_monitor_restarted_collector_during_intentional_backup_stop"
|
||||
)
|
||||
for false_field in (
|
||||
"consistent_archive_valid",
|
||||
"clickhouse_archive_committed",
|
||||
"sqlite_stage_reached",
|
||||
"restic_stage_reached",
|
||||
"retention_stage_reached",
|
||||
"success_stage_reached",
|
||||
):
|
||||
assert attempt[false_field] is False
|
||||
|
||||
rollback = attempt["rollback"]
|
||||
assert rollback["collector_running"] is True
|
||||
assert rollback["collector_restart_count"] == 0
|
||||
assert rollback["grpc_4317_listening"] is True
|
||||
assert rollback["http_4318_listening"] is True
|
||||
assert rollback["canary_processes_absent"] is True
|
||||
assert rollback["canary_container_absent"] is True
|
||||
assert rollback["current_temp_dir_absent"] is True
|
||||
assert rollback["partial_artifacts_absent"] is True
|
||||
assert not attempt["terminal"].startswith("completed")
|
||||
assert "learning_recorded" not in attempt
|
||||
|
||||
retry2 = backup["attempts"][1]
|
||||
assert retry2["run_id"] == ("P0-OBS-002-backup-canary-20260714T221230Z-retry2")
|
||||
assert retry2["terminal"] == "failed_rolled_back"
|
||||
assert retry2["backup_script_exit_code"] == 1
|
||||
assert retry2["archive_container_exit_code"] == "unknown_not_durably_captured"
|
||||
assert retry2["archive_stderr_receipt"] == "suppressed_by_legacy_pipeline"
|
||||
assert retry2["root_cause"] == (
|
||||
"raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract"
|
||||
)
|
||||
assert retry2["exact_tar_error_proven"] is False
|
||||
assert retry2["monitor_window"]["cron_intervals_crossed"] == 2
|
||||
assert retry2["monitor_window"]["auto_repair_count"] == 0
|
||||
for false_field in (
|
||||
"consistent_archive_valid",
|
||||
"clickhouse_archive_committed",
|
||||
"sqlite_stage_reached",
|
||||
"restic_stage_reached",
|
||||
"retention_stage_reached",
|
||||
"success_stage_reached",
|
||||
):
|
||||
assert retry2[false_field] is False
|
||||
retry2_rollback = retry2["rollback"]
|
||||
assert retry2_rollback["cooldown_lease_restored"] is True
|
||||
assert retry2_rollback["collector_running"] is True
|
||||
assert retry2_rollback["collector_restart_count"] == 0
|
||||
assert retry2_rollback["grpc_4317_listening"] is True
|
||||
assert retry2_rollback["http_4318_listening"] is True
|
||||
assert retry2_rollback["current_temp_dir_absent"] is True
|
||||
assert retry2_rollback["partial_artifacts_absent"] is True
|
||||
assert retry2["residual_cleanup_debt"] == []
|
||||
|
||||
|
||||
def test_clickhouse_native_backup_disk_preserves_failed_attempt_and_closes_retry() -> (
|
||||
None
|
||||
):
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
native_disk = receipt["controlled_applies"]["clickhouse_native_backup_disk"]
|
||||
|
||||
assert native_disk["source"]["override_sha256"] == (
|
||||
"cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488"
|
||||
)
|
||||
assert native_disk["source"]["config_sha256"] == (
|
||||
"c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9"
|
||||
)
|
||||
assert native_disk["check_receipt"]["terminal"] == "check_pass_no_write"
|
||||
assert len(native_disk["attempts"]) == 2
|
||||
|
||||
apply1, retry2 = native_disk["attempts"]
|
||||
assert apply1["terminal"] == "failed_rolled_back"
|
||||
assert apply1["exit_code"] == 2
|
||||
assert apply1["rollback"]["terminal"] == "pass"
|
||||
assert apply1["rollback"]["original_data_volume_preserved"] is True
|
||||
assert apply1["rollback"]["residue_count"] == 0
|
||||
|
||||
assert retry2["run_id"] == (
|
||||
"P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
|
||||
)
|
||||
assert retry2["terminal"] == "pass"
|
||||
assert retry2["execution"]["image_digest_unchanged"] is True
|
||||
assert retry2["execution"]["data_volume_unchanged"] is True
|
||||
assert retry2["post_verifier"]["independent"] is True
|
||||
assert retry2["post_verifier"]["server_write_access"] is True
|
||||
assert retry2["post_verifier"]["dependent_restart_count_delta"] == 0
|
||||
assert retry2["post_verifier"]["api_http_status"] == 200
|
||||
assert retry2["post_verifier"]["active_backup_or_restore_count"] == 0
|
||||
assert retry2["post_verifier"]["stage_candidate_process_residue_count"] == 0
|
||||
assert native_disk["terminal"] == "deployed_verified"
|
||||
|
||||
|
||||
def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"]
|
||||
assert canary["completion_scope"] == "clickhouse_native_only"
|
||||
assert canary["source"]["post_canary_focused_tests_passed"] == 94
|
||||
assert canary["source"]["post_format_focused_tests_passed"] == 94
|
||||
assert canary["source"]["ruff_format"] == "pass"
|
||||
assert canary["toolchain_controlled_apply"]["apply_terminal"] == "pass"
|
||||
assert canary["toolchain_controlled_apply"]["postcheck_diff"] == (
|
||||
"matching_7_drift_0_absent_0_active_operations_0"
|
||||
)
|
||||
assert len(canary["attempts"]) == 5
|
||||
_, apply2, apply3, apply4, apply5 = canary["attempts"]
|
||||
assert apply2["artifact_retention"] == "preserved_failed_run_evidence"
|
||||
assert apply2["cleanup_verified"] is True
|
||||
assert apply3["artifact_retention"] == "preserved_failed_run_evidence"
|
||||
assert apply3["cleanup_verified"] is True
|
||||
|
||||
assert apply4["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
|
||||
assert apply4["check_terminal"] == "check_pass_no_write"
|
||||
assert apply4["terminal"] == "pass"
|
||||
assert apply4["backup"]["terminal"] == "BACKUP_CREATED"
|
||||
assert apply4["backup"]["error"] == ""
|
||||
assert apply4["backup"]["artifact_sha256"] == (
|
||||
"4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc"
|
||||
)
|
||||
|
||||
restore = apply4["isolated_restore"]
|
||||
assert restore["terminal"] == "verified"
|
||||
assert restore["clickhouse_restore_terminal"] == "RESTORED"
|
||||
assert restore["production_network_attached"] is False
|
||||
assert restore["production_restore_performed"] is False
|
||||
assert restore["source_staged_clickhouse_artifact_sha_match"] is True
|
||||
assert restore["database_count"] == restore["restored_database_count"] == 6
|
||||
assert restore["table_count"] == restore["restored_table_count"] == 100
|
||||
assert restore["table_engine_schema_manifest_parity"] is True
|
||||
assert restore["check_table_count"] == restore["check_table_pass_count"] == 44
|
||||
assert restore["critical_nonzero_count"] == 4
|
||||
assert restore["aggregate_counter_exact_parity_gate"] is False
|
||||
|
||||
restic = apply4["restic"]
|
||||
assert restic["snapshot_id"] == "574e2a1a"
|
||||
assert restic["snapshot_readback"] == "pass"
|
||||
assert restic["repository_scope"] == "local_same_failure_domain"
|
||||
assert restic["offsite_verified"] is False
|
||||
|
||||
post = apply4["independent_post_verifier"]
|
||||
assert post["production_restart_count_delta"] == 0
|
||||
assert post["api_http_status"] == 200
|
||||
assert post["new_server_artifact_removed"] is True
|
||||
assert post["apply2_apply3_preserved_artifact_hashes_unchanged"] is True
|
||||
assert (
|
||||
post["exact_ephemeral_container_volume_network_process_tmp_residue_count"] == 0
|
||||
)
|
||||
assert post["cleanup_verified"] is True
|
||||
assert post["secret_values_collected"] is False
|
||||
|
||||
assert apply5["run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert apply5["check_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
||||
)
|
||||
assert apply5["wrapper_terminal"] == "partial_degraded"
|
||||
assert apply5["terminal"] == "partial_degraded"
|
||||
assert apply5["runtime_operation"]["terminal"] == "pass"
|
||||
assert apply5["nested_machine_receipt_verifier"] == {
|
||||
"stage": "native_restore_receipt_verifier",
|
||||
"terminal": "pass",
|
||||
}
|
||||
assert apply5["backup"]["terminal"] == "BACKUP_CREATED"
|
||||
assert apply5["backup"]["file_count"] == 4885
|
||||
assert apply5["backup"]["total_size_bytes"] == 108006575
|
||||
assert apply5["backup"]["compressed_size_bytes"] == 99461897
|
||||
assert apply5["backup"]["artifact_sha256"] == (
|
||||
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
)
|
||||
current_restore = apply5["isolated_restore"]
|
||||
assert (
|
||||
current_restore["database_count"]
|
||||
== current_restore["restored_database_count"]
|
||||
== 6
|
||||
)
|
||||
assert (
|
||||
current_restore["table_count"] == current_restore["restored_table_count"] == 100
|
||||
)
|
||||
assert (
|
||||
current_restore["check_table_count"]
|
||||
== current_restore["check_table_pass_count"]
|
||||
== 44
|
||||
)
|
||||
assert current_restore["critical_nonzero_count"] == 4
|
||||
assert apply5["restic"]["snapshot_id"] == "135485ab"
|
||||
current_post = apply5["independent_post_verifier"]
|
||||
assert current_post["independent_monitor"] == "pass"
|
||||
assert current_post["deployed_source_hash_drift_count"] == 0
|
||||
assert current_post["runtime_drift_count"] == 0
|
||||
assert (
|
||||
current_post[
|
||||
"exact_ephemeral_container_volume_network_process_tmp_residue_count"
|
||||
]
|
||||
== 0
|
||||
)
|
||||
assert current_post["cleanup_verified"] is True
|
||||
assert canary["terminal"] == "production_closed_clickhouse_native_scope"
|
||||
|
||||
|
||||
def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt() -> (
|
||||
None
|
||||
):
|
||||
module = _load_preflight_module()
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"]
|
||||
historical = canary["source"]
|
||||
current = canary["current_source"]
|
||||
|
||||
historical_hashes = {
|
||||
field: historical[field] for field in module.TOOLCHAIN_SOURCE_PATHS
|
||||
}
|
||||
assert historical["evidence_scope"] == "historical_production_receipt"
|
||||
assert historical["deployment_status"] == "deployed_verified"
|
||||
assert historical_hashes == module.HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES
|
||||
assert historical["inventory_verifier_sha256"] == (
|
||||
"79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c"
|
||||
)
|
||||
|
||||
current_hashes = {
|
||||
field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest()
|
||||
for field, relative in module.TOOLCHAIN_SOURCE_PATHS.items()
|
||||
}
|
||||
assert current["hashes"] == current_hashes
|
||||
assert current["hashes"]["inventory_verifier_sha256"] == (
|
||||
"c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863"
|
||||
)
|
||||
assert current["deployed_source_hashes"] == current_hashes
|
||||
assert current["status"] == "deployed_verified"
|
||||
assert current["deployment_status"] == "deployed"
|
||||
assert current["current_source_deployed"] is True
|
||||
assert current["runtime_verifier_terminal"] == "pass"
|
||||
assert current["historical_production_receipt_preserved"] is True
|
||||
assert current["drift_from_historical_production_receipt"] == [
|
||||
"backup_orchestrator_sha256",
|
||||
"canary_wrapper_sha256",
|
||||
"inventory_verifier_sha256",
|
||||
]
|
||||
assert current["toolchain_check_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015040Z-check6"
|
||||
)
|
||||
assert current["toolchain_apply_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
||||
)
|
||||
assert current["toolchain_postcheck_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
|
||||
)
|
||||
assert current["toolchain_postcanary_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
|
||||
)
|
||||
assert current["default_canary_check_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
||||
)
|
||||
assert current["runtime_canary_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert current["wrapper_terminal"] == "partial_degraded"
|
||||
assert current["runtime_operation_terminal"] == "pass"
|
||||
assert current["nested_machine_receipt_verifier_terminal"] == "pass"
|
||||
|
||||
|
||||
def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
module = _load_preflight_module()
|
||||
real_sha256_file = module._sha256_file
|
||||
|
||||
def stale_inventory_hash(path: Path) -> str:
|
||||
if path.name == "clickhouse-restore-inventory.py":
|
||||
return "0" * 64
|
||||
return real_sha256_file(path)
|
||||
|
||||
monkeypatch.setattr(module, "_sha256_file", stale_inventory_hash)
|
||||
payload = module.evaluate(ROOT, CONTRACT)
|
||||
|
||||
assert payload["source_ready"] is False
|
||||
assert payload["terminal"] == "failed"
|
||||
assert payload["checks"]["native_backup_current_source_hashes_bound"] is False
|
||||
assert payload["checks"]["native_backup_historical_source_receipt_preserved"]
|
||||
assert payload["backup_toolchain_current_source_status"] == "deployed_verified"
|
||||
|
||||
|
||||
def test_preflight_fails_closed_when_current_runtime_receipt_is_incomplete(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
module = _load_preflight_module()
|
||||
real_load_yaml = module._load_yaml
|
||||
|
||||
def load_without_operation_identity(path: Path):
|
||||
payload = real_load_yaml(path)
|
||||
if path.name == POST_CLOSURE_REGRESSION.name:
|
||||
attempts = payload["controlled_applies"][
|
||||
"clickhouse_native_backup_restore_canary"
|
||||
]["attempts"]
|
||||
attempts[-1]["backup"].pop("operation_identity_hash")
|
||||
return payload
|
||||
|
||||
monkeypatch.setattr(module, "_load_yaml", load_without_operation_identity)
|
||||
payload = module.evaluate(ROOT, CONTRACT)
|
||||
|
||||
assert payload["source_ready"] is False
|
||||
assert payload["terminal"] == "failed"
|
||||
assert payload["checks"]["native_backup_current_source_runtime_closed"] is False
|
||||
assert payload["runtime_status"] == "partial_degraded"
|
||||
|
||||
|
||||
def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None:
|
||||
receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8"))
|
||||
verifier = receipt["post_release_runtime_verifier"]
|
||||
assert verifier["run_id"] == "P0-OBS-002-post-release-20260714T215500Z"
|
||||
assert verifier["window_seconds"] == 600
|
||||
assert verifier["terminal"] == "pass"
|
||||
assert verifier["restart_delta"] == 0
|
||||
assert verifier["exporter_trace_errors"] == 0
|
||||
assert verifier["exporter_metric_errors"] == 0
|
||||
assert verifier["trace_count"] == 4026
|
||||
assert verifier["metric_sample_count"] == 76369
|
||||
assert verifier["api_runtime_cooldown_closed"] is True
|
||||
assert (
|
||||
receipt["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is True
|
||||
)
|
||||
|
||||
closed = receipt["completed_verifiers"]["clickhouse_native_backup_migration"]
|
||||
assert closed["prior_run_id"] == (
|
||||
"P0-OBS-002-backup-canary-20260714T221230Z-retry2"
|
||||
)
|
||||
assert closed["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4")
|
||||
assert closed["status"] == "production_closed_clickhouse_native_scope"
|
||||
assert closed["native_backup_terminal"] == "BACKUP_CREATED"
|
||||
assert closed["isolated_restore_terminal"] == "verified"
|
||||
assert closed["restic_snapshot_id"] == "574e2a1a"
|
||||
assert closed["independent_verifier"] == "pass"
|
||||
assert closed["exact_cleanup"] == "pass"
|
||||
sqlite_pending = receipt["pending_verifiers"][
|
||||
"signoz_sqlite_application_consistent_backup"
|
||||
]
|
||||
assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False
|
||||
assert sqlite_pending["sqlite_cli_present_in_runtime"] is False
|
||||
offsite = receipt["pending_verifiers"]["signoz_backup_offsite_dr"]
|
||||
assert offsite["current_repository_scope"] == "local_same_failure_domain"
|
||||
assert offsite["offsite_verified"] is False
|
||||
artifacts = receipt["pending_verifiers"][
|
||||
"clickhouse_native_failed_artifact_retention"
|
||||
]
|
||||
assert artifacts["destructive_cleanup_authorized"] is False
|
||||
assert artifacts["active_backup_or_restore_count"] == 0
|
||||
assert len(artifacts["preserved_artifacts"]) == 2
|
||||
resume = receipt["pending_verifiers"][
|
||||
"clickhouse_native_resume_submitted_inventory"
|
||||
]
|
||||
assert resume["same_run_resume_safe"] is False
|
||||
assert len(resume["required_preconditions"]) == 4
|
||||
current_source = receipt["completed_verifiers"][
|
||||
"clickhouse_native_current_source_toolchain"
|
||||
]
|
||||
assert current_source["status"] == "deployed_verified"
|
||||
assert current_source["current_source_deployed"] is True
|
||||
assert current_source["runtime_verifier_terminal"] == "pass"
|
||||
assert current_source["historical_production_receipt_preserved"] is True
|
||||
assert current_source["toolchain_apply_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
||||
)
|
||||
assert current_source["runtime_canary_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert current_source["wrapper_terminal"] == "partial_degraded"
|
||||
assert current_source["runtime_operation_terminal"] == "pass"
|
||||
assert current_source["nested_machine_receipt_verifier_terminal"] == "pass"
|
||||
assert current_source["restic_snapshot_id"] == "135485ab"
|
||||
assert current_source["artifact_sha256"] == (
|
||||
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
)
|
||||
assert current_source["independent_monitor"] == "pass"
|
||||
assert current_source["runtime_drift_count"] == 0
|
||||
assert current_source["residue_count"] == 0
|
||||
assert current_source["exact_cleanup"] == "pass"
|
||||
assert (
|
||||
"clickhouse_native_current_source_toolchain" not in receipt["pending_verifiers"]
|
||||
)
|
||||
assets = receipt["asset_reconciliation"]
|
||||
assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001"
|
||||
assert assets["source_service_count"] == 27
|
||||
assert assets["runtime_mirror_service_count"] == 24
|
||||
assert assets["exact_shared_service_count"] == 24
|
||||
assert assets["shared_service_drift_count"] == 0
|
||||
assert assets["source_only_services"] == ["bitan-app", "sentry", "signoz"]
|
||||
assert assets["signoz_clickhouse_exact_match"] is True
|
||||
assert assets["live_signoz_query_host"] == "192.168.0.110"
|
||||
assert assets["source_signoz_host"] == "192.168.0.188"
|
||||
github_drift = assets["github_freeze_legacy_asset_drift"]
|
||||
assert github_drift["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-002"
|
||||
assert github_drift["superseded_by"] == "global_product_governance_v2"
|
||||
assert github_drift["active_github_use_in_this_run"] is False
|
||||
assert len(github_drift["source_references"]) == 3
|
||||
assert github_drift["p0_p1_github_recovery_scheduled"] is False
|
||||
duplicate_identity = assets["monitoring_generator_duplicate_identity_drift"]
|
||||
assert duplicate_identity["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-003"
|
||||
assert duplicate_identity["status"] == "pending_source_deduplication"
|
||||
assert duplicate_identity["awoooi_api_registry_entry_count"] == 2
|
||||
assert duplicate_identity["generated_scrape_config_count"] == 24
|
||||
assert duplicate_identity["generated_awoooi_api_job_count"] == 2
|
||||
assert duplicate_identity["generated_blackbox_target_count"] == 20
|
||||
assert duplicate_identity["runtime_apply_performed"] is False
|
||||
assert receipt["terminal"]["status"] == "partial_degraded"
|
||||
assert receipt["terminal"]["blockers"] == [
|
||||
"signoz_sqlite_application_consistent_backup_pending",
|
||||
"signoz_backup_offsite_dr_pending",
|
||||
"clickhouse_native_failed_artifact_retention_pending",
|
||||
"clickhouse_native_resume_submitted_inventory_pending",
|
||||
"service_registry_runtime_mirror_reconciliation_pending",
|
||||
"github_freeze_legacy_asset_retirement_pending",
|
||||
"monitoring_generator_duplicate_awoooi_api_identity_pending",
|
||||
]
|
||||
|
||||
|
||||
def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> None:
|
||||
contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8"))
|
||||
post_closure = contract["runtime_observations"]["post_closure_regression"]
|
||||
assert post_closure["snapshot_schema"] == (
|
||||
"awoooi_signoz_post_closure_regression_v2"
|
||||
)
|
||||
assert post_closure["gitea_cd_run_id"] == 5140
|
||||
assert post_closure["api_runtime_cooldown_closed"] is True
|
||||
assert post_closure["cd_probe_safety_fix_deployed"] is True
|
||||
assert post_closure["config_route_patch_deployed"] is True
|
||||
assert post_closure["signoz_control_plane_health_route_deployed"] is True
|
||||
assert post_closure["backup_collector_restore_guard_deployed"] is True
|
||||
assert post_closure["backup_canary_scope"] == "clickhouse_native_only"
|
||||
assert post_closure["backup_canary_verified"] is True
|
||||
assert post_closure["backup_canary_last_terminal"] == "partial_degraded"
|
||||
assert post_closure["backup_canary_last_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert post_closure["backup_canary_runtime_operation_terminal"] == "pass"
|
||||
assert post_closure["backup_canary_machine_receipt_verifier_terminal"] == "pass"
|
||||
assert post_closure["backup_canary_cooldown_retry_status"] == (
|
||||
"superseded_by_online_native_no_stop_contract"
|
||||
)
|
||||
assert post_closure["clickhouse_native_backup_migration_status"] == (
|
||||
"production_closed"
|
||||
)
|
||||
assert post_closure["clickhouse_native_backup_disk_run_id"] == (
|
||||
"P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2"
|
||||
)
|
||||
assert post_closure["clickhouse_native_backup_disk_status"] == "deployed_verified"
|
||||
assert post_closure["clickhouse_native_backup_toolchain_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T001252Z-apply4"
|
||||
)
|
||||
assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4"
|
||||
)
|
||||
assert (
|
||||
post_closure["clickhouse_native_current_source_status"] == "deployed_verified"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_deployed"] is True
|
||||
assert (
|
||||
post_closure["clickhouse_native_current_source_runtime_verifier_terminal"]
|
||||
== "pass"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_toolchain_check_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015040Z-check6"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_toolchain_apply_run_id"] == (
|
||||
"P0-OBS-002-native-toolchain-20260715T015059Z-apply5"
|
||||
)
|
||||
assert (
|
||||
post_closure["clickhouse_native_current_source_toolchain_postcheck_run_id"]
|
||||
== "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5"
|
||||
)
|
||||
assert (
|
||||
post_closure["clickhouse_native_current_source_toolchain_postcanary_run_id"]
|
||||
== "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_canary_check_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015131Z-check5"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_canary_apply_run_id"] == (
|
||||
"P0-OBS-002-native-canary-default-20260715T015149Z-apply5"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_wrapper_terminal"] == (
|
||||
"partial_degraded"
|
||||
)
|
||||
assert post_closure[
|
||||
"clickhouse_native_current_source_runtime_operation_terminal"
|
||||
] == ("pass")
|
||||
assert (
|
||||
post_closure[
|
||||
"clickhouse_native_current_source_machine_receipt_verifier_terminal"
|
||||
]
|
||||
== "pass"
|
||||
)
|
||||
assert post_closure["clickhouse_native_current_source_artifact_sha256"] == (
|
||||
"08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83"
|
||||
)
|
||||
assert post_closure["clickhouse_native_restore_terminal"] == "verified"
|
||||
assert post_closure["clickhouse_native_restic_snapshot_id"] == "135485ab"
|
||||
assert post_closure["clickhouse_native_offsite_status"] == "pending"
|
||||
assert post_closure["clickhouse_native_failed_artifact_retention_status"] == (
|
||||
"pending_bounded_retention_decision"
|
||||
)
|
||||
assert post_closure["clickhouse_native_resume_inventory_status"] == (
|
||||
"pending_durable_submitted_inventory_contract"
|
||||
)
|
||||
assert (
|
||||
post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending"
|
||||
)
|
||||
assert post_closure["service_registry_runtime_mirror_status"] == (
|
||||
"partial_degraded"
|
||||
)
|
||||
assert post_closure["service_registry_runtime_mirror_work_item_id"] == (
|
||||
"P0-OBS-002-ASSET-DRIFT-001"
|
||||
)
|
||||
assert post_closure["github_freeze_legacy_asset_status"] == (
|
||||
"pending_controlled_retirement"
|
||||
)
|
||||
assert post_closure["github_freeze_legacy_asset_work_item_id"] == (
|
||||
"P0-OBS-002-ASSET-DRIFT-002"
|
||||
)
|
||||
assert post_closure["monitoring_generator_duplicate_identity_status"] == (
|
||||
"pending_source_deduplication"
|
||||
)
|
||||
assert post_closure["monitoring_generator_duplicate_identity_work_item_id"] == (
|
||||
"P0-OBS-002-ASSET-DRIFT-003"
|
||||
)
|
||||
assert post_closure["post_release_otlp_grpc_600s_verifier_status"] == "pass"
|
||||
assert contract["terminal"]["status"] == "partial_degraded"
|
||||
assert contract["terminal"]["blockers"] == [
|
||||
"organization_not_initialized",
|
||||
"direct_ingress_policy_not_closed",
|
||||
"http_bridge_not_supervised",
|
||||
"filelog_normalization_degraded",
|
||||
"wave_b_not_authorized",
|
||||
]
|
||||
|
||||
|
||||
def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None:
|
||||
env = os.environ.copy()
|
||||
env.pop("MANAGED_PROMOTION_RECEIPT", None)
|
||||
|
||||
Reference in New Issue
Block a user