From 322e812d930ee35afc422202e08aaaf8a3c68b3e Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 08:55:08 +0800 Subject: [PATCH 1/9] feat(observability): close native SigNoz backup restore canary --- .agents/skills/04-awoooi-devops-commander.md | 16 +- .agents/skills/05-awoooi-sre-qa.md | 9 +- .agents/skills/07-tool-integration-expert.md | 7 +- .awoooi-agent-rules.md | 21 +- .../test_signoz_canonical_health_route.py | 179 ++- apps/sensor/agent.py | 5 +- apps/web/src/app/[locale]/classic/page.tsx | 5 +- .../tabs/automation-inventory-tab.tsx | 4 +- .../components/dashboard/live-dashboard.tsx | 2 +- infra/ansible/playbooks/110-devops.yml | 34 + .../15-service-registry-configmap.yaml | 4 +- k8s/monitoring/deploy-prometheus-config.sh | 128 +- k8s/monitoring/k3s-alerts.yaml | 14 +- k8s/monitoring/prometheus.yml | 2 +- ops/config/service-registry.yaml | 4 +- ops/monitoring/alerts-unified.yml | 12 +- ops/monitoring/alerts.yml | 12 +- ops/monitoring/generate_monitoring.py | 5 +- ops/monitoring/service-registry.yaml | 10 +- ops/signoz/alerting/log-rules.md | 8 +- .../clickhouse/config.d/backup_disk.xml | 24 + .../config.d/isolated-cluster.xml | 35 + ...er-compose.clickhouse-backup.override.yaml | 13 + ...rganization-canonical-route-migration.yaml | 48 +- .../p0-obs-002-post-closure-regression.yaml | 617 ++++++++- scripts/alert_chain_smoke_test.py | 8 +- scripts/backup/backup-signoz.sh | 409 +++--- scripts/backup/clickhouse-native-backup.sh | 628 +++++++++ .../backup/clickhouse-native-restore-drill.sh | 1152 +++++++++++++++++ .../backup/clickhouse-restore-inventory.py | 284 ++++ scripts/backup/run-signoz-backup-canary.sh | 665 ++++++++++ .../test_backup_signoz_collector_restore.py | 547 +++----- .../tests/test_clickhouse_native_backup.py | 478 +++++++ .../test_clickhouse_native_restore_drill.py | 848 ++++++++++++ .../test_signoz_backup_canary_wrapper.py | 528 ++++++++ scripts/ops/ansible-validate.sh | 4 + .../deploy-signoz-clickhouse-backup-disk.sh | 594 +++++++++ .../deploy-signoz-native-backup-toolchain.sh | 801 ++++++++++++ scripts/ops/deploy-signoz-prometheus-route.sh | 218 ++++ ...st_signoz_clickhouse_backup_disk_deploy.py | 231 ++++ ...t_signoz_native_backup_toolchain_deploy.py | 339 +++++ .../reboot-recovery-readiness-audit.sh | 6 + .../signoz-canonical-route-preflight.py | 582 ++++++++- .../test_signoz_canonical_route_migration.py | 511 +++++++- 44 files changed, 9209 insertions(+), 842 deletions(-) create mode 100644 ops/signoz/clickhouse/config.d/backup_disk.xml create mode 100644 ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml create mode 100644 ops/signoz/docker-compose.clickhouse-backup.override.yaml create mode 100755 scripts/backup/clickhouse-native-backup.sh create mode 100755 scripts/backup/clickhouse-native-restore-drill.sh create mode 100755 scripts/backup/clickhouse-restore-inventory.py create mode 100755 scripts/backup/run-signoz-backup-canary.sh create mode 100644 scripts/backup/tests/test_clickhouse_native_backup.py create mode 100644 scripts/backup/tests/test_clickhouse_native_restore_drill.py create mode 100644 scripts/backup/tests/test_signoz_backup_canary_wrapper.py create mode 100755 scripts/ops/deploy-signoz-clickhouse-backup-disk.sh create mode 100755 scripts/ops/deploy-signoz-native-backup-toolchain.sh create mode 100755 scripts/ops/deploy-signoz-prometheus-route.sh create mode 100644 scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py create mode 100644 scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py diff --git a/.agents/skills/04-awoooi-devops-commander.md b/.agents/skills/04-awoooi-devops-commander.md index 1676868a0..5dbd79d3a 100644 --- a/.agents/skills/04-awoooi-devops-commander.md +++ b/.agents/skills/04-awoooi-devops-commander.md @@ -3,6 +3,7 @@ > **管轄範圍**: Docker, K3s, Nginx, NetworkPolicy, CI/CD > **觸發條件**: 修改 `k8s/`, `Dockerfile`, `.github/workflows/`, `nginx` +> **治理覆寫**: `superseded_by=global_product_governance_v2`；本文 GitHub/GitHub Actions 段落只保留歷史背景，不得執行。現行 source truth 與 workflow 一律使用 Gitea；舊 SigNoz `192.168.0.188:3301` UI/query 路由同樣失效。 --- @@ -190,12 +191,12 @@ echo "${{ secrets.SUDO_PASSWORD }}" | sudo -S kubectl ... | 主機 | IP | 角色 | 部署內容 | |------|-----|------|----------| -| DevOps | 192.168.0.110 | 金庫 | Harbor:5000, GitHub Runner, Sentry:9000, Langfuse:3100 | +| DevOps | 192.168.0.110 | 金庫 | Harbor:5000, Gitea Actions Runner, Sentry:9000, Langfuse:3100, SigNoz health:8080 | | Security | 192.168.0.112 | 安全 | Kali Scanner:8080 | | K3s Server #1 | 192.168.0.120 | 叢集 | Control-Plane MASTER (keepalived priority=101) | | K3s Server #2 | 192.168.0.121 | 叢集 | Control-Plane BACKUP (keepalived priority=100) | | **VIP** | **192.168.0.125** | **HA 入口** | **K3s API:6443 + NodePort (32334/32335)** | -| AI/Web | 192.168.0.188 | 大腦 | Nginx, **PostgreSQL:5432 (K3s Datastore)**, Redis:6380, Ollama:11434, OpenClaw:8089, SigNoz:3301 | +| AI/Web | 192.168.0.188 | 大腦 | Nginx, **PostgreSQL:5432 (K3s Datastore)**, Redis:6380, Ollama:11434, OpenClaw:8089, OTLP bridge:24317/24318 | ### 絕對禁令 @@ -421,7 +422,8 @@ kubectl --server=https://192.168.0.125:6443 get nodes | 服務 | 端點 | 用途 | |------|------|------| -| **SignOz Web UI** | `http://192.168.0.188:3301` | APM Dashboard | +| **SigNoz machine health** | `http://192.168.0.110:8080/api/v1/health` | service-to-service health/readback | +| **SigNoz Web UI** | `https://signoz.wooo.work` | human APM dashboard / deep links | | **OTEL gRPC** | `192.168.0.188:24317` | K8s/應用程式 Traces | | **OTEL HTTP** | `http://192.168.0.188:24318` | CI/CD workflows | | **Sentry** | `http://192.168.0.110:9000` | Error Tracking | @@ -451,8 +453,8 @@ OTEL_EXPORTER_OTLP_ENDPOINT: http://192.168.0.188:24318 ### 驗證指令 ```bash -# 檢查 SignOz 容器運行狀態 -ssh ollama@192.168.0.188 "docker ps | grep signoz" +# 檢查 SigNoz machine health +curl -fsS http://192.168.0.110:8080/api/v1/health | jq . # 測試 OTEL 端點 curl -s http://192.168.0.188:24318/v1/traces -X POST | head -c 100 @@ -1165,7 +1167,7 @@ def verify_github_signature(payload: bytes, signature: str, secret: str) -> bool ┌─────────────┐ trace_id ┌─────────────┐ trace_id ┌─────────────┐ │ Sentry │ ◄────────────────► │ SignOz │ ◄────────────────► │ Langfuse │ │ Errors │ │ Traces │ │ LLMOps │ -│ :9000 (.110)│ │ :3301 (.188)│ │ :3100 (.110)│ +│ :9000 (.110)│ │ signoz.wooo │ │ :3100 (.110)│ └─────────────┘ └─────────────┘ └─────────────┘ ``` @@ -1173,7 +1175,7 @@ def verify_github_signature(payload: bytes, signature: str, secret: str) -> bool | 系統 | URL 格式 | |------|----------| -| SignOz Trace | `http://192.168.0.188:3301/trace/{trace_id}` | +| SigNoz Trace | `https://signoz.wooo.work/trace/{trace_id}` | | Langfuse Trace | `http://192.168.0.110:3100/project/awoooi-openclaw/traces/{id}` | | Sentry Issue | `http://192.168.0.110:9000/organizations/sentry/issues/{id}/` | diff --git a/.agents/skills/05-awoooi-sre-qa.md b/.agents/skills/05-awoooi-sre-qa.md index b8e0cac10..732afc551 100644 --- a/.agents/skills/05-awoooi-sre-qa.md +++ b/.agents/skills/05-awoooi-sre-qa.md @@ -3,6 +3,7 @@ > **管轄範圍**: Playwright, API Testing, Health Monitoring > **觸發條件**: 懷疑 UI 異常、API 錯誤、部署驗證 +> **治理覆寫**: `superseded_by=global_product_governance_v2`；本文 `.github/workflows` 與舊 SigNoz `192.168.0.188:3301` 測試僅屬歷史背景。現行 CI/CD 使用 Gitea，SigNoz health/UI/OTLP 依 canonical route 分流。 --- @@ -494,8 +495,8 @@ kubectl rollout undo deployment/awoooi-api -n awoooi-prod ### 觀測系統健康檢查 ```bash -# SignOz (Traces) -curl -f http://192.168.0.188:3301/api/v1/services | jq '.data[] | .serviceName' +# SigNoz control-plane health (machine readback) +curl -fsS http://192.168.0.110:8080/api/v1/health | jq . # Langfuse (LLMOps) curl -f http://192.168.0.110:3100/api/health @@ -511,7 +512,7 @@ curl -f http://192.168.0.110:9000/api/0/internal/health/ from src.core.deep_linking import DeepLinking # 應返回有效 URL -assert DeepLinking.signoz_trace_url("abc123") == "http://192.168.0.188:3301/trace/abc123" +assert DeepLinking.signoz_trace_url("abc123") == "https://signoz.wooo.work/trace/abc123" assert DeepLinking.langfuse_trace_url("lf-123") != "" assert DeepLinking.sentry_issue_url("456") != "" ``` @@ -539,7 +540,7 @@ with restore_trace_context({"trace_id": "", "span_id": ""}) as span: | 系統 | URL | 狀態 | |------|-----|------| -| SignOz | http://192.168.0.188:3301 | ✅ | +| SigNoz | https://signoz.wooo.work (machine health: `192.168.0.110:8080/api/v1/health`) | ✅ | | Langfuse | http://192.168.0.110:3100 | ✅ | | Sentry | http://192.168.0.110:9000 | ✅ | | Deep Linking | N/A (內建) | ✅ | diff --git a/.agents/skills/07-tool-integration-expert.md b/.agents/skills/07-tool-integration-expert.md index af8df8103..87a427466 100644 --- a/.agents/skills/07-tool-integration-expert.md +++ b/.agents/skills/07-tool-integration-expert.md @@ -3,6 +3,7 @@ > **管轄範圍**: MCP Bridge, 外部系統連接, RAG 向量化 > **觸發條件**: 修改 `plugins/mcp/`, `services/*_tool.py`, 向量資料庫 +> **治理覆寫**: `superseded_by=global_product_governance_v2`；舊 SigNoz `192.168.0.188:3301` query/UI route 已失效，machine health、public query/UI 與 OTLP ingestion 必須使用下列 canonical endpoints。 --- @@ -100,7 +101,11 @@ Phase 13.2 Tool 實作 (P0 最優先): ``` ClickHouse HTTP API: http://192.168.0.188:8123 -SignOz Query API: http://192.168.0.188:3301/api/v3 +SigNoz machine health: http://192.168.0.110:8080/api/v1/health +SigNoz Query API (machine/MCP): http://192.168.0.110:8080/api/v3 +SigNoz UI / deep links: https://signoz.wooo.work +OTLP gRPC: http://192.168.0.188:24317 +OTLP HTTP: http://192.168.0.188:24318 ``` ### 查詢範例 diff --git a/.awoooi-agent-rules.md b/.awoooi-agent-rules.md index 6269c6ed3..23b88dc54 100644 --- a/.awoooi-agent-rules.md +++ b/.awoooi-agent-rules.md @@ -4,6 +4,7 @@ > **生效日期**: 2026-03-21 > **適用對象**: Claude Code / Cursor / GitHub Copilot > **強制等級**: 絕對遵守 (違反等同事故) +> **治理覆寫**: `superseded_by=global_product_governance_v2`；本文內以 GitHub 為 source truth、以 `192.168.0.188:3301` 存取 SigNoz UI/query，或以未受控人工 gate 作為終局的舊指令均已失效。現行 source truth/CI/CD 僅使用 Gitea；SigNoz machine health、human deep link 與 OTLP ingestion 必須分流。 --- @@ -310,9 +311,9 @@ import { Eye, Sparkles } from 'lucide-react' | IP | 名稱 | 職責 | |-----|------|------| -| **192.168.0.110** | DevOps 金庫 | Harbor:5000 (`awoooi/` Project), GH Runner (`awoooi-runner`) | +| **192.168.0.110** | DevOps 金庫 | Harbor:5000 (`awoooi/` Project), Gitea:3001, **SigNoz machine health:8080 (`/api/v1/health`)** | | **192.168.0.112** | Kali Security | Scanner API:8080 (Header 區分: `X-Source: awoooi`) | -| **192.168.0.188** | AI+Web 中心 | Nginx SSL, Ollama:11434, **OpenClaw:8089**, **SigNoz:3301 (OTEL 收集器)** | +| **192.168.0.188** | AI+Web 中心 | Nginx SSL, Ollama:11434, **OpenClaw:8089**, **OTLP bridge:24317/24318** | | **192.168.0.120** | K3s Master | **awoooi-prod (API:32334, Frontend:32335)** ⚠️ 唯一部署目標 | | **192.168.0.121** | K3s Worker | HA 備援 | @@ -344,10 +345,10 @@ import { Eye, Sparkles } from 'lucide-react' | 1 | **kustomization 禁寫 newTag** | 導致 Ghost Rollback，Tag 由 CI 動態注入 | | 2 | **CI/CD 禁止 git push 回 main** | GitOps Lite 已永久停用 | | 3 | **映像標籤必須唯一不可變** | 格式: `{sha}-{run_id}`，禁用 latest/main/dev | -| 4 | **禁止 Gitea ↔ GitHub 雙向同步** | 唯一來源: GitHub | +| 4 | **禁止 Gitea ↔ GitHub 雙向同步** | 唯一來源: Gitea；GitHub 全面停用 | | 5 | **LLM 調用必須走快取** | 使用 `_call_with_cache`，禁止直接調用 | | 6 | **Markdown 產出必須遵守格式** | 頭部元數據 + 更新 LOGBOOK | -| 7 | **可觀測性必須打向 SigNoz** | OTEL Traces/Logs → 192.168.0.188:3301 | +| 7 | **SigNoz 流量必須依用途分流** | machine health → `192.168.0.110:8080/api/v1/health`；human UI → `https://signoz.wooo.work`；OTLP → `192.168.0.188:24317/24318` | | 8 | **UI 文字禁止硬編碼** | 100% next-intl 字典檔 (zh-TW/en) | --- @@ -366,12 +367,14 @@ from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor -# SigNoz 端點 (絕對禁止寫死其他地址) -SIGNOZ_ENDPOINT = "http://192.168.0.188:4317" +# SigNoz canonical endpoints：健康、UI、遙測攝取不可混用 +SIGNOZ_HEALTH_URL = "http://192.168.0.110:8080/api/v1/health" +SIGNOZ_PUBLIC_URL = "https://signoz.wooo.work" +OTEL_GRPC_ENDPOINT = "http://192.168.0.188:24317" def setup_telemetry(app): provider = TracerProvider() - processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=SIGNOZ_ENDPOINT)) + processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=OTEL_GRPC_ENDPOINT)) provider.add_span_processor(processor) trace.set_tracer_provider(provider) FastAPIInstrumentor.instrument_app(app) @@ -440,9 +443,9 @@ AI 可跨資料夾讀取以下舊專案文檔 (唯讀)： | IP | 職責 | 允許操作 | |-----|------|----------| -| **192.168.0.110** | DevOps 金庫 (Harbor) | Docker Registry 操作 | +| **192.168.0.110** | DevOps 金庫 (Harbor + SigNoz control plane) | Docker Registry、SigNoz machine health (`:8080/api/v1/health`) | | **192.168.0.112** | Kali Security | 安全掃描 API 呼叫 | -| **192.168.0.188** | AI+Web 中心 | **Nginx 部署**、Ollama、SigNoz | +| **192.168.0.188** | AI+Web 中心 | **Nginx 部署**、Ollama、OTLP bridge (`:24317/:24318`) | | **192.168.0.120** | K3s Master | kubectl 操作、NodePort 存取 | | **192.168.0.121** | K3s Worker | HA 備援 (禁止直接操作) | diff --git a/apps/api/tests/test_signoz_canonical_health_route.py b/apps/api/tests/test_signoz_canonical_health_route.py index 80946b9fa..cc83758b2 100644 --- a/apps/api/tests/test_signoz_canonical_health_route.py +++ b/apps/api/tests/test_signoz_canonical_health_route.py @@ -1,5 +1,7 @@ from __future__ import annotations +import subprocess +import sys from datetime import UTC, datetime from pathlib import Path from types import SimpleNamespace @@ -15,9 +17,15 @@ from src.services import signoz_client as signoz_client_module from src.services.host_aggregator import HOST_CONFIGS REPO_ROOT = Path(__file__).resolve().parents[3] +ALERT_CHAIN_SMOKE = REPO_ROOT / "scripts/alert_chain_smoke_test.py" +PROMETHEUS_ROUTE_DEPLOY = ( + REPO_ROOT / "scripts/ops/deploy-signoz-prometheus-route.sh" +) +LEGACY_PROMETHEUS_DEPLOY = REPO_ROOT / "k8s/monitoring/deploy-prometheus-config.sh" INTERNAL_URL = "http://192.168.0.110:8080" PUBLIC_URL = "https://signoz.wooo.work" LEGACY_URL = "192.168.0.188:3301" +INTERNAL_HEALTH_URL = f"{INTERNAL_URL}/api/v1/health" def _configmap(relative_path: str) -> dict: @@ -81,7 +89,9 @@ def test_user_facing_deep_links_use_public_route( ) -> None: monkeypatch.setattr(deep_linking_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL) monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL) - monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL) + monkeypatch.setattr( + signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL + ) trace_id = "0af7651916cd43dd8448eb211c80319c" assert deep_linking_module.DeepLinking.signoz_trace_url(trace_id) == ( @@ -165,9 +175,176 @@ def test_active_api_source_has_no_legacy_signoz_ui_route() -> None: assert offenders == [] +def test_cd_alert_chain_smoke_uses_canonical_public_signoz_route() -> None: + text = ALERT_CHAIN_SMOKE.read_text(encoding="utf-8") + + assert '"SIGNOZ_PUBLIC_URL"' in text + assert PUBLIC_URL in text + assert LEGACY_URL not in text + + def test_host_asset_map_places_signoz_on_the_canonical_query_host() -> None: canonical_services = HOST_CONFIGS["192.168.0.110"]["services"] legacy_services = HOST_CONFIGS["192.168.0.188"]["services"] assert ("SigNoz", 8080, "http", "/api/v1/health") in canonical_services assert not any(service[0] == "SigNoz" for service in legacy_services) + + +def test_monitoring_registry_splits_query_health_from_otlp_transport() -> None: + registry = _configmap("ops/monitoring/service-registry.yaml") + services = {service["name"]: service for service in registry["services"]} + + query = services["signoz-ui"] + assert query["host"] == "192.168.0.110" + assert query["port"] == 8080 + assert query["health_endpoint"] == "/api/v1/health" + assert query["health_type"] == "http" + assert query["monitoring"]["prometheus"] is True + assert query["monitoring"]["prometheus_scrape"] is False + + collector = services["signoz-collector"] + assert collector["host"] == "192.168.0.188" + assert collector["port"] == 24317 + assert collector["health_type"] == "grpc" + assert collector["monitoring"]["prometheus_scrape"] is False + assert collector["monitoring"]["external_verifier"] == ( + "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh" + ) + + +def test_stateful_registry_places_clickhouse_on_canonical_host() -> None: + source_registry = _configmap("ops/config/service-registry.yaml") + configmap = _configmap("k8s/awoooi-prod/15-service-registry-configmap.yaml") + runtime_registry = yaml.safe_load(configmap["data"]["service-registry.yaml"]) + + for registry in (source_registry, runtime_registry): + services = {service["name"]: service for service in registry["services"]} + clickhouse = services["signoz-clickhouse"] + assert clickhouse["host"] == "192.168.0.110" + assert clickhouse["stateful_level"] == "BLOCK" + assert clickhouse["alert_only"] is True + + +def test_active_prometheus_targets_canonical_http_health_not_legacy_tcp() -> None: + config = _configmap("k8s/monitoring/prometheus.yml") + jobs = {job["job_name"]: job for job in config["scrape_configs"]} + http_job = jobs["blackbox-http"] + http_targets = http_job["static_configs"][0]["targets"] + tcp_targets = jobs["blackbox-tcp"]["static_configs"][0]["targets"] + + assert INTERNAL_HEALTH_URL in http_targets + assert LEGACY_URL not in tcp_targets + assert http_job["relabel_configs"] == [ + { + "source_labels": ["__address__"], + "target_label": "__param_target", + }, + { + "source_labels": ["__param_target"], + "target_label": "instance", + }, + { + "target_label": "__address__", + "replacement": "192.168.0.110:9115", + }, + ] + + +def test_signoz_prometheus_route_deploy_is_target_first_and_rollback_safe() -> None: + text = PROMETHEUS_ROUTE_DEPLOY.read_text(encoding="utf-8") + + assert 'MODE="${1:-apply}"' in text + assert "promtool check config" in text + assert "cp -p \"${REMOTE_CONFIG}\" \"${BACKUP_CONFIG}\"" in text + assert 'action=target_first_config_apply' in text + assert text.index("promtool check config") < text.index( + 'action=target_first_config_apply' + ) + assert text.index('action=target_first_config_apply') < text.index( + "terminal=target_up_two_scrapes" + ) + assert "rollback_deploy" in text + assert INTERNAL_HEALTH_URL in text + assert LEGACY_URL in text + + +def test_legacy_prometheus_deployer_routes_to_bounded_host110_replacement() -> None: + text = LEGACY_PROMETHEUS_DEPLOY.read_text(encoding="utf-8") + + assert "superseded_by=global_product_governance_v2" in text + assert "expiry=2026-07-15" in text + assert "scripts/ops/deploy-signoz-prometheus-route.sh" in text + assert "192.168.0.188" not in text + + +@pytest.mark.parametrize( + "relative_path", + [ + "ops/monitoring/alerts-unified.yml", + "ops/monitoring/alerts.yml", + ], +) +def test_signoz_down_alert_is_exact_and_fails_closed_on_missing_series( + relative_path: str, +) -> None: + payload = _configmap(relative_path) + rules = [ + rule + for group in payload["groups"] + for rule in group.get("rules", []) + if rule.get("alert") == "SignOzDown" + ] + + assert len(rules) == 1 + rule = rules[0] + assert 'job="blackbox-http"' in rule["expr"] + assert INTERNAL_HEALTH_URL in rule["expr"] + assert "absent(" in rule["expr"] + assert LEGACY_URL not in rule["expr"] + assert rule["labels"]["host"] == "110" + assert rule["labels"]["target_host"] == "192.168.0.110" + + +def test_legacy_188_rule_plane_does_not_duplicate_canonical_signoz_alert() -> None: + payload = _configmap("k8s/monitoring/k3s-alerts.yaml") + signoz_rules = [ + rule + for group in payload["groups"] + for rule in group.get("rules", []) + if rule.get("alert") == "SignOzDown" + ] + text = (REPO_ROOT / "k8s/monitoring/k3s-alerts.yaml").read_text(encoding="utf-8") + + assert signoz_rules == [] + assert LEGACY_URL not in text + + +def test_monitoring_generator_uses_blackbox_without_scraping_signoz_spa( + tmp_path: Path, +) -> None: + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "ops/monitoring/generate_monitoring.py"), + "--output-dir", + str(tmp_path), + ], + cwd=REPO_ROOT, + check=True, + capture_output=True, + text=True, + ) + + scrape = _configmap(str(tmp_path / "prometheus-scrape-generated.yaml")) + blackbox = yaml.safe_load( + (tmp_path / "blackbox-targets-generated.yaml").read_text(encoding="utf-8") + ) + generated_jobs = {job["job_name"] for job in scrape["scrape_configs"]} + assert "signoz-ui" not in generated_jobs + assert "signoz-collector" not in generated_jobs + assert any( + target["labels"].get("service") == "signoz-ui" + and target["url"] == INTERNAL_HEALTH_URL + for target in blackbox + ) diff --git a/apps/sensor/agent.py b/apps/sensor/agent.py index a873d31dc..7473bbcae 100644 --- a/apps/sensor/agent.py +++ b/apps/sensor/agent.py @@ -66,7 +66,6 @@ HOST_CONFIGS: dict[str, dict] = { {"name": "Redis", "host": "127.0.0.1", "port": 6380}, {"name": "Ollama", "host": "127.0.0.1", "port": 11434}, {"name": "Nginx", "host": "127.0.0.1", "port": 80}, - {"name": "SigNoz", "host": "127.0.0.1", "port": 3301}, # OpenClaw 跑在 K3s (awoooi-prod)，非 188 本機服務，不在此探測 ], }, @@ -77,7 +76,9 @@ HOST_CONFIGS: dict[str, dict] = { "services": [ {"name": "Harbor", "host": "127.0.0.1", "port": 5000}, {"name": "Gitea", "host": "127.0.0.1", "port": 3001}, - {"name": "GH-Runner", "host": "127.0.0.1", "port": 8080}, + # TCP availability for canonical machine health endpoint: + # http://192.168.0.110:8080/api/v1/health + {"name": "SigNoz", "host": "127.0.0.1", "port": 8080}, ], }, } diff --git a/apps/web/src/app/[locale]/classic/page.tsx b/apps/web/src/app/[locale]/classic/page.tsx index 003feaa3e..091de2970 100644 --- a/apps/web/src/app/[locale]/classic/page.tsx +++ b/apps/web/src/app/[locale]/classic/page.tsx @@ -31,6 +31,7 @@ import { HostGrid, type HostInfo, type HostService } from '@/components/infra/ho import { AppLayout } from '@/components/layout' const API_BASE = process.env.NEXT_PUBLIC_API_URL ?? '' +const SIGNOZ_PUBLIC_URL = 'https://signoz.wooo.work' // ============================================================================= // Types @@ -152,7 +153,7 @@ function MonitoringTools() { const statusText = isUp ? (hasFiring ? `${tool.firing_count} ${tDash('monitoringStatus.firing')}` : tDash('monitoringStatus.up')) : tDash('monitoringStatus.down') const accentColor = TOOL_ACCENT_COLOR[tool.name] ?? '#b0ad9f' const icon = TOOL_ICON[tool.name] ?? - const link = safeExternalMonitoringUrl(tool.url) + const link = tool.name === 'SigNoz' ? SIGNOZ_PUBLIC_URL : safeExternalMonitoringUrl(tool.url) const timeStr = (() => { try { return new Date(tool.checked_at).toLocaleTimeString('zh-TW', { timeZone: 'Asia/Taipei', hour: '2-digit', minute: '2-digit' }) } catch { return '--' } @@ -294,6 +295,7 @@ const HOST_CATALOG: Record = { '110:3002': 'host:public-gateway/grafana', '110:3100': 'host:public-gateway/langfuse', '110:5000': 'host:public-gateway/registry', + '110:8080': 'host:public-gateway/signoz-health', '110:9000': 'host:public-gateway/sentry', '112:8080': 'host:kali-readonly/scanner', '188:11434': 'host:observability-a/ollama', '188:8088': 'host:observability-a/openclaw-legacy', '188:8089': 'host:observability-a/openclaw', - '188:3301': 'host:observability-a/signoz', + // Historical scrub-only alias; never use this legacy route for live access. + '188:3301': 'host:observability-a/signoz-legacy', '188:5432': 'host:observability-a/postgres', '188:6380': 'host:observability-a/redis', } diff --git a/apps/web/src/components/dashboard/live-dashboard.tsx b/apps/web/src/components/dashboard/live-dashboard.tsx index b4c096bef..5122e979d 100644 --- a/apps/web/src/components/dashboard/live-dashboard.tsx +++ b/apps/web/src/components/dashboard/live-dashboard.tsx @@ -81,6 +81,7 @@ export function LiveDashboard({ locale: _locale }: LiveDashboardProps) { services: [ { name: 'Harbor', status: 'idle', port: 5000 }, { name: 'GH Runner', status: 'idle' }, + { name: 'SigNoz', status: 'idle', port: 8080 }, { name: 'Docker', status: 'idle', port: 2375 }, ], }, @@ -111,7 +112,6 @@ export function LiveDashboard({ locale: _locale }: LiveDashboardProps) { { name: 'Redis', status: 'idle', port: 6380 }, { name: 'Ollama', status: 'idle', port: 11434 }, { name: 'OpenClaw', status: 'idle', port: 8089 }, - { name: 'SigNoz', status: 'idle', port: 3301 }, ], }, } diff --git a/infra/ansible/playbooks/110-devops.yml b/infra/ansible/playbooks/110-devops.yml index a0db8eb98..9130aefc1 100644 --- a/infra/ansible/playbooks/110-devops.yml +++ b/infra/ansible/playbooks/110-devops.yml @@ -242,6 +242,10 @@ - backup-awoooi.sh - backup-awoooi-frequent.sh - backup-signoz.sh + - clickhouse-native-backup.sh + - clickhouse-native-restore-drill.sh + - clickhouse-restore-inventory.py + - run-signoz-backup-canary.sh - backup-configs.sh - check-backup-integrity.sh - verify-gitea-backup.sh @@ -250,6 +254,36 @@ - verify-offsite-full-sync.sh tags: backup_jobs + - name: "Backup | 建立 SigNoz native backup/restore config 目錄" + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: wooo + group: wooo + mode: "0755" + loop: + - /backup/config/signoz/clickhouse/config.d + - /backup/config/signoz/clickhouse/restore-drill/config.d + tags: backup_jobs + + - name: "Backup | 部署 SigNoz native backup disk config" + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../../ops/signoz/clickhouse/config.d/backup_disk.xml" + dest: /backup/config/signoz/clickhouse/config.d/backup_disk.xml + owner: wooo + group: wooo + mode: "0644" + tags: backup_jobs + + - name: "Backup | 部署 SigNoz isolated restore cluster config" + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../../ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + dest: /backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml + owner: wooo + group: wooo + mode: "0644" + tags: backup_jobs + - name: "Backup | 安裝 AWOOOI daily backup Telegram heartbeat" ansible.builtin.cron: name: "AWOOOI daily backup Telegram heartbeat" diff --git a/k8s/awoooi-prod/15-service-registry-configmap.yaml b/k8s/awoooi-prod/15-service-registry-configmap.yaml index fefaaaf2d..7a040908d 100644 --- a/k8s/awoooi-prod/15-service-registry-configmap.yaml +++ b/k8s/awoooi-prod/15-service-registry-configmap.yaml @@ -70,9 +70,9 @@ data: - name: signoz-clickhouse display_name: "ClickHouse (SignOz)" - host: "192.168.0.188" + host: "192.168.0.110" stateful_level: BLOCK - reason: "列欄式 OLAP 資料庫，寫入中重啟可能損壞列欄檔案" + reason: "列欄式 OLAP 資料庫；只允許具備 native backup、bounded apply、rollback 與 post-verifier 的受控重建" alert_only: true containers: ["signoz-clickhouse"] diff --git a/k8s/monitoring/deploy-prometheus-config.sh b/k8s/monitoring/deploy-prometheus-config.sh index 18ed07200..ee398e098 100755 --- a/k8s/monitoring/deploy-prometheus-config.sh +++ b/k8s/monitoring/deploy-prometheus-config.sh @@ -1,118 +1,18 @@ -#!/bin/bash -# ============================================================================= -# Prometheus Config GitOps 部署腳本 -# ============================================================================= -# 用途: 將 Git 管理的 Prometheus 配置同步到 192.168.0.188 -# 建立者: Claude Code (首席架構師) -# 日期: 2026-03-29 (台北時間) +#!/usr/bin/env bash +# Legacy compatibility entrypoint. +# superseded_by=global_product_governance_v2 +# owner=observability-platform +# expiry=2026-07-15T00:00:00+08:00 +# replacement=scripts/ops/deploy-signoz-prometheus-route.sh +# verifier=promtool_then_two_successful_blackbox_scrapes # -# 使用方式: -# ./deploy-prometheus-config.sh [--dry-run] -# -# 依賴: -# - SSH 免密碼登入 ollama@192.168.0.188 -# - 遠端主機已安裝 Prometheus -# ============================================================================= +# The historical implementation appended partial config to the retired host188 +# Prometheus plane. Full-file replacement is also unsafe while independently +# governed production targets differ from the repository snapshot. Route this +# entrypoint to the bounded, target-only host110 deployer instead. set -euo pipefail -# 配置 -REMOTE_HOST="ollama@192.168.0.188" -REMOTE_CONFIG="/home/ollama/momo-pro/monitoring/prometheus.yml" -LOCAL_ADDITIONS="$(dirname "$0")/prometheus-config-additions.yaml" -DRY_RUN="${1:-}" - -# 顏色輸出 -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; } - -# 檢查本地配置檔案 -if [[ ! -f "$LOCAL_ADDITIONS" ]]; then - log_error "找不到配置檔案: $LOCAL_ADDITIONS" - exit 1 -fi - -log_info "Prometheus GitOps 部署開始" -log_info "目標主機: $REMOTE_HOST" -log_info "遠端配置: $REMOTE_CONFIG" - -# 備份遠端配置 -BACKUP_FILE="${REMOTE_CONFIG}.bak.$(date +%Y%m%d_%H%M%S)" -log_info "備份遠端配置至: $BACKUP_FILE" - -if [[ "$DRY_RUN" == "--dry-run" ]]; then - log_warn "DRY RUN 模式 - 不會實際執行" - echo "" - echo "將執行以下操作:" - echo " 1. ssh $REMOTE_HOST \"cp $REMOTE_CONFIG $BACKUP_FILE\"" - echo " 2. 檢查配置中是否已包含 argocd job" - echo " 3. 如需更新，附加新配置" - echo " 4. ssh $REMOTE_HOST \"docker exec prometheus kill -SIGHUP 1\"" - exit 0 -fi - -# 執行備份 -ssh "$REMOTE_HOST" "cp $REMOTE_CONFIG $BACKUP_FILE" -log_info "備份完成" - -# 檢查是否已有 argocd job -ARGOCD_EXISTS=$(ssh "$REMOTE_HOST" "grep -c 'job_name.*argocd' $REMOTE_CONFIG || true") - -if [[ "$ARGOCD_EXISTS" -gt 0 ]]; then - log_info "ArgoCD job 已存在於配置中，跳過新增" -else - log_info "新增 ArgoCD 和 Blackbox scrape configs..." - - # 提取配置內容 (去除註解標題) - CONFIG_CONTENT=$(grep -A 100 "^- job_name: argocd" "$LOCAL_ADDITIONS" | head -35) - - # 附加到遠端配置 - ssh "$REMOTE_HOST" "cat >> $REMOTE_CONFIG << 'EOF' - -# === GitOps 新增 ($(date +%Y-%m-%d)) === -$CONFIG_CONTENT -EOF" - - log_info "配置已更新" -fi - -# 驗證配置語法 -log_info "驗證 Prometheus 配置語法..." -SYNTAX_CHECK=$(ssh "$REMOTE_HOST" "docker exec prometheus promtool check config /etc/prometheus/prometheus.yml 2>&1 || true") - -if echo "$SYNTAX_CHECK" | grep -q "SUCCESS"; then - log_info "配置語法正確" -else - log_error "配置語法錯誤!" - echo "$SYNTAX_CHECK" - log_warn "正在還原備份..." - ssh "$REMOTE_HOST" "cp $BACKUP_FILE $REMOTE_CONFIG" - exit 1 -fi - -# 重載 Prometheus -log_info "重載 Prometheus 配置..." -ssh "$REMOTE_HOST" "docker exec prometheus kill -SIGHUP 1 || sudo systemctl reload prometheus" - -# 等待並驗證 -sleep 3 -log_info "驗證 targets..." - -TARGETS=$(curl -s "http://192.168.0.188:9090/api/v1/targets" 2>/dev/null | \ - jq -r '.data.activeTargets[] | "\(.labels.job): \(.health)"' 2>/dev/null || echo "驗證失敗") - -echo "" -echo "═══════════════════════════════════════════════════════" -echo " Prometheus Targets 狀態" -echo "═══════════════════════════════════════════════════════" -echo "$TARGETS" | grep -E "argocd|blackbox" || echo " (無 argocd/blackbox targets)" -echo "═══════════════════════════════════════════════════════" -echo "" - -log_info "部署完成!" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +printf 'NOTICE: deploy-prometheus-config.sh is superseded; using the bounded host110 route deployer\n' >&2 +exec "${REPO_ROOT}/scripts/ops/deploy-signoz-prometheus-route.sh" "${1:-apply}" diff --git a/k8s/monitoring/k3s-alerts.yaml b/k8s/monitoring/k3s-alerts.yaml index b6bc71302..6682f6bcd 100644 --- a/k8s/monitoring/k3s-alerts.yaml +++ b/k8s/monitoring/k3s-alerts.yaml @@ -92,17 +92,9 @@ groups: # ===== 可觀測性服務告警 ===== - name: observability_alerts rules: - # SignOz 離線 - - alert: SignOzDown - expr: probe_success{job="blackbox-tcp", instance="192.168.0.188:3301"} == 0 - for: 2m - labels: - severity: warning - team: ops - component: signoz - annotations: - summary: "⚠️ SignOz 服務離線" - description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘" + # SigNoz query/UI health belongs to the canonical 110 Prometheus plane in + # ops/monitoring/alerts-unified.yml. This legacy 188 rule mirror has no + # matching canonical blackbox-http series and must not duplicate it. # Sentry 離線 - alert: SentryDown diff --git a/k8s/monitoring/prometheus.yml b/k8s/monitoring/prometheus.yml index 23db89f82..067b868dd 100644 --- a/k8s/monitoring/prometheus.yml +++ b/k8s/monitoring/prometheus.yml @@ -56,6 +56,7 @@ scrape_configs: - https://aiops.wooo.work - https://mo.wooo.work - http://192.168.0.110:3001 + - http://192.168.0.110:8080/api/v1/health - http://192.168.0.125:32334/api/v1/health - http://192.168.0.125:32335 - https://www.tsenyang.com @@ -87,7 +88,6 @@ scrape_configs: - 192.168.0.110:5000 - 192.168.0.110:3100 # 188 服務（Ollama 已於 2026-05-06 自 AWOOOI provider pool 退役） - - 192.168.0.188:3301 - 192.168.0.188:5432 - 192.168.0.188:6380 - 192.168.0.188:8089 diff --git a/ops/config/service-registry.yaml b/ops/config/service-registry.yaml index 710df45b6..262520d69 100644 --- a/ops/config/service-registry.yaml +++ b/ops/config/service-registry.yaml @@ -54,9 +54,9 @@ services: - name: signoz-clickhouse display_name: "ClickHouse (SignOz)" - host: "192.168.0.188" + host: "192.168.0.110" stateful_level: BLOCK - reason: "列欄式 OLAP 資料庫，寫入中重啟可能損壞列欄檔案" + reason: "列欄式 OLAP 資料庫；只允許具備 native backup、bounded apply、rollback 與 post-verifier 的受控重建" alert_only: true containers: ["signoz-clickhouse"] diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index f30809f58..c453077fe 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -716,21 +716,23 @@ groups: description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘" - alert: SignOzDown - expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0 + expr: >- + (probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"} == 0) + or absent(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"}) for: 2m labels: severity: warning - layer: docker-188 + layer: docker-110 component: signoz - host: "188" + host: "110" team: ops auto_repair: "true" mcp_provider: "ssh_host" - target_host: "192.168.0.188" + target_host: "192.168.0.110" alert_category: "devops_tool" annotations: summary: "SignOz 服務離線" - description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘" + description: "SignOz (http://192.168.0.110:8080/api/v1/health) 已離線或監控序列缺失超過 2 分鐘" # ---- 110 Docker 層 ---- - alert: SentryDown diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml index 3e21b64f2..b437fc24f 100644 --- a/ops/monitoring/alerts.yml +++ b/ops/monitoring/alerts.yml @@ -400,21 +400,23 @@ groups: description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘" - alert: SignOzDown - expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0 + expr: >- + (probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"} == 0) + or absent(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"}) for: 2m labels: severity: warning - layer: docker-188 + layer: docker-110 component: signoz - host: "188" + host: "110" team: ops auto_repair: "true" mcp_provider: "ssh_host" - target_host: "192.168.0.188" + target_host: "192.168.0.110" alert_category: "devops_tool" annotations: summary: "SignOz 服務離線" - description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘" + description: "SignOz (http://192.168.0.110:8080/api/v1/health) 已離線或監控序列缺失超過 2 分鐘" # ---- 110 Docker 層 ---- - alert: SentryDown diff --git a/ops/monitoring/generate_monitoring.py b/ops/monitoring/generate_monitoring.py index 1488cc519..6c8c3014a 100755 --- a/ops/monitoring/generate_monitoring.py +++ b/ops/monitoring/generate_monitoring.py @@ -54,7 +54,10 @@ def generate_prometheus_scrape_configs(registry: dict) -> list[dict]: # K8s 服務 for svc in registry.get("services", []): - if svc.get("monitoring", {}).get("prometheus"): + monitoring = svc.get("monitoring", {}) + if monitoring.get("prometheus") and monitoring.get( + "prometheus_scrape", True + ): if svc.get("type") == "k8s-deployment": # K8s ServiceMonitor 風格 config = { diff --git a/ops/monitoring/service-registry.yaml b/ops/monitoring/service-registry.yaml index 10bafca81..ffc41e6fe 100644 --- a/ops/monitoring/service-registry.yaml +++ b/ops/monitoring/service-registry.yaml @@ -335,6 +335,8 @@ services: health_type: grpc monitoring: prometheus: true + prometheus_scrape: false + external_verifier: scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh sentry: false alerts: - service_down @@ -345,11 +347,13 @@ services: # --- SignOz UI --- - name: signoz-ui type: docker - host: 192.168.0.188 - port: 3301 - health_endpoint: / + host: 192.168.0.110 + port: 8080 + health_endpoint: /api/v1/health + health_type: http monitoring: prometheus: true + prometheus_scrape: false sentry: false alerts: - service_down diff --git a/ops/signoz/alerting/log-rules.md b/ops/signoz/alerting/log-rules.md index 4c1a418eb..3f9344d4d 100644 --- a/ops/signoz/alerting/log-rules.md +++ b/ops/signoz/alerting/log-rules.md @@ -1,11 +1,13 @@ # SigNoz Log-Based Alert Rules # 2026-04-05 Claude Code: Sprint 2 — 日誌告警 -# 設定位置: http://192.168.0.188:3301/alerts (SigNoz UI) +# superseded_by=global_product_governance_v2: legacy 192.168.0.188:3301 UI/query route +# 設定位置: https://signoz.wooo.work/alerts (SigNoz human UI) +# Machine health: http://192.168.0.110:8080/api/v1/health # Webhook: http://192.168.0.121:32334/api/v1/webhooks/signoz ## 設定步驟 -1. 開啟 http://192.168.0.188:3301/alerts +1. 開啟 https://signoz.wooo.work/alerts 2. 點擊 "New Alert Rule" → "Logs Based Alert" 3. 依照下表填入各欄位 4. Notification Channel: 選擇 awoooi-api (指向 /api/v1/webhooks/signoz) @@ -13,6 +15,8 @@ 驗證 Webhook 鏈路： ```bash +curl -fsS http://192.168.0.110:8080/api/v1/health | jq . + curl -X POST http://192.168.0.121:32334/api/v1/webhooks/signoz \ -H 'Content-Type: application/json' \ -d '{"alerts":[{"labels":{"alertname":"SignozWebhookTest","severity":"info","layer":"k8s"},"annotations":{"summary":"Sprint 2 webhook 驗證，請忽略"},"status":"firing"}],"version":"4"}' diff --git a/ops/signoz/clickhouse/config.d/backup_disk.xml b/ops/signoz/clickhouse/config.d/backup_disk.xml new file mode 100644 index 000000000..b4d5e4523 --- /dev/null +++ b/ops/signoz/clickhouse/config.d/backup_disk.xml @@ -0,0 +1,24 @@ + + + + + + + local + /backups/ + + + + + backups + /backups/ + false + false + + diff --git a/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml b/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml new file mode 100644 index 000000000..390587092 --- /dev/null +++ b/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml @@ -0,0 +1,35 @@ + + + + + + restore-zookeeper + 2181 + + + + + + + true + + restore-clickhouse + 9000 + + + + + + + 01 + + + diff --git a/ops/signoz/docker-compose.clickhouse-backup.override.yaml b/ops/signoz/docker-compose.clickhouse-backup.override.yaml new file mode 100644 index 000000000..db51a7bed --- /dev/null +++ b/ops/signoz/docker-compose.clickhouse-backup.override.yaml @@ -0,0 +1,13 @@ +# P0-OBS-002: additive override for the host110 SigNoz ClickHouse service. +# Always pass this file explicitly after the upstream compose file with -f. +services: + clickhouse: + volumes: + - type: bind + source: /backup/staging/signoz-clickhouse + target: /backups + read_only: false + - type: bind + source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml + target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml + read_only: true diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index e9ee97bc5..3a94d61c3 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -2,7 +2,7 @@ schema: awoooi_signoz_organization_canonical_route_migration_v1 work_item_id: P0-OBS-002 owner_lane: platform-observability phase: wave_a_dual_allow -generated_at: "2026-07-15T03:51:00+08:00" +generated_at: "2026-07-15T06:06:46+08:00" policy: superseded_by: global_product_governance_v2 controlled_apply_allowed: true @@ -154,11 +154,47 @@ runtime_observations: promotion_authorized: false post_closure_regression: snapshot: ops/signoz/p0-obs-002-post-closure-regression.yaml + snapshot_schema: awoooi_signoz_post_closure_regression_v2 + gitea_cd_run_id: 5140 + release_source_sha: e4da6570a645e504cc93647d396f45f167b3172a + release_terminal: success grpc_bridge_rollback_indicated: false canonical_collector_recovered: true - api_runtime_cooldown_closed: false - cd_probe_safety_fix_deployed: false - backup_collector_restore_guard_deployed: false + api_runtime_cooldown_closed: true + cd_probe_safety_fix_deployed: true + config_route_patch_run_id: P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1 + config_route_patch_deployed: true + signoz_control_plane_health_route_deployed: true + prometheus_target_route_run_id: P0-OBS-002-prometheus-route-20260714T215106Z-apply1 + prometheus_target_route_terminal: completed_target_first + alert_rules_run_id: P0-OBS-002-alert-rules-20260714T215328Z-apply1 + alert_rules_terminal: completed_verified + backup_collector_restore_guard_deployed: true + backup_canary_scope: clickhouse_native_only + backup_canary_verified: true + backup_canary_last_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4 + backup_canary_last_terminal: pass + backup_canary_prior_root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract + backup_canary_cooldown_retry_status: superseded_by_online_native_no_stop_contract + clickhouse_native_backup_migration_status: production_closed + clickhouse_native_backup_disk_run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2 + clickhouse_native_backup_disk_status: deployed_verified + clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4 + clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4 + clickhouse_native_restore_terminal: verified + clickhouse_native_restic_snapshot_id: 574e2a1a + clickhouse_native_offsite_status: pending + clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision + clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract + signoz_sqlite_application_consistent_backup_status: pending + service_registry_runtime_mirror_status: partial_degraded + service_registry_runtime_mirror_work_item_id: P0-OBS-002-ASSET-DRIFT-001 + github_freeze_legacy_asset_status: pending_controlled_retirement + github_freeze_legacy_asset_work_item_id: P0-OBS-002-ASSET-DRIFT-002 + monitoring_generator_duplicate_identity_status: pending_source_deduplication + monitoring_generator_duplicate_identity_work_item_id: P0-OBS-002-ASSET-DRIFT-003 + post_release_otlp_grpc_run_id: P0-OBS-002-post-release-20260714T215500Z + post_release_otlp_grpc_600s_verifier_status: pass terminal: status: partial_degraded @@ -167,8 +203,4 @@ terminal: - direct_ingress_policy_not_closed - http_bridge_not_supervised - filelog_normalization_degraded - - api_runtime_cooldown_pending - - cd_probe_safety_fix_not_deployed - - backup_collector_restore_guard_not_deployed - - signoz_control_plane_health_route_not_deployed - wave_b_not_authorized diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 7934161f4..0bb28a860 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -1,7 +1,7 @@ -schema: awoooi_signoz_post_closure_regression_v1 +schema: awoooi_signoz_post_closure_regression_v2 work_item_id: P0-OBS-002 trace_id: P0-OBS-002 -observed_at: "2026-07-15T02:20:00+08:00" +observed_at: "2026-07-15T08:16:07+08:00" baseline: grpc_runtime_run_id: P0-OBS-002-20260715T013800+0800 @@ -11,6 +11,91 @@ baseline: trace_count: 1098 metric_sample_count: 60180 +delivery_validation: + backup_restore_deployer_governance_focused_tests: + passed: 94 + failed: 0 + post_format_rerun: pass + api_monitoring_alert_adjacent_tests: + passed: 115 + failed: 0 + environment: explicit_dummy_local_database_and_redis_urls_no_env_file_read + web: + dependency_install: offline_frozen_lockfile + packages_reused: 851 + packages_downloaded: 0 + typecheck: pass + prettier: legacy_file_style_not_enforced + bulk_format_churn_removed: true + eslint_errors: 0 + eslint_existing_warnings: 6 + production_build: pass + build_environment: explicit_local_public_api_and_signoz_urls_no_env_file_read + monitoring: + generator_validate_only: pass + registry_service_count: 26 + registry_monitored_count: 25 + registry_coverage_percent: 96.2 + generated_scrape_config_count: 24 + generated_blackbox_target_count: 20 + promtool_rules: + alerts: 81 + alerts_unified: 147 + k3s_alerts: 14 + prometheus_config_syntax: pass + infrastructure: + configmap_client_dry_run: pass + ansible_validation_script: pass_with_local_ansible_syntax_check_unavailable + ansible_110_playbook_isolated_syntax_check: pass + shell_syntax: pass + python_compile: pass + documentation_secret_sanity_files_scanned: 1090 + github_used: false + secret_read: false + +asset_reconciliation: + drift_work_item_id: P0-OBS-002-ASSET-DRIFT-001 + source: ops/config/service-registry.yaml + runtime_mirror: k8s/awoooi-prod/15-service-registry-configmap.yaml + source_service_count: 27 + runtime_mirror_service_count: 24 + exact_shared_service_count: 24 + shared_service_drift_count: 0 + source_only_services: + - bitan-app + - sentry + - signoz + signoz_clickhouse_exact_match: true + live_signoz_query_host: 192.168.0.110 + source_signoz_host: 192.168.0.188 + terminal: partial_degraded + safe_next_action: >- + Reconcile the three source-only services and the stale SignOz query host + against production runtime before atomically regenerating the ConfigMap; + do not copy the stale 188 SignOz row into the runtime mirror. + github_freeze_legacy_asset_drift: + drift_work_item_id: P0-OBS-002-ASSET-DRIFT-002 + superseded_by: global_product_governance_v2 + status: pending_controlled_retirement + active_github_use_in_this_run: false + source_references: + - ops/monitoring/service-registry.yaml:github-runner + - infra/ansible/playbooks/110-devops.yml:github_runner_management + - apps/web/src/components/dashboard/live-dashboard.tsx:gh_runner_fallback + replacement: gitea_runner_readiness_and_queue_receipts + exit_condition: legacy_runner_assets_absent_and_gitea_replacement_verified + p0_p1_github_recovery_scheduled: false + monitoring_generator_duplicate_identity_drift: + drift_work_item_id: P0-OBS-002-ASSET-DRIFT-003 + status: pending_source_deduplication + registry_service_count: 26 + awoooi_api_registry_entry_count: 2 + generated_scrape_config_count: 24 + generated_awoooi_api_job_count: 2 + generated_blackbox_target_count: 20 + runtime_apply_performed: false + exit_condition: one_canonical_awoooi_api_asset_and_one_generated_scrape_job + regressions: api_runtime: first_probe_timeout_at: "2026-07-15T02:11:22+08:00" @@ -21,7 +106,7 @@ regressions: expected_replicas: 2 restart_count_per_pod_at_recovery: 4 last_observed_ready_replicas: 2 - last_observed_cooldown_closed: false + last_observed_cooldown_closed: true primary_classification: db_connection_budget_and_request_pressure structural_risk_confidence: 0.98 direct_application_commit_regression_confidence: 0.12 @@ -54,50 +139,508 @@ controlled_recovery: - 192.168.0.120 - 192.168.0.121 -source_candidates: - cd_probe: - path: .gitea/workflows/cd.yaml - connection_budget_precheck_required: true - max_workers: 4 - max_attempts: 1 - request_timeout_seconds: 5 - safety_reserve_connections: 4 - receipt_fields_added: true - deployment_status: pending - backup_restore_guard: - path: scripts/backup/backup-signoz.sh - deployment_playbook: infra/ansible/playbooks/110-devops.yml - deployment_tag: backup_jobs - deployment_contract_present: true - trap_exit_hup_int_term: true - restore_armed_before_stop: true - bounded_exit_retry: true - deployment_status: pending - canonical_health_route: - internal_url: http://192.168.0.110:8080 - public_url: https://signoz.wooo.work - production_networkpolicy_tcp_port: 8080 - legacy_query_route_removed: http://192.168.0.188:3301 - otlp_producer_route_changed: false - deployment_status: pending +release_receipts: + gitea_cd_5140: + gitea_run_id: 5140 + source_sha: e4da6570a645e504cc93647d396f45f167b3172a + workflow_terminal: success + focused_tests_passed: 6 + unit_tests: + passed: 4817 + skipped: 23 + warnings: 69 + integration_tests_passed: 5 + production_readback: + build_tag_matched: true + desired_tag_matched: true + attempt: 1 + rollout_terminal: success + transient_rollout_risk_observed: true + postdeploy: + alert_chain_terminal: pass + alert_chain_checks_passed: 8 + alert_chain_checks_total: 9 + playwright_passed: 5 + notification_mirrored: true + cd_probe: + receipt_provenance: gitea_run_5140_job_log + passed: true + workers: 4 + safety_reserve_connections: 4 + max_attempts: 1 + request_timeout_seconds: 5 + skip_reason: "" + restart_free: true + restart_count_before: 0 + restart_count_after: 0 + ready_replicas: 2 + desired_replicas: 2 + +controlled_applies: + config_route_patch: + trace_id: P0-OBS-002 + run_id: P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1 + work_item_id: P0-OBS-002 + source_commit: 9a9d1464a586ba172d1613db5bc285def43eb3e8 + included_in_release_sha: e4da6570a645e504cc93647d396f45f167b3172a + normalized_asset: + canonical_id: signoz-110-query-health-route + internal_url: http://192.168.0.110:8080 + public_url: https://signoz.wooo.work + health_path: /api/v1/health + production_networkpolicy_tcp_port: 8080 + source_truth_diff: + legacy_query_route_removed: http://192.168.0.188:3301 + otlp_producer_route_changed: false + check_receipt: + source_tests_passed: true + execution_receipt: + gitea_run_id: 5140 + deployment_status: deployed_verified + post_verifier: + release_readback_matched: true + ready_replicas: 2 + desired_replicas: 2 + rollback: not_required + terminal: applied_post_verifier_pass + + prometheus_target_route: + trace_id: P0-OBS-002 + run_id: P0-OBS-002-prometheus-route-20260714T215106Z-apply1 + work_item_id: P0-OBS-002 + risk: medium + check_receipt: + promtool: pass + source_sha: 462cb5c2397d1a6ec0542f2fde83e06e0baa05d82df299ac9ae9464d08d9947d + candidate_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c + execution_receipt: + action: target_first_config_apply + active_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c + runtime_sha: 08db2f8207b416b6f1fbed9e4102579fab9207b5de8cace7bdfa9a9ea2601d2c + ready: true + backup_path: /home/wooo/monitoring/prometheus.yml.bak.P0-OBS-002-prometheus-route-20260714T215106Z-apply1 + backup_sha: 462cb5c2397d1a6ec0542f2fde83e06e0baa05d82df299ac9ae9464d08d9947d + post_verifier: + terminal: target_up_two_scrapes + target_count: 1 + target_health: up + first_scrape_at: "2026-07-15T05:52:38.146935853+08:00" + second_scrape_at: "2026-07-15T05:52:53.146935853+08:00" + probe_success: 1 + legacy_target_count: 0 + independent_verifier: + mode: independent_readback + observed_at: "2026-07-15T05:58:09+08:00" + active_sha_matches_runtime: true + target_count: 1 + target_health: up + probe_success: 1 + legacy_target_count: 0 + terminal: pass + rollback: + required: false + available: true + terminal: completed_target_first + + alert_rules: + trace_id: P0-OBS-002 + run_id: P0-OBS-002-alert-rules-20260714T215328Z-apply1 + work_item_id: P0-OBS-002 + check_receipt: + promtool: pass + source_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58 + execution_receipt: + remote_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58 + canonical_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58 + runtime_sha: 1d9ed0d07055a5f7253d7e6653a901af873c1da04a00fa89ea585bdbd7c38c58 + hashes_match: true + prometheus_ready: true + post_verifier: + window_seconds: 130 + terminal: pass + signoz_rule_count: 1 + signoz_rule_health: ok + signoz_rule_state: inactive + exact_query: '(probe_success{instance="http://192.168.0.110:8080/api/v1/health",job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",job="blackbox-http"})' + rollback: + required: false + available: true + terminal: completed_verified + + clickhouse_native_backup_disk: + trace_id: P0-OBS-002 + work_item_id: P0-OBS-002 + risk: high + normalized_asset: + canonical_id: host110-signoz-clickhouse-backup-disk + compose_project: signoz + compose_service: clickhouse + clickhouse_disk: backups + container_path: /backups/ + host_path: /backup/staging/signoz-clickhouse + source: + override_sha256: cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488 + config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9 + focused_tests_passed: 12 + bash_syntax: pass + check_receipt: + run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064800P0800-check2 + terminal: check_pass_no_write + prior_disk_count: 0 + prior_backup_mount_count: 0 + collector_running: true + collector_restart_count: 0 + attempts: + - run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064300P0800-apply1 + terminal: failed_rolled_back + failed_stage: post_recreate_server_identity_verifier + exit_code: 2 + root_cause: remote_heredoc_expanded_awk_positional_parameter + source_fix: direct_awk_arguments_without_remote_shell_expansion + rollback: + terminal: pass + prior_compose_state_restored: true + managed_files_absent: true + host_backup_directory_absent: true + clickhouse_healthy: true + clickhouse_restart_count: 0 + original_data_volume_preserved: true + dependent_container_identity_preserved: true + active_backup_or_restore_count: 0 + residue_count: 0 + - run_id: P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2 + terminal: pass + action: clickhouse_only_force_recreate_no_pull + execution: + image_digest_unchanged: true + data_volume_name: signoz-clickhouse + data_volume_unchanged: true + clickhouse_restart_count: 0 + post_verifier: + independent: true + disk_type: Local + disk_read_only: false + disk_remote: false + disk_broken: false + disk_free_space_positive: true + server_uid: 101 + server_gid: 101 + server_write_access: true + concurrent_backups_allowed: false + concurrent_restores_allowed: false + collector_identity_unchanged: true + signoz_identity_unchanged: true + zookeeper_identity_unchanged: true + dependent_restart_count_delta: 0 + grpc_4317_listening: true + http_4318_listening: true + health_8080_listening: true + api_http_status: 200 + active_backup_or_restore_count: 0 + backup_directory_entry_count: 0 + stage_candidate_process_residue_count: 0 + terminal: deployed_verified + + clickhouse_native_backup_restore_canary: + trace_id: P0-OBS-002 + work_item_id: P0-OBS-002 + risk: high + completion_scope: clickhouse_native_only + source: + backup_orchestrator_sha256: 2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520 + native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a + isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7 + inventory_verifier_sha256: 79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c + canary_wrapper_sha256: 64d23d9f7a64d1e4e1347b98d45e50e26eadea1741d0e28a5e2e5d74b22322d8 + backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9 + isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b + latest_restore_focused_tests_passed: 12 + post_canary_focused_tests_passed: 94 + post_format_focused_tests_passed: 94 + bash_syntax: pass + ruff: pass + ruff_format: pass + yaml_xml_parse: pass + toolchain_controlled_apply: + check_run_id: P0-OBS-002-native-toolchain-20260715T001124Z-check5 + check_terminal: pass + check_diff: matching_6_drift_1_absent_0_active_operations_0 + apply_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4 + apply_terminal: pass + action: seven_candidates_atomically_replaced + rollback_available: true + runtime_backup_restore_restart_or_pull: false + postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4 + postcheck_terminal: check_pass_no_write + postcheck_diff: matching_7_drift_0_absent_0_active_operations_0 + attempts: + - run_id: P0-OBS-002-native-canary-20260714T233524Z-apply1 + terminal: failed_cleanup_verified + failed_stage: backup_submit_parser + exit_code: 62 + root_cause: clickhouse_25_5_async_settings_id_parameter_syntax_invalid + backup_submitted: false + restore_submitted: false + server_artifact_created: false + ephemeral_residue_count: 0 + - run_id: P0-OBS-002-native-canary-20260714T234436Z-apply2 + terminal: failed_cleanup_verified_artifact_preserved + failed_stage: isolated_restore_startup + root_cause: + - read_only_source_artifact_was_chowned_by_clickhouse_entrypoint + - isolated_zookeeper_anonymous_login_contract_missing + backup_terminal: BACKUP_CREATED + backup_file_count: 3690 + backup_total_size_bytes: 88261384 + artifact_size_bytes: 81474109 + artifact_sha256: 139230bc6aa9f53b7b351b5d8f37eed87a68cfbba31a0b4ff4caa31906dfdaac + restore_submitted: false + cleanup_verified: true + artifact_retention: preserved_failed_run_evidence + - run_id: P0-OBS-002-native-canary-20260715T000638Z-apply3 + terminal: failed_cleanup_verified_artifact_preserved + failed_stage: macro_contract + exit_code: 47 + root_cause: clickhouse_25_5_system_macros_uses_macro_and_substitution_columns + backup_terminal: BACKUP_CREATED + backup_file_count: 4414 + backup_total_size_bytes: 92901923 + artifact_size_bytes: 85561114 + artifact_sha256: e4da648d36462e3362ff6532b8bee8f5c6745c2de4bf188fe95fe68946287993 + staging_artifact_verified: true + restore_submitted: false + cleanup_verified: true + artifact_retention: preserved_failed_run_evidence + - run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4 + check_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4-check + check_terminal: check_pass_no_write + terminal: pass + exit_code: 0 + backup: + operation_id: awoooi-6d64f408f85ad25cd78a351c0c851db8a29bea958afaad5ba6e07f03ec7a55dc + terminal: BACKUP_CREATED + error: "" + file_count: 4228 + total_size_bytes: 93405832 + compressed_size_bytes: 86033128 + artifact_sha256: 4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc + source_inventory_sha256: 1c290facb734d59443eaeeec0d778772f5fd8e06e832a096c0d4f82a4a7210ef + isolated_restore: + terminal: verified + clickhouse_restore_terminal: RESTORED + exact_clickhouse_image: true + exact_zookeeper_image: true + staging_network_mode: none + restore_network_mode: docker_internal_only + production_network_attached: false + production_restore_performed: false + artifact_source_mount: read_only_staging_only + backup_volume_mount: writable_ephemeral + source_staged_clickhouse_artifact_sha_match: true + database_count: 6 + restored_database_count: 6 + table_count: 100 + restored_table_count: 100 + table_engine_schema_manifest_parity: true + check_table_count: 44 + check_table_pass_count: 44 + critical_nonzero_count: 4 + source_total_rows: 4772292 + restored_total_rows: 4752914 + row_delta: -19378 + source_total_bytes: 93042212 + restored_total_bytes: 93243468 + byte_delta: 201256 + aggregate_counter_exact_parity_gate: false + aggregate_counter_reason: online_ingestion_can_advance_between_source_inventory_and_backup_snapshot + restic: + snapshot_id: 574e2a1a + snapshot_readback: pass + repository_check_read_data_subset: 1% + repository_scope: local_same_failure_domain + offsite_verified: false + independent_post_verifier: + independent_monitor: pass + collector_identity_started_at_restart_health_listeners_unchanged: true + production_container_identity_count_unchanged: 4 + production_restart_count_delta: 0 + api_http_status: 200 + new_server_artifact_removed: true + apply2_apply3_preserved_artifact_hashes_unchanged: true + exact_ephemeral_container_volume_network_process_tmp_residue_count: 0 + cleanup_verified: true + secret_values_collected: false + closure_writeback: durable_local_receipts_acknowledged + terminal: production_closed_clickhouse_native_scope + + backup_canary: + guard_source: + path: scripts/backup/run-signoz-backup-canary.sh + monitor_cooldown_contract_present: true + deployment_status: deployed_verified + attempts: + - trace_id: P0-OBS-002 + run_id: P0-OBS-002-backup-canary-20260714T213210Z-retry1 + work_item_id: P0-OBS-002 + terminal: failed_rolled_back + classification: failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention + failed_stage: clickhouse_archive_create + archive_container_exit_code: 137 + root_cause: docker_health_monitor_restarted_collector_during_intentional_backup_stop + consistent_archive_valid: false + clickhouse_archive_committed: false + sqlite_stage_reached: false + restic_stage_reached: false + retention_stage_reached: false + success_stage_reached: false + rollback: + collector_running: true + collector_pid_at_verifier: 1518279 + collector_started_at: "2026-07-14T21:35:01Z" + collector_restart_count: 0 + grpc_4317_listening: true + http_4318_listening: true + canary_processes_absent: true + canary_container_absent: true + current_temp_dir_absent: true + partial_artifacts_absent: true + residual_cleanup_debt: + - /tmp/signoz-backup-3723877 + - trace_id: P0-OBS-002 + run_id: P0-OBS-002-backup-canary-20260714T221230Z-retry2 + work_item_id: P0-OBS-002 + terminal: failed_rolled_back + classification: failed_live_rw_volume_archive_native_backup_required_no_restic_no_retention + failed_stage: clickhouse_archive_create + backup_script_exit_code: 1 + archive_container_exit_code: unknown_not_durably_captured + archive_stderr_receipt: suppressed_by_legacy_pipeline + root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract + exact_tar_error_proven: false + likely_tar_file_changed_during_read: true + consistent_archive_valid: false + clickhouse_archive_committed: false + sqlite_stage_reached: false + restic_stage_reached: false + retention_stage_reached: false + success_stage_reached: false + monitor_window: + cron_intervals_crossed: 2 + collector_detected_during_intentional_stop: true + cooldown_receipt_observed: true + auto_repair_count: 0 + rollback: + cooldown_lease_restored: true + collector_running: true + collector_started_at: "2026-07-14T22:20:53Z" + collector_restart_count: 0 + grpc_4317_listening: true + http_4318_listening: true + health_8080_listening: true + canary_processes_absent: true + canary_container_absent: true + current_temp_dir_absent: true + partial_artifacts_absent: true + residual_cleanup_debt: [] + +post_release_runtime_verifier: + trace_id: P0-OBS-002 + run_id: P0-OBS-002-post-release-20260714T215500Z + work_item_id: P0-OBS-002 + window_seconds: 600 + terminal: pass + restart_delta: 0 + exporter_trace_errors: 0 + exporter_metric_errors: 0 + trace_count: 4026 + metric_sample_count: 76369 + api_runtime_cooldown_closed: true + rollback: not_required + +completed_verifiers: + clickhouse_native_backup_migration: + prior_run_id: P0-OBS-002-backup-canary-20260714T221230Z-retry2 + run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4 + status: production_closed_clickhouse_native_scope + native_backup_terminal: BACKUP_CREATED + isolated_restore_terminal: verified + restic_snapshot_id: 574e2a1a + independent_verifier: pass + exact_cleanup: pass + +pending_verifiers: + signoz_sqlite_application_consistent_backup: + status: pending_supported_non_raw_export_or_coordinated_snapshot + raw_sqlite_read_or_sync_allowed: false + sqlite_cli_present_in_runtime: false + required_preconditions: + - supported_metadata_export_or_application_consistent_snapshot_design + - independent_restore_verifier_without_raw_session_or_secret_read + signoz_backup_offsite_dr: + status: pending_offsite_snapshot_and_restore_verifier + current_repository_scope: local_same_failure_domain + latest_local_snapshot_id: 574e2a1a + offsite_verified: false + required_preconditions: + - immutable_or_versioned_offsite_target + - bounded_snapshot_copy_receipt + - independent_offsite_restore_drill + clickhouse_native_failed_artifact_retention: + status: pending_bounded_retention_decision + destructive_cleanup_authorized: false + active_backup_or_restore_count: 0 + preserved_artifacts: + - run_id: P0-OBS-002-native-canary-20260714T234436Z-apply2 + path: /backups/signoz-35003fe7ee97b8f74547b5fb2c4b1df01b9f472137e3b8299fd8e95ded6d220e.zip + sha256: 139230bc6aa9f53b7b351b5d8f37eed87a68cfbba31a0b4ff4caa31906dfdaac + size_bytes: 81474109 + mode: "0640" + - run_id: P0-OBS-002-native-canary-20260715T000638Z-apply3 + path: /backups/signoz-a011c242647d8a58cf47c38e90605b5325fa0800871256a04e0fbcdeb9b9a289.zip + sha256: e4da648d36462e3362ff6532b8bee8f5c6745c2de4bf188fe95fe68946287993 + size_bytes: 85561114 + mode: "0640" + clickhouse_native_resume_submitted_inventory: + status: pending_durable_submitted_inventory_contract + same_run_resume_safe: false + reason: live_inventory_drift_must_not_rebind_an_existing_backup_artifact + required_preconditions: + - atomically_persist_inventory_before_first_backup_submit + - bind_inventory_hash_operation_id_artifact_hash_and_run_identity + - resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered + - regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits scope_boundaries: grpc_bridge_rollback_indicated: false api_manual_rollback_performed: false firewall_changed: false database_changed: false - stateful_volume_read: false + stateful_volume_read: true + stateful_volume_write: true + clickhouse_container_recreated: true + clickhouse_data_volume_identity_changed: false secret_read: false github_used: false terminal: status: partial_degraded blockers: - - api_runtime_cooldown_pending - - cd_probe_safety_fix_not_deployed - - backup_collector_restore_guard_not_deployed - - newer_main_cd_nonterminal + - signoz_sqlite_application_consistent_backup_pending + - signoz_backup_offsite_dr_pending + - clickhouse_native_failed_artifact_retention_pending + - clickhouse_native_resume_submitted_inventory_pending + - service_registry_runtime_mirror_reconciliation_pending + - github_freeze_legacy_asset_retirement_pending + - monitoring_generator_duplicate_awoooi_api_identity_pending safe_next_action: >- - Merge current Gitea main, validate and deploy the bounded CD probe and backup - restore guard, then require a fresh CD terminal plus ten-minute API and OTLP - runtime verifier with zero restart and exporter-error deltas. + Preserve the verified native ClickHouse backup and isolated restore lane. + Add a supported application-consistent SigNoz metadata export and restore + verifier, copy a bounded snapshot to an immutable or versioned offsite + target and restore it independently, then apply an explicit failed-artifact + retention decision. Persist the submitted source inventory before BACKUP so + same-run resume cannot bind an old artifact to newer live counters. Keep raw + SQLite reads and syncs disabled. Reconcile the source-only service registry + rows against live runtime before regenerating its K8s ConfigMap mirror, and + retire legacy GitHub runner assets only after the Gitea replacement verifier + is ready. Deduplicate the monitoring registry's awoooi-api identity before + applying regenerated scrape configuration. diff --git a/scripts/alert_chain_smoke_test.py b/scripts/alert_chain_smoke_test.py index 3cd6050de..a469c5e2e 100644 --- a/scripts/alert_chain_smoke_test.py +++ b/scripts/alert_chain_smoke_test.py @@ -9,7 +9,7 @@ Wave A.6 (ADR-037): 驗證告警鏈路 E2E 完整性 2. Alert Chain Metric — awoooi_alert_chain_last_success_timestamp 不超過 2h 3. Webhook 可達性 — /api/v1/webhooks/alertmanager, /signoz, /sentry health 4. Telegram Secret — K8s Secret 存在且非空 - 5. SigNoz 可達 — 192.168.0.188:3301 + 5. SigNoz 可達 — canonical public route (https://signoz.wooo.work) 6. Prometheus Alertmanager — 192.168.0.188:9093 (可選) 使用方式: @@ -56,7 +56,11 @@ def _env_float(name: str, default: float) -> float: DEFAULT_API_URL = "https://awoooi.wooo.work" -SIGNOZ_URL = "http://192.168.0.188:3301" +DEFAULT_SIGNOZ_PUBLIC_URL = "https://signoz.wooo.work" +SIGNOZ_URL = ( + os.environ.get("SIGNOZ_PUBLIC_URL", DEFAULT_SIGNOZ_PUBLIC_URL).strip().rstrip("/") + or DEFAULT_SIGNOZ_PUBLIC_URL +) ALERTMANAGER_URL = "http://192.168.0.188:9093" PROMETHEUS_URL = "http://192.168.0.110:9090" diff --git a/scripts/backup/backup-signoz.sh b/scripts/backup/backup-signoz.sh index a942e4957..406a9ceab 100755 --- a/scripts/backup/backup-signoz.sh +++ b/scripts/backup/backup-signoz.sh @@ -1,12 +1,12 @@ #!/bin/bash # ============================================================================= -# WOOO AIOps - SignOz 備份腳本 (ClickHouse + SQLite) -# 版本: 1.3.0 -# 建立日期: 2026-04-05 -# 2026-04-05 Claude Code: 新增 SignOz 分散式追蹤備份 — 首席架構師備份審計 -# 2026-04-05 Claude Code: v1.1 修正 tar pipeline exit code 處理 + || true -# 2026-07-15 Codex: v1.2 確保任何 exit/signal/failure 都會受控還原 OTEL Collector -# 2026-07-15 Codex: v1.3 加入初始狀態、全 run 有界還原與 success 前獨立 verifier +# WOOO AIOps - SigNoz ClickHouse native backup orchestrator +# Version: 2.0.0 +# +# ClickHouse is backed up online through its native BACKUP engine. This script +# never reads or archives the live ClickHouse/SQLite Docker volumes. SigNoz +# metadata/SQLite remains an explicit application-consistent export gap and is +# recorded in the Restic payload without being misreported as completed. # ============================================================================= set -euo pipefail @@ -16,17 +16,31 @@ source "$(dirname "$0")/common.sh" SERVICE="signoz" LOCAL_REPO="${BACKUP_BASE}/signoz" DUMP_DIR="/tmp/signoz-backup-$$" -COLLECTOR_NAME="signoz-otel-collector" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}" +COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}" COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}" -COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}" -COLLECTOR_RESTORE_RETRY_DELAY_SECONDS="${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS:-2}" -ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}" -ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}" +COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}" TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}" BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}" -COLLECTOR_WAS_RUNNING=0 -COLLECTOR_RESTORE_PENDING=0 -COLLECTOR_RESTORE_ATTEMPTS_USED=0 +RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}" +RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}" +RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}" +RESTIC_READBACK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_READBACK_TIMEOUT_SECONDS:-120}" +RESTIC_CHECK_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_CHECK_TIMEOUT_SECONDS:-1800}" + +BACKUP_ID_SEED="$(date -u '+%Y%m%dT%H%M%SZ')-$$" +TRACE_ID="${TRACE_ID:-signoz-backup-${BACKUP_ID_SEED}}" +RUN_ID="${RUN_ID:-signoz-${BACKUP_ID_SEED}}" +WORK_ITEM_ID="${WORK_ITEM_ID:-SIGNOZ-BACKUP}" +export TRACE_ID RUN_ID WORK_ITEM_ID + +COLLECTOR_ID_BEFORE="" +COLLECTOR_STARTED_AT_BEFORE="" +COLLECTOR_RESTARTS_BEFORE="" +COLLECTOR_HEALTH_BEFORE="" +COLLECTOR_OBSERVER_PID="" +COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv" CLEANUP_COMPLETE=0 FAILURE_NOTIFIED=0 @@ -35,27 +49,9 @@ run_collector_docker() { "${COLLECTOR_DOCKER_TIMEOUT_SECONDS}" docker "$@" } -collector_running_state() { - local running - if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then - return 1 - fi - - case "${running}" in - true) - printf '%s\n' "running" - ;; - false) - printf '%s\n' "stopped" - ;; - *) - return 1 - ;; - esac -} - -collector_is_running() { - [ "$(collector_running_state)" = "running" ] +collector_identity_state() { + run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \ + "${COLLECTOR_NAME}" 2>/dev/null } notify_failure_once() { @@ -67,50 +63,63 @@ notify_failure_once() { notify_clawbot "failed" "${SERVICE}" "${detail}" || true } -restore_collector() { - if [ "${COLLECTOR_RESTORE_PENDING}" -ne 1 ]; then - return 0 - fi +observe_collector_continuity() { + local epoch observation - while [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; do - COLLECTOR_RESTORE_ATTEMPTS_USED=$((COLLECTOR_RESTORE_ATTEMPTS_USED + 1)) - log_info "重啟 ${COLLECTOR_NAME} (${COLLECTOR_RESTORE_ATTEMPTS_USED}/${COLLECTOR_RESTORE_MAX_ATTEMPTS})..." - - if run_collector_docker start "${COLLECTOR_NAME}" >/dev/null 2>&1 \ - && collector_is_running; then - COLLECTOR_RESTORE_PENDING=0 - return 0 - fi - - log_warn "${COLLECTOR_NAME} 重啟或 running readback 失敗" - if [ "${COLLECTOR_RESTORE_ATTEMPTS_USED}" -lt "${COLLECTOR_RESTORE_MAX_ATTEMPTS}" ]; then - sleep "${COLLECTOR_RESTORE_RETRY_DELAY_SECONDS}" + while true; do + epoch="$(date +%s)" + if observation="$(collector_identity_state)"; then + printf '%s\t%s\n' "${epoch}" "${observation}" >> "${COLLECTOR_OBSERVATIONS}" + else + printf '%s\tunknown\tunknown\tunknown\tunknown\tunknown\n' "${epoch}" >> "${COLLECTOR_OBSERVATIONS}" fi + sleep "${COLLECTOR_CONTINUITY_POLL_SECONDS}" done - - log_error "${COLLECTOR_NAME} 在有界重試後仍無法恢復" - return 1 } -verify_collector_final_state() { - if [ "${COLLECTOR_WAS_RUNNING}" -ne 1 ]; then - log_info "${COLLECTOR_NAME} 原本即為 stopped，不執行自動啟動" - return 0 - fi +start_collector_observer() { + : > "${COLLECTOR_OBSERVATIONS}" + observe_collector_continuity & + COLLECTOR_OBSERVER_PID=$! +} - if collector_is_running; then - log_success "${COLLECTOR_NAME} final running verifier 通過" - return 0 +stop_collector_observer() { + if [ -n "${COLLECTOR_OBSERVER_PID}" ] \ + && kill -0 "${COLLECTOR_OBSERVER_PID}" 2>/dev/null; then + kill -TERM "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true + wait "${COLLECTOR_OBSERVER_PID}" 2>/dev/null || true fi + COLLECTOR_OBSERVER_PID="" +} - log_warn "${COLLECTOR_NAME} final running verifier 首次失敗，使用剩餘有界還原額度" - COLLECTOR_RESTORE_PENDING=1 - if ! restore_collector; then +verify_collector_continuity() { + local observation final_id final_running final_started_at final_restarts final_health + + stop_collector_observer + [ -s "${COLLECTOR_OBSERVATIONS}" ] || { + log_error "${COLLECTOR_NAME} continuity receipt 為空" + return 1 + } + if awk -F '\t' -v expected_id="${COLLECTOR_ID_BEFORE}" \ + -v expected_started="${COLLECTOR_STARTED_AT_BEFORE}" \ + -v expected_restarts="${COLLECTOR_RESTARTS_BEFORE}" \ + -v expected_health="${COLLECTOR_HEALTH_BEFORE}" \ + '$2 == "unknown" || $3 != "true" || $2 != expected_id || \ + $4 != expected_started || $5 != expected_restarts || $6 != expected_health { exit 1 }' \ + "${COLLECTOR_OBSERVATIONS}"; then + : + else + log_error "${COLLECTOR_NAME} 在 native backup window 發生停止或 identity drift" return 1 fi - # 與 docker start 的回傳值與 restore 內部 readback 分離，作為 success 前獨立 verifier。 - collector_is_running + observation="$(collector_identity_state)" || return 1 + IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${observation}" + [ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \ + && [ "${final_running}" = "true" ] \ + && [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \ + && [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \ + && [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ] } cleanup() { @@ -120,13 +129,9 @@ cleanup() { return "${exit_code}" fi CLEANUP_COMPLETE=1 - - # cleanup 期間忽略重複 signal，避免還原與刪除流程重入。 trap '' HUP INT TERM - if ! restore_collector; then - notify_failure_once "SignOz 備份結束時無法恢復 ${COLLECTOR_NAME}" - fi - rm -rf "${DUMP_DIR}" + stop_collector_observer + rm -rf -- "${DUMP_DIR}" return "${exit_code}" } @@ -135,166 +140,198 @@ on_signal() { exit "$((128 + signal_number))" } -create_volume_archive() { - local volume_name="$1" - local output_file="$2" - local extra_exclude="${3:-}" - local partial_file="${output_file}.partial" +run_native_clickhouse_backup() { + local mode="$1" - log_info "備份 volume: ${volume_name}" - rm -f "${partial_file}" "${output_file}" - - if [ -n "${extra_exclude}" ]; then - if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ - "${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \ - docker run --rm -v "${volume_name}:/data" alpine \ - tar czf - "${extra_exclude}" /data \ - > "${partial_file}" 2>/dev/null; then - log_error " Volume ${volume_name} archive 建立失敗" - rm -f "${partial_file}" "${output_file}" + [ -f "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \ + && [ -x "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \ + && [ ! -L "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" ] \ + || { + log_error "ClickHouse native backup adapter 缺失、不可執行或為 symlink" return 1 - fi - else - if ! timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ - "${ARCHIVE_CREATE_TIMEOUT_SECONDS}" \ - docker run --rm -v "${volume_name}:/data" alpine \ - tar czf - /data \ - > "${partial_file}" 2>/dev/null; then - log_error " Volume ${volume_name} archive 建立失敗" - rm -f "${partial_file}" "${output_file}" - return 1 - fi - fi + } - if [ ! -s "${partial_file}" ]; then - log_error " Volume ${volume_name} 備份失敗 (空檔案)" - rm -f "${partial_file}" "${output_file}" - return 1 - fi - - mv "${partial_file}" "${output_file}" + CLICKHOUSE_NATIVE_OUTPUT_DIR="${DUMP_DIR}" \ + TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ + "${CLICKHOUSE_NATIVE_BACKUP_SCRIPT}" "${mode}" } -verify_volume_archive() { - local volume_name="$1" - local output_file="$2" +write_metadata_gap_receipt() { + cat > "${DUMP_DIR}/signoz-metadata-gap.json" </dev/null 2>&1; then - log_error " Volume ${volume_name} archive integrity verifier 失敗" - rm -f "${output_file}" "${output_file}.partial" - return 1 - fi +write_restic_receipt() { + local snapshot_id="$1" + local artifact_hash="$2" + local manifest_hash="$3" + local inventory_hash="$4" + local receipt_dir receipt_tmp receipt_file - local size - size="$(du -h "${output_file}" | cut -f1)" - log_success " Volume ${volume_name} 備份與 integrity verifier 完成 (${size})" + [[ "${RESTIC_RECEIPT_ROOT}" == /* ]] && [ ! -L "${RESTIC_RECEIPT_ROOT}" ] || return 1 + mkdir -p "${RESTIC_RECEIPT_ROOT}" + receipt_dir="${RESTIC_RECEIPT_ROOT}/${RUN_ID}" + [ ! -L "${receipt_dir}" ] || return 1 + mkdir -m 700 -p "${receipt_dir}" + receipt_file="${receipt_dir}/restic-snapshot.json" + receipt_tmp="${receipt_file}.partial.$$" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","service":"signoz","snapshot_id":"%s","artifact_sha256":"%s","manifest_sha256":"%s","source_inventory_sha256":"%s","repository_scope":"local_same_failure_domain","restic_check_read_data_subset":"1%%","offsite_verified":false,"metadata_sqlite_terminal":"pending_application_consistent_export","completion_claim":"clickhouse_native_only"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${snapshot_id}" \ + "${artifact_hash}" "${manifest_hash}" "${inventory_hash}" > "${receipt_tmp}" + chmod 600 "${receipt_tmp}" + mv "${receipt_tmp}" "${receipt_file}" } main() { - local start_time=$(date +%s) + local start_time end_time duration snapshot_id snapshot_json tags + local initial_observation initial_running final_id final_running + local final_started_at final_restarts final_health + local artifact_file manifest_file inventory_file artifact_hash manifest_hash inventory_hash + start_time="$(date +%s)" trap cleanup EXIT trap 'on_signal 1' HUP trap 'on_signal 2' INT trap 'on_signal 15' TERM - log_info "========== 開始 SignOz 備份 ==========" + log_info "========== 開始 SigNoz ClickHouse native 備份 ==========" + [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" =~ ^[0-9]+([.][0-9]+)?$ ]] \ + && [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0" ]] \ + && [[ "${COLLECTOR_CONTINUITY_POLL_SECONDS}" != "0.0" ]] || { + notify_failure_once "COLLECTOR_CONTINUITY_POLL_SECONDS 無效" + exit 1 + } + for timeout_value in "${RESTIC_INIT_TIMEOUT_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \ + "${RESTIC_READBACK_TIMEOUT_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}"; do + [[ "${timeout_value}" =~ ^[0-9]+$ ]] && [ "${timeout_value}" -gt 0 ] || { + notify_failure_once "Restic timeout contract 無效" + exit 1 + } + done mkdir -p "${DUMP_DIR}" - local timestamp=$(date "+%Y%m%d_%H%M%S") - - # Step 1: 只在 Collector 原本 running 時暫停，並在 stop 前 arm 還原。 - local collector_initial_state - if ! collector_initial_state="$(collector_running_state)"; then - log_error "無法在有界 timeout 內讀取 ${COLLECTOR_NAME} 初始狀態" - notify_failure_once "SignOz 備份無法驗證 ${COLLECTOR_NAME} 初始狀態" + # Check-mode must pass before the backup window. It writes only durable + # receipts and does not submit BACKUP or create a server artifact. + if ! run_native_clickhouse_backup --check; then + log_error "ClickHouse native BACKUP check-mode 失敗；沒有 volume tar fallback" + notify_failure_once "SigNoz ClickHouse native BACKUP runtime contract 未就緒" exit 1 fi - if [ "${collector_initial_state}" = "running" ]; then - COLLECTOR_WAS_RUNNING=1 - COLLECTOR_RESTORE_PENDING=1 - log_info "暫停 ${COLLECTOR_NAME} 以確保數據一致性..." - if ! run_collector_docker stop "${COLLECTOR_NAME}" >/dev/null 2>&1; then - log_error "${COLLECTOR_NAME} 在有界 timeout 內無法停止" - notify_failure_once "SignOz 備份無法受控暫停 ${COLLECTOR_NAME}" - exit 1 - fi - else - log_error "${COLLECTOR_NAME} 初始狀態為 stopped，停止備份且不自動啟動" - notify_failure_once "SignOz 備份發現 ${COLLECTOR_NAME} 原本已停止" - exit 1 - fi - docker stop signoz-telemetrystore-migrator 2>/dev/null || true - - # Step 2: 備份 ClickHouse volume (排除 tmp 目錄降低體積) - local clickhouse_archive="${DUMP_DIR}/clickhouse_${timestamp}.tar.gz" - local sqlite_archive="${DUMP_DIR}/sqlite_${timestamp}.tar.gz" - create_volume_archive "signoz-clickhouse" "${clickhouse_archive}" "--exclude=/data/tmp" || { - log_error "ClickHouse volume 備份失敗" - notify_failure_once "SignOz ClickHouse 備份失敗" + initial_observation="$(collector_identity_state)" || { + notify_failure_once "無法讀取 ${COLLECTOR_NAME} 初始 identity/state" exit 1 } - - # Step 3: 備份 SQLite volume (SignOz metadata) - create_volume_archive "signoz-sqlite" "${sqlite_archive}" || { - log_error "SQLite volume 備份失敗" - notify_failure_once "SignOz SQLite 備份失敗" + IFS=$'\t' read -r COLLECTOR_ID_BEFORE initial_running COLLECTOR_STARTED_AT_BEFORE \ + COLLECTOR_RESTARTS_BEFORE COLLECTOR_HEALTH_BEFORE <<< "${initial_observation}" + [ -n "${COLLECTOR_ID_BEFORE}" ] && [ "${initial_running}" = "true" ] || { + notify_failure_once "${COLLECTOR_NAME} 初始狀態不是 running" exit 1 } + [ -n "${COLLECTOR_STARTED_AT_BEFORE}" ] \ + && [[ "${COLLECTOR_RESTARTS_BEFORE}" =~ ^[0-9]+$ ]] \ + && [[ "${COLLECTOR_HEALTH_BEFORE}" =~ ^(healthy|none)$ ]] || { + notify_failure_once "${COLLECTOR_NAME} 初始 runtime metadata 不完整" + exit 1 + } + start_collector_observer - # Step 4: 在全 run 共用的有界 retry budget 內恢復 Collector。 - if ! restore_collector; then - notify_failure_once "SignOz 備份無法恢復 ${COLLECTOR_NAME}" + # Online native backup: no collector stop and no live-volume read. + if ! run_native_clickhouse_backup --apply; then + notify_failure_once "SigNoz ClickHouse native BACKUP 失敗" exit 1 fi - # Step 5: Collector 恢復後才在 host 做有界 archive integrity verifier。 - verify_volume_archive "signoz-clickhouse" "${clickhouse_archive}" || { - notify_failure_once "SignOz ClickHouse archive integrity verifier 失敗" + if ! verify_collector_continuity; then + notify_failure_once "${COLLECTOR_NAME} no-downtime continuity verifier 失敗" exit 1 - } - verify_volume_archive "signoz-sqlite" "${sqlite_archive}" || { - notify_failure_once "SignOz SQLite archive integrity verifier 失敗" - exit 1 - } + fi + log_success "${COLLECTOR_NAME} no-downtime continuity verifier 通過" + + write_metadata_gap_receipt + + artifact_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip' -print)" + manifest_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.zip.manifest.json' -print)" + inventory_file="$(find "${DUMP_DIR}" -maxdepth 1 -type f -name 'clickhouse-native-*.source-inventory.tsv' -print)" + [ "$(printf '%s\n' "${artifact_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \ + && [ "$(printf '%s\n' "${manifest_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] \ + && [ "$(printf '%s\n' "${inventory_file}" | awk 'NF {count++} END {print count+0}')" -eq 1 ] || { + log_error "ClickHouse native artifact set 不唯一或不完整" + exit 1 + } + artifact_hash="$(sha256sum "${artifact_file}" | awk '{print $1}')" + manifest_hash="$(sha256sum "${manifest_file}" | awk '{print $1}')" + inventory_hash="$(sha256sum "${inventory_file}" | awk '{print $1}')" - # Step 6: 初始化 Restic 倉庫 if [ ! -d "${LOCAL_REPO}/data" ]; then log_info "初始化 Restic 倉庫: ${LOCAL_REPO}" - restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || { + timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_INIT_TIMEOUT_SECONDS}" \ + restic -r "${LOCAL_REPO}" init --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || { log_error "Restic 倉庫初始化失敗" exit 1 } fi - # Step 7: Restic 備份 - log_info "建立 Restic 備份..." - local tags=$(build_tags "${SERVICE}") - restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" --password-file "${RESTIC_PASSWORD_FILE}" ${tags} 2>&1 + log_info "建立 ClickHouse native artifact 的 Restic 備份..." + tags="$(build_tags "${SERVICE}")" + timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_BACKUP_TIMEOUT_SECONDS}" \ + restic -r "${LOCAL_REPO}" backup "${DUMP_DIR}" \ + --password-file "${RESTIC_PASSWORD_FILE}" ${tags} \ + --tag "trace:${TRACE_ID}" --tag "run:${RUN_ID}" \ + --tag "work-item:${WORK_ITEM_ID}" 2>&1 - local snapshot_id=$(restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | grep -oP '"short_id":"\K[^"]+' | head -1) + snapshot_json="$(timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${RESTIC_READBACK_TIMEOUT_SECONDS}" restic -r "${LOCAL_REPO}" snapshots \ + --tag "run:${RUN_ID}" --latest 1 --json \ + --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)" + snapshot_id="$(printf '%s\n' "${snapshot_json}" \ + | sed -n 's/.*"short_id":"\([^"]*\)".*/\1/p' | head -1)" + [[ "${snapshot_id}" =~ ^[A-Za-z0-9]+$ ]] || { + log_error "Restic same-run snapshot durable readback 缺失" + exit 1 + } + timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${RESTIC_CHECK_TIMEOUT_SECONDS}" \ + restic -r "${LOCAL_REPO}" check --read-data-subset=1% \ + --password-file "${RESTIC_PASSWORD_FILE}" 2>&1 || { + log_error "Restic repository/data-subset verifier 失敗" + exit 1 + } + write_restic_receipt "${snapshot_id}" "${artifact_hash}" \ + "${manifest_hash}" "${inventory_hash}" || { + log_error "Restic durable receipt writeback 失敗" + exit 1 + } log_success "Restic 備份完成: ${snapshot_id}" - # Step 8: GFS 清理；real canary 可明確跳過 destructive retention。 if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)" else cleanup_old_backups "${LOCAL_REPO}" fi - # Step 9: success 前獨立檢查 runtime state；失敗時不得發送 success。 - if ! verify_collector_final_state; then - notify_failure_once "SignOz 備份完成前 ${COLLECTOR_NAME} final running verifier 失敗" + # Final readback is separate from the continuity observer receipt. + initial_observation="$(collector_identity_state)" || { + notify_failure_once "${COLLECTOR_NAME} final readback 失敗" exit 1 - fi + } + IFS=$'\t' read -r final_id final_running final_started_at final_restarts final_health <<< "${initial_observation}" + [ "${final_id}" = "${COLLECTOR_ID_BEFORE}" ] \ + && [ "${final_running}" = "true" ] \ + && [ "${final_started_at}" = "${COLLECTOR_STARTED_AT_BEFORE}" ] \ + && [ "${final_restarts}" = "${COLLECTOR_RESTARTS_BEFORE}" ] \ + && [ "${final_health}" = "${COLLECTOR_HEALTH_BEFORE}" ] || { + notify_failure_once "${COLLECTOR_NAME} final identity/start/restart/health verifier 失敗" + exit 1 + } - local end_time=$(date +%s) - local duration=$((end_time - start_time)) - log_success "========== SignOz 備份完成 (${duration}s) ==========" - notify_clawbot "success" "${SERVICE}" "SignOz 備份完成 (ClickHouse+SQLite)" "${duration}" + end_time="$(date +%s)" + duration=$((end_time - start_time)) + log_success "========== SigNoz ClickHouse native 備份完成 (${duration}s) ==========" + notify_clawbot "warning" "${SERVICE}" \ + "ClickHouse native backup 完成；metadata/SQLite application-consistent export 仍 pending" \ + "${duration}" } main "$@" diff --git a/scripts/backup/clickhouse-native-backup.sh b/scripts/backup/clickhouse-native-backup.sh new file mode 100755 index 000000000..256273a90 --- /dev/null +++ b/scripts/backup/clickhouse-native-backup.sh @@ -0,0 +1,628 @@ +#!/usr/bin/env bash +# ============================================================================= +# P0-OBS-002 - ClickHouse native BACKUP adapter for SigNoz +# +# This adapter deliberately has no live-volume tar fallback. It requires a +# ClickHouse Disk configured and allowed for BACKUP, records every bounded +# command's stdout/stderr/exit status, and copies only the completed native +# backup artifact into the caller's staging directory. +# ============================================================================= + +set -euo pipefail + +CONTAINER_NAME="${CLICKHOUSE_NATIVE_CONTAINER_NAME:-signoz-clickhouse}" +BACKUP_DISK="${CLICKHOUSE_NATIVE_BACKUP_DISK:-backups}" +EXPECTED_DISK_PATH="${CLICKHOUSE_NATIVE_EXPECTED_DISK_PATH:-/backups/}" +OUTPUT_DIR="${CLICKHOUSE_NATIVE_OUTPUT_DIR:-}" +RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}" +COMMAND_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS:-30}" +BACKUP_TIMEOUT_SECONDS="${CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS:-1800}" +POLL_INTERVAL_SECONDS="${CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS:-5}" +KILL_AFTER_SECONDS="${CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS:-30}" +POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}" +EXPECTED_DATABASES_CSV="signoz_analytics,signoz_logs,signoz_metadata,signoz_meter,signoz_metrics,signoz_traces" +EXPECTED_DATABASES_JSON='["signoz_analytics","signoz_logs","signoz_metadata","signoz_meter","signoz_metrics","signoz_traces"]' +BACKUP_SCOPE_SQL="DATABASE signoz_analytics,DATABASE signoz_logs,DATABASE signoz_metadata,DATABASE signoz_meter,DATABASE signoz_metrics,DATABASE signoz_traces" + +TRACE_ID="${TRACE_ID:-}" +RUN_ID="${RUN_ID:-}" +WORK_ITEM_ID="${WORK_ITEM_ID:-}" + +MODE="" +IDENTITY_HASH="" +OPERATION_ID="" +ARTIFACT_NAME="" +SERVER_ARTIFACT_PATH="" +OUTPUT_FILE="" +MANIFEST_FILE="" +HASH_FILE="" +INVENTORY_FILE="" +OUTPUT_PARTIAL="" +MANIFEST_PARTIAL="" +INVENTORY_PARTIAL="" +RECEIPT_DIR="" +RECEIPT_LOG="" +IDENTITY_FILE="" +INVOCATION_ID="" + +LAST_STDOUT="" +LAST_STDERR="" +LAST_STATUS_FILE="" +LAST_EXIT_CODE=0 +COMMAND_SEQUENCE=0 + +CONTAINER_ID="" +CLICKHOUSE_VERSION="" +DATABASES_HASH="" +SOURCE_INVENTORY_RECEIPT="" +SOURCE_INVENTORY_HASH="" +STATUS_PRESENT=0 +BACKUP_NAME="" +BACKUP_STATUS="" +BACKUP_NUM_FILES="0" +BACKUP_TOTAL_SIZE="0" +BACKUP_UNCOMPRESSED_SIZE="0" +BACKUP_COMPRESSED_SIZE="0" +BACKUP_ERROR_HEX="none" +BACKUP_ERROR_HASH="" + +RECEIPTS_READY=0 +CHECK_PASS=0 +APPLY_PASS=0 +OUTPUT_CREATED=0 +SERVER_ARTIFACT_ACTIVE=0 + +log_info() { printf '[INFO] %s\n' "$*"; } +log_error() { printf '[ERROR] %s\n' "$*" >&2; } + +valid_identity() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] +} + +valid_name() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] +} + +valid_positive_integer() { + [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ] +} + +emit_receipt() { + local stage="$1" + local terminal="$2" + local detail="$3" + local observed_at + + [ "${RECEIPTS_READY}" -eq 1 ] || return 0 + observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${INVOCATION_ID}" \ + "${observed_at}" "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}" +} + +fail_closed() { + local detail="$1" + log_error "${detail}" + emit_receipt "failure" "failed" "${detail}" + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}" +} + +cleanup() { + local exit_code=$? + local terminal="failed" + + trap - EXIT HUP INT TERM + set +e + [ -z "${OUTPUT_PARTIAL}" ] || rm -f -- "${OUTPUT_PARTIAL}" + [ -z "${MANIFEST_PARTIAL}" ] || rm -f -- "${MANIFEST_PARTIAL}" + [ -z "${INVENTORY_PARTIAL}" ] || rm -f -- "${INVENTORY_PARTIAL}" + + if [ "${OUTPUT_CREATED}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then + rm -f -- "${OUTPUT_FILE}" "${MANIFEST_FILE}" "${HASH_FILE}" "${INVENTORY_FILE}" + emit_receipt "cleanup" "removed" "unverified_local_artifact_removed" + fi + + if [ "${SERVER_ARTIFACT_ACTIVE}" -eq 1 ] && [ "${APPLY_PASS}" -ne 1 ]; then + emit_receipt "cleanup" "preserved" \ + "active_or_unverified_server_artifact_preserved_for_safe_followup" + fi + + if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_PASS}" -eq 1 ]; then + terminal="check_pass_no_write" + elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "apply" ] && [ "${APPLY_PASS}" -eq 1 ]; then + terminal="pass" + fi + + emit_receipt "terminal" "${terminal}" "clickhouse_native_backup_${terminal}" + printf 'CLICKHOUSE_NATIVE_BACKUP_TERMINAL trace_id=%s run_id=%s work_item_id=%s mode=%s terminal=%s exit_code=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" "${terminal}" "${exit_code}" + exit "${exit_code}" +} + +on_signal() { + local signal_number="$1" + emit_receipt "signal" "failed" "signal_${signal_number}" + exit "$((128 + signal_number))" +} + +setup() { + local identity_content identity_tmp + + [ "$#" -eq 1 ] || { log_error "exactly one mode is required: --check or --apply"; exit 2; } + case "$1" in + --check) MODE="check" ;; + --apply) MODE="apply" ;; + *) log_error "unsupported mode: use --check or --apply"; exit 2 ;; + esac + + valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and path-safe"; exit 2; } + valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and path-safe"; exit 2; } + valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and path-safe"; exit 2; } + valid_name "${CONTAINER_NAME}" || { log_error "invalid container name"; exit 2; } + valid_name "${BACKUP_DISK}" || { log_error "invalid backup disk name"; exit 2; } + valid_positive_integer "${COMMAND_TIMEOUT_SECONDS}" || { log_error "invalid command timeout"; exit 2; } + valid_positive_integer "${BACKUP_TIMEOUT_SECONDS}" || { log_error "invalid backup timeout"; exit 2; } + valid_positive_integer "${POLL_INTERVAL_SECONDS}" || { log_error "invalid poll interval"; exit 2; } + valid_positive_integer "${KILL_AFTER_SECONDS}" || { log_error "invalid kill-after timeout"; exit 2; } + [[ "${EXPECTED_DISK_PATH}" == /*/ ]] && [[ "${EXPECTED_DISK_PATH}" != *".."* ]] \ + || { log_error "expected disk path must be an absolute trailing-slash path without dot-dot"; exit 2; } + [ -n "${OUTPUT_DIR}" ] && [[ "${OUTPUT_DIR}" == /* ]] \ + || { log_error "CLICKHOUSE_NATIVE_OUTPUT_DIR must be absolute"; exit 2; } + [ -d "${OUTPUT_DIR}" ] && [ -w "${OUTPUT_DIR}" ] && [ ! -L "${OUTPUT_DIR}" ] \ + || { log_error "output directory missing, unwritable, or symlink"; exit 2; } + if [ -n "${POST_VERIFY_HOOK}" ]; then + [[ "${POST_VERIFY_HOOK}" == /* ]] && [ -f "${POST_VERIFY_HOOK}" ] \ + && [ -x "${POST_VERIFY_HOOK}" ] && [ ! -L "${POST_VERIFY_HOOK}" ] \ + || { log_error "post verifier hook must be an absolute executable regular file"; exit 2; } + fi + + for command_name in awk basename cat chmod cmp cp date docker env flock grep mkdir mktemp mv rm sha256sum sleep timeout touch tr unzip wc; do + command -v "${command_name}" >/dev/null 2>&1 \ + || { log_error "required command missing: ${command_name}"; exit 2; } + done + + IDENTITY_HASH="$(printf '%s\000%s\000%s' "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" | sha256sum | awk '{print $1}')" + OPERATION_ID="awoooi-${IDENTITY_HASH}" + ARTIFACT_NAME="signoz-${IDENTITY_HASH}.zip" + SERVER_ARTIFACT_PATH="${EXPECTED_DISK_PATH%/}/${ARTIFACT_NAME}" + OUTPUT_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.zip" + MANIFEST_FILE="${OUTPUT_FILE}.manifest.json" + HASH_FILE="${OUTPUT_FILE}.sha256" + INVENTORY_FILE="${OUTPUT_DIR}/clickhouse-native-${IDENTITY_HASH}.source-inventory.tsv" + OUTPUT_PARTIAL="${OUTPUT_FILE}.partial" + MANIFEST_PARTIAL="${MANIFEST_FILE}.partial" + INVENTORY_PARTIAL="${INVENTORY_FILE}.partial" + + [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root symlink is unsafe"; exit 2; } + mkdir -p "${RECEIPT_ROOT}" + RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" + [ ! -L "${RECEIPT_DIR}" ] || { log_error "receipt directory symlink is unsafe"; exit 2; } + mkdir -m 700 -p "${RECEIPT_DIR}" + RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl" + IDENTITY_FILE="${RECEIPT_DIR}/identity.json" + INVOCATION_ID="${MODE}-$(date -u '+%Y%m%dT%H%M%SZ')-$$" + + identity_content="{\"trace_id\":\"${TRACE_ID}\",\"run_id\":\"${RUN_ID}\",\"work_item_id\":\"${WORK_ITEM_ID}\",\"identity_hash\":\"${IDENTITY_HASH}\",\"operation_id\":\"${OPERATION_ID}\",\"artifact_name\":\"${ARTIFACT_NAME}\"}" + if [ -e "${IDENTITY_FILE}" ]; then + [ -f "${IDENTITY_FILE}" ] && [ ! -L "${IDENTITY_FILE}" ] \ + || { log_error "existing identity receipt is unsafe"; exit 2; } + [ "$(cat "${IDENTITY_FILE}")" = "${identity_content}" ] \ + || { log_error "run identity conflicts with durable receipt"; exit 2; } + else + identity_tmp="$(mktemp "${RECEIPT_DIR}/.identity.XXXXXX")" + printf '%s\n' "${identity_content}" > "${identity_tmp}" + chmod 600 "${identity_tmp}" + mv "${identity_tmp}" "${IDENTITY_FILE}" + fi + touch "${RECEIPT_LOG}" + RECEIPTS_READY=1 + + [ ! -L "${RECEIPT_DIR}/run.lock" ] || fail_closed "run_lock_symlink_unsafe" + exec 9>>"${RECEIPT_DIR}/run.lock" + flock -n 9 || fail_closed "same_run_native_backup_is_active" + + trap cleanup EXIT + trap 'on_signal 1' HUP + trap 'on_signal 2' INT + trap 'on_signal 15' TERM +} + +run_captured() { + local step="$1" + local stdout_hash stderr_hash terminal + shift + + valid_name "${step}" || fail_closed "invalid_command_step" + COMMAND_SEQUENCE=$((COMMAND_SEQUENCE + 1)) + LAST_STDOUT="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stdout" + LAST_STDERR="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stderr" + LAST_STATUS_FILE="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.status" + + set +e + "$@" >"${LAST_STDOUT}" 2>"${LAST_STDERR}" + LAST_EXIT_CODE=$? + set -e + printf '%s\n' "${LAST_EXIT_CODE}" > "${LAST_STATUS_FILE}" + stdout_hash="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')" + stderr_hash="$(sha256sum "${LAST_STDERR}" | awk '{print $1}')" + terminal="pass" + [ "${LAST_EXIT_CODE}" -eq 0 ] || terminal="failed" + emit_receipt "command" "${terminal}" \ + "step_${step}_exit_${LAST_EXIT_CODE}_stdout_${stdout_hash}_stderr_${stderr_hash}" + return "${LAST_EXIT_CODE}" +} + +run_sql() { + local step="$1" + local query="$2" + shift 2 + run_captured "${step}" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \ + clickhouse-client --format TSVRaw "$@" --query "${query}" +} + +perform_check() { + local inspect_line running disk_path system_backups_count database_count backup_grant + local expected_databases_file + + run_captured "container" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker inspect \ + --format $'{{.Id}}\t{{.State.Running}}' "${CONTAINER_NAME}" \ + || fail_closed "clickhouse_container_inspect_failed" + inspect_line="$(tr -d '\r\n' < "${LAST_STDOUT}")" + IFS=$'\t' read -r CONTAINER_ID running <<< "${inspect_line}" + [ -n "${CONTAINER_ID}" ] && [ "${running}" = "true" ] \ + || fail_closed "clickhouse_container_not_running_or_identity_ambiguous" + + run_sql "version" 'SELECT version()' || fail_closed "clickhouse_version_query_failed" + CLICKHOUSE_VERSION="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ -n "${CLICKHOUSE_VERSION}" ] || fail_closed "clickhouse_version_empty" + + run_sql "system_backups" \ + "SELECT count() FROM system.tables WHERE database='system' AND name='backups'" \ + || fail_closed "system_backups_capability_query_failed" + system_backups_count="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${system_backups_count}" = "1" ] || fail_closed "native_backup_capability_absent" + + run_sql "backup_grant" 'CHECK GRANT BACKUP ON *.*' \ + || fail_closed "native_backup_grant_check_failed" + backup_grant="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${backup_grant}" = "1" ] || fail_closed "native_backup_grant_absent" + + run_sql "backup_disk" \ + 'SELECT path FROM system.disks WHERE name={disk:String}' \ + --param_disk "${BACKUP_DISK}" || fail_closed "backup_disk_query_failed" + disk_path="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${disk_path}" = "${EXPECTED_DISK_PATH}" ] \ + || fail_closed "backup_disk_missing_or_path_mismatch" + + run_captured "disk_readable" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \ + test -d "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_absent" + run_captured "disk_writable" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \ + test -w "${EXPECTED_DISK_PATH}" || fail_closed "backup_disk_directory_unwritable" + + run_sql "databases" \ + "SELECT name FROM system.databases WHERE name NOT IN ('system','information_schema','INFORMATION_SCHEMA','default') ORDER BY name" \ + || fail_closed "user_database_inventory_query_failed" + database_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")" + expected_databases_file="${RECEIPT_DIR}/${INVOCATION_ID}-expected-databases.tsv" + printf '%s\n' signoz_analytics signoz_logs signoz_metadata signoz_meter signoz_metrics signoz_traces \ + > "${expected_databases_file}" + cmp -s "${expected_databases_file}" "${LAST_STDOUT}" \ + || fail_closed "signoz_database_allowlist_drift" + [ "${database_count}" -eq 6 ] || fail_closed "signoz_database_count_drift" + DATABASES_HASH="$(sha256sum "${LAST_STDOUT}" | awk '{print $1}')" + + run_sql "table_inventory" \ + "SELECT database,name,engine,toString(ifNull(total_rows,0)),toString(ifNull(total_bytes,0)),hex(SHA256(create_table_query)) FROM system.tables WHERE database IN ('signoz_analytics','signoz_logs','signoz_metadata','signoz_meter','signoz_metrics','signoz_traces') ORDER BY database,name" \ + || fail_closed "source_table_inventory_query_failed" + [ "$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")" -gt 0 ] \ + || fail_closed "source_table_inventory_empty" + SOURCE_INVENTORY_RECEIPT="${LAST_STDOUT}" + SOURCE_INVENTORY_HASH="$(sha256sum "${SOURCE_INVENTORY_RECEIPT}" | awk '{print $1}')" + + emit_receipt "sensor_source" "pass" \ + "container_${CONTAINER_ID}_version_${CLICKHOUSE_VERSION}_disk_${BACKUP_DISK}" + emit_receipt "normalized_asset_identity" "pass" \ + "identity_${IDENTITY_HASH}_operation_${OPERATION_ID}_artifact_${ARTIFACT_NAME}" + emit_receipt "source_of_truth_diff" "pass" \ + "configured_disk_path_exact_six_databases_hash_${DATABASES_HASH}_table_inventory_${SOURCE_INVENTORY_HASH}" + emit_receipt "risk_policy" "pass" \ + "risk_medium_native_backup_async_bounded_poll_exact_allowlist_no_volume_tar_fallback" +} + +query_backup_status() { + local line_count status_line + + run_sql "backup_status" \ + "SELECT name,toString(status),toString(num_files),toString(total_size),toString(uncompressed_size),toString(compressed_size),if(empty(error),'none',hex(substring(error,1,2048))),hex(SHA256(error)) FROM system.backups WHERE id={operation_id:String}" \ + --param_operation_id "${OPERATION_ID}" || fail_closed "system_backups_status_query_failed" + line_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")" + if [ "${line_count}" -eq 0 ]; then + STATUS_PRESENT=0 + return 0 + fi + [ "${line_count}" -eq 1 ] || fail_closed "system_backups_operation_id_not_unique" + STATUS_PRESENT=1 + status_line="$(tr -d '\r\n' < "${LAST_STDOUT}")" + IFS=$'\t' read -r BACKUP_NAME BACKUP_STATUS BACKUP_NUM_FILES BACKUP_TOTAL_SIZE \ + BACKUP_UNCOMPRESSED_SIZE BACKUP_COMPRESSED_SIZE BACKUP_ERROR_HEX \ + BACKUP_ERROR_HASH <<< "${status_line}" + [[ "${BACKUP_NUM_FILES}" =~ ^[0-9]+$ ]] \ + && [[ "${BACKUP_TOTAL_SIZE}" =~ ^[0-9]+$ ]] \ + && [[ "${BACKUP_UNCOMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \ + && [[ "${BACKUP_COMPRESSED_SIZE}" =~ ^[0-9]+$ ]] \ + || fail_closed "system_backups_metrics_ambiguous" +} + +server_artifact_absent() { + run_captured "artifact_absent" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \ + test ! -e "${SERVER_ARTIFACT_PATH}" +} + +cleanup_server_artifact() { + run_captured "artifact_cleanup" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CONTAINER_NAME}" \ + rm -f -- "${SERVER_ARTIFACT_PATH}" || return 1 + server_artifact_absent || return 1 + SERVER_ARTIFACT_ACTIVE=0 + emit_receipt "cleanup" "pass" "exact_server_artifact_removed_and_absent" +} + +poll_backup_terminal() { + local deadline now expected_name + expected_name="Disk('${BACKUP_DISK}', '${ARTIFACT_NAME}')" + deadline=$(( $(date +%s) + BACKUP_TIMEOUT_SECONDS )) + + while true; do + query_backup_status + [ "${STATUS_PRESENT}" -eq 1 ] || fail_closed "async_backup_status_disappeared" + [ "${BACKUP_NAME}" = "${expected_name}" ] || fail_closed "async_backup_name_mismatch" + emit_receipt "backup_status" "observed" \ + "status_${BACKUP_STATUS}_files_${BACKUP_NUM_FILES}_total_${BACKUP_TOTAL_SIZE}_error_hash_${BACKUP_ERROR_HASH}" + + case "${BACKUP_STATUS}" in + BACKUP_CREATED) + [ "${BACKUP_NUM_FILES}" -gt 0 ] \ + && [ "${BACKUP_TOTAL_SIZE}" -gt 0 ] \ + && [ "${BACKUP_UNCOMPRESSED_SIZE}" -gt 0 ] \ + && [ "${BACKUP_COMPRESSED_SIZE}" -gt 0 ] \ + || fail_closed "native_backup_terminal_metrics_empty" + SERVER_ARTIFACT_ACTIVE=1 + return 0 + ;; + CREATING_BACKUP) + SERVER_ARTIFACT_ACTIVE=1 + ;; + BACKUP_FAILED|BACKUP_CANCELLED) + printf '%s\n' "${BACKUP_ERROR_HEX}" \ + > "${RECEIPT_DIR}/${INVOCATION_ID}-backup-error-bounded.hex" + emit_receipt "backup_error" "retained" \ + "bounded_hex_2048_bytes_hash_${BACKUP_ERROR_HASH}" + SERVER_ARTIFACT_ACTIVE=1 + if ! cleanup_server_artifact; then + emit_receipt "cleanup" "failed" "failed_terminal_exact_artifact_cleanup_failed" + fi + fail_closed "native_backup_terminal_${BACKUP_STATUS}" + ;; + *) fail_closed "unexpected_native_backup_status_${BACKUP_STATUS}" ;; + esac + + now="$(date +%s)" + if [ "${now}" -ge "${deadline}" ]; then + fail_closed "native_backup_poll_timeout_artifact_preserved" + fi + sleep "${POLL_INTERVAL_SECONDS}" + done +} + +verify_existing_local_artifact() { + local recorded_hash actual_hash inventory_hash artifact_size + local manifest_hash hook_manifest_hash hook_inventory_hash + + if [ ! -e "${OUTPUT_FILE}" ] && [ ! -e "${MANIFEST_FILE}" ] \ + && [ ! -e "${HASH_FILE}" ] && [ ! -e "${INVENTORY_FILE}" ]; then + return 1 + fi + [ -f "${OUTPUT_FILE}" ] && [ ! -L "${OUTPUT_FILE}" ] \ + && [ -f "${MANIFEST_FILE}" ] && [ ! -L "${MANIFEST_FILE}" ] \ + && [ -f "${HASH_FILE}" ] && [ ! -L "${HASH_FILE}" ] \ + && [ -f "${INVENTORY_FILE}" ] && [ ! -L "${INVENTORY_FILE}" ] \ + || fail_closed "local_artifact_identity_incomplete_or_unsafe" + grep -F -q "\"identity_hash\":\"${IDENTITY_HASH}\"" "${MANIFEST_FILE}" \ + || fail_closed "local_artifact_manifest_identity_mismatch" + recorded_hash="$(awk 'NR == 1 { print $1 }' "${HASH_FILE}")" + [[ "${recorded_hash}" =~ ^[0-9a-f]{64}$ ]] || fail_closed "local_artifact_hash_receipt_invalid" + actual_hash="$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')" + [ "${actual_hash}" = "${recorded_hash}" ] || fail_closed "local_artifact_hash_mismatch" + artifact_size="$(wc -c < "${OUTPUT_FILE}" | tr -d ' ')" + inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')" + manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')" + grep -F -q "\"operation_id\":\"${OPERATION_ID}\"" "${MANIFEST_FILE}" \ + || fail_closed "local_artifact_manifest_operation_mismatch" + grep -F -q "\"artifact_name\":\"${ARTIFACT_NAME}\"" "${MANIFEST_FILE}" \ + || fail_closed "local_artifact_manifest_name_mismatch" + grep -F -q "\"databases\":${EXPECTED_DATABASES_JSON}" "${MANIFEST_FILE}" \ + || fail_closed "local_artifact_manifest_database_allowlist_mismatch" + grep -F -q "\"source_inventory_sha256\":\"${inventory_hash}\"" "${MANIFEST_FILE}" \ + || fail_closed "local_source_inventory_hash_mismatch" + grep -F -q "\"sha256\":\"${actual_hash}\"" "${MANIFEST_FILE}" \ + || fail_closed "local_artifact_manifest_hash_mismatch" + grep -F -q "\"size_bytes\":${artifact_size}" "${MANIFEST_FILE}" \ + || fail_closed "local_artifact_manifest_size_mismatch" + grep -F -q '"status":"BACKUP_CREATED"' "${MANIFEST_FILE}" \ + || fail_closed "local_artifact_manifest_terminal_mismatch" + run_captured "local_zip_reuse" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_FILE}" \ + || fail_closed "local_native_archive_reuse_verifier_failed" + if [ -n "${POST_VERIFY_HOOK}" ]; then + run_captured "reuse_verify_hook_check" env \ + TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ + CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \ + CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \ + CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \ + CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \ + CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \ + CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \ + "${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_reuse_check_failed" + run_captured "reuse_verify_hook" env \ + TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ + CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_FILE}" \ + CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${actual_hash}" \ + CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_FILE}" \ + CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \ + CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \ + CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_FILE}" \ + "${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_reuse_failed" + [ "$(sha256sum "${OUTPUT_FILE}" | awk '{print $1}')" = "${actual_hash}" ] \ + || fail_closed "post_verify_hook_modified_reused_artifact" + hook_inventory_hash="$(sha256sum "${INVENTORY_FILE}" | awk '{print $1}')" + [ "${hook_inventory_hash}" = "${inventory_hash}" ] \ + || fail_closed "post_verify_hook_modified_reused_source_inventory" + hook_manifest_hash="$(sha256sum "${MANIFEST_FILE}" | awk '{print $1}')" + [ "${hook_manifest_hash}" = "${manifest_hash}" ] \ + || fail_closed "post_verify_hook_modified_reused_manifest" + fi + emit_receipt "idempotency" "reused" "verified_local_artifact_hash_${actual_hash}" + APPLY_PASS=1 + return 0 +} + +copy_and_verify_artifact() { + local artifact_hash artifact_size hook_hash inventory_hash hook_inventory_hash + local manifest_hash hook_manifest_hash + + rm -f -- "${OUTPUT_PARTIAL}" "${MANIFEST_PARTIAL}" "${INVENTORY_PARTIAL}" + run_captured "artifact_copy" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${BACKUP_TIMEOUT_SECONDS}" docker cp \ + "${CONTAINER_NAME}:${SERVER_ARTIFACT_PATH}" "${OUTPUT_PARTIAL}" \ + || fail_closed "native_backup_artifact_copy_failed" + [ -s "${OUTPUT_PARTIAL}" ] || fail_closed "native_backup_artifact_copy_empty" + + run_captured "local_zip_verify" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${BACKUP_TIMEOUT_SECONDS}" unzip -tq "${OUTPUT_PARTIAL}" \ + || fail_closed "native_backup_archive_integrity_failed" + artifact_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')" + artifact_size="$(wc -c < "${OUTPUT_PARTIAL}" | tr -d ' ')" + [[ "${artifact_size}" =~ ^[0-9]+$ ]] && [ "${artifact_size}" -gt 0 ] \ + || fail_closed "native_backup_local_size_invalid" + + cp "${SOURCE_INVENTORY_RECEIPT}" "${INVENTORY_PARTIAL}" + inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')" + [ "${inventory_hash}" = "${SOURCE_INVENTORY_HASH}" ] \ + || fail_closed "copied_source_inventory_hash_mismatch" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","identity_hash":"%s","operation_id":"%s","artifact_name":"%s","databases":%s,"source_inventory_sha256":"%s","sha256":"%s","size_bytes":%s,"status":"BACKUP_CREATED","num_files":%s,"total_size":%s,"uncompressed_size":%s,"compressed_size":%s}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${IDENTITY_HASH}" \ + "${OPERATION_ID}" "${ARTIFACT_NAME}" "${EXPECTED_DATABASES_JSON}" \ + "${SOURCE_INVENTORY_HASH}" "${artifact_hash}" "${artifact_size}" \ + "${BACKUP_NUM_FILES}" "${BACKUP_TOTAL_SIZE}" "${BACKUP_UNCOMPRESSED_SIZE}" \ + "${BACKUP_COMPRESSED_SIZE}" > "${MANIFEST_PARTIAL}" + + if [ -n "${POST_VERIFY_HOOK}" ]; then + manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')" + run_captured "post_verify_hook_check" env \ + TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ + CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \ + CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \ + CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \ + CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \ + CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \ + CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \ + "${POST_VERIFY_HOOK}" --check || fail_closed "post_verify_hook_check_failed" + run_captured "post_verify_hook" env \ + TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ + CLICKHOUSE_NATIVE_ARTIFACT_PATH="${OUTPUT_PARTIAL}" \ + CLICKHOUSE_NATIVE_ARTIFACT_SHA256="${artifact_hash}" \ + CLICKHOUSE_NATIVE_MANIFEST_PATH="${MANIFEST_PARTIAL}" \ + CLICKHOUSE_NATIVE_OPERATION_ID="${OPERATION_ID}" \ + CLICKHOUSE_NATIVE_EXPECTED_DATABASES="${EXPECTED_DATABASES_CSV}" \ + CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH="${INVENTORY_PARTIAL}" \ + "${POST_VERIFY_HOOK}" --apply || fail_closed "post_verify_hook_failed" + hook_hash="$(sha256sum "${OUTPUT_PARTIAL}" | awk '{print $1}')" + [ "${hook_hash}" = "${artifact_hash}" ] || fail_closed "post_verify_hook_modified_artifact" + hook_inventory_hash="$(sha256sum "${INVENTORY_PARTIAL}" | awk '{print $1}')" + [ "${hook_inventory_hash}" = "${inventory_hash}" ] \ + || fail_closed "post_verify_hook_modified_source_inventory" + hook_manifest_hash="$(sha256sum "${MANIFEST_PARTIAL}" | awk '{print $1}')" + [ "${hook_manifest_hash}" = "${manifest_hash}" ] \ + || fail_closed "post_verify_hook_modified_manifest" + fi + + mv "${OUTPUT_PARTIAL}" "${OUTPUT_FILE}" + OUTPUT_CREATED=1 + mv "${INVENTORY_PARTIAL}" "${INVENTORY_FILE}" + printf '%s %s\n' "${artifact_hash}" "$(basename "${OUTPUT_FILE}")" > "${HASH_FILE}" + mv "${MANIFEST_PARTIAL}" "${MANIFEST_FILE}" + + cleanup_server_artifact || fail_closed "completed_server_artifact_cleanup_failed" + emit_receipt "independent_verifier" "pass" \ + "system_status_archive_integrity_hash_${artifact_hash}_size_${artifact_size}" + APPLY_PASS=1 +} + +apply_backup() { + local submission_line submitted_id submitted_status + + grep -F -q '"stage":"check","terminal":"check_pass_no_write"' "${RECEIPT_LOG}" \ + || fail_closed "same_run_check_pass_receipt_required_before_apply" + perform_check + emit_receipt "check" "pass" "apply_revalidation_pass" + + if verify_existing_local_artifact; then + return 0 + fi + + query_backup_status + if [ "${STATUS_PRESENT}" -eq 0 ]; then + server_artifact_absent || fail_closed "orphan_server_artifact_without_system_status" + emit_receipt "ai_decision" "candidate" \ + "submit_native_async_backup_exact_six_database_allowlist_to_exact_artifact" + SERVER_ARTIFACT_ACTIVE=1 + run_sql "backup_submit" \ + "BACKUP ${BACKUP_SCOPE_SQL} TO Disk({disk:String},{artifact:String}) SETTINGS id='${OPERATION_ID}' ASYNC" \ + --param_disk "${BACKUP_DISK}" --param_artifact "${ARTIFACT_NAME}" \ + || fail_closed "native_backup_submit_failed" + submission_line="$(tr -d '\r\n' < "${LAST_STDOUT}")" + IFS=$'\t' read -r submitted_id submitted_status <<< "${submission_line}" + [ "${submitted_id}" = "${OPERATION_ID}" ] \ + || fail_closed "native_backup_submission_id_mismatch" + case "${submitted_status}" in + CREATING_BACKUP|BACKUP_CREATED) ;; + *) fail_closed "native_backup_submission_status_${submitted_status:-empty}" ;; + esac + SERVER_ARTIFACT_ACTIVE=1 + emit_receipt "execution" "submitted" "operation_${OPERATION_ID}_status_${submitted_status}" + else + emit_receipt "idempotency" "resumed" "existing_operation_status_${BACKUP_STATUS}" + fi + + poll_backup_terminal + copy_and_verify_artifact +} + +main() { + setup "$@" + + if [ "${MODE}" = "check" ]; then + perform_check + emit_receipt "ai_decision" "candidate" \ + "native_backup_apply_requires_same_run_check_receipt" + emit_receipt "check" "check_pass_no_write" \ + "runtime_contract_valid_no_backup_or_artifact_write" + CHECK_PASS=1 + log_info "ClickHouse native backup check passed without runtime write" + return 0 + fi + + apply_backup + log_info "ClickHouse native backup verified: ${OUTPUT_FILE}" +} + +main "$@" diff --git a/scripts/backup/clickhouse-native-restore-drill.sh b/scripts/backup/clickhouse-native-restore-drill.sh new file mode 100755 index 000000000..afb54ef3c --- /dev/null +++ b/scripts/backup/clickhouse-native-restore-drill.sh @@ -0,0 +1,1152 @@ +#!/usr/bin/env bash +# ============================================================================= +# P0-OBS-002 - isolated ClickHouse native RESTORE drill +# +# This verifier never joins a production network and never restores into an +# existing ClickHouse instance. Apply mode stages the read-only source artifact +# through an exact-image, network-none container into a fresh writable volume, +# then creates one Docker internal network, one exact local ZooKeeper container, +# one exact local ClickHouse container and fresh ephemeral volumes. Every owned +# resource is removed on every terminal. +# ============================================================================= + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-${ROOT_DIR}/scripts/backup/clickhouse-restore-inventory.py}" +BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml}" +ISOLATED_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml}" + +BACKUP_DISK="backups" +CONTAINER_ARTIFACT_NAME="native-backup.zip" +CONTAINER_ARTIFACT_PATH="/backups/${CONTAINER_ARTIFACT_NAME}" +STAGING_SOURCE_ARTIFACT_PATH="/tmp/awoooi-source-${CONTAINER_ARTIFACT_NAME}" +RESTORE_SCOPE_SQL="DATABASE signoz_analytics,DATABASE signoz_logs,DATABASE signoz_metadata,DATABASE signoz_meter,DATABASE signoz_metrics,DATABASE signoz_traces" +DATABASE_IN_SQL="'signoz_analytics','signoz_logs','signoz_metadata','signoz_meter','signoz_metrics','signoz_traces'" +EMPTY_SHA256="E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" + +COMMAND_TIMEOUT_SECONDS="${CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS:-60}" +STARTUP_TIMEOUT_SECONDS="${CLICKHOUSE_RESTORE_STARTUP_TIMEOUT_SECONDS:-180}" +RESTORE_TIMEOUT_SECONDS="${CLICKHOUSE_RESTORE_TIMEOUT_SECONDS:-1800}" +POLL_INTERVAL_SECONDS="${CLICKHOUSE_RESTORE_POLL_INTERVAL_SECONDS:-2}" +KILL_AFTER_SECONDS="${CLICKHOUSE_RESTORE_KILL_AFTER_SECONDS:-15}" +MAX_LOG_BYTES="${CLICKHOUSE_RESTORE_MAX_LOG_BYTES:-262144}" +LOG_TAIL_LINES="${CLICKHOUSE_RESTORE_LOG_TAIL_LINES:-200}" + +MODE="" +TRACE_ID="${TRACE_ID:-}" +RUN_ID="${RUN_ID:-}" +WORK_ITEM_ID="${WORK_ITEM_ID:-}" +ARTIFACT_PATH="${CLICKHOUSE_NATIVE_ARTIFACT_PATH:-}" +MANIFEST_PATH="${CLICKHOUSE_NATIVE_MANIFEST_PATH:-}" +SOURCE_INVENTORY_PATH="${CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH:-}" +CLICKHOUSE_IMAGE="${CLICKHOUSE_RESTORE_CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:25.5.6}" +CLICKHOUSE_IMAGE_ID_EXPECTED="${CLICKHOUSE_RESTORE_CLICKHOUSE_IMAGE_ID:-sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6}" +ZOOKEEPER_IMAGE="${CLICKHOUSE_RESTORE_ZOOKEEPER_IMAGE:-signoz/zookeeper:3.7.1}" +ZOOKEEPER_IMAGE_ID_EXPECTED="${CLICKHOUSE_RESTORE_ZOOKEEPER_IMAGE_ID:-sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0}" +RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-/backup/logs/clickhouse-native-restore-drill}" +HOOK_ARTIFACT_SHA256="${CLICKHOUSE_NATIVE_ARTIFACT_SHA256:-}" +HOOK_OPERATION_ID="${CLICKHOUSE_NATIVE_OPERATION_ID:-}" +HOOK_EXPECTED_DATABASES="${CLICKHOUSE_NATIVE_EXPECTED_DATABASES:-}" + +IDENTITY_HASH="" +SHORT_ID="" +RESTORE_REPLICA="" +RESTORE_OPERATION_ID="" +NETWORK_NAME="" +STAGING_CONTAINER="" +CLICKHOUSE_CONTAINER="" +ZOOKEEPER_CONTAINER="" +BACKUP_VOLUME="" +CLICKHOUSE_VOLUME="" +ZOOKEEPER_DATA_VOLUME="" +ZOOKEEPER_LOG_VOLUME="" + +RECEIPT_DIR="" +RECEIPT_LOG="" +STATUS_FILE="" +PREFLIGHT_FILE="" +PREFLIGHT_CANDIDATE="" +LOCK_DIR="" +INVOCATION_ID="" +RECEIPTS_READY=0 +COMMAND_SEQUENCE=0 +LAST_STDOUT="" +LAST_STDERR="" +LAST_EXIT_CODE=0 + +ARTIFACT_SHA256="" +STAGED_ARTIFACT_SHA256="" +CLICKHOUSE_ARTIFACT_SHA256="" +SOURCE_INVENTORY_SHA256="" +BACKUP_OPERATION_ID="" +SOURCE_TABLE_COUNT=0 +SOURCE_DATABASE_COUNT=0 +REPLICATED_TABLE_COUNT=0 +DISTRIBUTED_TABLE_COUNT=0 +PHYSICAL_TABLE_COUNT=0 +CLICKHOUSE_IMAGE_ID="" +ZOOKEEPER_IMAGE_ID="" +BACKUP_CONFIG_SHA256="" +CLUSTER_CONFIG_SHA256="" + +RESTORE_TERMINAL="not_started" +RESTORE_NUM_FILES=0 +RESTORE_TOTAL_SIZE=0 +MANIFEST_PARITY=0 +RESTORED_DATABASE_COUNT=0 +RESTORED_TABLE_COUNT=0 +CRITICAL_NONZERO_COUNT=0 +CHECK_TABLE_COUNT=0 +CHECK_TABLE_PASS_COUNT=0 +SOURCE_TOTAL_ROWS=0 +RESTORED_TOTAL_ROWS=0 +ROW_DELTA=0 +SOURCE_TOTAL_BYTES=0 +RESTORED_TOTAL_BYTES=0 +BYTE_DELTA=0 + +NETWORK_CREATED=0 +STAGING_CONTAINER_CREATED=0 +CLICKHOUSE_CONTAINER_CREATED=0 +ZOOKEEPER_CONTAINER_CREATED=0 +BACKUP_VOLUME_CREATED=0 +CLICKHOUSE_VOLUME_CREATED=0 +ZOOKEEPER_DATA_VOLUME_CREATED=0 +ZOOKEEPER_LOG_VOLUME_CREATED=0 +EPHEMERAL_RUNTIME_CREATED=0 +ARTIFACT_STAGING_VERIFIED=0 +CLICKHOUSE_ARTIFACT_READBACK_VERIFIED=0 +APPLY_RESOURCE_SCOPE=0 +CHECK_PASS=0 +APPLY_PASS=0 +CLEANUP_PASS=1 +FINAL_REASON="not_started" + +usage() { + cat <<'USAGE' +Usage: + clickhouse-native-restore-drill.sh --check|--apply \ + --trace-id ID --run-id ID --work-item-id ID \ + --artifact ABSOLUTE_PATH --manifest ABSOLUTE_PATH \ + --source-inventory ABSOLUTE_PATH \ + [--clickhouse-image REF --clickhouse-image-id sha256:...] \ + [--zookeeper-image signoz/zookeeper:3.7.1] \ + [--zookeeper-image-id sha256:...] [--receipt-root ABSOLUTE_PATH] + +--check performs local artifact/config validation plus read-only Docker image +inspection. It creates no container, volume or network. --apply requires the +same run's check receipt, stages the source through an exact-image network-none +container into a writable owned backup volume, then restores only into fresh +ephemeral volumes on an internal-only Docker network. It never attaches to +production and publishes no ports. All ephemeral resources are removed on +every terminal. + +When invoked as CLICKHOUSE_NATIVE_POST_VERIFY_HOOK, TRACE_ID/RUN_ID/ +WORK_ITEM_ID and CLICKHOUSE_NATIVE_{ARTIFACT,MANIFEST,SOURCE_INVENTORY}_PATH +are accepted directly from the adapter's bounded environment. Deployments may +set CLICKHOUSE_RESTORE_{INVENTORY_HELPER,BACKUP_DISK_CONFIG,CLUSTER_CONFIG} to +reviewed absolute host paths; no env file is read. +USAGE +} + +log_info() { printf '[INFO] %s\n' "$*"; } +log_error() { printf '[ERROR] %s\n' "$*" >&2; } + +valid_identity() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] +} + +valid_positive_integer() { + [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ] +} + +valid_image_ref() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._/@:-]{0,255}$ ]] +} + +valid_image_id() { + [[ "$1" =~ ^sha256:[0-9a-f]{64}$ ]] +} + +require_value() { + local option="$1" + local value="${2:-}" + [ -n "${value}" ] || { log_error "${option} requires a value"; exit 64; } +} + +set_mode() { + local requested="$1" + [ -z "${MODE}" ] || { log_error "exactly one of --check or --apply is required"; exit 64; } + MODE="${requested}" +} + +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + --check) set_mode check ;; + --apply) set_mode apply ;; + --trace-id) + require_value "$1" "${2:-}" + TRACE_ID="$2" + shift + ;; + --run-id) + require_value "$1" "${2:-}" + RUN_ID="$2" + shift + ;; + --work-item-id) + require_value "$1" "${2:-}" + WORK_ITEM_ID="$2" + shift + ;; + --artifact) + require_value "$1" "${2:-}" + ARTIFACT_PATH="$2" + shift + ;; + --manifest) + require_value "$1" "${2:-}" + MANIFEST_PATH="$2" + shift + ;; + --source-inventory) + require_value "$1" "${2:-}" + SOURCE_INVENTORY_PATH="$2" + shift + ;; + --clickhouse-image) + require_value "$1" "${2:-}" + CLICKHOUSE_IMAGE="$2" + shift + ;; + --clickhouse-image-id) + require_value "$1" "${2:-}" + CLICKHOUSE_IMAGE_ID_EXPECTED="$2" + shift + ;; + --zookeeper-image) + require_value "$1" "${2:-}" + ZOOKEEPER_IMAGE="$2" + shift + ;; + --zookeeper-image-id) + require_value "$1" "${2:-}" + ZOOKEEPER_IMAGE_ID_EXPECTED="$2" + shift + ;; + --receipt-root) + require_value "$1" "${2:-}" + RECEIPT_ROOT="$2" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + log_error "unsupported argument: $1" + usage >&2 + exit 64 + ;; + esac + shift + done +} + +validate_regular_file() { + local label="$1" + local path="$2" + [ -n "${path}" ] && [[ "${path}" == /* ]] && [[ "${path}" != *$'\n'* ]] \ + || { log_error "${label} must be an absolute single-line path"; exit 64; } + [ -f "${path}" ] && [ ! -L "${path}" ] \ + || { log_error "${label} must be a non-symlink regular file"; exit 64; } +} + +validate_contract() { + [ -n "${MODE}" ] || { log_error "exactly one of --check or --apply is required"; exit 64; } + valid_identity "${TRACE_ID}" || { log_error "--trace-id is required and path-safe"; exit 64; } + valid_identity "${RUN_ID}" || { log_error "--run-id is required and path-safe"; exit 64; } + valid_identity "${WORK_ITEM_ID}" || { log_error "--work-item-id is required and path-safe"; exit 64; } + valid_image_ref "${CLICKHOUSE_IMAGE}" || { log_error "invalid ClickHouse image reference"; exit 64; } + valid_image_id "${CLICKHOUSE_IMAGE_ID_EXPECTED}" || { log_error "invalid expected ClickHouse image ID"; exit 64; } + valid_image_ref "${ZOOKEEPER_IMAGE}" || { log_error "invalid ZooKeeper image reference"; exit 64; } + [ "${ZOOKEEPER_IMAGE}" = "signoz/zookeeper:3.7.1" ] \ + || { log_error "ZooKeeper image must be the reviewed signoz/zookeeper:3.7.1 reference"; exit 64; } + valid_image_id "${ZOOKEEPER_IMAGE_ID_EXPECTED}" || { log_error "invalid expected ZooKeeper image ID"; exit 64; } + [[ "${RECEIPT_ROOT}" == /* ]] && [[ "${RECEIPT_ROOT}" != *$'\n'* ]] \ + || { log_error "receipt root must be an absolute single-line path"; exit 64; } + [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root symlink is unsafe"; exit 64; } + validate_regular_file artifact "${ARTIFACT_PATH}" + validate_regular_file manifest "${MANIFEST_PATH}" + validate_regular_file source_inventory "${SOURCE_INVENTORY_PATH}" + validate_regular_file backup_disk_config "${BACKUP_DISK_CONFIG}" + validate_regular_file isolated_cluster_config "${ISOLATED_CLUSTER_CONFIG}" + validate_regular_file inventory_helper "${INVENTORY_HELPER}" + if [ -n "${HOOK_EXPECTED_DATABASES}" ]; then + [ "${HOOK_EXPECTED_DATABASES}" = "signoz_analytics,signoz_logs,signoz_metadata,signoz_meter,signoz_metrics,signoz_traces" ] \ + || { log_error "hook database allowlist mismatch"; exit 64; } + fi + + for value in "${COMMAND_TIMEOUT_SECONDS}" "${STARTUP_TIMEOUT_SECONDS}" \ + "${RESTORE_TIMEOUT_SECONDS}" "${POLL_INTERVAL_SECONDS}" \ + "${KILL_AFTER_SECONDS}" "${MAX_LOG_BYTES}" "${LOG_TAIL_LINES}"; do + valid_positive_integer "${value}" || { log_error "timeout/log limits must be positive integers"; exit 64; } + done + for command_name in awk basename cat cmp date docker grep head mkdir mktemp mv python3 rm \ + rmdir sed sha256sum sleep timeout touch tr unzip wc; do + command -v "${command_name}" >/dev/null 2>&1 \ + || { log_error "required command missing: ${command_name}"; exit 64; } + done + + grep -F -q 'backups' "${BACKUP_DISK_CONFIG}" \ + || { log_error "repo-owned backup disk config does not allow backups disk"; exit 64; } + grep -F -q '/backups/' "${BACKUP_DISK_CONFIG}" \ + || { log_error "repo-owned backup disk config path mismatch"; exit 64; } + grep -F -q 'restore-zookeeper' "${ISOLATED_CLUSTER_CONFIG}" \ + || { log_error "isolated ZooKeeper host contract missing"; exit 64; } + grep -F -q '' "${ISOLATED_CLUSTER_CONFIG}" \ + || { log_error "isolated cluster name contract missing"; exit 64; } + grep -F -q '01' "${ISOLATED_CLUSTER_CONFIG}" \ + || { log_error "isolated shard macro contract missing"; exit 64; } + grep -F -q 'from_env="CLICKHOUSE_RESTORE_REPLICA"' "${ISOLATED_CLUSTER_CONFIG}" \ + || { log_error "run-derived replica macro contract missing"; exit 64; } +} + +sha256_file() { + sha256sum "$1" | awk '{print $1}' +} + +emit_receipt() { + local stage="$1" + local terminal="$2" + local detail="$3" + local observed_at + [ "${RECEIPTS_READY}" -eq 1 ] || return 0 + observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${INVOCATION_ID}" \ + "${observed_at}" "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}" +} + +fail_closed() { + FINAL_REASON="$1" + log_error "${FINAL_REASON}" + emit_receipt failure failed "${FINAL_REASON}" + exit 1 +} + +setup_receipts() { + local identity_file identity_content identity_tmp + + IDENTITY_HASH="$(printf '%s\000%s\000%s' "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" | sha256sum | awk '{print $1}')" + SHORT_ID="${IDENTITY_HASH:0:16}" + RESTORE_REPLICA="restore-${SHORT_ID}" + RESTORE_OPERATION_ID="restore-${IDENTITY_HASH}" + NETWORK_NAME="awoooi-ch-restore-${SHORT_ID}-net" + STAGING_CONTAINER="awoooi-ch-restore-${SHORT_ID}-stage" + CLICKHOUSE_CONTAINER="awoooi-ch-restore-${SHORT_ID}" + ZOOKEEPER_CONTAINER="awoooi-zk-restore-${SHORT_ID}" + BACKUP_VOLUME="awoooi-ch-restore-${SHORT_ID}-backup" + CLICKHOUSE_VOLUME="awoooi-ch-restore-${SHORT_ID}-data" + ZOOKEEPER_DATA_VOLUME="awoooi-zk-restore-${SHORT_ID}-data" + ZOOKEEPER_LOG_VOLUME="awoooi-zk-restore-${SHORT_ID}-log" + + mkdir -p "${RECEIPT_ROOT}" + RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" + [ ! -L "${RECEIPT_DIR}" ] || { log_error "receipt directory symlink is unsafe"; exit 64; } + mkdir -m 700 -p "${RECEIPT_DIR}" + LOCK_DIR="${RECEIPT_DIR}/active.lock" + mkdir "${LOCK_DIR}" 2>/dev/null \ + || { log_error "same restore-drill run is active or has a stale lock"; exit 1; } + + RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl" + STATUS_FILE="${RECEIPT_DIR}/status.json" + PREFLIGHT_FILE="${RECEIPT_DIR}/preflight.contract" + INVOCATION_ID="${MODE}-$(date -u '+%Y%m%dT%H%M%SZ')-$$" + PREFLIGHT_CANDIDATE="${RECEIPT_DIR}/${INVOCATION_ID}.preflight.candidate" + identity_file="${RECEIPT_DIR}/identity.json" + identity_content="{\"trace_id\":\"${TRACE_ID}\",\"run_id\":\"${RUN_ID}\",\"work_item_id\":\"${WORK_ITEM_ID}\",\"identity_hash\":\"${IDENTITY_HASH}\"}" + if [ -e "${identity_file}" ]; then + [ -f "${identity_file}" ] && [ ! -L "${identity_file}" ] \ + || { rmdir "${LOCK_DIR}"; log_error "existing identity receipt is unsafe"; exit 1; } + [ "$(sed -n '1p' "${identity_file}")" = "${identity_content}" ] \ + || { rmdir "${LOCK_DIR}"; log_error "run identity conflicts with durable receipt"; exit 1; } + else + identity_tmp="$(mktemp "${RECEIPT_DIR}/.identity.XXXXXX")" + printf '%s\n' "${identity_content}" > "${identity_tmp}" + mv "${identity_tmp}" "${identity_file}" + fi + touch "${RECEIPT_LOG}" + RECEIPTS_READY=1 + trap cleanup EXIT + trap 'on_signal 1' HUP + trap 'on_signal 2' INT + trap 'on_signal 15' TERM +} + +run_captured() { + local step="$1" + local stdout_tmp stderr_tmp stdout_size stderr_size stdout_hash stderr_hash truncated=0 + shift + [[ "${step}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]] || fail_closed invalid_command_step + + COMMAND_SEQUENCE=$((COMMAND_SEQUENCE + 1)) + LAST_STDOUT="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stdout" + LAST_STDERR="${RECEIPT_DIR}/${INVOCATION_ID}-$(printf '%03d' "${COMMAND_SEQUENCE}")-${step}.stderr" + stdout_tmp="$(mktemp "${RECEIPT_DIR}/.stdout.XXXXXX")" + stderr_tmp="$(mktemp "${RECEIPT_DIR}/.stderr.XXXXXX")" + + set +e + "$@" >"${stdout_tmp}" 2>"${stderr_tmp}" + LAST_EXIT_CODE=$? + set -e + stdout_size="$(wc -c < "${stdout_tmp}" | tr -d ' ')" + stderr_size="$(wc -c < "${stderr_tmp}" | tr -d ' ')" + head -c "${MAX_LOG_BYTES}" "${stdout_tmp}" > "${LAST_STDOUT}" + head -c "${MAX_LOG_BYTES}" "${stderr_tmp}" > "${LAST_STDERR}" + rm -f "${stdout_tmp}" "${stderr_tmp}" + if [ "${stdout_size}" -gt "${MAX_LOG_BYTES}" ] || [ "${stderr_size}" -gt "${MAX_LOG_BYTES}" ]; then + truncated=1 + [ "${LAST_EXIT_CODE}" -ne 0 ] || LAST_EXIT_CODE=70 + fi + stdout_hash="$(sha256_file "${LAST_STDOUT}")" + stderr_hash="$(sha256_file "${LAST_STDERR}")" + printf '%s\n' "${LAST_EXIT_CODE}" > "${LAST_STDOUT}.status" + emit_receipt command "$([ "${LAST_EXIT_CODE}" -eq 0 ] && printf pass || printf failed)" \ + "step_${step}_exit_${LAST_EXIT_CODE}_stdout_bytes_${stdout_size}_stderr_bytes_${stderr_size}_truncated_${truncated}_stdout_${stdout_hash}_stderr_${stderr_hash}" + return "${LAST_EXIT_CODE}" +} + +run_sql() { + local step="$1" + local query="$2" + shift 2 + run_captured "${step}" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CLICKHOUSE_CONTAINER}" \ + clickhouse-client --format TSVRaw "$@" --query "${query}" +} + +read_key() { + local key="$1" + local file="$2" + awk -F= -v wanted="${key}" '$1 == wanted {sub(/^[^=]*=/, ""); print; exit}' "${file}" +} + +validate_preflight() { + local image_line + + run_captured artifact_zip timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" unzip -tq "${ARTIFACT_PATH}" \ + || fail_closed native_backup_zip_integrity_failed + run_captured manifest_inventory python3 "${INVENTORY_HELPER}" preflight \ + --manifest "${MANIFEST_PATH}" --artifact "${ARTIFACT_PATH}" \ + --source-inventory "${SOURCE_INVENTORY_PATH}" \ + || fail_closed native_backup_manifest_or_source_inventory_failed + + ARTIFACT_SHA256="$(read_key artifact_sha256 "${LAST_STDOUT}")" + SOURCE_INVENTORY_SHA256="$(read_key source_inventory_sha256 "${LAST_STDOUT}")" + BACKUP_OPERATION_ID="$(read_key backup_operation_id "${LAST_STDOUT}")" + SOURCE_TABLE_COUNT="$(read_key source_table_count "${LAST_STDOUT}")" + SOURCE_DATABASE_COUNT="$(read_key source_database_count "${LAST_STDOUT}")" + REPLICATED_TABLE_COUNT="$(read_key replicated_table_count "${LAST_STDOUT}")" + DISTRIBUTED_TABLE_COUNT="$(read_key distributed_table_count "${LAST_STDOUT}")" + PHYSICAL_TABLE_COUNT="$(read_key physical_table_count "${LAST_STDOUT}")" + [[ "${ARTIFACT_SHA256}" =~ ^[0-9a-f]{64}$ ]] || fail_closed artifact_hash_receipt_invalid + [[ "${SOURCE_INVENTORY_SHA256}" =~ ^[0-9a-f]{64}$ ]] || fail_closed inventory_hash_receipt_invalid + valid_identity "${BACKUP_OPERATION_ID}" || fail_closed backup_operation_receipt_invalid + if [ -n "${HOOK_ARTIFACT_SHA256}" ]; then + [ "${HOOK_ARTIFACT_SHA256}" = "${ARTIFACT_SHA256}" ] \ + || fail_closed hook_artifact_sha256_mismatch + fi + if [ -n "${HOOK_OPERATION_ID}" ]; then + [ "${HOOK_OPERATION_ID}" = "${BACKUP_OPERATION_ID}" ] \ + || fail_closed hook_backup_operation_id_mismatch + fi + + run_captured clickhouse_image timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker image inspect --format '{{.Id}}' "${CLICKHOUSE_IMAGE}" \ + || fail_closed clickhouse_local_image_missing + image_line="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${image_line}" = "${CLICKHOUSE_IMAGE_ID_EXPECTED}" ] \ + || fail_closed clickhouse_local_image_id_mismatch + CLICKHOUSE_IMAGE_ID="${image_line}" + + run_captured zookeeper_image timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker image inspect --format '{{.Id}}' "${ZOOKEEPER_IMAGE}" \ + || fail_closed zookeeper_local_image_missing + image_line="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${image_line}" = "${ZOOKEEPER_IMAGE_ID_EXPECTED}" ] \ + || fail_closed zookeeper_local_image_id_mismatch + ZOOKEEPER_IMAGE_ID="${image_line}" + + BACKUP_CONFIG_SHA256="$(sha256_file "${BACKUP_DISK_CONFIG}")" + CLUSTER_CONFIG_SHA256="$(sha256_file "${ISOLATED_CLUSTER_CONFIG}")" + { + printf 'identity_hash=%s\n' "${IDENTITY_HASH}" + printf 'artifact_path=%s\n' "${ARTIFACT_PATH}" + printf 'artifact_sha256=%s\n' "${ARTIFACT_SHA256}" + printf 'manifest_path=%s\n' "${MANIFEST_PATH}" + printf 'source_inventory_path=%s\n' "${SOURCE_INVENTORY_PATH}" + printf 'source_inventory_sha256=%s\n' "${SOURCE_INVENTORY_SHA256}" + printf 'backup_operation_id=%s\n' "${BACKUP_OPERATION_ID}" + printf 'clickhouse_image=%s\n' "${CLICKHOUSE_IMAGE}" + printf 'clickhouse_image_id=%s\n' "${CLICKHOUSE_IMAGE_ID}" + printf 'zookeeper_image=%s\n' "${ZOOKEEPER_IMAGE}" + printf 'zookeeper_image_id=%s\n' "${ZOOKEEPER_IMAGE_ID}" + printf 'backup_config_sha256=%s\n' "${BACKUP_CONFIG_SHA256}" + printf 'cluster_config_sha256=%s\n' "${CLUSTER_CONFIG_SHA256}" + printf 'restore_replica=%s\n' "${RESTORE_REPLICA}" + printf 'source_database_count=%s\n' "${SOURCE_DATABASE_COUNT}" + printf 'source_table_count=%s\n' "${SOURCE_TABLE_COUNT}" + printf 'replicated_table_count=%s\n' "${REPLICATED_TABLE_COUNT}" + printf 'distributed_table_count=%s\n' "${DISTRIBUTED_TABLE_COUNT}" + printf 'physical_table_count=%s\n' "${PHYSICAL_TABLE_COUNT}" + } > "${PREFLIGHT_CANDIDATE}" + emit_receipt sensor_source pass \ + "backup_${BACKUP_OPERATION_ID}_artifact_${ARTIFACT_SHA256}_inventory_${SOURCE_INVENTORY_SHA256}" + emit_receipt normalized_asset_identity pass \ + "restore_${IDENTITY_HASH}_clickhouse_${CLICKHOUSE_IMAGE_ID}_zookeeper_${ZOOKEEPER_IMAGE_ID}" + emit_receipt risk_policy pass \ + "risk_high_isolated_internal_network_fresh_keeper_fresh_data_volumes_no_production_attachment" +} + +persist_check_contract() { + if [ -e "${PREFLIGHT_FILE}" ]; then + [ -f "${PREFLIGHT_FILE}" ] && [ ! -L "${PREFLIGHT_FILE}" ] \ + || fail_closed existing_preflight_contract_unsafe + cmp -s "${PREFLIGHT_CANDIDATE}" "${PREFLIGHT_FILE}" \ + || fail_closed same_run_preflight_contract_drift + rm -f "${PREFLIGHT_CANDIDATE}" + else + mv "${PREFLIGHT_CANDIDATE}" "${PREFLIGHT_FILE}" + fi + emit_receipt source_of_truth_diff pass \ + "exact_six_databases_tables_${SOURCE_TABLE_COUNT}_replicated_${REPLICATED_TABLE_COUNT}_distributed_${DISTRIBUTED_TABLE_COUNT}" + emit_receipt ai_decision candidate \ + "isolated_keeper_clickhouse_restore_requires_same_run_apply" + emit_receipt check check_pass_no_runtime_write \ + "local_artifact_config_and_exact_local_images_valid_no_container_volume_or_network_created" + CHECK_PASS=1 + FINAL_REASON="check_pass_no_runtime_write" +} + +require_check_contract() { + [ -f "${PREFLIGHT_FILE}" ] && [ ! -L "${PREFLIGHT_FILE}" ] \ + || fail_closed same_run_check_contract_missing + cmp -s "${PREFLIGHT_CANDIDATE}" "${PREFLIGHT_FILE}" \ + || fail_closed apply_preflight_contract_drift + rm -f "${PREFLIGHT_CANDIDATE}" + grep -F -q '"stage":"terminal","terminal":"check_pass_no_runtime_write"' "${RECEIPT_LOG}" \ + || fail_closed same_run_check_terminal_receipt_missing + CHECK_PASS=1 + emit_receipt check pass apply_revalidation_matches_same_run_check_contract +} + +ensure_resources_absent() { + docker_daemon_ready || fail_closed docker_daemon_unavailable_before_restore_create + exact_container_absent "${CLICKHOUSE_CONTAINER}" \ + || fail_closed isolated_clickhouse_container_name_collision_or_query_failed + exact_container_absent "${ZOOKEEPER_CONTAINER}" \ + || fail_closed isolated_zookeeper_container_name_collision_or_query_failed + exact_container_absent "${STAGING_CONTAINER}" \ + || fail_closed isolated_staging_container_name_collision_or_query_failed + exact_volume_absent "${BACKUP_VOLUME}" \ + || fail_closed isolated_backup_volume_name_collision_or_query_failed + exact_volume_absent "${CLICKHOUSE_VOLUME}" \ + || fail_closed isolated_clickhouse_volume_name_collision_or_query_failed + exact_volume_absent "${ZOOKEEPER_DATA_VOLUME}" \ + || fail_closed isolated_zookeeper_data_volume_name_collision_or_query_failed + exact_volume_absent "${ZOOKEEPER_LOG_VOLUME}" \ + || fail_closed isolated_zookeeper_log_volume_name_collision_or_query_failed + exact_network_absent "${NETWORK_NAME}" \ + || fail_closed isolated_network_name_collision_or_query_failed + emit_receipt source_of_truth_diff pass exact_ephemeral_resource_names_absent +} + +docker_daemon_ready() { + timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker info --format '{{.ServerVersion}}' >/dev/null 2>&1 +} + +exact_container_absent() { + local name="$1" observed + observed="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker container ls -a --format '{{.Names}}')" || return 1 + ! grep -F -x -q -- "${name}" <<< "${observed}" +} + +exact_volume_absent() { + local name="$1" observed + observed="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker volume ls -q)" || return 1 + ! grep -F -x -q -- "${name}" <<< "${observed}" +} + +exact_network_absent() { + local name="$1" observed + observed="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker network ls --format '{{.Name}}')" || return 1 + ! grep -F -x -q -- "${name}" <<< "${observed}" +} + +wait_for_container_running() { + local label="$1" + local container="$2" + local deadline state + deadline=$(( $(date +%s) + STARTUP_TIMEOUT_SECONDS )) + while true; do + if run_captured "${label}_running" timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.State.Running}}' "${container}"; then + state="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${state}" = "true" ] && return 0 + fi + [ "$(date +%s)" -lt "${deadline}" ] || fail_closed "${label}_startup_timeout" + sleep "${POLL_INTERVAL_SECONDS}" + done +} + +wait_for_clickhouse() { + local deadline + deadline=$(( $(date +%s) + STARTUP_TIMEOUT_SECONDS )) + while true; do + if run_sql clickhouse_ready 'SELECT 1'; then + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "1" ] && return 0 + fi + [ "$(date +%s)" -lt "${deadline}" ] || fail_closed clickhouse_startup_timeout + sleep "${POLL_INTERVAL_SECONDS}" + done +} + +stage_artifact_into_backup_volume() { + local staged_hash + + run_captured artifact_stage timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker run --pull=never \ + --name "${STAGING_CONTAINER}" --network none --restart=no --user 0:0 \ + --label "com.awoooi.restore.identity=${IDENTITY_HASH}" \ + --env "EXPECTED_ARTIFACT_SHA256=${ARTIFACT_SHA256}" \ + --mount "type=bind,src=${ARTIFACT_PATH},dst=${STAGING_SOURCE_ARTIFACT_PATH},readonly" \ + --mount "type=volume,src=${BACKUP_VOLUME},dst=/backups" \ + --entrypoint /bin/sh "${CLICKHOUSE_IMAGE}" -ceu \ + 'cp -- /tmp/awoooi-source-native-backup.zip /backups/native-backup.zip + sync /backups/native-backup.zip + set -- $(sha256sum /backups/native-backup.zip) + [ "$1" = "$EXPECTED_ARTIFACT_SHA256" ] + printf "%s\n" "$1"' \ + || fail_closed isolated_artifact_staging_failed + STAGING_CONTAINER_CREATED=1 + staged_hash="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [[ "${staged_hash}" =~ ^[0-9a-f]{64}$ ]] \ + || fail_closed staged_artifact_hash_readback_invalid + [ "${staged_hash}" = "${ARTIFACT_SHA256}" ] \ + || fail_closed staged_artifact_hash_mismatch + STAGED_ARTIFACT_SHA256="${staged_hash}" + + run_captured staging_image_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.Image}}' "${STAGING_CONTAINER}" \ + || fail_closed isolated_staging_image_readback_failed + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "${CLICKHOUSE_IMAGE_ID_EXPECTED}" ] \ + || fail_closed isolated_staging_runtime_image_id_mismatch + run_captured staging_network_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.HostConfig.NetworkMode}}' \ + "${STAGING_CONTAINER}" \ + || fail_closed isolated_staging_network_readback_failed + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = none ] \ + || fail_closed isolated_staging_network_not_none + ARTIFACT_STAGING_VERIFIED=1 + emit_receipt artifact_staging pass \ + "read_only_source_to_writable_owned_volume_sha256_${STAGED_ARTIFACT_SHA256}_network_none_exact_clickhouse_image" +} + +create_isolated_runtime() { + local clickhouse_artifact_hash + + # From this point onward cleanup inspects every exact resource name even if + # a collision or Docker CLI timeout occurs before a *_CREATED flag is set. + APPLY_RESOURCE_SCOPE=1 + ensure_resources_absent + + run_captured network_create timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker network create --internal \ + --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${NETWORK_NAME}" \ + || fail_closed isolated_internal_network_create_failed + NETWORK_CREATED=1 + EPHEMERAL_RUNTIME_CREATED=1 + + run_captured backup_volume timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker volume create \ + --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${BACKUP_VOLUME}" \ + || fail_closed isolated_backup_volume_create_failed + BACKUP_VOLUME_CREATED=1 + run_captured clickhouse_volume timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker volume create \ + --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${CLICKHOUSE_VOLUME}" \ + || fail_closed isolated_clickhouse_volume_create_failed + CLICKHOUSE_VOLUME_CREATED=1 + run_captured zookeeper_data_volume timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker volume create \ + --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${ZOOKEEPER_DATA_VOLUME}" \ + || fail_closed isolated_zookeeper_data_volume_create_failed + ZOOKEEPER_DATA_VOLUME_CREATED=1 + run_captured zookeeper_log_volume timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker volume create \ + --label "com.awoooi.restore.identity=${IDENTITY_HASH}" "${ZOOKEEPER_LOG_VOLUME}" \ + || fail_closed isolated_zookeeper_log_volume_create_failed + ZOOKEEPER_LOG_VOLUME_CREATED=1 + + stage_artifact_into_backup_volume + + run_captured zookeeper_run timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker run --detach --pull=never \ + --name "${ZOOKEEPER_CONTAINER}" --network "${NETWORK_NAME}" \ + --network-alias restore-zookeeper --restart=no \ + --label "com.awoooi.restore.identity=${IDENTITY_HASH}" \ + --env "ALLOW_ANONYMOUS_LOGIN=yes" \ + --mount "type=volume,src=${ZOOKEEPER_DATA_VOLUME},dst=/data" \ + --mount "type=volume,src=${ZOOKEEPER_LOG_VOLUME},dst=/datalog" \ + "${ZOOKEEPER_IMAGE}" \ + || fail_closed isolated_zookeeper_start_failed + ZOOKEEPER_CONTAINER_CREATED=1 + wait_for_container_running zookeeper "${ZOOKEEPER_CONTAINER}" + + run_captured clickhouse_run timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker run --detach --pull=never \ + --name "${CLICKHOUSE_CONTAINER}" --network "${NETWORK_NAME}" \ + --network-alias restore-clickhouse --restart=no \ + --label "com.awoooi.restore.identity=${IDENTITY_HASH}" \ + --env "CLICKHOUSE_RESTORE_REPLICA=${RESTORE_REPLICA}" \ + --mount "type=volume,src=${CLICKHOUSE_VOLUME},dst=/var/lib/clickhouse" \ + --mount "type=volume,src=${BACKUP_VOLUME},dst=/backups" \ + --mount "type=bind,src=${BACKUP_DISK_CONFIG},dst=/etc/clickhouse-server/config.d/awoooi-backup-disk.xml,readonly" \ + --mount "type=bind,src=${ISOLATED_CLUSTER_CONFIG},dst=/etc/clickhouse-server/config.d/awoooi-restore-isolation.xml,readonly" \ + "${CLICKHOUSE_IMAGE}" \ + || fail_closed isolated_clickhouse_start_failed + CLICKHOUSE_CONTAINER_CREATED=1 + wait_for_container_running clickhouse "${CLICKHOUSE_CONTAINER}" + wait_for_clickhouse + + run_captured zookeeper_image_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.Image}}' "${ZOOKEEPER_CONTAINER}" \ + || fail_closed isolated_zookeeper_image_readback_failed + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "${ZOOKEEPER_IMAGE_ID_EXPECTED}" ] \ + || fail_closed isolated_zookeeper_runtime_image_id_mismatch + run_captured clickhouse_image_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}" \ + || fail_closed isolated_clickhouse_image_readback_failed + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "${CLICKHOUSE_IMAGE_ID_EXPECTED}" ] \ + || fail_closed isolated_clickhouse_runtime_image_id_mismatch + run_captured clickhouse_artifact_hash_readback timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker exec "${CLICKHOUSE_CONTAINER}" \ + sha256sum "${CONTAINER_ARTIFACT_PATH}" \ + || fail_closed isolated_clickhouse_artifact_hash_readback_failed + clickhouse_artifact_hash="$(awk 'NR == 1 {print $1}' "${LAST_STDOUT}")" + [[ "${clickhouse_artifact_hash}" =~ ^[0-9a-f]{64}$ ]] \ + || fail_closed isolated_clickhouse_artifact_hash_readback_invalid + CLICKHOUSE_ARTIFACT_SHA256="${clickhouse_artifact_hash}" + [ "${CLICKHOUSE_ARTIFACT_SHA256}" = "${ARTIFACT_SHA256}" ] \ + || fail_closed isolated_clickhouse_artifact_hash_mismatch + CLICKHOUSE_ARTIFACT_READBACK_VERIFIED=1 + emit_receipt execution created \ + "artifact_${ARTIFACT_SHA256}_staged_read_only_source_writable_backup_volume_internal_network_ephemeral_keeper_and_clickhouse_started_pull_never_exact_image_ids_no_published_ports" +} + +verify_isolation_and_empty_target() { + local observed + + run_captured network_internal timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker network inspect --format '{{.Internal}}' "${NETWORK_NAME}" \ + || fail_closed isolated_network_inspect_failed + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "true" ] \ + || fail_closed restore_network_not_internal + for container in "${ZOOKEEPER_CONTAINER}" "${CLICKHOUSE_CONTAINER}"; do + run_captured container_network timeout --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker inspect --format '{{.HostConfig.NetworkMode}}' "${container}" \ + || fail_closed isolated_container_network_inspect_failed + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "${NETWORK_NAME}" ] \ + || fail_closed isolated_container_has_unexpected_network + done + + run_sql disk_contract \ + "SELECT path FROM system.disks WHERE name={disk:String}" --param_disk "${BACKUP_DISK}" \ + || fail_closed isolated_backup_disk_missing + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "/backups/" ] \ + || fail_closed isolated_backup_disk_path_mismatch + + run_sql macro_contract \ + 'SELECT macro,substitution FROM system.macros ORDER BY macro' \ + || fail_closed isolated_macro_query_failed + grep -F -x -q $'replica\t'"${RESTORE_REPLICA}" "${LAST_STDOUT}" \ + || fail_closed isolated_replica_macro_mismatch + grep -F -x -q $'shard\t01' "${LAST_STDOUT}" \ + || fail_closed isolated_shard_macro_mismatch + + run_sql cluster_contract \ + "SELECT cluster,host_name,toString(port) FROM system.clusters WHERE cluster='cluster' ORDER BY shard_num,replica_num" \ + || fail_closed isolated_cluster_query_failed + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = $'cluster\trestore-clickhouse\t9000' ] \ + || fail_closed isolated_cluster_contract_mismatch + + run_sql keeper_contract "SELECT count() > 0 FROM system.zookeeper WHERE path='/'" \ + || fail_closed isolated_keeper_connection_failed + [ "$(tr -d '\r\n' < "${LAST_STDOUT}")" = "1" ] \ + || fail_closed isolated_keeper_connection_not_ready + + run_sql empty_databases \ + "SELECT count() FROM system.databases WHERE name IN (${DATABASE_IN_SQL})" \ + || fail_closed isolated_database_empty_check_failed + observed="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${observed}" = "0" ] || fail_closed isolated_restore_target_databases_not_empty + run_sql empty_tables \ + "SELECT count() FROM system.tables WHERE database IN (${DATABASE_IN_SQL})" \ + || fail_closed isolated_table_empty_check_failed + observed="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${observed}" = "0" ] || fail_closed isolated_restore_target_tables_not_empty + emit_receipt independent_pre_verifier pass \ + "artifact_${STAGED_ARTIFACT_SHA256}_readback_internal_network_only_fresh_keeper_exact_macros_cluster_and_empty_signoz_target" +} + +query_restore_status() { + local line_count status_line restore_name error_hash + run_sql restore_status \ + "SELECT name,toString(status),toString(num_files),toString(total_size),hex(SHA256(error)) FROM system.backups WHERE id={operation_id:String}" \ + --param_operation_id "${RESTORE_OPERATION_ID}" \ + || fail_closed restore_system_backups_query_failed + line_count="$(awk 'NF { count++ } END { print count + 0 }' "${LAST_STDOUT}")" + [ "${line_count}" -eq 1 ] || fail_closed restore_system_backups_terminal_not_unique + status_line="$(tr -d '\r\n' < "${LAST_STDOUT}")" + IFS=$'\t' read -r restore_name RESTORE_TERMINAL RESTORE_NUM_FILES RESTORE_TOTAL_SIZE error_hash <<< "${status_line}" + [ "${restore_name}" = "Disk('${BACKUP_DISK}', '${CONTAINER_ARTIFACT_NAME}')" ] \ + || fail_closed restore_system_backups_artifact_mismatch + [[ "${RESTORE_NUM_FILES}" =~ ^[0-9]+$ ]] && [[ "${RESTORE_TOTAL_SIZE}" =~ ^[0-9]+$ ]] \ + || fail_closed restore_system_backups_metrics_invalid + [ "${error_hash}" = "${EMPTY_SHA256}" ] \ + || fail_closed restore_system_backups_error_present +} + +restore_native_backup() { + local submission_line submitted_id submitted_status deadline + emit_receipt ai_decision candidate \ + "restore_exact_six_signoz_databases_to_empty_isolated_clickhouse_with_fresh_keeper" + run_sql restore_submit \ + "RESTORE ${RESTORE_SCOPE_SQL} FROM Disk({disk:String},{artifact:String}) SETTINGS id='${RESTORE_OPERATION_ID}' ASYNC" \ + --param_disk "${BACKUP_DISK}" --param_artifact "${CONTAINER_ARTIFACT_NAME}" \ + || fail_closed native_restore_submission_failed + submission_line="$(tr -d '\r\n' < "${LAST_STDOUT}")" + IFS=$'\t' read -r submitted_id submitted_status <<< "${submission_line}" + [ "${submitted_id}" = "${RESTORE_OPERATION_ID}" ] \ + || fail_closed native_restore_submission_id_mismatch + case "${submitted_status}" in + RESTORING|RESTORED) ;; + *) fail_closed "native_restore_submission_status_${submitted_status:-empty}" ;; + esac + emit_receipt execution submitted \ + "restore_operation_${RESTORE_OPERATION_ID}_status_${submitted_status}" + + deadline=$(( $(date +%s) + RESTORE_TIMEOUT_SECONDS )) + while true; do + query_restore_status + emit_receipt restore_status observed \ + "status_${RESTORE_TERMINAL}_files_${RESTORE_NUM_FILES}_total_${RESTORE_TOTAL_SIZE}" + case "${RESTORE_TERMINAL}" in + RESTORED) return 0 ;; + RESTORING) ;; + RESTORE_FAILED|RESTORE_CANCELLED) fail_closed "native_restore_terminal_${RESTORE_TERMINAL}" ;; + *) fail_closed "native_restore_terminal_unexpected_${RESTORE_TERMINAL}" ;; + esac + [ "$(date +%s)" -lt "${deadline}" ] || fail_closed native_restore_poll_timeout + sleep "${POLL_INTERVAL_SECONDS}" + done +} + +verify_restored_inventory_and_tables() { + local restored_inventory compare_receipt check_list database table output step + + run_sql restored_inventory \ + "SELECT database,name,engine,toString(ifNull(total_rows,0)),toString(ifNull(total_bytes,0)),hex(SHA256(create_table_query)) FROM system.tables WHERE database IN (${DATABASE_IN_SQL}) ORDER BY database,name" \ + || fail_closed restored_table_inventory_query_failed + restored_inventory="${LAST_STDOUT}" + [ -s "${restored_inventory}" ] || fail_closed restored_table_inventory_empty + + run_captured inventory_compare python3 "${INVENTORY_HELPER}" compare \ + --source-inventory "${SOURCE_INVENTORY_PATH}" \ + --restored-inventory "${restored_inventory}" \ + || fail_closed restored_database_table_engine_manifest_mismatch + compare_receipt="${LAST_STDOUT}" + MANIFEST_PARITY="$(read_key manifest_parity "${compare_receipt}")" + RESTORED_DATABASE_COUNT="$(read_key restored_database_count "${compare_receipt}")" + RESTORED_TABLE_COUNT="$(read_key restored_table_count "${compare_receipt}")" + PHYSICAL_TABLE_COUNT="$(read_key physical_table_count "${compare_receipt}")" + CRITICAL_NONZERO_COUNT="$(read_key critical_nonzero_count "${compare_receipt}")" + SOURCE_TOTAL_ROWS="$(read_key source_total_rows "${compare_receipt}")" + RESTORED_TOTAL_ROWS="$(read_key restored_total_rows "${compare_receipt}")" + ROW_DELTA="$(read_key row_delta "${compare_receipt}")" + SOURCE_TOTAL_BYTES="$(read_key source_total_bytes "${compare_receipt}")" + RESTORED_TOTAL_BYTES="$(read_key restored_total_bytes "${compare_receipt}")" + BYTE_DELTA="$(read_key byte_delta "${compare_receipt}")" + [ "${MANIFEST_PARITY}" = "1" ] || fail_closed restored_manifest_parity_not_proven + + run_captured physical_table_list python3 "${INVENTORY_HELPER}" check-list \ + --inventory "${restored_inventory}" \ + || fail_closed restored_physical_table_list_failed + check_list="${LAST_STDOUT}" + CHECK_TABLE_COUNT="$(awk 'NF { count++ } END { print count + 0 }' "${check_list}")" + [ "${CHECK_TABLE_COUNT}" -eq "${PHYSICAL_TABLE_COUNT}" ] \ + || fail_closed restored_physical_table_count_mismatch + + while IFS=$'\t' read -r database table; do + [ -n "${database}" ] && [ -n "${table}" ] || continue + step="check_table_$(printf '%03d' "$((CHECK_TABLE_PASS_COUNT + 1))")" + run_sql "${step}" \ + "CHECK TABLE \`${database}\`.\`${table}\` SETTINGS check_query_single_value_result=1" \ + || fail_closed "check_table_command_failed_${database}_${table}" + output="$(tr -d '\r\n' < "${LAST_STDOUT}")" + [ "${output}" = "1" ] || fail_closed "check_table_not_passed_${database}_${table}" + CHECK_TABLE_PASS_COUNT=$((CHECK_TABLE_PASS_COUNT + 1)) + done < "${check_list}" + [ "${CHECK_TABLE_PASS_COUNT}" -eq "${CHECK_TABLE_COUNT}" ] \ + || fail_closed check_table_pass_count_mismatch + + emit_receipt independent_post_verifier pass \ + "restore_RESTORED_manifest_parity_${MANIFEST_PARITY}_tables_${RESTORED_TABLE_COUNT}_check_tables_${CHECK_TABLE_PASS_COUNT}_critical_nonzero_${CRITICAL_NONZERO_COUNT}_row_delta_${ROW_DELTA}_byte_delta_${BYTE_DELTA}" + APPLY_PASS=1 + FINAL_REASON="isolated_native_restore_verified" +} + +capture_container_log() { + local container="$1" + local label="$2" + local raw target + raw="$(mktemp "${RECEIPT_DIR}/.${label}.XXXXXX")" + target="${RECEIPT_DIR}/${INVOCATION_ID}-${label}.log" + set +e + timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker logs --tail "${LOG_TAIL_LINES}" "${container}" >"${raw}" 2>&1 + set -e + head -c "${MAX_LOG_BYTES}" "${raw}" > "${target}" + rm -f "${raw}" +} + +cleanup_container() { + local container="$1" + local label inspect_status remove_status final_status + set +e + label="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker container inspect --format '{{index .Config.Labels "com.awoooi.restore.identity"}}' \ + "${container}" 2>/dev/null)" + inspect_status=$? + set -e + if [ "${inspect_status}" -ne 0 ]; then + exact_container_absent "${container}" + return $? + fi + [ "${label}" = "${IDENTITY_HASH}" ] || return 1 + capture_container_log "${container}" "$(basename "${container}")" + set +e + timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker rm -f "${container}" >/dev/null 2>&1 + remove_status=$? + exact_container_absent "${container}" + final_status=$? + set -e + [ "${remove_status}" -eq 0 ] && [ "${final_status}" -eq 0 ] \ + && docker_daemon_ready +} + +cleanup_volume() { + local volume="$1" + local label inspect_status remove_status final_status + set +e + label="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker volume inspect --format '{{index .Labels "com.awoooi.restore.identity"}}' \ + "${volume}" 2>/dev/null)" + inspect_status=$? + set -e + if [ "${inspect_status}" -ne 0 ]; then + exact_volume_absent "${volume}" + return $? + fi + [ "${label}" = "${IDENTITY_HASH}" ] || return 1 + set +e + timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker volume rm "${volume}" >/dev/null 2>&1 + remove_status=$? + exact_volume_absent "${volume}" + final_status=$? + set -e + [ "${remove_status}" -eq 0 ] && [ "${final_status}" -eq 0 ] \ + && docker_daemon_ready +} + +cleanup_network() { + local label inspect_status remove_status final_status + set +e + label="$(timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker network inspect --format '{{index .Labels "com.awoooi.restore.identity"}}' \ + "${NETWORK_NAME}" 2>/dev/null)" + inspect_status=$? + set -e + if [ "${inspect_status}" -ne 0 ]; then + exact_network_absent "${NETWORK_NAME}" + return $? + fi + [ "${label}" = "${IDENTITY_HASH}" ] || return 1 + set +e + timeout --kill-after="${KILL_AFTER_SECONDS}" "${COMMAND_TIMEOUT_SECONDS}" \ + docker network rm "${NETWORK_NAME}" >/dev/null 2>&1 + remove_status=$? + exact_network_absent "${NETWORK_NAME}" + final_status=$? + set -e + [ "${remove_status}" -eq 0 ] && [ "${final_status}" -eq 0 ] \ + && docker_daemon_ready +} + +verify_all_resources_absent() { + docker_daemon_ready || return 1 + exact_container_absent "${STAGING_CONTAINER}" \ + && exact_container_absent "${CLICKHOUSE_CONTAINER}" \ + && exact_container_absent "${ZOOKEEPER_CONTAINER}" \ + && exact_volume_absent "${BACKUP_VOLUME}" \ + && exact_volume_absent "${CLICKHOUSE_VOLUME}" \ + && exact_volume_absent "${ZOOKEEPER_DATA_VOLUME}" \ + && exact_volume_absent "${ZOOKEEPER_LOG_VOLUME}" \ + && exact_network_absent "${NETWORK_NAME}" \ + && docker_daemon_ready +} + +write_status() { + local terminal="$1" + local exit_code="$2" + local cleanup_json runtime_json staging_json clickhouse_hash_json check_json apply_json status_tmp + cleanup_json=false + runtime_json=false + staging_json=false + clickhouse_hash_json=false + check_json=false + apply_json=false + [ "${CLEANUP_PASS}" -eq 1 ] && cleanup_json=true + [ "${EPHEMERAL_RUNTIME_CREATED}" -eq 1 ] && runtime_json=true + [ "${ARTIFACT_STAGING_VERIFIED}" -eq 1 ] && staging_json=true + [ "${CLICKHOUSE_ARTIFACT_READBACK_VERIFIED}" -eq 1 ] && clickhouse_hash_json=true + [ "${CHECK_PASS}" -eq 1 ] && check_json=true + [ "${APPLY_PASS}" -eq 1 ] && apply_json=true + status_tmp="${STATUS_FILE}.tmp.$$" + printf '%s\n' \ + '{' \ + ' "schema_version":"clickhouse_native_restore_drill_status_v1",' \ + " \"trace_id\":\"${TRACE_ID}\"," \ + " \"run_id\":\"${RUN_ID}\"," \ + " \"work_item_id\":\"${WORK_ITEM_ID}\"," \ + " \"identity_hash\":\"${IDENTITY_HASH}\"," \ + " \"mode\":\"${MODE}\"," \ + " \"terminal\":\"${terminal}\"," \ + " \"reason\":\"${FINAL_REASON}\"," \ + " \"exit_code\":${exit_code}," \ + " \"artifact_sha256\":\"${ARTIFACT_SHA256}\"," \ + " \"staged_artifact_sha256\":\"${STAGED_ARTIFACT_SHA256}\"," \ + " \"clickhouse_artifact_sha256\":\"${CLICKHOUSE_ARTIFACT_SHA256}\"," \ + " \"source_inventory_sha256\":\"${SOURCE_INVENTORY_SHA256}\"," \ + " \"clickhouse_image_id\":\"${CLICKHOUSE_IMAGE_ID}\"," \ + " \"zookeeper_image_id\":\"${ZOOKEEPER_IMAGE_ID}\"," \ + ' "network_mode":"docker_internal_only",' \ + ' "staging_network_mode":"none",' \ + ' "production_network_attached":false,' \ + ' "production_restore_performed":false,' \ + ' "allow_non_empty_tables":false,' \ + ' "isolated_keeper":true,' \ + ' "artifact_source_mount":"read_only_staging_only",' \ + ' "backup_volume_mount":"writable_ephemeral",' \ + " \"artifact_staging_verified\":${staging_json}," \ + " \"clickhouse_artifact_readback_verified\":${clickhouse_hash_json}," \ + " \"ephemeral_runtime_created\":${runtime_json}," \ + " \"check_pass\":${check_json}," \ + " \"apply_pass\":${apply_json}," \ + " \"restore_terminal\":\"${RESTORE_TERMINAL}\"," \ + " \"manifest_parity\":${MANIFEST_PARITY}," \ + " \"source_database_count\":${SOURCE_DATABASE_COUNT}," \ + " \"restored_database_count\":${RESTORED_DATABASE_COUNT}," \ + " \"source_table_count\":${SOURCE_TABLE_COUNT}," \ + " \"restored_table_count\":${RESTORED_TABLE_COUNT}," \ + " \"check_table_count\":${CHECK_TABLE_COUNT}," \ + " \"check_table_pass_count\":${CHECK_TABLE_PASS_COUNT}," \ + " \"critical_nonzero_count\":${CRITICAL_NONZERO_COUNT}," \ + " \"source_total_rows\":${SOURCE_TOTAL_ROWS}," \ + " \"restored_total_rows\":${RESTORED_TOTAL_ROWS}," \ + " \"row_delta\":${ROW_DELTA}," \ + " \"source_total_bytes\":${SOURCE_TOTAL_BYTES}," \ + " \"restored_total_bytes\":${RESTORED_TOTAL_BYTES}," \ + " \"byte_delta\":${BYTE_DELTA}," \ + " \"cleanup_verified\":${cleanup_json}," \ + ' "secret_values_collected":false' \ + '}' > "${status_tmp}" + mv "${status_tmp}" "${STATUS_FILE}" +} + +cleanup() { + local exit_code=$? + local terminal="failed" + trap - EXIT HUP INT TERM + set +e + + [ -z "${PREFLIGHT_CANDIDATE}" ] || rm -f "${PREFLIGHT_CANDIDATE}" + if [ "${APPLY_RESOURCE_SCOPE}" -eq 1 ]; then + cleanup_container "${CLICKHOUSE_CONTAINER}" || CLEANUP_PASS=0 + cleanup_container "${ZOOKEEPER_CONTAINER}" || CLEANUP_PASS=0 + cleanup_container "${STAGING_CONTAINER}" || CLEANUP_PASS=0 + cleanup_volume "${BACKUP_VOLUME}" || CLEANUP_PASS=0 + cleanup_volume "${CLICKHOUSE_VOLUME}" || CLEANUP_PASS=0 + cleanup_volume "${ZOOKEEPER_DATA_VOLUME}" || CLEANUP_PASS=0 + cleanup_volume "${ZOOKEEPER_LOG_VOLUME}" || CLEANUP_PASS=0 + cleanup_network || CLEANUP_PASS=0 + verify_all_resources_absent || CLEANUP_PASS=0 + fi + [ -z "${LOCK_DIR}" ] || rmdir "${LOCK_DIR}" 2>/dev/null || CLEANUP_PASS=0 + + if [ "${CLEANUP_PASS}" -ne 1 ]; then + FINAL_REASON="ephemeral_resource_cleanup_failed" + exit_code=1 + elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = check ] && [ "${CHECK_PASS}" -eq 1 ]; then + terminal="check_pass_no_runtime_write" + elif [ "${exit_code}" -eq 0 ] && [ "${MODE}" = apply ] && [ "${APPLY_PASS}" -eq 1 ]; then + terminal="verified" + fi + emit_receipt cleanup "$([ "${CLEANUP_PASS}" -eq 1 ] && printf pass || printf failed)" \ + "staging_keeper_clickhouse_containers_backup_data_volumes_and_internal_network_absence_verified_${CLEANUP_PASS}" + emit_receipt terminal "${terminal}" "${FINAL_REASON}" + write_status "${terminal}" "${exit_code}" + printf 'CLICKHOUSE_NATIVE_RESTORE_DRILL_TERMINAL trace_id=%s run_id=%s work_item_id=%s mode=%s terminal=%s cleanup=%s exit_code=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" "${terminal}" \ + "${CLEANUP_PASS}" "${exit_code}" + exit "${exit_code}" +} + +on_signal() { + local signal_number="$1" + FINAL_REASON="signal_${signal_number}" + emit_receipt signal failed "${FINAL_REASON}" + exit "$((128 + signal_number))" +} + +main() { + parse_args "$@" + validate_contract + setup_receipts + validate_preflight + + if [ "${MODE}" = check ]; then + persist_check_contract + log_info "isolated native RESTORE drill check passed without runtime mutation" + return 0 + fi + + require_check_contract + create_isolated_runtime + verify_isolation_and_empty_target + restore_native_backup + verify_restored_inventory_and_tables + log_info "isolated native RESTORE drill verified; ephemeral runtime will be removed" +} + +main "$@" diff --git a/scripts/backup/clickhouse-restore-inventory.py b/scripts/backup/clickhouse-restore-inventory.py new file mode 100755 index 000000000..42b970b1f --- /dev/null +++ b/scripts/backup/clickhouse-restore-inventory.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Validate SigNoz ClickHouse native-backup inventories without data access. + +The shell restore-drill driver uses this helper for two narrow jobs: + +* prove that a local artifact is a completed, hashed native BACKUP with the + exact six-database source inventory expected by P0-OBS-002; and +* compare the restored database/table/engine set with that source inventory. + +Row and byte values are recorded as bounded aggregate evidence, not exact +parity gates, because the source inventory is captured while online ingestion +can still advance before the native BACKUP snapshot is committed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +EXPECTED_DATABASES = ( + "signoz_analytics", + "signoz_logs", + "signoz_metadata", + "signoz_meter", + "signoz_metrics", + "signoz_traces", +) +CRITICAL_TABLES = ( + ("signoz_logs", "logs_v2"), + ("signoz_metrics", "time_series_v4"), + ("signoz_metrics", "samples_v4"), + ("signoz_traces", "signoz_index_v3"), +) +SAFE_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +SAFE_OPERATION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +SHA256 = re.compile(r"^[0-9a-f]{64}$") +SCHEMA_SHA256 = re.compile(r"^[0-9A-F]{64}$") + + +class ContractError(ValueError): + """A bounded, non-secret backup or inventory contract failure.""" + + +@dataclass(frozen=True) +class TableReceipt: + database: str + table: str + engine: str + rows: int + bytes: int + schema_hash: str + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def parse_inventory(path: Path) -> dict[tuple[str, str], TableReceipt]: + receipts: dict[tuple[str, str], TableReceipt] = {} + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeError) as exc: + raise ContractError("inventory_read_failed") from exc + + if not lines: + raise ContractError("inventory_empty") + + for line_number, line in enumerate(lines, start=1): + fields = line.split("\t") + if len(fields) != 6: + raise ContractError(f"inventory_field_count_invalid_line_{line_number}") + database, table, engine, rows_text, bytes_text, schema_hash = fields + if not SAFE_IDENTIFIER.fullmatch(database): + raise ContractError( + f"inventory_database_identifier_unsafe_line_{line_number}" + ) + if not SAFE_IDENTIFIER.fullmatch(table): + raise ContractError(f"inventory_table_identifier_unsafe_line_{line_number}") + if not SAFE_IDENTIFIER.fullmatch(engine): + raise ContractError( + f"inventory_engine_identifier_unsafe_line_{line_number}" + ) + if not rows_text.isdecimal() or not bytes_text.isdecimal(): + raise ContractError(f"inventory_counter_invalid_line_{line_number}") + if not SCHEMA_SHA256.fullmatch(schema_hash): + raise ContractError(f"inventory_schema_hash_invalid_line_{line_number}") + key = (database, table) + if key in receipts: + raise ContractError(f"inventory_duplicate_table_line_{line_number}") + receipts[key] = TableReceipt( + database=database, + table=table, + engine=engine, + rows=int(rows_text), + bytes=int(bytes_text), + schema_hash=schema_hash, + ) + return receipts + + +def require_database_set(receipts: dict[tuple[str, str], TableReceipt]) -> None: + observed = tuple(sorted({receipt.database for receipt in receipts.values()})) + if observed != EXPECTED_DATABASES: + raise ContractError("signoz_database_set_mismatch") + + +def require_critical_tables( + receipts: dict[tuple[str, str], TableReceipt], *, require_nonzero: bool +) -> None: + for key in CRITICAL_TABLES: + receipt = receipts.get(key) + if receipt is None: + raise ContractError(f"critical_table_missing_{key[0]}_{key[1]}") + if require_nonzero and receipt.rows <= 0: + raise ContractError(f"critical_table_empty_{key[0]}_{key[1]}") + + +def emit(values: list[tuple[str, object]]) -> None: + for key, value in values: + print(f"{key}={value}") + + +def preflight(args: argparse.Namespace) -> None: + manifest_path = Path(args.manifest) + artifact_path = Path(args.artifact) + inventory_path = Path(args.source_inventory) + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ContractError("backup_manifest_read_failed") from exc + if not isinstance(manifest, dict): + raise ContractError("backup_manifest_not_object") + if manifest.get("status") != "BACKUP_CREATED": + raise ContractError("backup_terminal_not_BACKUP_CREATED") + + artifact_sha256 = sha256_file(artifact_path) + inventory_sha256 = sha256_file(inventory_path) + artifact_size = artifact_path.stat().st_size + manifest_sha256 = manifest.get("sha256") + manifest_inventory_sha256 = manifest.get("source_inventory_sha256") + manifest_size = manifest.get("size_bytes") + if not isinstance(manifest_sha256, str) or not SHA256.fullmatch(manifest_sha256): + raise ContractError("backup_manifest_sha256_invalid") + if manifest_sha256 != artifact_sha256: + raise ContractError("backup_artifact_sha256_mismatch") + if manifest_inventory_sha256 != inventory_sha256: + raise ContractError("source_inventory_sha256_mismatch") + if not isinstance(manifest_size, int) or manifest_size <= 0: + raise ContractError("backup_manifest_size_invalid") + if manifest_size != artifact_size: + raise ContractError("backup_artifact_size_mismatch") + if tuple(manifest.get("databases", ())) != EXPECTED_DATABASES: + raise ContractError("backup_manifest_database_allowlist_mismatch") + + operation_id = manifest.get("operation_id") + if not isinstance(operation_id, str) or not SAFE_OPERATION_ID.fullmatch( + operation_id + ): + raise ContractError("backup_operation_id_invalid") + + receipts = parse_inventory(inventory_path) + require_database_set(receipts) + require_critical_tables(receipts, require_nonzero=True) + replicated_count = sum( + receipt.engine.startswith("Replicated") for receipt in receipts.values() + ) + distributed_count = sum( + receipt.engine == "Distributed" for receipt in receipts.values() + ) + physical_count = sum("MergeTree" in receipt.engine for receipt in receipts.values()) + if replicated_count <= 0: + raise ContractError("replicated_table_precondition_missing") + if distributed_count <= 0: + raise ContractError("distributed_table_precondition_missing") + if physical_count <= 0: + raise ContractError("physical_table_precondition_missing") + + emit( + [ + ("artifact_sha256", artifact_sha256), + ("artifact_size_bytes", artifact_size), + ("source_inventory_sha256", inventory_sha256), + ("source_database_count", len(EXPECTED_DATABASES)), + ("source_table_count", len(receipts)), + ("replicated_table_count", replicated_count), + ("distributed_table_count", distributed_count), + ("physical_table_count", physical_count), + ("backup_operation_id", operation_id), + ] + ) + + +def compare(args: argparse.Namespace) -> None: + source = parse_inventory(Path(args.source_inventory)) + restored = parse_inventory(Path(args.restored_inventory)) + require_database_set(source) + require_database_set(restored) + require_critical_tables(source, require_nonzero=True) + require_critical_tables(restored, require_nonzero=True) + + source_keys = set(source) + restored_keys = set(restored) + if source_keys != restored_keys: + raise ContractError("database_table_manifest_set_mismatch") + for key in sorted(source_keys): + if source[key].engine != restored[key].engine: + raise ContractError(f"table_engine_mismatch_{key[0]}_{key[1]}") + if source[key].schema_hash != restored[key].schema_hash: + raise ContractError(f"table_schema_mismatch_{key[0]}_{key[1]}") + + source_rows = sum(receipt.rows for receipt in source.values()) + restored_rows = sum(receipt.rows for receipt in restored.values()) + source_bytes = sum(receipt.bytes for receipt in source.values()) + restored_bytes = sum(receipt.bytes for receipt in restored.values()) + physical_count = sum("MergeTree" in receipt.engine for receipt in restored.values()) + emit( + [ + ("manifest_parity", 1), + ("restored_database_count", len(EXPECTED_DATABASES)), + ("restored_table_count", len(restored)), + ("physical_table_count", physical_count), + ("critical_nonzero_count", len(CRITICAL_TABLES)), + ("source_total_rows", source_rows), + ("restored_total_rows", restored_rows), + ("row_delta", restored_rows - source_rows), + ("source_total_bytes", source_bytes), + ("restored_total_bytes", restored_bytes), + ("byte_delta", restored_bytes - source_bytes), + ] + ) + + +def check_list(args: argparse.Namespace) -> None: + receipts = parse_inventory(Path(args.inventory)) + require_database_set(receipts) + for key in sorted(receipts): + receipt = receipts[key] + if "MergeTree" in receipt.engine: + print(f"{receipt.database}\t{receipt.table}") + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + commands = root.add_subparsers(dest="command", required=True) + + preflight_parser = commands.add_parser("preflight") + preflight_parser.add_argument("--manifest", required=True) + preflight_parser.add_argument("--artifact", required=True) + preflight_parser.add_argument("--source-inventory", required=True) + preflight_parser.set_defaults(handler=preflight) + + compare_parser = commands.add_parser("compare") + compare_parser.add_argument("--source-inventory", required=True) + compare_parser.add_argument("--restored-inventory", required=True) + compare_parser.set_defaults(handler=compare) + + check_parser = commands.add_parser("check-list") + check_parser.add_argument("--inventory", required=True) + check_parser.set_defaults(handler=check_list) + return root + + +def main() -> int: + args = parser().parse_args() + try: + args.handler(args) + except (ContractError, OSError) as exc: + print(f"ERROR={exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/backup/run-signoz-backup-canary.sh b/scripts/backup/run-signoz-backup-canary.sh new file mode 100755 index 000000000..c6bdddfe4 --- /dev/null +++ b/scripts/backup/run-signoz-backup-canary.sh @@ -0,0 +1,665 @@ +#!/usr/bin/env bash +# ============================================================================= +# P0-OBS-002 - SigNoz online backup canary and legacy maintenance lease verifier +# +# The default native-backup path never stops the collector and therefore does +# not depend on, read, or mutate the legacy monitor/cooldown contract. The +# bounded lease path remains available only when an explicit legacy stop-mode +# canary is requested. Neither mode reads or sources a secret/.env file. +# ============================================================================= + +set -euo pipefail + +COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}" +BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}" +MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}" +MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}" +MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}" +COOLDOWN_FILE="${MONITOR_COOLDOWN_DIR}/${COLLECTOR_NAME}.cooldown" +RECEIPT_ROOT="${SIGNOZ_CANARY_RECEIPT_ROOT:-/backup/logs/signoz-canary}" +LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}" + +CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}" +TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}" +LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}" +MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}" +STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}" +EXPECT_COLLECTOR_STOP="${SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP:-0}" + +TRACE_ID="${TRACE_ID:-}" +RUN_ID="${RUN_ID:-}" +WORK_ITEM_ID="${WORK_ITEM_ID:-}" + +RECEIPT_DIR="" +RECEIPT_LOG="" +BACKUP_OUTPUT="" +MONITOR_WINDOW="" +STATE_OBSERVATIONS="" +ROLLBACK_FILE="" +ROLLBACK_METADATA="" + +RECEIPTS_READY=0 +LEASE_ROLLBACK_READY=0 +LEASE_MUTATION_STARTED=0 +LEASE_ARMED=0 +PRIOR_LEASE_PRESENT=0 +PRIOR_LEASE_MODE="" +PRIOR_LEASE_UID="" +PRIOR_LEASE_GID="" +PRIOR_LEASE_HASH="" +PRIOR_LEASE_VALUE="" +LEASE_TMP="" +BACKUP_PID="" +OBSERVER_PID="" +CANARY_VERIFIED=0 +CHECK_VERIFIED=0 +MODE="" + +log_info() { printf '[INFO] %s\n' "$*"; } +log_error() { printf '[ERROR] %s\n' "$*" >&2; } + +valid_identity() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] +} + +valid_positive_integer() { + [[ "$1" =~ ^[0-9]+$ ]] && [ "$1" -gt 0 ] +} + +valid_poll_interval() { + [[ "$1" =~ ^[0-9]+([.][0-9]+)?$ ]] && [[ "$1" != "0" ]] && [[ "$1" != "0.0" ]] +} + +emit_receipt() { + local stage="$1" + local terminal="$2" + local detail="$3" + local observed_at + + [ "${RECEIPTS_READY}" -eq 1 ] || return 0 + observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","observed_at":"%s","stage":"%s","terminal":"%s","detail":"%s"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${observed_at}" \ + "${stage}" "${terminal}" "${detail}" >> "${RECEIPT_LOG}" +} + +fail_closed() { + local detail="$1" + log_error "${detail}" + emit_receipt "failure" "failed" "${detail}" + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}" +} + +require_monitor_fragment() { + local fragment="$1" + local count + count="$(grep -F -c -- "${fragment}" "${MONITOR_SCRIPT}" || true)" + [ "${count}" -eq 1 ] || fail_closed "monitor_contract_ambiguous_fragment_count_${count}" +} + +require_monitor_function_fragment() { + local function_name="$1" + local fragment="$2" + local count + + count="$(awk -v header="${function_name}() {" -v needle="${fragment}" ' + $0 == header { + inside = 1 + depth = 0 + } + inside { + original = $0 + if (index(original, needle)) { + count++ + } + open_line = original + close_line = original + depth += gsub(/\{/, "", open_line) + depth -= gsub(/\}/, "", close_line) + if (depth == 0) { + print count + 0 + printed = 1 + exit + } + } + END { + if (!printed) { + print count + 0 + } + } + ' "${MONITOR_SCRIPT}")" + [ "${count}" -eq 1 ] \ + || fail_closed "monitor_${function_name}_contract_ambiguous_fragment_count_${count}" +} + +collector_running_state() { + local running + if ! running="$(docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then + return 1 + fi + case "${running}" in + true) printf '%s\n' "running" ;; + false) printf '%s\n' "stopped" ;; + *) return 1 ;; + esac +} + +collector_runtime_metadata() { + docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \ + "${COLLECTOR_NAME}" 2>/dev/null +} + +listener_is_up() { + local port="$1" + ss -H -lnt | awk -v port="${port}" ' + $4 ~ (":" port "$") { found = 1 } + END { exit(found ? 0 : 1) } + ' +} + +verify_collector_runtime() { + local expected_id="$1" + local expected_started_at="$2" + local expected_restart_count="$3" + local expected_health="$4" + local metadata actual_id actual_running actual_started_at actual_restart_count actual_health + + metadata="$(collector_runtime_metadata)" || return 1 + IFS=$'\t' read -r actual_id actual_running actual_started_at actual_restart_count actual_health <<< "${metadata}" + [ "${actual_id}" = "${expected_id}" ] || return 1 + [ "${actual_running}" = "true" ] || return 1 + [ "${actual_started_at}" = "${expected_started_at}" ] || return 1 + [ "${actual_restart_count}" = "${expected_restart_count}" ] || return 1 + [ "${actual_health}" = "${expected_health}" ] || return 1 + listener_is_up 4317 || return 1 + listener_is_up 4318 || return 1 +} + +restore_lease() { + local restore_tmp restore_hash current_value + + [ "${LEASE_MUTATION_STARTED}" -eq 1 ] || return 0 + [ "${LEASE_ROLLBACK_READY}" -eq 1 ] || return 1 + + if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then + [ -f "${ROLLBACK_FILE}" ] || return 1 + restore_tmp="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.restore.${RUN_ID}.XXXXXX")" || return 1 + cp -p "${ROLLBACK_FILE}" "${restore_tmp}" || return 1 + chmod "${PRIOR_LEASE_MODE}" "${restore_tmp}" || return 1 + mv -f "${restore_tmp}" "${COOLDOWN_FILE}" || return 1 + restore_tmp="" + + restore_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" || return 1 + [ "${restore_hash}" = "${PRIOR_LEASE_HASH}" ] || return 1 + [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] || return 1 + [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] || return 1 + [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] || return 1 + current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")" + [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] || return 1 + else + rm -f "${COOLDOWN_FILE}" || return 1 + [ ! -e "${COOLDOWN_FILE}" ] || return 1 + fi + + LEASE_ARMED=0 + LEASE_MUTATION_STARTED=0 + return 0 +} + +terminate_children() { + if [ -n "${BACKUP_PID}" ] && kill -0 "${BACKUP_PID}" 2>/dev/null; then + kill -TERM "${BACKUP_PID}" 2>/dev/null || true + wait "${BACKUP_PID}" 2>/dev/null || true + fi + if [ -n "${OBSERVER_PID}" ] && kill -0 "${OBSERVER_PID}" 2>/dev/null; then + kill -TERM "${OBSERVER_PID}" 2>/dev/null || true + wait "${OBSERVER_PID}" 2>/dev/null || true + fi +} + +cleanup() { + local exit_code=$? + local lease_terminal="not_armed" + local overall_terminal="failed" + + trap - EXIT HUP INT TERM + set +e + terminate_children + rm -f "${LEASE_TMP}" 2>/dev/null || true + + if [ "${LEASE_MUTATION_STARTED}" -eq 1 ]; then + if restore_lease; then + lease_terminal="restored" + else + lease_terminal="restore_failed" + exit_code=91 + fi + elif [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then + lease_terminal="no_write" + elif [ "${MODE}" = "apply" ] && [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then + lease_terminal="no_write" + fi + + if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then + overall_terminal="check_pass_no_write" + elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then + overall_terminal="pass" + fi + + emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}" + if [ "${overall_terminal}" = "pass" ]; then + emit_receipt "closure_writeback" "pass" \ + "no_incident_notification_or_learning_delta_local_durable_receipt_ack" + elif [ "${overall_terminal}" = "check_pass_no_write" ]; then + emit_receipt "closure_writeback" "not_applicable" \ + "check_mode_no_runtime_change_local_durable_receipt_ack" + else + emit_receipt "closure_writeback" "deferred" \ + "failed_terminal_requires_followup_no_completion_claim" + fi + emit_receipt "terminal" "${overall_terminal}" "backup_canary_${overall_terminal}" + printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \ + "${lease_terminal}" "${exit_code}" + exit "${exit_code}" +} + +on_signal() { + local signal_number="$1" + emit_receipt "signal" "failed" "signal_${signal_number}" + exit "$((128 + signal_number))" +} + +setup_receipts() { + valid_identity "${TRACE_ID}" || { log_error "TRACE_ID is required and must be path-safe"; exit 2; } + valid_identity "${RUN_ID}" || { log_error "RUN_ID is required and must be path-safe"; exit 2; } + valid_identity "${WORK_ITEM_ID}" || { log_error "WORK_ITEM_ID is required and must be path-safe"; exit 2; } + + [ ! -L "${RECEIPT_ROOT}" ] || { log_error "receipt root must not be a symlink"; exit 2; } + mkdir -p "${RECEIPT_ROOT}" + RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" + [ ! -e "${RECEIPT_DIR}" ] || { log_error "receipt directory already exists: ${RECEIPT_DIR}"; exit 2; } + mkdir -m 700 "${RECEIPT_DIR}" + + RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl" + BACKUP_OUTPUT="${RECEIPT_DIR}/backup-output.log" + MONITOR_WINDOW="${RECEIPT_DIR}/monitor-window.log" + STATE_OBSERVATIONS="${RECEIPT_DIR}/collector-state.tsv" + ROLLBACK_FILE="${RECEIPT_DIR}/cooldown.rollback" + ROLLBACK_METADATA="${RECEIPT_DIR}/cooldown.rollback.json" + : > "${RECEIPT_LOG}" + RECEIPTS_READY=1 + + trap cleanup EXIT + trap 'on_signal 1' HUP + trap 'on_signal 2' INT + trap 'on_signal 15' TERM +} + +verify_monitor_contract() { + local cron_output cron_count cron_line exclude_count exclude_line + + [ -f "${MONITOR_SCRIPT}" ] && [ -r "${MONITOR_SCRIPT}" ] && [ ! -L "${MONITOR_SCRIPT}" ] \ + || fail_closed "monitor_script_missing_unreadable_or_symlink" + [ -f "${MONITOR_LOG}" ] && [ -r "${MONITOR_LOG}" ] && [ ! -L "${MONITOR_LOG}" ] \ + || fail_closed "monitor_log_missing_unreadable_or_symlink" + [ -d "${MONITOR_COOLDOWN_DIR}" ] && [ -w "${MONITOR_COOLDOWN_DIR}" ] && [ ! -L "${MONITOR_COOLDOWN_DIR}" ] \ + || fail_closed "monitor_cooldown_directory_missing_unwritable_or_symlink" + + require_monitor_fragment "COOLDOWN_DIR:=${MONITOR_COOLDOWN_DIR}" + require_monitor_fragment 'local cooldown_file="${COOLDOWN_DIR}/${container}.cooldown"' + require_monitor_fragment 'last_sent=$(cat "$cooldown_file")' + # The live monitor has another cooldown path with the same elapsed + # calculation. Bind this assertion to is_in_cooldown instead of requiring + # a globally unique string. + require_monitor_function_fragment is_in_cooldown 'elapsed=$(( now - last_sent ))' + require_monitor_fragment 'if (( elapsed < ACTION_COOLDOWN_SECONDS )); then' + require_monitor_fragment 'COOLDOWN: ${container}' + require_monitor_fragment 'is_in_cooldown "$container_name" && continue' + require_monitor_fragment 'set_cooldown "$container_name"' + require_monitor_fragment 'log "AUTO_REPAIR: docker restart ${container}"' + require_monitor_fragment 'if docker restart "$container"' + + exclude_count="$(grep -F -c 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}" || true)" + [ "${exclude_count}" -eq 1 ] || fail_closed "monitor_exclude_contract_ambiguous" + exclude_line="$(grep -F 'EXCLUDE_CONTAINERS:=' "${MONITOR_SCRIPT}")" + [[ "${exclude_line}" != *"${COLLECTOR_NAME}"* ]] || fail_closed "collector_already_excluded_monitor_contract_changed" + [ "$(grep -F -c "${COLLECTOR_NAME}" "${MONITOR_SCRIPT}" || true)" -eq 0 ] \ + || fail_closed "collector_literal_present_monitor_contract_changed" + + cron_output="$(crontab -l 2>/dev/null)" || fail_closed "monitor_crontab_unreadable" + cron_count="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" ' + $0 !~ /^[[:space:]]*#/ && index($0, script) { count++ } + END { print count + 0 } + ')" + [ "${cron_count}" -eq 1 ] || fail_closed "monitor_cron_contract_ambiguous_count_${cron_count}" + cron_line="$(printf '%s\n' "${cron_output}" | awk -v script="${MONITOR_SCRIPT}" ' + $0 !~ /^[[:space:]]*#/ && index($0, script) { print } + ')" + [[ "${cron_line}" == "*/5 * * * * "* ]] || fail_closed "monitor_cron_schedule_not_every_five_minutes" + [[ "${cron_line}" == *">> ${MONITOR_LOG}"* ]] || fail_closed "monitor_cron_log_route_mismatch" + [[ "${cron_line}" != *"COOLDOWN_DIR="* ]] || fail_closed "monitor_cron_cooldown_override_ambiguous" + +} + +inspect_prior_lease() { + local current_uid current_gid + current_uid="$(id -u)" + current_gid="$(id -g)" + + if [ -e "${COOLDOWN_FILE}" ]; then + [ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \ + || fail_closed "existing_cooldown_receipt_unsafe" + PRIOR_LEASE_PRESENT=1 + PRIOR_LEASE_MODE="$(stat -c '%a' "${COOLDOWN_FILE}")" + PRIOR_LEASE_UID="$(stat -c '%u' "${COOLDOWN_FILE}")" + PRIOR_LEASE_GID="$(stat -c '%g' "${COOLDOWN_FILE}")" + [ "${PRIOR_LEASE_UID}" = "${current_uid}" ] && [ "${PRIOR_LEASE_GID}" = "${current_gid}" ] \ + || fail_closed "existing_cooldown_receipt_owner_mismatch" + PRIOR_LEASE_VALUE="$(tr -d '\r\n' < "${COOLDOWN_FILE}")" + [[ "${PRIOR_LEASE_VALUE}" =~ ^[0-9]+$ ]] || fail_closed "existing_cooldown_receipt_not_epoch" + PRIOR_LEASE_HASH="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" + else + PRIOR_LEASE_PRESENT=0 + PRIOR_LEASE_MODE="664" + PRIOR_LEASE_UID="${current_uid}" + PRIOR_LEASE_GID="${current_gid}" + PRIOR_LEASE_HASH="absent" + PRIOR_LEASE_VALUE="absent" + fi + +} + +capture_prior_lease() { + local current_hash current_value + + if [ "${PRIOR_LEASE_PRESENT}" -eq 1 ]; then + [ -f "${COOLDOWN_FILE}" ] && [ ! -L "${COOLDOWN_FILE}" ] && [ -r "${COOLDOWN_FILE}" ] && [ -w "${COOLDOWN_FILE}" ] \ + || fail_closed "cooldown_changed_between_check_and_rollback_capture" + current_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" + current_value="$(tr -d '\r\n' < "${COOLDOWN_FILE}")" + [ "${current_hash}" = "${PRIOR_LEASE_HASH}" ] \ + && [ "$(stat -c '%a' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_MODE}" ] \ + && [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_UID}" ] \ + && [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "${PRIOR_LEASE_GID}" ] \ + && [ "${current_value}" = "${PRIOR_LEASE_VALUE}" ] \ + || fail_closed "cooldown_changed_between_check_and_rollback_capture" + cp -p "${COOLDOWN_FILE}" "${ROLLBACK_FILE}" + [ "$(sha256sum "${ROLLBACK_FILE}" | awk '{print $1}')" = "${PRIOR_LEASE_HASH}" ] \ + || fail_closed "cooldown_changed_between_check_and_rollback_capture" + else + [ ! -e "${COOLDOWN_FILE}" ] \ + || fail_closed "cooldown_appeared_between_check_and_rollback_capture" + fi + + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","prior_present":%s,"prior_mode":"%s","prior_uid":"%s","prior_gid":"%s","prior_hash":"%s","prior_value":"%s"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${PRIOR_LEASE_PRESENT}" \ + "${PRIOR_LEASE_MODE}" "${PRIOR_LEASE_UID}" "${PRIOR_LEASE_GID}" \ + "${PRIOR_LEASE_HASH}" "${PRIOR_LEASE_VALUE}" > "${ROLLBACK_METADATA}" + LEASE_ROLLBACK_READY=1 + emit_receipt "rollback_capture" "pass" "prior_cooldown_receipt_captured" +} + +arm_lease() { + local now lease_epoch lease_hash + now="$(date +%s)" + lease_epoch=$((now + CANARY_TIMEOUT_SECONDS + LEASE_MARGIN_SECONDS)) + LEASE_TMP="$(mktemp "${MONITOR_COOLDOWN_DIR}/.${COLLECTOR_NAME}.lease.${RUN_ID}.XXXXXX")" + printf '%s\n' "${lease_epoch}" > "${LEASE_TMP}" + chmod "${PRIOR_LEASE_MODE}" "${LEASE_TMP}" + + LEASE_MUTATION_STARTED=1 + mv -f "${LEASE_TMP}" "${COOLDOWN_FILE}" + LEASE_TMP="" + LEASE_ARMED=1 + + [ "$(tr -d '\r\n' < "${COOLDOWN_FILE}")" = "${lease_epoch}" ] \ + || fail_closed "cooldown_lease_epoch_readback_mismatch" + [ "$(stat -c '%u' "${COOLDOWN_FILE}")" = "$(id -u)" ] \ + || fail_closed "cooldown_lease_owner_readback_mismatch" + [ "$(stat -c '%g' "${COOLDOWN_FILE}")" = "$(id -g)" ] \ + || fail_closed "cooldown_lease_group_readback_mismatch" + lease_hash="$(sha256sum "${COOLDOWN_FILE}" | awk '{print $1}')" + emit_receipt "lease" "armed" "future_epoch_${lease_epoch}_hash_${lease_hash}" +} + +observe_collector() { + local epoch state + while kill -0 "${BACKUP_PID}" 2>/dev/null; do + epoch="$(date +%s)" + if state="$(collector_running_state)"; then + printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}" + else + printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}" + fi + sleep "${STATE_POLL_SECONDS}" + done + + epoch="$(date +%s)" + if state="$(collector_running_state)"; then + printf '%s\t%s\n' "${epoch}" "${state}" >> "${STATE_OBSERVATIONS}" + else + printf '%s\tunknown\n' "${epoch}" >> "${STATE_OBSERVATIONS}" + fi +} + +verify_monitor_window() { + local log_identity_before="$1" + local log_size_before="$2" + local log_identity_after log_size_after start_byte + local auto_repair_count detected_count cooldown_count + local stop_epoch restore_epoch next_cron_epoch cron_crossed=0 + + log_identity_after="$(stat -c '%d:%i' "${MONITOR_LOG}")" + log_size_after="$(stat -c '%s' "${MONITOR_LOG}")" + [ "${log_identity_after}" = "${log_identity_before}" ] \ + || fail_closed "monitor_log_rotated_during_canary" + [ "${log_size_after}" -ge "${log_size_before}" ] \ + || fail_closed "monitor_log_shrank_during_canary" + + start_byte=$((log_size_before + 1)) + tail -c "+${start_byte}" "${MONITOR_LOG}" > "${MONITOR_WINDOW}" + auto_repair_count="$(grep -a -F -c "AUTO_REPAIR: docker restart ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)" + detected_count="$(grep -a -F -c "DETECTED: ${COLLECTOR_NAME} state=exited" "${MONITOR_WINDOW}" || true)" + cooldown_count="$(grep -a -F -c "COOLDOWN: ${COLLECTOR_NAME}" "${MONITOR_WINDOW}" || true)" + + [ "${auto_repair_count}" -eq 0 ] || fail_closed "monitor_auto_repair_contaminated_canary" + [ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \ + || fail_closed "collector_state_observer_ambiguous" + + if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then + [ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \ + || fail_closed "online_native_backup_collector_continuity_failed" + [ "${detected_count}" -eq 0 ] \ + || fail_closed "online_native_backup_monitor_detected_collector" + emit_receipt "monitor_verifier" "pass" \ + "online_native_collector_continuous_auto_repair_0_detected_0" + return 0 + fi + + stop_epoch="$(awk '$2 == "stopped" { print $1; exit }' "${STATE_OBSERVATIONS}")" + [ -n "${stop_epoch}" ] || fail_closed "collector_stop_transition_not_observed" + restore_epoch="$(awk ' + $2 == "stopped" { stopped_seen = 1; next } + stopped_seen && $2 == "running" { print $1; exit } + ' "${STATE_OBSERVATIONS}")" + [ -n "${restore_epoch}" ] || fail_closed "collector_restore_transition_not_observed" + + next_cron_epoch=$(( ((stop_epoch / MONITOR_CRON_INTERVAL_SECONDS) + 1) * MONITOR_CRON_INTERVAL_SECONDS )) + if [ "${next_cron_epoch}" -le "${restore_epoch}" ]; then + cron_crossed=1 + fi + + if [ "${detected_count}" -gt 0 ]; then + [ "${cooldown_count}" -ge "${detected_count}" ] \ + || fail_closed "monitor_detected_without_matching_cooldown" + fi + if [ "${cron_crossed}" -eq 1 ]; then + [ "${detected_count}" -gt 0 ] && [ "${cooldown_count}" -gt 0 ] \ + || fail_closed "cron_crossed_without_collector_cooldown_receipt" + fi + + emit_receipt "monitor_verifier" "pass" \ + "auto_repair_${auto_repair_count}_detected_${detected_count}_cooldown_${cooldown_count}_cron_crossed_${cron_crossed}" +} + +verify_online_collector_window() { + [ "$(awk '$2 == "unknown" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \ + || fail_closed "collector_state_observer_ambiguous" + [ "$(awk '$2 != "running" { count++ } END { print count + 0 }' "${STATE_OBSERVATIONS}")" -eq 0 ] \ + || fail_closed "online_native_backup_collector_continuity_failed" + emit_receipt "monitor_verifier" "pass" \ + "online_native_collector_continuous_monitor_contract_not_applicable" +} + +main() { + local container_id_before container_started_at_before container_restart_count_before + local container_health_before collector_metadata_before + local monitor_log_identity="" monitor_log_size="" + local source_diff_detail ai_decision_detail + local backup_status + + setup_receipts + + [ "$#" -eq 1 ] || fail_closed "exactly_one_mode_required_use_check_or_apply" + case "$1" in + --check) MODE="check" ;; + --apply) MODE="apply" ;; + *) fail_closed "unsupported_mode_use_check_or_apply" ;; + esac + + for command_name in awk date docker env flock grep kill pgrep sha256sum sleep ss timeout tr; do + require_command "${command_name}" + done + valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout" + valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after" + valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval" + case "${EXPECT_COLLECTOR_STOP}" in + 0|1) ;; + *) fail_closed "invalid_expect_collector_stop_mode" ;; + esac + + [ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \ + || fail_closed "backup_script_missing_unexecutable_or_symlink" + if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then + for command_name in cp crontab id mktemp mv rm stat tail; do + require_command "${command_name}" + done + valid_positive_integer "${LEASE_MARGIN_SECONDS}" || fail_closed "invalid_lease_margin" + valid_positive_integer "${MONITOR_CRON_INTERVAL_SECONDS}" || fail_closed "invalid_monitor_interval" + [ "${MONITOR_CRON_INTERVAL_SECONDS}" -le 300 ] || fail_closed "monitor_interval_exceeds_cron_contract" + verify_monitor_contract + fi + + [ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe" + if [ -e "${LOCK_FILE}" ]; then + [ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file" + fi + exec 9>>"${LOCK_FILE}" + flock -n 9 || fail_closed "another_signoz_canary_holds_lock" + if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then + fail_closed "another_signoz_backup_is_running" + fi + + collector_metadata_before="$(collector_runtime_metadata)" \ + || fail_closed "collector_runtime_metadata_unreadable" + IFS=$'\t' read -r container_id_before _container_running_before \ + container_started_at_before container_restart_count_before container_health_before \ + <<< "${collector_metadata_before}" + [ -n "${container_id_before}" ] \ + && [ -n "${container_started_at_before}" ] \ + && [[ "${container_restart_count_before}" =~ ^[0-9]+$ ]] \ + && [[ "${container_health_before}" =~ ^(healthy|none)$ ]] \ + || fail_closed "collector_runtime_metadata_invalid" + verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \ + "${container_restart_count_before}" "${container_health_before}" \ + || fail_closed "collector_preflight_runtime_failed" + + if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then + monitor_log_identity="$(stat -c '%d:%i' "${MONITOR_LOG}")" + monitor_log_size="$(stat -c '%s' "${MONITOR_LOG}")" + inspect_prior_lease + emit_receipt "sensor_source" "pass" \ + "monitor_hash_$(sha256sum "${MONITOR_SCRIPT}" | awk '{print $1}')_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}" + source_diff_detail="legacy_monitor_and_cron_contract_exact_cooldown_prior_present_${PRIOR_LEASE_PRESENT}_hash_${PRIOR_LEASE_HASH}" + ai_decision_detail="arm_exact_container_future_epoch_run_legacy_stop_backup_then_restore" + else + PRIOR_LEASE_PRESENT=0 + PRIOR_LEASE_HASH="not_applicable" + emit_receipt "sensor_source" "pass" \ + "online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}" + source_diff_detail="online_native_backup_monitor_and_cooldown_contract_not_applicable" + ai_decision_detail="run_online_native_backup_without_collector_or_monitor_mutation" + fi + emit_receipt "normalized_asset_identity" "pass" \ + "asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker" + emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}" + emit_receipt "ai_decision" "candidate" "${ai_decision_detail}" + emit_receipt "risk_policy" "pass" \ + "risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_independent_verifier" + emit_receipt "check" "pass" \ + "mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent" + + if [ "${MODE}" = "check" ]; then + CHECK_VERIFIED=1 + log_info "SigNoz backup canary check passed without arming lease or launching backup" + return 0 + fi + + if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then + capture_prior_lease + arm_lease + else + emit_receipt "lease" "not_required" \ + "online_native_backup_has_no_collector_stop_or_cooldown_write" + fi + + : > "${STATE_OBSERVATIONS}" + set +e + timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \ + env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ + BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \ + > "${BACKUP_OUTPUT}" 2>&1 & + BACKUP_PID=$! + observe_collector & + OBSERVER_PID=$! + wait "${BACKUP_PID}" + backup_status=$? + BACKUP_PID="" + wait "${OBSERVER_PID}" + OBSERVER_PID="" + set -e + + emit_receipt "execution" "$([ "${backup_status}" -eq 0 ] && printf pass || printf failed)" \ + "backup_exit_${backup_status}" + [ "${backup_status}" -eq 0 ] || fail_closed "backup_terminal_nonzero_${backup_status}" + grep -a -F -q '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)' "${BACKUP_OUTPUT}" \ + || fail_closed "retention_skip_receipt_missing" + if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then + grep -a -F -q '========== SignOz 備份完成 (' "${BACKUP_OUTPUT}" \ + || fail_closed "backup_success_terminal_missing" + else + grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \ + || fail_closed "native_backup_success_terminal_missing" + fi + + if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then + verify_monitor_window "${monitor_log_identity}" "${monitor_log_size}" + else + verify_online_collector_window + fi + verify_collector_runtime "${container_id_before}" "${container_started_at_before}" \ + "${container_restart_count_before}" "${container_health_before}" \ + || fail_closed "collector_post_backup_runtime_failed" + emit_receipt "post_verifier" "pass" \ + "collector_running_same_identity_started_at_restart_count_health_listeners_4317_4318" + + CANARY_VERIFIED=1 + log_info "SigNoz backup canary verified; cooldown lease will now roll back" +} + +main "$@" diff --git a/scripts/backup/tests/test_backup_signoz_collector_restore.py b/scripts/backup/tests/test_backup_signoz_collector_restore.py index 01d9635e8..71113b96c 100644 --- a/scripts/backup/tests/test_backup_signoz_collector_restore.py +++ b/scripts/backup/tests/test_backup_signoz_collector_restore.py @@ -5,6 +5,7 @@ import shutil import subprocess from pathlib import Path + ROOT = Path(__file__).resolve().parents[3] SCRIPT = ROOT / "scripts" / "backup" / "backup-signoz.sh" BACKUP_ALL = ROOT / "scripts" / "backup" / "backup-all.sh" @@ -20,27 +21,24 @@ def _run_backup( tmp_path: Path, *, initial_running: bool = True, - docker_run_mode: str = "success", - docker_run_target: str = "signoz-clickhouse", - archive_timeout_target: str = "", - tar_verify_mode: str = "success", - tar_verify_target: str = "clickhouse_", - tar_timeout_target: str = "", - docker_start_failures: int = 0, + native_check_status: int = 0, + native_apply_status: int = 0, + continuity_mode: str = "stable", restic_backup_status: int = 0, - restore_max_attempts: int = 3, + skip_retention: bool = False, ) -> tuple[subprocess.CompletedProcess[str], list[str], Path]: harness = tmp_path / "backup" fake_bin = tmp_path / "bin" backup_base = tmp_path / "repository" event_log = tmp_path / "events.log" + collector_id = tmp_path / "collector.id" collector_state = tmp_path / "collector.state" - start_count = tmp_path / "start.count" harness.mkdir() fake_bin.mkdir() (backup_base / "signoz" / "data").mkdir(parents=True) shutil.copy2(SCRIPT, harness / SCRIPT.name) event_log.touch() + collector_id.write_text("sha256:collector-stable\n", encoding="utf-8") collector_state.write_text( "true\n" if initial_running else "false\n", encoding="utf-8", @@ -50,37 +48,62 @@ def _run_backup( """\ BACKUP_BASE="${TEST_BACKUP_BASE:?}" RESTIC_PASSWORD_FILE="${TEST_BACKUP_BASE}/password" -log_info() { :; } -log_warn() { :; } -log_error() { :; } -log_success() { :; } +log_info() { printf 'info %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } +log_warn() { printf 'warn %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } +log_error() { printf 'error %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } +log_success() { printf 'success %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } -build_tags() { :; } -cleanup_old_backups() { :; } +build_tags() { printf '%s' '--tag signoz'; } +cleanup_old_backups() { printf 'retention %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } """, encoding="utf-8", ) - + _write_executable( + harness / "clickhouse-native-backup.sh", + """\ +#!/bin/bash +set -eu +printf 'native %s trace=%s run=%s work=%s\\n' \ + "${1:-}" "${TRACE_ID:-}" "${RUN_ID:-}" "${WORK_ITEM_ID:-}" \ + >> "${TEST_EVENT_LOG:?}" +case "${1:-}" in + --check) + exit "${FAKE_NATIVE_CHECK_STATUS:-0}" + ;; + --apply) + status="${FAKE_NATIVE_APPLY_STATUS:-0}" + [ "$status" -eq 0 ] || exit "$status" + case "${FAKE_CONTINUITY_MODE:-stable}" in + stopped) + printf 'false\\n' > "${TEST_COLLECTOR_STATE:?}" + sleep 0.08 + printf 'true\\n' > "${TEST_COLLECTOR_STATE:?}" + ;; + identity) + printf 'sha256:collector-replaced\\n' > "${TEST_COLLECTOR_ID:?}" + sleep 0.08 + ;; + *) sleep 0.08 ;; + esac + printf 'native-zip\\n' \ + > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip" + printf '{}\\n' \ + > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.zip.manifest.json" + printf 'db\\ttable\\tengine\\trows\\tbytes\\t%s\\n' \ + "$(printf 'A%.0s' {1..64})" \ + > "${CLICKHOUSE_NATIVE_OUTPUT_DIR:?}/clickhouse-native-test.source-inventory.tsv" + ;; + *) exit 64 ;; +esac +""", + ) _write_executable( fake_bin / "timeout", """\ #!/bin/bash set -eu -printf 'timeout %s\n' "$*" >> "${TEST_EVENT_LOG:?}" -while [[ "${1:-}" == --kill-after=* ]]; do - shift -done +while [[ "${1:-}" == --kill-after=* ]]; do shift; done shift -if [ -n "${FAKE_ARCHIVE_TIMEOUT_TARGET:-}" ] \ - && [ "${1:-}" = "docker" ] \ - && [[ " $* " == *"${FAKE_ARCHIVE_TIMEOUT_TARGET}"* ]]; then - exit 124 -fi -if [ -n "${FAKE_TAR_TIMEOUT_TARGET:-}" ] \ - && [ "${1:-}" = "tar" ] \ - && [[ " $* " == *"${FAKE_TAR_TIMEOUT_TARGET}"* ]]; then - exit 124 -fi exec "$@" """, ) @@ -89,61 +112,12 @@ exec "$@" """\ #!/bin/bash set -eu -printf 'docker %s\n' "$*" >> "${TEST_EVENT_LOG:?}" -case "${1:-}" in - inspect) - cat "${TEST_COLLECTOR_STATE:?}" - ;; - stop) - if [ "${2:-}" = "signoz-otel-collector" ]; then - printf 'false\n' > "${TEST_COLLECTOR_STATE:?}" - fi - ;; - start) - count=0 - if [ -e "${TEST_START_COUNT:?}" ]; then - count=$(cat "${TEST_START_COUNT}") - fi - count=$((count + 1)) - printf '%s\n' "$count" > "${TEST_START_COUNT}" - if [ "$count" -le "${FAKE_DOCKER_START_FAILURES:-0}" ]; then - exit 1 - fi - printf 'true\n' > "${TEST_COLLECTOR_STATE:?}" - ;; - run) - mode=success - if [[ " $* " == *"${FAKE_DOCKER_RUN_TARGET:-signoz-clickhouse}"* ]]; then - mode="${FAKE_DOCKER_RUN_MODE:-success}" - fi - if [ "$mode" = "signal" ]; then - kill -TERM "${PPID}" - sleep 0.1 - exit 0 - fi - if [ "$mode" = "empty" ]; then - exit 0 - fi - if [ "$mode" = "partial_nonzero" ]; then - printf 'partial-archive-data\n' - exit 23 - fi - printf 'archive-data\n' - ;; -esac -""", - ) - _write_executable( - fake_bin / "tar", - """\ -#!/bin/bash -set -eu -printf 'tar %s\n' "$*" >> "${TEST_EVENT_LOG:?}" -if [[ " $* " == *"${FAKE_TAR_VERIFY_TARGET:-signoz-clickhouse}"* ]] \ - && [ "${FAKE_TAR_VERIFY_MODE:-success}" = "corrupt" ]; then - exit 2 -fi -exit 0 +printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}" +[ "${1:-}" = "inspect" ] || exit 70 +printf '%s\\t%s\\t%s\\t%s\\t%s\\n' \ + "$(tr -d '\\r\\n' < "${TEST_COLLECTOR_ID:?}")" \ + "$(tr -d '\\r\\n' < "${TEST_COLLECTOR_STATE:?}")" \ + '2026-07-15T00:00:00Z' '0' 'healthy' """, ) _write_executable( @@ -151,18 +125,20 @@ exit 0 """\ #!/bin/bash set -eu -printf 'restic %s\n' "$*" >> "${TEST_EVENT_LOG:?}" +printf 'restic %s\\n' "$*" >> "${TEST_EVENT_LOG:?}" case " $* " in - *" backup "*) exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" ;; - *" snapshots "*) printf '[{"short_id":"test123"}]\n' ;; + *" backup "*) + dump_dir="${4:?}" + if [ -f "$dump_dir/signoz-metadata-gap.json" ]; then + printf 'gap %s\\n' "$(cat "$dump_dir/signoz-metadata-gap.json")" \ + >> "${TEST_EVENT_LOG:?}" + fi + exit "${FAKE_RESTIC_BACKUP_STATUS:-0}" + ;; + *" snapshots "*) + printf '[{"short_id":"native123"}]\\n' + ;; esac -""", - ) - _write_executable( - fake_bin / "du", - """\ -#!/bin/bash -printf '4K\t%s\n' "${2:-unknown}" """, ) @@ -171,22 +147,20 @@ printf '4K\t%s\n' "${2:-unknown}" "PATH": f"{fake_bin}:{os.environ['PATH']}", "TEST_BACKUP_BASE": str(backup_base), "TEST_EVENT_LOG": str(event_log), + "TEST_COLLECTOR_ID": str(collector_id), "TEST_COLLECTOR_STATE": str(collector_state), - "TEST_START_COUNT": str(start_count), - "FAKE_DOCKER_RUN_MODE": docker_run_mode, - "FAKE_DOCKER_RUN_TARGET": docker_run_target, - "FAKE_ARCHIVE_TIMEOUT_TARGET": archive_timeout_target, - "FAKE_TAR_VERIFY_MODE": tar_verify_mode, - "FAKE_TAR_VERIFY_TARGET": tar_verify_target, - "FAKE_TAR_TIMEOUT_TARGET": tar_timeout_target, - "FAKE_DOCKER_START_FAILURES": str(docker_start_failures), + "FAKE_NATIVE_CHECK_STATUS": str(native_check_status), + "FAKE_NATIVE_APPLY_STATUS": str(native_apply_status), + "FAKE_CONTINUITY_MODE": continuity_mode, "FAKE_RESTIC_BACKUP_STATUS": str(restic_backup_status), - "COLLECTOR_RESTORE_MAX_ATTEMPTS": str(restore_max_attempts), - "COLLECTOR_RESTORE_RETRY_DELAY_SECONDS": "0", + "TRACE_ID": "trace-P0-OBS-002", + "RUN_ID": f"run-{tmp_path.name}", + "WORK_ITEM_ID": "P0-OBS-002", + "SIGNOZ_BACKUP_RECEIPT_ROOT": str(tmp_path / "receipts"), + "COLLECTOR_CONTINUITY_POLL_SECONDS": "0.01", "COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1", - "ARCHIVE_CREATE_TIMEOUT_SECONDS": "1", - "ARCHIVE_VERIFY_TIMEOUT_SECONDS": "1", "TIMEOUT_KILL_AFTER_SECONDS": "1", + "BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0", } process = subprocess.Popen( ["bash", str(harness / SCRIPT.name)], @@ -210,274 +184,155 @@ def _events(events: list[str], prefix: str) -> list[str]: return [event for event in events if event.startswith(prefix)] -def _run_backup_all_with_signoz_failure( +def test_source_sunsets_raw_volumes_and_collector_mutation() -> None: + source = SCRIPT.read_text(encoding="utf-8") + + assert "signoz-clickhouse" not in source + assert "signoz-sqlite" not in source + assert "docker run" not in source + assert "docker stop" not in source + assert "docker start" not in source + assert "create_volume_archive" not in source + assert "pending_application_consistent_export" in source + assert '"raw_volume_read":false' in source + assert "{{.State.StartedAt}}" in source + assert "{{.RestartCount}}" in source + assert '{{if (index .State "Health")}}' in source + assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source + assert 'notify_clawbot "success"' not in source + assert "no-downtime continuity verifier" in source + + +def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime( tmp_path: Path, -) -> tuple[subprocess.CompletedProcess[str], list[str]]: - harness = tmp_path / "backup-all" - fake_scripts = tmp_path / "deployed-scripts" - event_log = tmp_path / "backup-all-events.log" - harness.mkdir() - fake_scripts.mkdir() - event_log.touch() +) -> None: + result, events, dump_dir = _run_backup(tmp_path) - source = BACKUP_ALL.read_text(encoding="utf-8").replace( - "/backup/scripts", - str(fake_scripts), + assert result.returncode == 0, result.stdout + result.stderr + assert len(_events(events, "native --check")) == 1 + assert len(_events(events, "native --apply")) == 1 + check_identity = _events(events, "native --check")[0].split(" trace=", 1)[1] + apply_identity = _events(events, "native --apply")[0].split(" trace=", 1)[1] + assert check_identity == apply_identity + assert not _events(events, "docker stop") + assert not _events(events, "docker start") + assert not _events(events, "docker run") + assert any("pending_application_consistent_export" in event for event in events) + assert len(_events(events, "notify warning signoz ")) == 1 + assert not _events(events, "notify success ") + assert any("SigNoz ClickHouse native 備份完成" in event for event in events) + assert any( + "--tag run:run-" in event for event in events if event.startswith("restic ") ) - backup_all = harness / "backup-all.sh" - _write_executable(backup_all, source) - (harness / "common.sh").write_text( - """\ -log_info() { :; } -log_warn() { :; } -log_error() { :; } -log_success() { :; } -notify_clawbot() { printf 'notify %s\\n' "$*" >> "${TEST_EVENT_LOG:?}"; } -""", - encoding="utf-8", - ) - - for name in ( - "backup-gitea.sh", - "backup-momo.sh", - "backup-harbor.sh", - "backup-awoooi.sh", - "backup-langfuse.sh", - "backup-monitoring.sh", - "backup-signoz.sh", - "backup-open-webui.sh", - "backup-clawbot.sh", - ): - status = 17 if name == "backup-signoz.sh" else 0 - _write_executable( - fake_scripts / name, - f"#!/bin/bash\nprintf 'run {name}\\n' >> \"${{TEST_EVENT_LOG:?}}\"\nexit {status}\n", - ) - - result = subprocess.run( - ["bash", str(backup_all)], - env={**os.environ, "TEST_EVENT_LOG": str(event_log)}, - text=True, - capture_output=True, - timeout=10, - ) - return result, event_log.read_text(encoding="utf-8").splitlines() + assert any("restic -r" in event and " check " in event for event in events) + receipts = list((tmp_path / "receipts").glob("*/restic-snapshot.json")) + assert len(receipts) == 1 + receipt = receipts[0].read_text(encoding="utf-8") + assert '"snapshot_id":"native123"' in receipt + assert '"offsite_verified":false' in receipt + assert not dump_dir.exists() -def test_backup_jobs_deploys_the_guarded_signoz_script() -> None: - playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8") - - assert 'dest: "/backup/scripts/{{ item }}"' in playbook - assert "- backup-signoz.sh" in playbook - assert "tags: backup_jobs" in playbook - - -def test_restore_policy_is_bounded_timed_and_independently_verified() -> None: - source = SCRIPT.read_text(encoding="utf-8") - - assert 'COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}"' in source - assert 'COLLECTOR_RESTORE_MAX_ATTEMPTS="${COLLECTOR_RESTORE_MAX_ATTEMPTS:-3}"' in source - assert 'ARCHIVE_CREATE_TIMEOUT_SECONDS="${ARCHIVE_CREATE_TIMEOUT_SECONDS:-1200}"' in source - assert 'ARCHIVE_VERIFY_TIMEOUT_SECONDS="${ARCHIVE_VERIFY_TIMEOUT_SECONDS:-1200}"' in source - assert 'TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}"' in source - assert 'timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}"' in source - assert 'tar -tzf "${output_file}"' in source - assert "docker run" in source - assert "|| true" not in source.split("create_volume_archive()", 1)[1].split( - "verify_volume_archive()", - 1, - )[0] - assert "verify_collector_final_state" in source - assert source.index("verify_collector_final_state") < source.rindex( - 'notify_clawbot "success"' - ) - - -def test_canary_retention_skip_is_explicit_and_defaults_off() -> None: - source = SCRIPT.read_text(encoding="utf-8") - - assert 'BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}"' in source - assert 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' in source - assert source.index('restic -r "${LOCAL_REPO}" backup') < source.index( - 'if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]' - ) - assert source.index('if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]') < source.index( - "if ! verify_collector_final_state" - ) - - -def test_collector_is_restored_when_backup_receives_term(tmp_path: Path) -> None: - result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="signal") +def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None: + result, events, dump_dir = _run_backup(tmp_path, native_check_status=19) assert result.returncode != 0 - assert _events(events, "docker stop signoz-otel-collector") - assert len(_events(events, "docker start signoz-otel-collector")) == 1 - assert not _events(events, "notify success ") + assert _events(events, "native --check") + assert not _events(events, "native --apply") + assert not _events(events, "restic ") + assert len(_events(events, "notify failed signoz ")) == 1 assert not dump_dir.exists() -def test_collector_is_restored_when_clickhouse_backup_fails(tmp_path: Path) -> None: - result, events, dump_dir = _run_backup(tmp_path, docker_run_mode="empty") +def test_native_apply_failure_never_reports_scoped_success(tmp_path: Path) -> None: + result, events, dump_dir = _run_backup(tmp_path, native_apply_status=23) - assert result.returncode == 1 - assert len(_events(events, "docker start signoz-otel-collector")) == 1 + assert result.returncode != 0 + assert _events(events, "native --apply") + assert not _events(events, "restic ") + assert not _events(events, "notify warning ") assert not _events(events, "notify success ") + assert len(_events(events, "notify failed signoz ")) == 1 assert not dump_dir.exists() -def test_exit_cleanup_does_not_restart_an_already_restored_collector( +def test_initially_stopped_collector_fails_without_apply(tmp_path: Path) -> None: + result, events, dump_dir = _run_backup(tmp_path, initial_running=False) + + assert result.returncode != 0 + assert _events(events, "native --check") + assert not _events(events, "native --apply") + assert not _events(events, "restic ") + assert not dump_dir.exists() + + +def test_continuity_observer_rejects_stopped_sample(tmp_path: Path) -> None: + result, events, dump_dir = _run_backup(tmp_path, continuity_mode="stopped") + + assert result.returncode != 0 + assert any("continuity verifier 失敗" in event for event in events) + assert not _events(events, "restic ") + assert not _events(events, "notify warning ") + assert not dump_dir.exists() + + +def test_continuity_observer_rejects_container_identity_drift( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup(tmp_path, continuity_mode="identity") + + assert result.returncode != 0 + assert any("continuity verifier 失敗" in event for event in events) + assert not _events(events, "restic ") + assert not dump_dir.exists() + + +def test_restic_failure_is_nonzero_and_no_completion_notification( tmp_path: Path, ) -> None: result, events, dump_dir = _run_backup(tmp_path, restic_backup_status=17) assert result.returncode == 17 - assert len(_events(events, "docker start signoz-otel-collector")) == 1 + assert _events(events, "restic ") + assert not _events(events, "notify warning ") assert not _events(events, "notify success ") assert not dump_dir.exists() -def test_fail_once_restores_within_budget_and_succeeds(tmp_path: Path) -> None: - result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=1) +def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None: + result, events, _dump_dir = _run_backup(tmp_path, skip_retention=True) assert result.returncode == 0 - assert len(_events(events, "docker start signoz-otel-collector")) == 2 - assert len(_events(events, "notify success ")) == 1 - assert not _events(events, "notify failed ") - assert not dump_dir.exists() + assert any("BACKUP_SKIP_RETENTION_CLEANUP=1" in event for event in events) + assert not _events(events, "retention ") -def test_permanent_restore_failure_is_nonzero_and_never_notifies_success( - tmp_path: Path, -) -> None: - result, events, dump_dir = _run_backup(tmp_path, docker_start_failures=99) +def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None: + playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8") - assert result.returncode != 0 - assert len(_events(events, "docker start signoz-otel-collector")) == 3 - assert len(_events(events, "notify failed ")) == 1 - assert not _events(events, "notify success ") - assert not dump_dir.exists() - - -def test_initially_stopped_fails_closed_without_stop_start_or_success( - tmp_path: Path, -) -> None: - result, events, dump_dir = _run_backup(tmp_path, initial_running=False) - - assert result.returncode != 0 - assert not _events(events, "docker stop signoz-otel-collector") - assert not _events(events, "docker start signoz-otel-collector") - assert len(_events(events, "notify failed ")) == 1 - assert not _events(events, "notify success ") - assert not dump_dir.exists() - - -def test_success_notification_follows_independent_final_running_readback( - tmp_path: Path, -) -> None: - result, events, _dump_dir = _run_backup(tmp_path) - - assert result.returncode == 0 - inspect_positions = [ - index - for index, event in enumerate(events) - if event.startswith("docker inspect --format") - ] - success_position = next( - index - for index, event in enumerate(events) - if event.startswith("notify success ") + assert 'dest: "/backup/scripts/{{ item }}"' in playbook + assert "- backup-signoz.sh" in playbook + assert "- clickhouse-native-backup.sh" in playbook + assert "- clickhouse-native-restore-drill.sh" in playbook + assert "- clickhouse-restore-inventory.py" in playbook + assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in playbook + assert ( + "/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + in playbook ) - assert len(inspect_positions) >= 3 - assert inspect_positions[-1] < success_position + assert "tags: backup_jobs" in playbook - start_position = next( - index - for index, event in enumerate(events) - if event.startswith("docker start signoz-otel-collector") + +def test_scripts_are_valid_bash() -> None: + result = subprocess.run( + [ + "bash", + "-n", + str(SCRIPT), + str(ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh"), + ], + text=True, + capture_output=True, + check=False, ) - tar_positions = [ - index for index, event in enumerate(events) if event.startswith("tar -tzf") - ] - assert len(tar_positions) == 2 - assert start_position < tar_positions[0] - - -def test_archive_create_timeout_is_bounded_restores_and_notifies_once( - tmp_path: Path, -) -> None: - result, events, dump_dir = _run_backup( - tmp_path, - archive_timeout_target="signoz-clickhouse", - ) - - assert result.returncode != 0 - assert len(_events(events, "docker start signoz-otel-collector")) == 1 - assert len(_events(events, "notify failed ")) == 1 - assert not _events(events, "notify success ") - assert not dump_dir.exists() - - -def test_partial_sqlite_archive_nonzero_fails_overall_and_removes_output( - tmp_path: Path, -) -> None: - result, events, dump_dir = _run_backup( - tmp_path, - docker_run_mode="partial_nonzero", - docker_run_target="signoz-sqlite", - ) - - assert result.returncode != 0 - assert len(_events(events, "docker start signoz-otel-collector")) == 1 - assert len(_events(events, "notify failed ")) == 1 - assert not _events(events, "notify success ") - assert not dump_dir.exists() - - -def test_corrupt_archive_fails_after_collector_restore_and_notifies_once( - tmp_path: Path, -) -> None: - result, events, dump_dir = _run_backup( - tmp_path, - tar_verify_mode="corrupt", - tar_verify_target="clickhouse_", - ) - - assert result.returncode != 0 - start_position = next( - index - for index, event in enumerate(events) - if event.startswith("docker start signoz-otel-collector") - ) - tar_position = next( - index for index, event in enumerate(events) if event.startswith("tar -tzf") - ) - assert start_position < tar_position - assert len(_events(events, "notify failed ")) == 1 - assert not _events(events, "notify success ") - assert not dump_dir.exists() - - -def test_integrity_verifier_timeout_is_bounded_and_never_reports_success( - tmp_path: Path, -) -> None: - result, events, dump_dir = _run_backup( - tmp_path, - tar_timeout_target="clickhouse_", - ) - - assert result.returncode != 0 - assert len(_events(events, "docker start signoz-otel-collector")) == 1 - assert len(_events(events, "notify failed ")) == 1 - assert not _events(events, "notify success ") - assert not dump_dir.exists() - - -def test_backup_all_propagates_signoz_failure_to_nonzero_terminal( - tmp_path: Path, -) -> None: - result, events = _run_backup_all_with_signoz_failure(tmp_path) - - assert result.returncode == 1 - assert "run backup-signoz.sh" in events - assert any(event.startswith("notify warning all ") for event in events) - assert not any(event.startswith("notify success all ") for event in events) + assert result.returncode == 0, result.stderr diff --git a/scripts/backup/tests/test_clickhouse_native_backup.py b/scripts/backup/tests/test_clickhouse_native_backup.py new file mode 100644 index 000000000..39e1ed922 --- /dev/null +++ b/scripts/backup/tests/test_clickhouse_native_backup.py @@ -0,0 +1,478 @@ +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "backup" / "clickhouse-native-backup.sh" +PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml" + + +def _write_executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + + +def _install_fakes(fake_bin: Path) -> None: + _write_executable( + fake_bin / "timeout", + """\ +#!/bin/bash +set -eu +while [[ "${1:-}" == --kill-after=* ]]; do shift; done +shift +exec "$@" +""", + ) + _write_executable(fake_bin / "flock", "#!/bin/bash\nexit 0\n") + _write_executable( + fake_bin / "sha256sum", + """\ +#!/usr/bin/env python3 +import hashlib +import sys + +if len(sys.argv) == 1: + payload = sys.stdin.buffer.read() + print(f"{hashlib.sha256(payload).hexdigest()} -") +else: + with open(sys.argv[1], "rb") as source: + payload = source.read() + print(f"{hashlib.sha256(payload).hexdigest()} {sys.argv[1]}") +""", + ) + _write_executable( + fake_bin / "unzip", + """\ +#!/bin/bash +set -eu +printf 'unzip %s\\n' "$*" >> "${TEST_EVENT_LOG:?}" +exit "${FAKE_UNZIP_STATUS:-0}" +""", + ) + _write_executable( + fake_bin / "docker", + """\ +#!/bin/bash +set -eu +printf 'docker %s\\n' "$*" >> "${TEST_EVENT_LOG:?}" + +find_arg() { + local wanted="$1" + shift + while [ "$#" -gt 0 ]; do + if [ "$1" = "$wanted" ]; then + printf '%s\\n' "${2:-}" + return 0 + fi + shift + done + return 1 +} + +case "${1:-}" in + inspect) + printf 'sha256:clickhouse-test-id\\ttrue\\n' + ;; + cp) + [ -e "${TEST_SERVER_ARTIFACT:?}" ] || exit 44 + printf 'clickhouse-native-zip-bytes\\n' > "${3:?}" + ;; + exec) + shift + container="${1:?}" + shift + case "${1:-}" in + test) + shift + if [ "${1:-}" = "!" ]; then + [ ! -e "${TEST_SERVER_ARTIFACT:?}" ] + elif [ "${1:-}" = "-d" ] || [ "${1:-}" = "-w" ]; then + [ "${FAKE_DISK_READY:-1}" = "1" ] + [ "${2:-}" = "${FAKE_DISK_PATH:-/backups/}" ] + else + exit 45 + fi + ;; + rm) + rm -f "${TEST_SERVER_ARTIFACT:?}" + ;; + clickhouse-client) + query="$(find_arg --query "$@")" + operation_id="$(find_arg --param_operation_id "$@" || true)" + artifact="$(find_arg --param_artifact "$@" || true)" + disk="$(find_arg --param_disk "$@" || true)" + case "$query" in + 'SELECT version()') + printf '25.5.1.1\\n' + ;; + *"database='system' AND name='backups'"*) + printf '%s\\n' "${FAKE_SYSTEM_BACKUPS_COUNT:-1}" + ;; + 'CHECK GRANT BACKUP ON *.*') + printf '%s\\n' "${FAKE_BACKUP_GRANT:-1}" + ;; + *"FROM system.disks"*) + printf '%s\\n' "${FAKE_DISK_PATH:-/backups/}" + ;; + *"FROM system.databases"*) + printf '%s\\n' signoz_analytics signoz_logs signoz_metadata \ + signoz_meter signoz_metrics signoz_traces + if [ "${FAKE_EXTRA_DATABASE:-0}" = "1" ]; then + printf 'unexpected_db\\n' + fi + ;; + *"FROM system.tables WHERE database IN"*) + schema_hash="$(printf 'A%.0s' {1..64})" + printf 'signoz_analytics\\tanalytics\\tReplicatedMergeTree\\t10\\t1000\\t%s\\n' "$schema_hash" + printf 'signoz_logs\\tlogs_v2\\tReplicatedMergeTree\\t20\\t2000\\t%s\\n' "$schema_hash" + printf 'signoz_metadata\\tmetadata\\tMergeTree\\t3\\t300\\t%s\\n' "$schema_hash" + printf 'signoz_meter\\tmeter\\tDistributed\\t0\\t0\\t%s\\n' "$schema_hash" + printf 'signoz_metrics\\tsamples_v4\\tReplicatedMergeTree\\t40\\t4000\\t%s\\n' "$schema_hash" + printf 'signoz_traces\\tsignoz_index_v3\\tReplicatedMergeTree\\t50\\t5000\\t%s\\n' "$schema_hash" + ;; + BACKUP*) + case "$query" in + *" SETTINGS id='${EXPECTED_FAKE_OPERATION_ID:?}' ASYNC") + operation_id="${EXPECTED_FAKE_OPERATION_ID}" + ;; + *) exit 49 ;; + esac + printf '%s\\n' "$operation_id" > "${TEST_OPERATION_ID:?}" + printf '%s\\n' "$artifact" > "${TEST_ARTIFACT_NAME:?}" + printf '%s\\n' "$disk" > "${TEST_DISK_NAME:?}" + : > "${TEST_SUBMITTED:?}" + : > "${TEST_SERVER_ARTIFACT:?}" + printf '%s\\tCREATING_BACKUP\\n' "$operation_id" + ;; + *"FROM system.backups WHERE id="*) + mode="${FAKE_STATUS_MODE:-success}" + if [ "$mode" = "existing_success" ] || [ -e "${TEST_SUBMITTED:?}" ]; then + artifact_name="$(cat "${TEST_ARTIFACT_NAME:?}" 2>/dev/null || true)" + [ -n "$artifact_name" ] || artifact_name="${EXPECTED_FAKE_ARTIFACT:?}" + backup_name="Disk('backups', '$artifact_name')" + case "$mode" in + failed) + printf '%s\\tBACKUP_FAILED\\t0\\t0\\t0\\t0\\t434F44453A20353938\\t%s\\n' \ + "$backup_name" "$(printf '1%.0s' {1..64})" + ;; + *) + : > "${TEST_SERVER_ARTIFACT:?}" + printf '%s\\tBACKUP_CREATED\\t12\\t1200\\t2400\\t1100\\tnone\\t%s\\n' \ + "$backup_name" "$(printf '0%.0s' {1..64})" + ;; + esac + fi + ;; + *) exit 46 ;; + esac + ;; + *) exit 47 ;; + esac + ;; + *) exit 48 ;; +esac +""", + ) + + +def _harness( + tmp_path: Path, + *, + status_mode: str = "success", + extra_database: bool = False, + disk_ready: bool = True, + unzip_status: int = 0, + with_hook: bool = False, +) -> tuple[dict[str, str], dict[str, Path]]: + fake_bin = tmp_path / "bin" + output = tmp_path / "output" + receipts = tmp_path / "receipts" + fake_bin.mkdir() + output.mkdir() + receipts.mkdir() + _install_fakes(fake_bin) + + event_log = tmp_path / "events.log" + event_log.touch() + paths = { + "output": output, + "receipts": receipts, + "events": event_log, + "submitted": tmp_path / "submitted", + "server_artifact": tmp_path / "server-artifact", + "operation_id": tmp_path / "operation-id", + "artifact_name": tmp_path / "artifact-name", + "disk_name": tmp_path / "disk-name", + "hook_events": tmp_path / "hook-events.log", + } + identity_hash = ( + subprocess.check_output( + [str(fake_bin / "sha256sum")], + input=b"trace-native\0run-native\0P0-OBS-002", + ) + .decode() + .split()[0] + ) + expected_artifact = f"signoz-{identity_hash}.zip" + + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "TRACE_ID": "trace-native", + "RUN_ID": "run-native", + "WORK_ITEM_ID": "P0-OBS-002", + "CLICKHOUSE_NATIVE_OUTPUT_DIR": str(output), + "CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(receipts), + "CLICKHOUSE_NATIVE_COMMAND_TIMEOUT_SECONDS": "2", + "CLICKHOUSE_NATIVE_BACKUP_TIMEOUT_SECONDS": "2", + "CLICKHOUSE_NATIVE_POLL_INTERVAL_SECONDS": "1", + "CLICKHOUSE_NATIVE_KILL_AFTER_SECONDS": "1", + "TEST_EVENT_LOG": str(event_log), + "TEST_SUBMITTED": str(paths["submitted"]), + "TEST_SERVER_ARTIFACT": str(paths["server_artifact"]), + "TEST_OPERATION_ID": str(paths["operation_id"]), + "TEST_ARTIFACT_NAME": str(paths["artifact_name"]), + "TEST_DISK_NAME": str(paths["disk_name"]), + "EXPECTED_FAKE_ARTIFACT": expected_artifact, + "EXPECTED_FAKE_OPERATION_ID": f"awoooi-{identity_hash}", + "FAKE_STATUS_MODE": status_mode, + "FAKE_EXTRA_DATABASE": "1" if extra_database else "0", + "FAKE_DISK_READY": "1" if disk_ready else "0", + "FAKE_UNZIP_STATUS": str(unzip_status), + } + if with_hook: + hook = tmp_path / "restore-verifier-hook.sh" + _write_executable( + hook, + """\ +#!/bin/bash +set -eu +if [ "$1" = "--apply" ]; then + test -f "$CLICKHOUSE_NATIVE_ARTIFACT_PATH" + test -f "$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH" + test -f "$CLICKHOUSE_NATIVE_MANIFEST_PATH" + grep -q '"status":"BACKUP_CREATED"' "$CLICKHOUSE_NATIVE_MANIFEST_PATH" +fi +printf '%s trace=%s run=%s work=%s dbs=%s inventory=%s manifest=%s\\n' \ + "$1" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \ + "$CLICKHOUSE_NATIVE_EXPECTED_DATABASES" \ + "$CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH" \ + "${CLICKHOUSE_NATIVE_MANIFEST_PATH:-none}" \ + >> "${TEST_HOOK_EVENTS:?}" +""", + ) + env["CLICKHOUSE_NATIVE_POST_VERIFY_HOOK"] = str(hook) + env["TEST_HOOK_EVENTS"] = str(paths["hook_events"]) + return env, paths + + +def _run(env: dict[str, str], mode: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(SCRIPT), mode], + env=env, + text=True, + capture_output=True, + timeout=15, + check=False, + ) + + +def _receipts(path: Path) -> list[dict[str, object]]: + receipt_file = path / "run-native" / "receipts.jsonl" + return [ + json.loads(line) + for line in receipt_file.read_text(encoding="utf-8").splitlines() + ] + + +def test_check_validates_exact_scope_without_backup_or_artifact_write( + tmp_path: Path, +) -> None: + env, paths = _harness(tmp_path) + result = _run(env, "--check") + + assert result.returncode == 0, result.stdout + result.stderr + events = paths["events"].read_text(encoding="utf-8") + assert "BACKUP DATABASE" not in events + assert "docker cp" not in events + assert "docker exec signoz-clickhouse rm" not in events + assert not list(paths["output"].iterdir()) + receipts = _receipts(paths["receipts"]) + assert receipts[-1]["terminal"] == "check_pass_no_write" + + +def test_apply_requires_same_run_check_receipt(tmp_path: Path) -> None: + env, paths = _harness(tmp_path) + result = _run(env, "--apply") + + assert result.returncode != 0 + assert "same_run_check_pass_receipt_required_before_apply" in result.stderr + assert paths["events"].read_text(encoding="utf-8") == "" + + +def test_apply_submits_exact_six_database_backup_and_verifies_zip( + tmp_path: Path, +) -> None: + env, paths = _harness(tmp_path) + assert _run(env, "--check").returncode == 0 + result = _run(env, "--apply") + + assert result.returncode == 0, result.stdout + result.stderr + events = paths["events"].read_text(encoding="utf-8") + assert "BACKUP DATABASE signoz_analytics,DATABASE signoz_logs" in events + assert "DATABASE signoz_metadata,DATABASE signoz_meter" in events + assert "DATABASE signoz_metrics,DATABASE signoz_traces" in events + assert "BACKUP ALL" not in events + backup_line = next( + line for line in events.splitlines() if "--query BACKUP " in line + ) + assert " SETTINGS id='awoooi-" in backup_line + assert "id={operation_id:String}" not in backup_line + assert "--param_operation_id" not in backup_line + assert "docker cp signoz-clickhouse:/backups/signoz-" in events + assert "unzip -tq" in events + assert "docker exec signoz-clickhouse rm -f -- /backups/signoz-" in events + assert not paths["server_artifact"].exists() + outputs = list(paths["output"].glob("clickhouse-native-*.zip")) + assert len(outputs) == 1 + manifest = json.loads(Path(str(outputs[0]) + ".manifest.json").read_text()) + assert manifest["databases"] == [ + "signoz_analytics", + "signoz_logs", + "signoz_metadata", + "signoz_meter", + "signoz_metrics", + "signoz_traces", + ] + inventory = list(paths["output"].glob("*.source-inventory.tsv")) + assert len(inventory) == 1 + assert "ReplicatedMergeTree" in inventory[0].read_text(encoding="utf-8") + assert "Distributed" in inventory[0].read_text(encoding="utf-8") + + +def test_database_allowlist_drift_fails_check(tmp_path: Path) -> None: + env, _paths = _harness(tmp_path, extra_database=True) + result = _run(env, "--check") + assert result.returncode != 0 + assert "signoz_database_allowlist_drift" in result.stderr + + +def test_missing_backup_grant_fails_check_before_submission(tmp_path: Path) -> None: + env, paths = _harness(tmp_path) + env["FAKE_BACKUP_GRANT"] = "0" + + result = _run(env, "--check") + + assert result.returncode != 0 + assert "native_backup_grant_absent" in result.stderr + assert not paths["submitted"].exists() + + +def test_missing_runtime_backup_disk_fails_check(tmp_path: Path) -> None: + env, _paths = _harness(tmp_path, disk_ready=False) + result = _run(env, "--check") + assert result.returncode != 0 + assert "backup_disk_directory_absent" in result.stderr + + +def test_failed_async_terminal_has_exact_cleanup_and_no_local_artifact( + tmp_path: Path, +) -> None: + env, paths = _harness(tmp_path, status_mode="failed") + assert _run(env, "--check").returncode == 0 + result = _run(env, "--apply") + + assert result.returncode != 0 + assert "native_backup_terminal_BACKUP_FAILED" in result.stderr + assert not paths["server_artifact"].exists() + assert not list(paths["output"].iterdir()) + receipt_dir = paths["receipts"] / "run-native" + assert list(receipt_dir.glob("*.stderr")) + assert list(receipt_dir.glob("*.status")) + bounded_errors = list(receipt_dir.glob("*-backup-error-bounded.hex")) + assert len(bounded_errors) == 1 + assert bounded_errors[0].read_text(encoding="utf-8") == ("434F44453A20353938\n") + + +def test_zip_failure_removes_local_partial_and_preserves_server_evidence( + tmp_path: Path, +) -> None: + env, paths = _harness(tmp_path, unzip_status=7) + assert _run(env, "--check").returncode == 0 + result = _run(env, "--apply") + + assert result.returncode != 0 + assert "native_backup_archive_integrity_failed" in result.stderr + assert paths["server_artifact"].exists() + assert not list(paths["output"].iterdir()) + receipts = _receipts(paths["receipts"]) + assert any( + item["stage"] == "cleanup" and item["terminal"] == "preserved" + for item in receipts + ) + + +def test_restore_verifier_hook_uses_check_and_apply_contract(tmp_path: Path) -> None: + env, paths = _harness(tmp_path, with_hook=True) + assert _run(env, "--check").returncode == 0 + result = _run(env, "--apply") + + assert result.returncode == 0, result.stdout + result.stderr + hook_events = paths["hook_events"].read_text(encoding="utf-8").splitlines() + assert sum(event.startswith("--check ") for event in hook_events) == 1 + assert sum(event.startswith("--apply ") for event in hook_events) == 1 + assert all("signoz_analytics,signoz_logs" in event for event in hook_events) + assert any( + "manifest=" in event and "manifest=none" not in event + for event in hook_events + if event.startswith("--apply ") + ) + + +def test_verified_local_artifact_is_idempotently_reused(tmp_path: Path) -> None: + env, paths = _harness(tmp_path) + assert _run(env, "--check").returncode == 0 + assert _run(env, "--apply").returncode == 0 + submissions_before = ( + paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE") + ) + + result = _run(env, "--apply") + + assert result.returncode == 0, result.stdout + result.stderr + submissions_after = ( + paths["events"].read_text(encoding="utf-8").count("BACKUP DATABASE") + ) + assert submissions_after == submissions_before + receipts = _receipts(paths["receipts"]) + assert any( + item["stage"] == "idempotency" and item["terminal"] == "reused" + for item in receipts + ) + + +def test_source_contract_has_no_raw_volume_fallback_and_is_deployed() -> None: + source = SCRIPT.read_text(encoding="utf-8") + playbook = PLAYBOOK.read_text(encoding="utf-8") + + assert "BACKUP ${BACKUP_SCOPE_SQL}" in source + assert "SETTINGS id='${OPERATION_ID}' ASYNC" in source + assert "SETTINGS id={operation_id:String}" not in source + assert "BACKUP ALL" not in source + assert ".zip" in source + assert "unzip -tq" in source + assert "--format $'{{.Id}}\\t{{.State.Running}}'" in source + assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source + assert "docker run" not in source + assert "clickhouse-native-backup.sh" in playbook + result = subprocess.run( + ["bash", "-n", str(SCRIPT)], + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr diff --git a/scripts/backup/tests/test_clickhouse_native_restore_drill.py b/scripts/backup/tests/test_clickhouse_native_restore_drill.py new file mode 100644 index 000000000..cd5a298f7 --- /dev/null +++ b/scripts/backup/tests/test_clickhouse_native_restore_drill.py @@ -0,0 +1,848 @@ +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "clickhouse-native-restore-drill.sh" +INVENTORY_HELPER = ROOT / "clickhouse-restore-inventory.py" +REPO_ROOT = ROOT.parents[1] +CLICKHOUSE_IMAGE_ID = ( + "sha256:8248e5926d7304400e44ecdf0c7181fbde28e6a9b1d647780eca839f34c00cc6" +) +ZOOKEEPER_IMAGE_ID = ( + "sha256:3ab0e8f032ab58f14ac1e929ac40305a4a464cf320bdfe4151d62d6922729ba0" +) +EMPTY_SHA256 = "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" +EXPECTED_DATABASES = [ + "signoz_analytics", + "signoz_logs", + "signoz_metadata", + "signoz_meter", + "signoz_metrics", + "signoz_traces", +] + + +def _inventory( + *, + restored_delta: int = 0, + engine_mismatch: bool = False, + schema_mismatch: bool = False, +) -> str: + rows = [ + ("signoz_analytics", "events", "ReplicatedMergeTree", 10, 100), + ("signoz_analytics", "events_all", "Distributed", 10, 100), + ("signoz_logs", "logs_v2", "ReplicatedMergeTree", 20, 200), + ("signoz_metadata", "resource_attrs", "ReplicatedMergeTree", 2, 20), + ("signoz_meter", "meter", "ReplicatedMergeTree", 3, 30), + ("signoz_metrics", "samples_v4", "ReplicatedMergeTree", 30, 300), + ("signoz_metrics", "time_series_v4", "ReplicatedMergeTree", 12, 120), + ("signoz_traces", "signoz_index_v3", "ReplicatedMergeTree", 40, 400), + ] + rendered: list[str] = [] + for index, (database, table, engine, row_count, byte_count) in enumerate(rows): + if engine_mismatch and table == "meter": + engine = "MergeTree" + schema_hash = ( + hashlib.sha256(f"{database}.{table}:{engine}".encode("utf-8")) + .hexdigest() + .upper() + ) + if schema_mismatch and table == "meter": + schema_hash = "F" * 64 + rendered.append( + "\t".join( + ( + database, + table, + engine, + str(row_count + (restored_delta if index == 0 else 0)), + str(byte_count + (restored_delta if index == 0 else 0)), + schema_hash, + ) + ) + ) + return "\n".join(rendered) + "\n" + + +def _write_backup_fixture(tmp_path: Path) -> tuple[Path, Path, Path]: + artifact = tmp_path / "clickhouse-native.zip.partial" + with zipfile.ZipFile(artifact, "w") as archive: + archive.writestr(".backup", "bounded native backup fixture\n") + inventory = tmp_path / "source-inventory.tsv.partial" + inventory.write_text(_inventory(), encoding="utf-8") + artifact_sha = hashlib.sha256(artifact.read_bytes()).hexdigest() + inventory_sha = hashlib.sha256(inventory.read_bytes()).hexdigest() + manifest = tmp_path / "manifest.json.partial" + manifest.write_text( + json.dumps( + { + "trace_id": "backup-trace", + "run_id": "backup-run", + "work_item_id": "P0-OBS-002", + "operation_id": "awoooi-backup-operation", + "databases": EXPECTED_DATABASES, + "source_inventory_sha256": inventory_sha, + "sha256": artifact_sha, + "size_bytes": artifact.stat().st_size, + "status": "BACKUP_CREATED", + }, + separators=(",", ":"), + ) + + "\n", + encoding="utf-8", + ) + return artifact, manifest, inventory + + +def _write_fake_docker(tmp_path: Path) -> tuple[Path, Path, Path]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + state_dir = tmp_path / "docker-state" + state_dir.mkdir() + log_path = tmp_path / "docker.log" + docker = bin_dir / "docker" + docker.write_text( + r"""#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >> "${FAKE_DOCKER_LOG:?}" +state="${FAKE_DOCKER_STATE:?}" +cmd="${1:?}" +shift +last_arg() { eval "printf '%s' \"\${$#}\""; } + +case "$cmd" in + info) + [ "${FAKE_DOCKER_INFO_FAIL:-0}" = 0 ] || exit 78 + printf '25.0.0\\n' + ;; + image) + sub="${1:?}" + shift + [ "$sub" = inspect ] || exit 90 + image="$(last_arg "$@")" + case "$image" in + clickhouse/clickhouse-server:25.5.6) printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" ;; + signoz/zookeeper:3.7.1) printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" ;; + *) exit 1 ;; + esac + ;; + container) + sub="${1:?}" + shift + case "$sub" in + inspect) + format="" + if [ "${1:-}" = --format ]; then + format="$2" + shift 2 + fi + name="${1:?}" + [ -f "$state/container-$name" ] || exit 1 + if [[ "$format" == *Labels* ]]; then + if [ "${FAKE_LABEL_INSPECT_HANG:-0}" = 1 ] && [[ "$name" == *ch-restore* ]]; then + sleep 10 + fi + cat "$state/container-label-$name" + fi + ;; + ls) + for item in "$state"/container-*; do + [ -e "$item" ] || continue + case "$item" in + *-label-*|*-network-*|*-image-*) continue ;; + esac + basename "$item" | sed 's/^container-//' + done + ;; + *) exit 90 ;; + esac + ;; + inspect) + format="" + if [ "${1:-}" = --format ]; then + format="$2" + shift 2 + fi + name="${1:?}" + [ -f "$state/container-$name" ] || exit 1 + case "$format" in + *State.Running*) printf 'true\n' ;; + *HostConfig.NetworkMode*) cat "$state/container-network-$name" ;; + *.Image*) cat "$state/container-image-$name" ;; + *) printf '%s\n' "fake-$name" ;; + esac + ;; + network) + sub="${1:?}" + shift + case "$sub" in + create) + name="$(last_arg "$@")" + label="" + while [ "$#" -gt 0 ]; do + if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi + done + touch "$state/network-$name" + printf '%s\n' "$label" > "$state/network-label-$name" + printf '%s\n' "$name" > "$state/network-name" + printf '%s\n' "$name" + ;; + inspect) + format="" + if [ "${1:-}" = --format ]; then + format="$2" + shift 2 + fi + name="${1:?}" + [ -f "$state/network-$name" ] || exit 1 + if [[ "$format" == *Internal* ]]; then printf 'true\n'; fi + if [[ "$format" == *Labels* ]]; then cat "$state/network-label-$name"; fi + ;; + ls) + for item in "$state"/network-*; do + [ -e "$item" ] || continue + case "$item" in *-label-*|*/network-name) continue ;; esac + basename "$item" | sed 's/^network-//' + done + ;; + rm) + name="${1:?}" + rm -f "$state/network-$name" "$state/network-label-$name" "$state/network-name" + printf '%s\n' "$name" + ;; + *) exit 90 ;; + esac + ;; + volume) + sub="${1:?}" + shift + case "$sub" in + create) + name="$(last_arg "$@")" + label="" + while [ "$#" -gt 0 ]; do + if [ "$1" = --label ]; then label="${2#*=}"; shift 2; else shift; fi + done + touch "$state/volume-$name" + printf '%s\n' "$label" > "$state/volume-label-$name" + printf '%s\n' "$name" + ;; + inspect) + format="" + if [ "${1:-}" = --format ]; then format="$2"; shift 2; fi + name="${1:?}" + [ -f "$state/volume-$name" ] || exit 1 + if [[ "$format" == *Labels* ]]; then cat "$state/volume-label-$name"; fi + ;; + ls) + for item in "$state"/volume-*; do + [ -e "$item" ] || continue + case "$item" in *-label-*) continue ;; esac + basename "$item" | sed 's/^volume-//' + done + ;; + rm) + name="${1:?}" + rm -f "$state/volume-$name" "$state/volume-label-$name" + if [[ "$name" == *-backup ]]; then + rm -f "$state/staged-artifact-sha256" + fi + printf '%s\n' "$name" + ;; + *) exit 90 ;; + esac + ;; + run) + name="" + replica="" + label="" + network="" + image="" + expected_artifact_sha256="" + while [ "$#" -gt 0 ]; do + case "$1" in + --name) + name="$2" + shift 2 + ;; + --env) + if [[ "$2" == CLICKHOUSE_RESTORE_REPLICA=* ]]; then + replica="${2#*=}" + fi + if [[ "$2" == EXPECTED_ARTIFACT_SHA256=* ]]; then + expected_artifact_sha256="${2#*=}" + fi + shift 2 + ;; + --label) + label="${2#*=}" + shift 2 + ;; + --network) + network="$2" + shift 2 + ;; + --network-alias|--restart|--mount|--user|--entrypoint) + shift 2 + ;; + --detach|--pull=never) + shift + ;; + clickhouse/clickhouse-server:25.5.6|signoz/zookeeper:3.7.1) + image="$1" + shift + ;; + *) + shift + ;; + esac + done + [ -n "$name" ] || exit 91 + touch "$state/container-$name" + printf '%s\n' "$label" > "$state/container-label-$name" + printf '%s\n' "$network" > "$state/container-network-$name" + case "$image" in + clickhouse/clickhouse-server:25.5.6) + printf '%s\n' "${FAKE_CLICKHOUSE_IMAGE_ID:?}" > "$state/container-image-$name" + ;; + signoz/zookeeper:3.7.1) + printf '%s\n' "${FAKE_ZOOKEEPER_IMAGE_ID:?}" > "$state/container-image-$name" + ;; + *) exit 96 ;; + esac + if [ -n "$replica" ]; then printf '%s\n' "$replica" > "$state/replica"; fi + if [ "$network" = none ]; then + if [ "${FAKE_STAGING_HANG:-0}" = 1 ]; then sleep 10; fi + [ "${FAKE_STAGING_FAIL:-0}" = 0 ] || exit 94 + staged_hash="$expected_artifact_sha256" + if [ "${FAKE_STAGING_HASH_MISMATCH:-0}" = 1 ]; then + staged_hash="$(printf '0%.0s' {1..64})" + fi + printf '%s\n' "$staged_hash" > "$state/staged-artifact-sha256" + printf '%s\n' "$staged_hash" + else + printf 'fake-container-%s\n' "$name" + fi + ;; + exec) + container="${1:?}" + shift + [ -f "$state/container-$container" ] || exit 1 + if [ "${1:-}" = sha256sum ]; then + [ -f "$state/staged-artifact-sha256" ] || exit 97 + artifact_hash="$(cat "$state/staged-artifact-sha256")" + if [ "${FAKE_CLICKHOUSE_HASH_MISMATCH:-0}" = 1 ]; then + artifact_hash="$(printf 'f%.0s' {1..64})" + fi + printf '%s /backups/native-backup.zip\n' "$artifact_hash" + exit 0 + fi + query="" + operation="" + while [ "$#" -gt 0 ]; do + case "$1" in + --query) + query="$2" + shift 2 + ;; + --param_operation_id) + operation="$2" + shift 2 + ;; + *) shift ;; + esac + done + case "$query" in + CHECK\ TABLE*) + if [ "${FAKE_CHECK_TABLE_FAIL:-0}" = 1 ]; then printf '0\n'; else printf '1\n'; fi + ;; + RESTORE\ *) + case "$query" in + *" SETTINGS id='${EXPECTED_FAKE_RESTORE_OPERATION_ID:?}' ASYNC") + operation="${EXPECTED_FAKE_RESTORE_OPERATION_ID}" + ;; + *) exit 93 ;; + esac + touch "$state/restored" + printf '%s\tRESTORING\n' "$operation" + ;; + *FROM\ system.backups*) + printf "Disk('backups', 'native-backup.zip')\tRESTORED\t100\t1000\t%s\n" "${EMPTY_SHA256:?}" + ;; + *FROM\ system.disks*) printf '/backups/\n' ;; + *FROM\ system.macros*) + printf 'replica\t%s\nshard\t01\n' "$(cat "$state/replica")" + ;; + *FROM\ system.clusters*) printf 'cluster\trestore-clickhouse\t9000\n' ;; + *FROM\ system.zookeeper*) printf '1\n' ;; + *"SELECT count() FROM system.databases"*) printf '0\n' ;; + *"SELECT count() FROM system.tables"*) printf '0\n' ;; + *"SELECT database,name,engine"*) cat "${FAKE_RESTORED_INVENTORY:?}" ;; + "SELECT 1") printf '1\n' ;; + *) printf 'UNHANDLED QUERY: %s\n' "$query" >&2; exit 92 ;; + esac + ;; + logs) + printf 'bounded fake container log\n' + ;; + rm) + if [ "${1:-}" = -f ]; then shift; fi + name="${1:?}" + rm -f "$state/container-$name" "$state/container-label-$name" \ + "$state/container-network-$name" "$state/container-image-$name" + printf '%s\n' "$name" + ;; + *) exit 99 ;; +esac +""", + encoding="utf-8", + ) + docker.chmod(0o755) + return bin_dir, state_dir, log_path + + +def _environment( + tmp_path: Path, + artifact: Path, + manifest: Path, + inventory: Path, + restored_inventory: Path, +) -> dict[str, str]: + bin_dir, state_dir, log_path = _write_fake_docker(tmp_path) + return os.environ | { + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "TRACE_ID": "trace-restore-test", + "RUN_ID": "run-restore-test", + "WORK_ITEM_ID": "P0-OBS-002", + "CLICKHOUSE_NATIVE_ARTIFACT_PATH": str(artifact), + "CLICKHOUSE_NATIVE_MANIFEST_PATH": str(manifest), + "CLICKHOUSE_NATIVE_SOURCE_INVENTORY_PATH": str(inventory), + "CLICKHOUSE_NATIVE_ARTIFACT_SHA256": hashlib.sha256( + artifact.read_bytes() + ).hexdigest(), + "CLICKHOUSE_NATIVE_OPERATION_ID": "awoooi-backup-operation", + "CLICKHOUSE_NATIVE_EXPECTED_DATABASES": ",".join(EXPECTED_DATABASES), + "CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(tmp_path / "receipts"), + "CLICKHOUSE_RESTORE_POLL_INTERVAL_SECONDS": "1", + "CLICKHOUSE_RESTORE_STARTUP_TIMEOUT_SECONDS": "5", + "CLICKHOUSE_RESTORE_TIMEOUT_SECONDS": "5", + "CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS": "5", + "FAKE_DOCKER_LOG": str(log_path), + "FAKE_DOCKER_STATE": str(state_dir), + "FAKE_CLICKHOUSE_IMAGE_ID": CLICKHOUSE_IMAGE_ID, + "FAKE_ZOOKEEPER_IMAGE_ID": ZOOKEEPER_IMAGE_ID, + "FAKE_RESTORED_INVENTORY": str(restored_inventory), + "EXPECTED_FAKE_RESTORE_OPERATION_ID": "restore-" + + hashlib.sha256( + b"trace-restore-test\0run-restore-test\0P0-OBS-002" + ).hexdigest(), + "EMPTY_SHA256": EMPTY_SHA256, + } + + +def _run(mode: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(SCRIPT), mode], + text=True, + capture_output=True, + env=env, + ) + + +def _status(tmp_path: Path) -> dict[str, object]: + return json.loads( + (tmp_path / "receipts" / "run-restore-test" / "status.json").read_text( + encoding="utf-8" + ) + ) + + +def _assert_ephemeral_resources_absent(state_dir: Path) -> None: + assert not list(state_dir.glob("container-*")) + assert not list(state_dir.glob("volume-*")) + assert not list(state_dir.glob("network-*")) + assert not (state_dir / "staged-artifact-sha256").exists() + + +def test_hook_check_uses_env_contract_without_runtime_creation(tmp_path: Path) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(restored_delta=3), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + + result = _run("--check", env) + + assert result.returncode == 0, result.stderr + assert "terminal=check_pass_no_runtime_write" in result.stdout + docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8") + assert "image inspect" in docker_log + assert "network create" not in docker_log + assert "volume create" not in docker_log + assert not any(line.startswith("run ") for line in docker_log.splitlines()) + status = _status(tmp_path) + assert status["check_pass"] is True + assert status["ephemeral_runtime_created"] is False + assert status["production_network_attached"] is False + assert status["production_restore_performed"] is False + + +def test_apply_restores_on_internal_pair_and_cleans_every_resource( + tmp_path: Path, +) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(restored_delta=3), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + check = _run("--check", env) + assert check.returncode == 0, check.stderr + + result = _run("--apply", env) + + assert result.returncode == 0, result.stderr + assert "terminal=verified" in result.stdout + docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines() + network_create = next( + line for line in docker_lines if line.startswith("network create") + ) + assert "--internal" in network_create + run_lines = [line for line in docker_lines if line.startswith("run ")] + assert len(run_lines) == 3 + assert all("--pull=never" in line for line in run_lines) + assert all("--publish" not in line and " -p " not in line for line in run_lines) + + staging_run = next(line for line in run_lines if "--network none" in line) + zookeeper_run = next(line for line in run_lines if "signoz/zookeeper:3.7.1" in line) + clickhouse_run = next( + line for line in run_lines if "--network-alias restore-clickhouse" in line + ) + assert "--user 0:0" in staging_run + assert "--entrypoint /bin/sh" in staging_run + assert ( + f"type=bind,src={artifact},dst=/tmp/awoooi-source-native-backup.zip,readonly" + in staging_run + ) + assert sum(f"src={artifact}" in line for line in run_lines) == 1 + assert "type=volume,src=awoooi-ch-restore-" in staging_run + assert "-backup,dst=/backups" in staging_run + assert "ALLOW_ANONYMOUS_LOGIN=yes" in zookeeper_run + assert "--network awoooi-ch-restore-" in zookeeper_run + assert "--network awoooi-ch-restore-" in clickhouse_run + assert str(artifact) not in clickhouse_run + assert "type=volume,src=awoooi-ch-restore-" in clickhouse_run + assert "-backup,dst=/backups" in clickhouse_run + assert "dst=/backups,readonly" not in clickhouse_run + assert ( + "dst=/etc/clickhouse-server/config.d/awoooi-backup-disk.xml,readonly" + in clickhouse_run + ) + assert ( + "dst=/etc/clickhouse-server/config.d/awoooi-restore-isolation.xml,readonly" + in clickhouse_run + ) + hash_readback = next( + line + for line in docker_lines + if line.startswith("exec ") and "sha256sum /backups/native-backup.zip" in line + ) + assert "exec awoooi-ch-restore-" in hash_readback + restore_line = next(line for line in docker_lines if "RESTORE DATABASE" in line) + assert "allow_non_empty_tables" not in restore_line + assert " SETTINGS id='restore-" in restore_line + assert "id={operation_id:String}" not in restore_line + macro_line = next(line for line in docker_lines if "FROM system.macros" in line) + assert "SELECT macro,substitution" in macro_line + assert "SELECT name,value" not in macro_line + state_dir = Path(env["FAKE_DOCKER_STATE"]) + _assert_ephemeral_resources_absent(state_dir) + status = _status(tmp_path) + assert status["terminal"] == "verified" + assert status["check_pass"] is True + assert status["restore_terminal"] == "RESTORED" + assert status["manifest_parity"] == 1 + assert status["check_table_count"] == status["check_table_pass_count"] == 7 + assert status["critical_nonzero_count"] == 4 + assert status["row_delta"] == 3 + expected_hash = hashlib.sha256(artifact.read_bytes()).hexdigest() + assert status["artifact_sha256"] == expected_hash + assert status["staged_artifact_sha256"] == expected_hash + assert status["clickhouse_artifact_sha256"] == expected_hash + assert status["artifact_staging_verified"] is True + assert status["clickhouse_artifact_readback_verified"] is True + assert status["staging_network_mode"] == "none" + assert status["artifact_source_mount"] == "read_only_staging_only" + assert status["backup_volume_mount"] == "writable_ephemeral" + assert status["cleanup_verified"] is True + + +def test_apply_failure_still_cleans_pair_network_and_volumes(tmp_path: Path) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + assert _run("--check", env).returncode == 0 + env["FAKE_CHECK_TABLE_FAIL"] = "1" + + result = _run("--apply", env) + + assert result.returncode == 1 + assert "terminal=failed" in result.stdout + state_dir = Path(env["FAKE_DOCKER_STATE"]) + _assert_ephemeral_resources_absent(state_dir) + status = _status(tmp_path) + assert status["terminal"] == "failed" + assert status["cleanup_verified"] is True + assert status["apply_pass"] is False + + +def test_artifact_staging_failure_cleans_exact_resources(tmp_path: Path) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + assert _run("--check", env).returncode == 0 + env["FAKE_STAGING_FAIL"] = "1" + + result = _run("--apply", env) + + assert result.returncode == 1 + _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"])) + status = _status(tmp_path) + assert status["terminal"] == "failed" + assert status["reason"] == "isolated_artifact_staging_failed" + assert status["artifact_staging_verified"] is False + assert status["cleanup_verified"] is True + + +def test_artifact_staging_hang_is_bounded_and_cleans_exact_resources( + tmp_path: Path, +) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1" + assert _run("--check", env).returncode == 0 + env["FAKE_STAGING_HANG"] = "1" + + result = subprocess.run( + ["bash", str(SCRIPT), "--apply"], + text=True, + capture_output=True, + env=env, + timeout=12, + ) + + assert result.returncode == 1 + _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"])) + status = _status(tmp_path) + assert status["terminal"] == "failed" + assert status["reason"] == "isolated_artifact_staging_failed" + assert status["artifact_staging_verified"] is False + assert status["cleanup_verified"] is True + + +def test_staged_artifact_hash_mismatch_fails_before_isolated_services( + tmp_path: Path, +) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + assert _run("--check", env).returncode == 0 + env["FAKE_STAGING_HASH_MISMATCH"] = "1" + + result = _run("--apply", env) + + assert result.returncode == 1 + docker_lines = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8").splitlines() + assert not any( + "signoz/zookeeper:3.7.1" in line + for line in docker_lines + if line.startswith("run ") + ) + _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"])) + status = _status(tmp_path) + assert status["reason"] == "staged_artifact_hash_mismatch" + assert status["cleanup_verified"] is True + + +def test_clickhouse_artifact_hash_readback_mismatch_fails_and_cleans( + tmp_path: Path, +) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + assert _run("--check", env).returncode == 0 + env["FAKE_CLICKHOUSE_HASH_MISMATCH"] = "1" + + result = _run("--apply", env) + + assert result.returncode == 1 + _assert_ephemeral_resources_absent(Path(env["FAKE_DOCKER_STATE"])) + status = _status(tmp_path) + assert status["reason"] == "isolated_clickhouse_artifact_hash_mismatch" + assert status["artifact_staging_verified"] is True + assert status["clickhouse_artifact_readback_verified"] is False + assert status["clickhouse_artifact_sha256"] == "f" * 64 + assert status["cleanup_verified"] is True + + +def test_docker_daemon_failure_cannot_claim_cleanup_verified(tmp_path: Path) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + assert _run("--check", env).returncode == 0 + env["FAKE_DOCKER_INFO_FAIL"] = "1" + + result = _run("--apply", env) + + assert result.returncode == 1 + status = _status(tmp_path) + assert status["terminal"] == "failed" + assert status["cleanup_verified"] is False + assert status["reason"] == "ephemeral_resource_cleanup_failed" + + +def test_cleanup_label_inspect_hang_is_bounded_and_fails_closed(tmp_path: Path) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + env["CLICKHOUSE_RESTORE_COMMAND_TIMEOUT_SECONDS"] = "1" + assert _run("--check", env).returncode == 0 + env["FAKE_CHECK_TABLE_FAIL"] = "1" + env["FAKE_LABEL_INSPECT_HANG"] = "1" + + result = subprocess.run( + ["bash", str(SCRIPT), "--apply"], + text=True, + capture_output=True, + env=env, + timeout=12, + ) + + assert result.returncode == 1 + status = _status(tmp_path) + assert status["terminal"] == "failed" + assert status["cleanup_verified"] is False + assert status["reason"] == "ephemeral_resource_cleanup_failed" + + +def test_check_fails_closed_on_local_image_id_drift(tmp_path: Path) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + env["FAKE_CLICKHOUSE_IMAGE_ID"] = "sha256:" + "0" * 64 + + result = _run("--check", env) + + assert result.returncode == 1 + assert "clickhouse_local_image_id_mismatch" in result.stderr + docker_log = Path(env["FAKE_DOCKER_LOG"]).read_text(encoding="utf-8") + assert "network create" not in docker_log + assert "volume create" not in docker_log + + +def test_flat_host_deploy_accepts_reviewed_absolute_helper_and_config_paths( + tmp_path: Path, +) -> None: + artifact, manifest, inventory = _write_backup_fixture(tmp_path) + restored = tmp_path / "restored.tsv" + restored.write_text(_inventory(), encoding="utf-8") + env = _environment(tmp_path, artifact, manifest, inventory, restored) + host_root = tmp_path / "host-backup" + host_root.mkdir() + helper = host_root / "clickhouse-restore-inventory.py" + backup_config = host_root / "backup_disk.xml" + cluster_config = host_root / "isolated-cluster.xml" + shutil.copyfile(INVENTORY_HELPER, helper) + shutil.copyfile( + REPO_ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml", backup_config + ) + shutil.copyfile( + REPO_ROOT / "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml", + cluster_config, + ) + env.update( + { + "CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(helper), + "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_config), + "CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config), + } + ) + + result = _run("--check", env) + + assert result.returncode == 0, result.stderr + assert "terminal=check_pass_no_runtime_write" in result.stdout + + +def test_inventory_compare_allows_online_row_delta_but_not_engine_drift( + tmp_path: Path, +) -> None: + source = tmp_path / "source.tsv" + restored = tmp_path / "restored.tsv" + source.write_text(_inventory(), encoding="utf-8") + restored.write_text(_inventory(restored_delta=9), encoding="utf-8") + + accepted = subprocess.run( + [ + "python3", + str(INVENTORY_HELPER), + "compare", + "--source-inventory", + str(source), + "--restored-inventory", + str(restored), + ], + text=True, + capture_output=True, + ) + assert accepted.returncode == 0, accepted.stderr + assert "manifest_parity=1" in accepted.stdout + assert "row_delta=9" in accepted.stdout + + restored.write_text(_inventory(engine_mismatch=True), encoding="utf-8") + rejected = subprocess.run( + [ + "python3", + str(INVENTORY_HELPER), + "compare", + "--source-inventory", + str(source), + "--restored-inventory", + str(restored), + ], + text=True, + capture_output=True, + ) + assert rejected.returncode == 1 + assert "table_engine_mismatch_signoz_meter_meter" in rejected.stderr + + restored.write_text(_inventory(schema_mismatch=True), encoding="utf-8") + schema_rejected = subprocess.run( + [ + "python3", + str(INVENTORY_HELPER), + "compare", + "--source-inventory", + str(source), + "--restored-inventory", + str(restored), + ], + text=True, + capture_output=True, + ) + assert schema_rejected.returncode == 1 + assert "table_schema_mismatch_signoz_meter_meter" in schema_rejected.stderr diff --git a/scripts/backup/tests/test_signoz_backup_canary_wrapper.py b/scripts/backup/tests/test_signoz_backup_canary_wrapper.py new file mode 100644 index 000000000..f2ba902af --- /dev/null +++ b/scripts/backup/tests/test_signoz_backup_canary_wrapper.py @@ -0,0 +1,528 @@ +from __future__ import annotations + +import json +import os +import stat +import subprocess +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh" +PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-devops.yml" +ANSIBLE_VALIDATE = ROOT / "scripts" / "ops" / "ansible-validate.sh" +READINESS_AUDIT = ( + ROOT / "scripts" / "reboot-recovery" / "reboot-recovery-readiness-audit.sh" +) + + +def _write_executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(0o755) + + +def _monitor_source(cooldown_dir: Path, *, omit_contract: bool = False) -> str: + cooldown_log = ( + 'log "COOLDOWN: ${container} skipped"' + if not omit_contract + else 'log "maintenance skip for ${container}"' + ) + return f"""\ +#!/bin/bash +: "${{ACTION_COOLDOWN_SECONDS:=${{SEND_COOLDOWN_SECONDS}}}}" +: "${{COOLDOWN_DIR:={cooldown_dir}}}" +: "${{EXCLUDE_CONTAINERS:=signoz-clickhouse}}" +is_in_cooldown() {{ + local container="$1" + local cooldown_file="${{COOLDOWN_DIR}}/${{container}}.cooldown" + local last_sent now elapsed + last_sent=$(cat "$cooldown_file") + now=$(date +%s) + elapsed=$(( now - last_sent )) + if (( elapsed < ACTION_COOLDOWN_SECONDS )); then + {cooldown_log} + return 0 + fi + return 1 +}} +set_cooldown() {{ :; }} +# A second monitor path may use the same elapsed expression. The wrapper must +# validate the expression inside is_in_cooldown, not require global uniqueness. +# elapsed=$(( now - last_sent )) +while read -r container_name; do + is_in_cooldown "$container_name" && continue + set_cooldown "$container_name" + container="$container_name" + log "AUTO_REPAIR: docker restart ${{container}}" + if docker restart "$container"; then + : + fi +done +""" + + +def _install_fake_commands(fake_bin: Path) -> None: + _write_executable( + fake_bin / "crontab", + """\ +#!/bin/bash +set -eu +[ "${1:-}" = "-l" ] +printf '*/5 * * * * %s >> %s 2>&1\n' \ + "${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}" +if [ "${FAKE_DUPLICATE_CRON:-0}" = "1" ]; then + printf '*/5 * * * * %s >> %s 2>&1\n' \ + "${TEST_MONITOR_SCRIPT:?}" "${TEST_MONITOR_LOG:?}" +fi +""", + ) + _write_executable( + fake_bin / "docker", + """\ +#!/bin/bash +set -eu +[ "${1:-}" = "inspect" ] || exit 70 +case " $* " in + *"{{.State.StartedAt}}"*) + printf 'sha256:collector-test-id\\t%s\\t2026-07-15T00:00:00Z\\t0\\thealthy\\n' \ + "$(cat "${TEST_COLLECTOR_STATE:?}")" + ;; + *"{{.Id}}"*) printf 'sha256:collector-test-id\n' ;; + *"{{.State.Running}}"*) cat "${TEST_COLLECTOR_STATE:?}" ;; + *) exit 71 ;; +esac +""", + ) + _write_executable( + fake_bin / "ss", + """\ +#!/bin/bash +set -eu +while IFS= read -r port; do + [ -n "$port" ] || continue + printf 'LISTEN 0 4096 *:%s *:*\n' "$port" +done < "${TEST_LISTENERS:?}" +""", + ) + _write_executable( + fake_bin / "timeout", + """\ +#!/bin/bash +set -eu +while [[ "${1:-}" == --kill-after=* ]]; do + shift +done +[ "$#" -gt 1 ] +shift +exec "$@" +""", + ) + _write_executable( + fake_bin / "flock", + """\ +#!/bin/bash +exit 0 +""", + ) + _write_executable( + fake_bin / "pgrep", + """\ +#!/bin/bash +exit 1 +""", + ) + _write_executable( + fake_bin / "stat", + """\ +#!/usr/bin/env python3 +import os +import stat +import sys + +if len(sys.argv) != 4 or sys.argv[1] != "-c": + raise SystemExit(64) +value = os.stat(sys.argv[3]) +formats = { + "%a": format(stat.S_IMODE(value.st_mode), "o"), + "%u": str(value.st_uid), + "%g": str(value.st_gid), + "%s": str(value.st_size), + "%d:%i": f"{value.st_dev}:{value.st_ino}", +} +if sys.argv[2] not in formats: + raise SystemExit(65) +print(formats[sys.argv[2]]) +""", + ) + _write_executable( + fake_bin / "sha256sum", + """\ +#!/usr/bin/env python3 +import hashlib +import sys + +if len(sys.argv) != 2: + raise SystemExit(64) +with open(sys.argv[1], "rb") as source: + digest = hashlib.sha256(source.read()).hexdigest() +print(f"{digest} {sys.argv[1]}") +""", + ) + + +def _backup_source() -> str: + return """\ +#!/bin/bash +set -eu +[ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80 +[ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ] +printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \ + > "${TEST_BACKUP_IDENTITIES:?}" +if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then + cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}" + printf 'false\n' > "${TEST_COLLECTOR_STATE:?}" + case "${FAKE_MONITOR_MODE:-cooldown}" in + cooldown) + printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \ + >> "${TEST_MONITOR_LOG:?}" + printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \ + >> "${TEST_MONITOR_LOG:?}" + ;; + auto_repair) + printf '[test] DETECTED: signoz-otel-collector state=exited health=none\n' \ + >> "${TEST_MONITOR_LOG:?}" + printf '[test] COOLDOWN: signoz-otel-collector skipped\n' \ + >> "${TEST_MONITOR_LOG:?}" + printf '[test] AUTO_REPAIR: docker restart signoz-otel-collector\n' \ + >> "${TEST_MONITOR_LOG:?}" + ;; + none) : ;; + *) exit 81 ;; + esac + sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}" + printf 'true\n' > "${TEST_COLLECTOR_STATE:?}" +else + sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}" +fi +if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then + printf '4317\n' > "${TEST_LISTENERS:?}" +fi +printf '略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)\n' +if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then + printf '[SUCCESS] ========== SignOz 備份完成 (1s) ==========\n' +else + printf '[SUCCESS] ========== SigNoz ClickHouse native 備份完成 (1s) ==========\n' +fi +""" + + +def _run_canary( + tmp_path: Path, + *, + mode: str = "apply", + prior_lease: bool = True, + monitor_mode: str = "cooldown", + backup_sleep_seconds: float = 0.2, + monitor_interval_seconds: int = 300, + omit_monitor_contract: bool = False, + duplicate_cron: bool = False, + drop_listener: bool = False, + expect_collector_stop: bool = True, +) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]: + fake_bin = tmp_path / "bin" + cooldown_dir = tmp_path / "cooldown" + receipt_root = tmp_path / "receipts" + fake_bin.mkdir() + cooldown_dir.mkdir() + receipt_root.mkdir() + + monitor_script = tmp_path / "docker-health-monitor.sh" + monitor_log = tmp_path / "monitor.log" + backup_script = tmp_path / "backup-signoz.sh" + collector_state = tmp_path / "collector.state" + listeners = tmp_path / "listeners" + identities = tmp_path / "backup-identities.tsv" + lease_observed = tmp_path / "lease-observed.txt" + cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown" + run_id = f"canary-{tmp_path.name}" + + _write_executable( + monitor_script, + _monitor_source(cooldown_dir, omit_contract=omit_monitor_contract), + ) + _write_executable(backup_script, _backup_source()) + _install_fake_commands(fake_bin) + monitor_log.write_text("[test] monitor seed\n", encoding="utf-8") + collector_state.write_text("true\n", encoding="utf-8") + listeners.write_text("4317\n4318\n", encoding="utf-8") + if prior_lease: + cooldown_file.write_text("123\n", encoding="utf-8") + cooldown_file.chmod(0o640) + + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "TRACE_ID": "trace-P0-OBS-002", + "RUN_ID": run_id, + "WORK_ITEM_ID": "P0-OBS-002", + "SIGNOZ_CANARY_BACKUP_SCRIPT": str(backup_script), + "SIGNOZ_CANARY_MONITOR_SCRIPT": str(monitor_script), + "SIGNOZ_CANARY_MONITOR_LOG": str(monitor_log), + "SIGNOZ_CANARY_COOLDOWN_DIR": str(cooldown_dir), + "SIGNOZ_CANARY_RECEIPT_ROOT": str(receipt_root), + "SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"), + "SIGNOZ_CANARY_TIMEOUT_SECONDS": "5", + "SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2", + "SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2", + "SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds), + "SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05", + "SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"), + "TEST_MONITOR_SCRIPT": str(monitor_script), + "TEST_MONITOR_LOG": str(monitor_log), + "TEST_COLLECTOR_STATE": str(collector_state), + "TEST_LISTENERS": str(listeners), + "TEST_BACKUP_IDENTITIES": str(identities), + "TEST_COOLDOWN_FILE": str(cooldown_file), + "TEST_LEASE_OBSERVED": str(lease_observed), + "FAKE_MONITOR_MODE": monitor_mode, + "FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds), + "FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0", + "FAKE_DROP_LISTENER": "1" if drop_listener else "0", + "FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0", + } + started_at = int(time.time()) + result = subprocess.run( + ["bash", str(WRAPPER), f"--{mode}"], + env=env, + text=True, + capture_output=True, + timeout=15, + check=False, + ) + paths = { + "cooldown": cooldown_file, + "receipts": receipt_root / run_id / "receipts.jsonl", + "rollback_metadata": receipt_root / run_id / "cooldown.rollback.json", + "identities": identities, + "lease_observed": lease_observed, + "collector_state": collector_state, + "listeners": listeners, + } + return result, paths, str(started_at) + + +def _receipts(path: Path) -> list[dict[str, object]]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + + +def _terminal(receipts: list[dict[str, object]], stage: str) -> str: + return str([item for item in receipts if item["stage"] == stage][-1]["terminal"]) + + +def test_existing_lease_is_atomically_replaced_and_exactly_restored( + tmp_path: Path, +) -> None: + result, paths, started_at = _run_canary(tmp_path) + + assert result.returncode == 0, result.stdout + result.stderr + assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" + assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640 + assert int(paths["lease_observed"].read_text(encoding="utf-8")) >= ( + int(started_at) + 7 + ) + identities = paths["identities"].read_text(encoding="utf-8").split("\t") + assert identities[0] == "trace-P0-OBS-002" + assert identities[1].startswith("canary-") + assert identities[2] == "P0-OBS-002\n" + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "rollback") == "restored" + assert _terminal(receipts, "terminal") == "pass" + metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8")) + assert metadata["trace_id"] == "trace-P0-OBS-002" + assert metadata["prior_present"] == 1 + + stages = [str(item["stage"]) for item in receipts] + ordered = [ + "sensor_source", + "normalized_asset_identity", + "source_of_truth_diff", + "ai_decision", + "risk_policy", + "check", + "execution", + "post_verifier", + "rollback", + "closure_writeback", + "terminal", + ] + assert [stages.index(stage) for stage in ordered] == sorted( + stages.index(stage) for stage in ordered + ) + + +def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None: + result, paths, _ = _run_canary(tmp_path, prior_lease=False) + + assert result.returncode == 0, result.stdout + result.stderr + assert not paths["cooldown"].exists() + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "rollback") == "restored" + assert _terminal(receipts, "terminal") == "pass" + + +def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease( + tmp_path: Path, +) -> None: + result, paths, _ = _run_canary(tmp_path, expect_collector_stop=False) + + assert result.returncode == 0, result.stdout + result.stderr + assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" + assert not paths["lease_observed"].exists() + assert paths["collector_state"].read_text(encoding="utf-8") == "true\n" + receipts = _receipts(paths["receipts"]) + assert any( + item["stage"] == "lease" and item["terminal"] == "not_required" + for item in receipts + ) + assert _terminal(receipts, "rollback") == "no_write" + assert _terminal(receipts, "terminal") == "pass" + + +def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract( + tmp_path: Path, +) -> None: + result, paths, _ = _run_canary( + tmp_path, + expect_collector_stop=False, + omit_monitor_contract=True, + duplicate_cron=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert not paths["lease_observed"].exists() + receipts = _receipts(paths["receipts"]) + source_diff = [ + item for item in receipts if item["stage"] == "source_of_truth_diff" + ][-1] + assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"] + assert _terminal(receipts, "terminal") == "pass" + + +def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None: + result, paths, _ = _run_canary(tmp_path, monitor_mode="auto_repair") + + assert result.returncode != 0 + assert "monitor_auto_repair_contaminated_canary" in result.stderr + assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "rollback") == "restored" + assert _terminal(receipts, "terminal") == "failed" + + +def test_cron_crossing_requires_monitor_cooldown_receipt(tmp_path: Path) -> None: + result, paths, _ = _run_canary( + tmp_path, + monitor_mode="none", + backup_sleep_seconds=1.2, + monitor_interval_seconds=1, + ) + + assert result.returncode != 0 + assert "cron_crossed_without_collector_cooldown_receipt" in result.stderr + assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" + + +def test_cron_crossing_with_detected_and_cooldown_receipts_passes( + tmp_path: Path, +) -> None: + result, paths, _ = _run_canary( + tmp_path, + monitor_mode="cooldown", + backup_sleep_seconds=1.2, + monitor_interval_seconds=1, + ) + + assert result.returncode == 0, result.stdout + result.stderr + receipts = _receipts(paths["receipts"]) + monitor_receipt = [ + item for item in receipts if item["stage"] == "monitor_verifier" + ][-1] + assert monitor_receipt["terminal"] == "pass" + assert "cron_crossed_1" in str(monitor_receipt["detail"]) + + +def test_missing_monitor_contract_fails_before_lease_or_backup(tmp_path: Path) -> None: + result, paths, _ = _run_canary(tmp_path, omit_monitor_contract=True) + + assert result.returncode != 0 + assert "monitor_contract_ambiguous_fragment_count_0" in result.stderr + assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" + assert not paths["lease_observed"].exists() + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "rollback") == "not_armed" + assert _terminal(receipts, "terminal") == "failed" + + +def test_ambiguous_cron_contract_fails_before_lease_or_backup( + tmp_path: Path, +) -> None: + result, paths, _ = _run_canary(tmp_path, duplicate_cron=True) + + assert result.returncode != 0 + assert "monitor_cron_contract_ambiguous_count_2" in result.stderr + assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" + assert not paths["lease_observed"].exists() + + +def test_post_backup_listener_failure_fails_and_restores_lease( + tmp_path: Path, +) -> None: + result, paths, _ = _run_canary(tmp_path, drop_listener=True) + + assert result.returncode != 0 + assert "collector_post_backup_runtime_failed" in result.stderr + assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "rollback") == "restored" + assert _terminal(receipts, "terminal") == "failed" + + +def test_wrapper_deployment_and_static_contracts_are_owned() -> None: + wrapper = WRAPPER.read_text(encoding="utf-8") + assert "BACKUP_SKIP_RETENTION_CLEANUP=1" in wrapper + assert 'TRACE_ID="${TRACE_ID:-}"' in wrapper + assert 'RUN_ID="${RUN_ID:-}"' in wrapper + assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper + assert "monitor_auto_repair_contaminated_canary" in wrapper + assert "collector_post_backup_runtime_failed" in wrapper + assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8") + assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text( + encoding="utf-8" + ) + assert str(WRAPPER.relative_to(ROOT)) in READINESS_AUDIT.read_text(encoding="utf-8") + + +def test_check_mode_proves_contract_without_lease_or_backup_write( + tmp_path: Path, +) -> None: + result, paths, _ = _run_canary(tmp_path, mode="check") + + assert result.returncode == 0, result.stdout + result.stderr + assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" + assert stat.S_IMODE(paths["cooldown"].stat().st_mode) == 0o640 + assert not paths["identities"].exists() + assert not paths["lease_observed"].exists() + check_receipts = _receipts(paths["receipts"]) + assert _terminal(check_receipts, "rollback") == "no_write" + assert _terminal(check_receipts, "terminal") == "check_pass_no_write" + + +def test_wrapper_is_valid_bash() -> None: + result = subprocess.run( + ["bash", "-n", str(WRAPPER)], + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr diff --git a/scripts/ops/ansible-validate.sh b/scripts/ops/ansible-validate.sh index c84cd6d90..36a1ef359 100755 --- a/scripts/ops/ansible-validate.sh +++ b/scripts/ops/ansible-validate.sh @@ -54,6 +54,9 @@ bash -n \ scripts/backup/backup-langfuse.sh \ scripts/backup/backup-monitoring.sh \ scripts/backup/backup-signoz.sh \ + scripts/backup/clickhouse-native-backup.sh \ + scripts/backup/clickhouse-native-restore-drill.sh \ + scripts/backup/run-signoz-backup-canary.sh \ scripts/backup/backup-open-webui.sh \ scripts/backup/backup-clawbot.sh \ scripts/backup/backup-configs.sh \ @@ -81,6 +84,7 @@ python3 -m py_compile \ scripts/ops/backup-alert-label-contract-check.py \ scripts/ops/backup-alert-live-visibility-check.py \ scripts/ops/recovery-scorecard-contract-check.py \ + scripts/backup/clickhouse-restore-inventory.py \ scripts/ops/doc-secrets-sanity-check.py echo "Python 語法 OK" diff --git a/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh new file mode 100755 index 000000000..22a79e705 --- /dev/null +++ b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh @@ -0,0 +1,594 @@ +#!/usr/bin/env bash +# P0-OBS-002 - controlled deployment of the host110 ClickHouse native BACKUP disk. +# +# This script owns only an additive Compose override and one ClickHouse config +# fragment. It never runs BACKUP/RESTORE, reads a dotenv/secret source, pulls an +# image, or recreates another service. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml" +LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml" + +TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}" +REMOTE_DIR="/home/wooo/signoz/deploy/docker" +REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml" +REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml" +REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml" +REMOTE_PROJECT="signoz" +REMOTE_SERVICE="clickhouse" +CLICKHOUSE_CONTAINER="signoz-clickhouse" +COLLECTOR_CONTAINER="signoz-otel-collector" +SIGNOZ_CONTAINER="signoz" +ZOOKEEPER_CONTAINER="signoz-zookeeper-1" +HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse" +CONTAINER_BACKUP_DIR="/backups" +CLICKHOUSE_DISK="backups" +CLICKHOUSE_UID="101" +CLICKHOUSE_GID="101" +RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk" +API_URL="http://127.0.0.1:8080/api/v1/health" +SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +SAFE_HOME="/home/wooo" + +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout=8 + -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=2 +) + +usage() { + cat <<'USAGE' +Usage: deploy-signoz-clickhouse-backup-disk.sh [--check|--apply] + +Required environment: + TRACE_ID Path-safe controlled-apply trace identifier + RUN_ID Unique path-safe execution identifier + WORK_ITEM_ID Path-safe work-item identifier + +--check performs local source validation and remote read-only Compose/runtime +checks. --apply atomically installs the two repo-owned artifacts, recreates only +the ClickHouse service with --pull never, independently verifies runtime state, +and restores the exact prior managed Compose state on failure. +USAGE +} + +MODE="${1:---check}" +case "${MODE}" in + --check|--apply) ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; exit 64 ;; +esac +[ "$#" -le 1 ] || { usage >&2; exit 64; } + +TRACE_ID="${TRACE_ID:-}" +RUN_ID="${RUN_ID:-}" +WORK_ITEM_ID="${WORK_ITEM_ID:-}" + +valid_identity() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] +} + +for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do + valid_identity "${value}" || { + printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2 + exit 64 + } +done + +for command_name in python3 scp ssh; do + command -v "${command_name}" >/dev/null 2>&1 || { + printf 'required local command missing: %s\n' "${command_name}" >&2 + exit 69 + } +done + +hash_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +validate_local_sources() { + [ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ] + [ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ] + + python3 - "${LOCAL_CONFIG}" <<'PY' +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + +path = Path(sys.argv[1]) +root = ET.parse(path).getroot() +assert root.tag == "clickhouse" +disk = root.find("./storage_configuration/disks/backups") +assert disk is not None +assert disk.findtext("type") == "local" +assert disk.findtext("path") == "/backups/" +backups = root.find("./backups") +assert backups is not None +assert backups.findtext("allowed_disk") == "backups" +assert backups.findtext("allowed_path") == "/backups/" +assert backups.findtext("allow_concurrent_backups") == "false" +assert backups.findtext("allow_concurrent_restores") == "false" +PY + + [ "$(grep -F -c ' clickhouse:' "${LOCAL_OVERRIDE}")" -eq 1 ] + [ "$(grep -F -c 'source: /backup/staging/signoz-clickhouse' "${LOCAL_OVERRIDE}")" -eq 1 ] + [ "$(grep -F -c 'target: /backups' "${LOCAL_OVERRIDE}")" -eq 1 ] + [ "$(grep -F -c 'target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml' "${LOCAL_OVERRIDE}")" -eq 1 ] + ! grep -Eq '^[[:space:]]*(image|build|command|environment):' "${LOCAL_OVERRIDE}" +} + +validate_local_sources +OVERRIDE_SHA="$(hash_file "${LOCAL_OVERRIDE}")" +CONFIG_SHA="$(hash_file "${LOCAL_CONFIG}")" + +local_receipt() { + local phase="$1" result="$2" detail="$3" + printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}" +} + +remote_candidate_compose_check() { + ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" \ + "env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \ + < "${LOCAL_OVERRIDE}" +} + +remote_read_only_check() { + ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ + "${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK' +set -euo pipefail + +TRACE_ID="$1" +RUN_ID="$2" +WORK_ITEM_ID="$3" +EXPECTED_OVERRIDE_SHA="$4" +EXPECTED_CONFIG_SHA="$5" +REMOTE_DIR="/home/wooo/signoz/deploy/docker" +REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml" +REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml" +REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml" +HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse" +SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +SAFE_HOME="/home/wooo" + +file_hash_or_absent() { + if [ -f "$1" ] && [ ! -L "$1" ]; then + sha256sum "$1" | awk '{print $1}' + elif [ ! -e "$1" ]; then + printf 'absent\n' + else + printf 'unsafe\n' + fi +} + +[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ] +for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do + docker inspect "${container}" >/dev/null +done +env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \ + config -q >/dev/null +env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \ + config --services | grep -qx clickhouse + +override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")" +config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")" +override_match=false +config_match=false +[ "${override_hash}" = "${EXPECTED_OVERRIDE_SHA}" ] && override_match=true +[ "${config_hash}" = "${EXPECTED_CONFIG_SHA}" ] && config_match=true + +host_dir=absent +if [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ]; then + host_dir=present +elif [ -e "${HOST_BACKUP_DIR}" ]; then + host_dir=unsafe +fi + +disk_count="$(docker exec signoz-clickhouse clickhouse-client --query \ + "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" +mount_count="$(docker inspect --format \ + '{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \ + signoz-clickhouse | tr -cd '1' | wc -c | xargs)" +collector_running="$(docker inspect --format '{{.State.Running}}' signoz-otel-collector)" +collector_restarts="$(docker inspect --format '{{.RestartCount}}' signoz-otel-collector)" + +printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \ + "${config_match}" "${host_dir}" "${disk_count}" "${mount_count}" \ + "${collector_running}" "${collector_restarts}" +REMOTE_CHECK +} + +local_receipt sensor_source pass "override_sha_${OVERRIDE_SHA}_config_sha_${CONFIG_SHA}" +local_receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_disk_backups" +remote_candidate_compose_check +remote_read_only_check +local_receipt check pass "compose_config_q_no_output_no_write" + +if [ "${MODE}" = "--check" ]; then + local_receipt terminal check_pass_no_write "source_and_remote_preflight_valid" + exit 0 +fi + +STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}" +STAGE_CREATED=0 + +cleanup_stage() { + if [ "${STAGE_CREATED}" -eq 1 ]; then + ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true +set -u +stage_dir="$1" +rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml" +rmdir "${stage_dir}" 2>/dev/null || true +REMOTE_CLEANUP + fi +} +trap cleanup_stage EXIT + +ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' +set -euo pipefail +stage_dir="$1" +[ ! -e "${stage_dir}" ] +umask 077 +mkdir -m 0700 "${stage_dir}" +REMOTE_STAGE +STAGE_CREATED=1 + +scp -q "${SSH_OPTS[@]}" "${LOCAL_OVERRIDE}" \ + "${TARGET_HOST}:${STAGE_DIR}/override.yaml" +scp -q "${SSH_OPTS[@]}" "${LOCAL_CONFIG}" \ + "${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml" + +ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ + "${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY' +set -euo pipefail + +TRACE_ID="$1" +RUN_ID="$2" +WORK_ITEM_ID="$3" +EXPECTED_OVERRIDE_SHA="$4" +EXPECTED_CONFIG_SHA="$5" +STAGE_DIR="$6" + +REMOTE_DIR="/home/wooo/signoz/deploy/docker" +REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml" +REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml" +REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml" +STAGE_OVERRIDE="${STAGE_DIR}/override.yaml" +STAGE_CONFIG="${STAGE_DIR}/backup_disk.xml" +REMOTE_PROJECT="signoz" +REMOTE_SERVICE="clickhouse" +CLICKHOUSE_CONTAINER="signoz-clickhouse" +COLLECTOR_CONTAINER="signoz-otel-collector" +SIGNOZ_CONTAINER="signoz" +ZOOKEEPER_CONTAINER="signoz-zookeeper-1" +HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse" +RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk" +RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" +RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl" +API_URL="http://127.0.0.1:8080/api/v1/health" +SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +SAFE_HOME="/home/wooo" +LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock" + +APPLY_STARTED=0 +DEPLOY_VERIFIED=0 +HOST_DIR_CREATED=0 +PRIOR_OVERRIDE_PRESENT=0 +PRIOR_CONFIG_PRESENT=0 +PRIOR_DISK_COUNT=0 +BEFORE_IMAGE_ID="" +BEFORE_DATA_VOLUME="" +BEFORE_COLLECTOR_ID="" +BEFORE_COLLECTOR_RESTARTS="" +BEFORE_SIGNOZ_ID="" +BEFORE_SIGNOZ_RESTARTS="" +BEFORE_ZOOKEEPER_ID="" +BEFORE_ZOOKEEPER_RESTARTS="" + +receipt() { + local phase="$1" result="$2" detail="$3" + local line + line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"%s","result":"%s","detail":"%s"}' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")" + printf '%s\n' "${line}" + printf '%s\n' "${line}" >> "${RECEIPT_LOG}" +} + +compose_base_config() { + env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ + -f "${REMOTE_BASE}" config -q >/dev/null 2>&1 +} + +compose_managed_config() { + env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ + -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" config -q >/dev/null 2>&1 +} + +compose_stage_config() { + env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ + -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config -q >/dev/null 2>&1 +} + +recreate_managed() { + timeout --signal=TERM --kill-after=30 300 \ + env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ + -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \ + up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \ + >/dev/null 2>&1 +} + +recreate_base() { + timeout --signal=TERM --kill-after=30 300 \ + env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ + -f "${REMOTE_BASE}" \ + up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \ + >/dev/null 2>&1 +} + +wait_clickhouse() { + local attempt running health + for attempt in $(seq 1 36); do + running="$(docker inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)" + health="$(docker inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \ + "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)" + if [ "${running}" = true ] && [ "${health}" = healthy ] \ + && timeout --kill-after=5 10 docker exec "${CLICKHOUSE_CONTAINER}" \ + clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then + return 0 + fi + sleep 5 + done + return 1 +} + +listener_up() { + local port="$1" + ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' +} + +wait_api_and_listeners() { + local attempt api_code + for attempt in $(seq 1 30); do + api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}" 2>/dev/null || true)" + if [ "${api_code}" = 200 ] && listener_up 4317 && listener_up 4318 && listener_up 8080; then + return 0 + fi + sleep 5 + done + return 1 +} + +restore_managed_files() { + local tmp + if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then + tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}" + install -m 0644 "${RECEIPT_DIR}/prior-override.yaml" "${tmp}" + mv -f "${tmp}" "${REMOTE_OVERRIDE}" + else + rm -f "${REMOTE_OVERRIDE}" + fi + + if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then + tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}" + install -m 0644 "${RECEIPT_DIR}/prior-backup-disk.xml" "${tmp}" + mv -f "${tmp}" "${REMOTE_CONFIG}" + else + rm -f "${REMOTE_CONFIG}" + fi +} + +rollback_deploy() { + local disk_count rollback_dir_terminal=retained + restore_managed_files || return 1 + + if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then + compose_managed_config || return 1 + recreate_managed || return 1 + else + compose_base_config || return 1 + recreate_base || return 1 + fi + wait_clickhouse || return 1 + wait_api_and_listeners || return 1 + + disk_count="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1 + [ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1 + [ "$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1 + [ "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1 + [ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1 + [ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1 + + if [ "${HOST_DIR_CREATED}" -eq 1 ]; then + if sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then + rollback_dir_terminal=removed_empty + else + rollback_dir_terminal=retained_nonempty_no_delete + fi + fi + receipt rollback pass "prior_compose_state_restored_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}" +} + +on_exit() { + local rc=$? rollback_rc=0 terminal=failed + trap - EXIT HUP INT TERM + set +e + if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then + rollback_deploy + rollback_rc=$? + if [ "${rollback_rc}" -ne 0 ]; then + receipt rollback failed "prior_compose_state_restore_failed" + rc=91 + fi + fi + if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then + terminal=pass + fi + receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}" + exit "${rc}" +} + +mkdir -p "${RECEIPT_ROOT}" +[ ! -e "${RECEIPT_DIR}" ] +mkdir -m 0700 "${RECEIPT_DIR}" +: > "${RECEIPT_LOG}" +trap on_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +exec 9>>"${LOCK_FILE}" +flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; } + +for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss sudo timeout; do + command -v "${command_name}" >/dev/null 2>&1 \ + || { receipt check failed "required_command_missing_${command_name}"; exit 69; } +done + +[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] +[ -f "${STAGE_OVERRIDE}" ] && [ ! -L "${STAGE_OVERRIDE}" ] +[ -f "${STAGE_CONFIG}" ] && [ ! -L "${STAGE_CONFIG}" ] +[ "$(sha256sum "${STAGE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ] +[ "$(sha256sum "${STAGE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ] +[ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ] +[ -d "${REMOTE_DIR}" ] && [ ! -L "${REMOTE_DIR}" ] && [ -w "${REMOTE_DIR}" ] +[ -d /backup ] && [ ! -L /backup ] + +compose_base_config +compose_stage_config +env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ + -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \ + | grep -qx "${REMOTE_SERVICE}" + +if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then + receipt check failed active_signoz_backup_detected + exit 75 +fi +active_native_backups="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" +[ "${active_native_backups}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; } + +for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do + [ "$(docker inspect --format '{{.State.Running}}' "${container}")" = true ] +done +wait_clickhouse +wait_api_and_listeners + +if [ -e "${REMOTE_OVERRIDE}" ]; then + [ -f "${REMOTE_OVERRIDE}" ] && [ ! -L "${REMOTE_OVERRIDE}" ] + PRIOR_OVERRIDE_PRESENT=1 +fi +if [ -e "${REMOTE_CONFIG}" ]; then + [ -f "${REMOTE_CONFIG}" ] && [ ! -L "${REMOTE_CONFIG}" ] + PRIOR_CONFIG_PRESENT=1 +fi +[ "${PRIOR_OVERRIDE_PRESENT}" -eq "${PRIOR_CONFIG_PRESENT}" ] \ + || { receipt check failed partial_managed_file_drift; exit 78; } +if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then + compose_managed_config + cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml" + cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml" +fi + +PRIOR_DISK_COUNT="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" +BEFORE_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" +BEFORE_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +[ -n "${BEFORE_DATA_VOLUME}" ] +BEFORE_COLLECTOR_ID="$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" +BEFORE_COLLECTOR_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" +BEFORE_SIGNOZ_ID="$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" +BEFORE_SIGNOZ_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" +BEFORE_ZOOKEEPER_ID="$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" +BEFORE_ZOOKEEPER_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" + +receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid" +receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}" +receipt source_of_truth_diff pass "prior_override_${PRIOR_OVERRIDE_PRESENT}_prior_config_${PRIOR_CONFIG_PRESENT}_prior_disk_count_${PRIOR_DISK_COUNT}" +receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only" +receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback" +receipt check pass "compose_config_q_stage_hashes_runtime_preflight_backup_idle" + +APPLY_STARTED=1 +if [ -e "${HOST_BACKUP_DIR}" ]; then + [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ] + [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ] +else + sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}" + HOST_DIR_CREATED=1 +fi + +install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" +install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}" +mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}" +mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}" +[ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ] +[ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ] +compose_managed_config + +recreate_managed +wait_clickhouse + +AFTER_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" +AFTER_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +[ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ] +[ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ] + +backup_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +config_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +[ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ] +[ "${config_mount}" = false ] +[ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ] +server_uid="$(docker exec "${CLICKHOUSE_CONTAINER}" \ + awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)" +server_gid="$(docker exec "${CLICKHOUSE_CONTAINER}" \ + awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)" +[ "${server_uid}" = 101 ] +[ "${server_gid}" = 101 ] +docker exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \ + 'test -d /backups && test -w /backups' + +disk_state="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + "SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" +[ "${disk_state}" = "Local|0|0|0|1" ] +[ "$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + "EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ] +docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \ + | grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)' + +[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] +[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] +[ "$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ] +[ "$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ] +[ "$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ] +[ "$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ] +wait_api_and_listeners + +receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged" +receipt post_verifier pass "disk_backups_local_rw_healthy_server_uid101_write_access_concurrency_disabled_collector_unchanged_listeners_4317_4318_8080_api_200" +receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_executed" +DEPLOY_VERIFIED=1 +REMOTE_APPLY + +trap - EXIT +cleanup_stage +local_receipt terminal pass "remote_controlled_apply_and_postverify_complete" diff --git a/scripts/ops/deploy-signoz-native-backup-toolchain.sh b/scripts/ops/deploy-signoz-native-backup-toolchain.sh new file mode 100755 index 000000000..02ae66a68 --- /dev/null +++ b/scripts/ops/deploy-signoz-native-backup-toolchain.sh @@ -0,0 +1,801 @@ +#!/usr/bin/env bash +# P0-OBS-002 - atomically deploy the host110 SigNoz native backup toolchain. +# +# This deployer owns seven repo-sourced files only. It does not run a backup or +# restore, restart a container, pull an image, or inspect a secret-bearing +# runtime source. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +TARGET_HOST="wooo@192.168.0.110" +SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +SAFE_HOME="/home/wooo" +API_URL="http://127.0.0.1:8080/api/v1/health" + +REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}" +REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}" +RECEIPT_ROOT="${SIGNOZ_TOOLCHAIN_RECEIPT_ROOT:-/backup/deploy-receipts/signoz-native-backup-toolchain}" +STAGE_ROOT="${SIGNOZ_TOOLCHAIN_STAGE_ROOT:-/tmp}" + +LABELS=( + backup-signoz.sh + clickhouse-native-backup.sh + clickhouse-native-restore-drill.sh + clickhouse-restore-inventory.py + run-signoz-backup-canary.sh + backup_disk.xml + isolated-cluster.xml +) +LOCAL_SOURCES=( + "${ROOT_DIR}/scripts/backup/backup-signoz.sh" + "${ROOT_DIR}/scripts/backup/clickhouse-native-backup.sh" + "${ROOT_DIR}/scripts/backup/clickhouse-native-restore-drill.sh" + "${ROOT_DIR}/scripts/backup/clickhouse-restore-inventory.py" + "${ROOT_DIR}/scripts/backup/run-signoz-backup-canary.sh" + "${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml" + "${ROOT_DIR}/ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" +) +MODES=(0755 0755 0755 0755 0755 0644 0644) + +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout=8 + -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=2 +) + +usage() { + cat <<'USAGE' +Usage: deploy-signoz-native-backup-toolchain.sh [--check|--apply] + +Required environment: + TRACE_ID Path-safe controlled-apply trace identifier + RUN_ID Unique path-safe execution identifier + WORK_ITEM_ID Path-safe work-item identifier + +Optional safe absolute path environment: + SIGNOZ_TOOLCHAIN_SCRIPT_DIR + SIGNOZ_TOOLCHAIN_CONFIG_ROOT + SIGNOZ_TOOLCHAIN_RECEIPT_ROOT + SIGNOZ_TOOLCHAIN_STAGE_ROOT + +--check validates local sources and performs a remote read-only target diff and +runtime readiness check. --apply stages all seven sources, validates hashes and +syntax, atomically replaces exact targets, and rolls every target back to its +prior hash/mode/owner if any bounded verifier fails. + +Flat restore-driver invocation must set: + CLICKHOUSE_RESTORE_INVENTORY_HELPER=/clickhouse-restore-inventory.py + CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG=/config.d/backup_disk.xml + CLICKHOUSE_RESTORE_CLUSTER_CONFIG=/restore-drill/config.d/isolated-cluster.xml +USAGE +} + +MODE="${1:---check}" +case "${MODE}" in + --check|--apply) ;; + -h|--help) usage; exit 0 ;; + *) usage >&2; exit 64 ;; +esac +[ "$#" -le 1 ] || { usage >&2; exit 64; } + +TRACE_ID="${TRACE_ID:-}" +RUN_ID="${RUN_ID:-}" +WORK_ITEM_ID="${WORK_ITEM_ID:-}" + +valid_identity() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] +} + +safe_absolute_path() { + local path="$1" + [[ "${path}" == /* ]] || return 1 + [[ "${path}" != / && "${path}" != */ ]] || return 1 + [[ "${path}" =~ ^/[A-Za-z0-9._/-]+$ ]] || return 1 + case "/${path#/}/" in + *'//'*|*'/../'*|*'/./'*) return 1 ;; + esac +} + +paths_overlap() { + local left="$1" right="$2" + [[ "${left}" = "${right}" || "${left}" == "${right}"/* || "${right}" == "${left}"/* ]] +} + +for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do + valid_identity "${value}" || { + printf 'TRACE_ID, RUN_ID, and WORK_ITEM_ID are required and must be path-safe\n' >&2 + exit 64 + } +done + +for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do + safe_absolute_path "${path}" || { + printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2 + exit 64 + } +done +case "${REMOTE_SCRIPT_DIR}" in /backup/*) ;; *) printf 'script dir must stay under /backup\n' >&2; exit 64 ;; esac +case "${REMOTE_CONFIG_ROOT}" in /backup/*) ;; *) printf 'config root must stay under /backup\n' >&2; exit 64 ;; esac +case "${RECEIPT_ROOT}" in /backup/*) ;; *) printf 'receipt root must stay under /backup\n' >&2; exit 64 ;; esac +case "${STAGE_ROOT}" in /tmp|/tmp/*|/backup/deploy-staging|/backup/deploy-staging/*) ;; + *) printf 'stage root must stay under /tmp or /backup/deploy-staging\n' >&2; exit 64 ;; +esac +if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \ + || paths_overlap "${REMOTE_SCRIPT_DIR}" "${RECEIPT_ROOT}" \ + || paths_overlap "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" \ + || paths_overlap "${STAGE_ROOT}" "${REMOTE_SCRIPT_DIR}" \ + || paths_overlap "${STAGE_ROOT}" "${REMOTE_CONFIG_ROOT}" \ + || paths_overlap "${STAGE_ROOT}" "${RECEIPT_ROOT}"; then + printf 'toolchain target, receipt, and stage paths must not overlap\n' >&2 + exit 64 +fi + +for command_name in bash python3 scp ssh; do + command -v "${command_name}" >/dev/null 2>&1 || { + printf 'required local command missing: %s\n' "${command_name}" >&2 + exit 69 + } +done + +hash_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +validate_xml_pair() { + python3 - "$1" "$2" <<'PY' +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + +backup_path = Path(sys.argv[1]) +cluster_path = Path(sys.argv[2]) +backup = ET.parse(backup_path).getroot() +cluster = ET.parse(cluster_path).getroot() +assert backup.tag == "clickhouse" +assert backup.findtext("./storage_configuration/disks/backups/type") == "local" +assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/" +assert backup.findtext("./backups/allowed_disk") == "backups" +assert backup.findtext("./backups/allowed_path") == "/backups/" +assert backup.findtext("./backups/allow_concurrent_backups") == "false" +assert backup.findtext("./backups/allow_concurrent_restores") == "false" +assert cluster.tag == "clickhouse" +assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper" +assert cluster.findtext("./zookeeper/node/port") == "2181" +assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse" +assert cluster.findtext("./remote_servers/cluster/shard/replica/port") == "9000" +assert cluster.findtext("./macros/shard") == "01" +replica = cluster.find("./macros/replica") +assert replica is not None +assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"} +PY +} + +validate_local_sources() { + local index source + for index in "${!LOCAL_SOURCES[@]}"; do + source="${LOCAL_SOURCES[index]}" + [ -f "${source}" ] && [ ! -L "${source}" ] + [ -s "${source}" ] + done + + for index in 0 1 2 4; do + bash -n "${LOCAL_SOURCES[index]}" + done + python3 - "${LOCAL_SOURCES[3]}" <<'PY' +from pathlib import Path +import sys + +path = Path(sys.argv[1]) +compile(path.read_text(encoding="utf-8"), str(path), "exec") +PY + validate_xml_pair "${LOCAL_SOURCES[5]}" "${LOCAL_SOURCES[6]}" + + python3 - "${LOCAL_SOURCES[2]}" <<'PY' +from pathlib import Path +import sys + +text = Path(sys.argv[1]).read_text(encoding="utf-8") +for token in ( + "CLICKHOUSE_RESTORE_INVENTORY_HELPER", + "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG", + "CLICKHOUSE_RESTORE_CLUSTER_CONFIG", +): + assert token in text +PY +} + +validate_local_sources +SOURCE_HASHES=() +SOURCE_DETAIL="" +for index in "${!LOCAL_SOURCES[@]}"; do + source_hash="$(hash_file "${LOCAL_SOURCES[index]}")" + SOURCE_HASHES+=("${source_hash}") + SOURCE_DETAIL+="${LABELS[index]}_${source_hash}_" +done +SOURCE_DETAIL="${SOURCE_DETAIL%_}" + +CONFIG_DIR="${REMOTE_CONFIG_ROOT}/config.d" +RESTORE_CONFIG_DIR="${REMOTE_CONFIG_ROOT}/restore-drill/config.d" +TARGET_PATHS=( + "${REMOTE_SCRIPT_DIR}/backup-signoz.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py" + "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh" + "${CONFIG_DIR}/backup_disk.xml" + "${RESTORE_CONFIG_DIR}/isolated-cluster.xml" +) +STAGE_DIR="${STAGE_ROOT}/awoooi-signoz-native-backup-toolchain.${RUN_ID}" + +local_receipt() { + local phase="$1" result="$2" detail="$3" + printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}" +} + +remote_read_only_check() { + ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ + "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \ + "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK' +set -euo pipefail + +TRACE_ID="$1" +RUN_ID="$2" +WORK_ITEM_ID="$3" +REMOTE_SCRIPT_DIR="$4" +REMOTE_CONFIG_ROOT="$5" +RECEIPT_ROOT="$6" +STAGE_ROOT="$7" +shift 7 +EXPECTED_HASHES=("$@") +LABELS=( + backup-signoz.sh + clickhouse-native-backup.sh + clickhouse-native-restore-drill.sh + clickhouse-restore-inventory.py + run-signoz-backup-canary.sh + backup_disk.xml + isolated-cluster.xml +) +MODES=(0755 0755 0755 0755 0755 0644 0644) +TARGETS=( + "${REMOTE_SCRIPT_DIR}/backup-signoz.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py" + "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh" + "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml" + "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml" +) +API_URL="http://127.0.0.1:8080/api/v1/health" + +[ "${#EXPECTED_HASHES[@]}" -eq 7 ] +[ -d /backup ] && [ ! -L /backup ] +for command_name in awk curl docker pgrep sha256sum ss stat; do + command -v "${command_name}" >/dev/null 2>&1 +done + +listener_up() { + local port="$1" + ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' +} + +validate_existing_ancestors() { + local requested="$1" current="" component + local -a components + IFS='/' read -r -a components <<<"${requested#/}" + for component in "${components[@]}"; do + [ -n "${component}" ] || continue + current="${current}/${component}" + if [ -e "${current}" ] || [ -L "${current}" ]; then + [ -d "${current}" ] && [ ! -L "${current}" ] + else + break + fi + done +} + +runtime_line() { + local container="$1" value running health + value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" + IFS='|' read -r _ _ _ running health <<<"${value}" + [ "${running}" = true ] + case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac + printf '%s' "${value}" +} + +if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then + printf 'active SigNoz backup toolchain process detected\n' >&2 + exit 75 +fi +validate_existing_ancestors "${REMOTE_SCRIPT_DIR}" +validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/config.d" +validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d" +validate_existing_ancestors "${RECEIPT_ROOT}" +validate_existing_ancestors "${STAGE_ROOT}" +[ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ] +active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \ + "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" +[ "${active_operations}" = 0 ] + +runtime_summary="" +for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do + runtime_summary+="${container}_$(runtime_line "${container}")_" +done +for port in 4317 4318 8080; do listener_up "${port}"; done +api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")" +[ "${api_code}" = 200 ] + +matching=0 +drift=0 +absent=0 +for index in "${!TARGETS[@]}"; do + target="${TARGETS[index]}" + if [ -e "${target}" ] || [ -L "${target}" ]; then + [ -f "${target}" ] && [ ! -L "${target}" ] + current_hash="$(sha256sum "${target}" | awk '{print $1}')" + current_mode="$(stat -c '%a' "${target}")" + if [ "${current_hash}" = "${EXPECTED_HASHES[index]}" ] && [ "0${current_mode}" = "${MODES[index]}" ]; then + matching=$((matching + 1)) + else + drift=$((drift + 1)) + fi + else + absent=$((absent + 1)) + fi +done + +printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"source_of_truth_diff","result":"pass","detail":"matching_%s_drift_%s_absent_%s_active_operations_0_api_200_ports_4317_4318_8080"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${matching}" "${drift}" "${absent}" +printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"runtime_readiness","result":"pass","detail":"container_identity_started_at_restart_count_health_captured_%s"}\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${runtime_summary%_}" +REMOTE_CHECK +} + +local_receipt sensor_source pass "${SOURCE_DETAIL}" +local_receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2" +remote_read_only_check +local_receipt check pass "local_syntax_xml_python_compile_and_remote_read_only_preflight" + +if [ "${MODE}" = --check ]; then + local_receipt terminal check_pass_no_write "remote_target_unchanged_no_stage_no_candidate" + exit 0 +fi + +STAGE_CREATED=0 +cleanup_stage() { + if [ "${STAGE_CREATED}" -eq 1 ]; then + ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true +set -u +stage_dir="$1" +if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then + rm -rf "${stage_dir}/pycache" "${stage_dir}/candidate-pycache" + rm -f \ + "${stage_dir}/backup-signoz.sh" \ + "${stage_dir}/clickhouse-native-backup.sh" \ + "${stage_dir}/clickhouse-native-restore-drill.sh" \ + "${stage_dir}/clickhouse-restore-inventory.py" \ + "${stage_dir}/run-signoz-backup-canary.sh" \ + "${stage_dir}/backup_disk.xml" \ + "${stage_dir}/isolated-cluster.xml" + rmdir "${stage_dir}" 2>/dev/null || true +fi +REMOTE_CLEANUP + fi +} +trap cleanup_stage EXIT + +ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' +set -euo pipefail +stage_dir="$1" +[ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ] +stage_root="${stage_dir%/*}" +[ -d "${stage_root}" ] && [ ! -L "${stage_root}" ] && [ -w "${stage_root}" ] +umask 077 +mkdir -m 0700 "${stage_dir}" +REMOTE_STAGE +STAGE_CREATED=1 + +for index in "${!LOCAL_SOURCES[@]}"; do + scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \ + "${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}" +done + +ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ + "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \ + "${SOURCE_HASHES[@]}" <<'REMOTE_APPLY' +set -euo pipefail + +TRACE_ID="$1" +RUN_ID="$2" +WORK_ITEM_ID="$3" +REMOTE_SCRIPT_DIR="$4" +REMOTE_CONFIG_ROOT="$5" +RECEIPT_ROOT="$6" +STAGE_DIR="$7" +shift 7 +EXPECTED_HASHES=("$@") +LABELS=( + backup-signoz.sh + clickhouse-native-backup.sh + clickhouse-native-restore-drill.sh + clickhouse-restore-inventory.py + run-signoz-backup-canary.sh + backup_disk.xml + isolated-cluster.xml +) +MODES=(0755 0755 0755 0755 0755 0644 0644) +STAGED=( + "${STAGE_DIR}/backup-signoz.sh" + "${STAGE_DIR}/clickhouse-native-backup.sh" + "${STAGE_DIR}/clickhouse-native-restore-drill.sh" + "${STAGE_DIR}/clickhouse-restore-inventory.py" + "${STAGE_DIR}/run-signoz-backup-canary.sh" + "${STAGE_DIR}/backup_disk.xml" + "${STAGE_DIR}/isolated-cluster.xml" +) +TARGETS=( + "${REMOTE_SCRIPT_DIR}/backup-signoz.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh" + "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py" + "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh" + "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml" + "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml" +) +TARGET_DIRS=( + "${REMOTE_SCRIPT_DIR}" + "${REMOTE_CONFIG_ROOT}/config.d" + "${REMOTE_CONFIG_ROOT}/restore-drill/config.d" +) +CONTAINERS=(signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1) +API_URL="http://127.0.0.1:8080/api/v1/health" +RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}" +RECEIPT_LOG="${RECEIPT_DIR}/receipts.jsonl" +PRIOR_DIR="${RECEIPT_DIR}/prior" +PRIOR_MANIFEST="${RECEIPT_DIR}/prior-manifest.tsv" +DEPLOYED_MANIFEST="${RECEIPT_DIR}/deployed-manifest.tsv" +RUNTIME_BEFORE="${RECEIPT_DIR}/runtime-before.tsv" +RUNTIME_AFTER="${RECEIPT_DIR}/runtime-after.tsv" +LOCK_FILE="/tmp/awoooi-signoz-native-backup-toolchain.lock" + +APPLY_STARTED=0 +DEPLOY_VERIFIED=0 +RUNTIME_BASELINE_CAPTURED=0 +LOCK_HELD=0 +REMOTE_UID="$(id -u)" +REMOTE_GID="$(id -g)" +CREATED_DIRS=() +PRIOR_PRESENT=() +PRIOR_MODES=() +PRIOR_UIDS=() +PRIOR_GIDS=() +PRIOR_HASHES=() + +for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat; do + command -v "${command_name}" >/dev/null 2>&1 || { + printf 'required remote command missing: %s\n' "${command_name}" >&2 + exit 69 + } +done +if [ "${REMOTE_UID}" -ne 0 ]; then + command -v sudo >/dev/null 2>&1 || { printf 'required remote command missing: sudo\n' >&2; exit 69; } + sudo -n true +fi +[ "${#EXPECTED_HASHES[@]}" -eq 7 ] +[ -d /backup ] && [ ! -L /backup ] + +run_root() { + if [ "${REMOTE_UID}" -eq 0 ]; then + "$@" + else + sudo -n "$@" + fi +} + +root_hash() { + if [ "${REMOTE_UID}" -eq 0 ]; then + sha256sum "$1" | awk '{print $1}' + else + sudo -n sha256sum "$1" | awk '{print $1}' + fi +} + +ensure_dir_tree() { + local requested="$1" record_created="$2" current="" component + local -a components + IFS='/' read -r -a components <<<"${requested#/}" + for component in "${components[@]}"; do + [ -n "${component}" ] || continue + current="${current}/${component}" + if [ -e "${current}" ] || [ -L "${current}" ]; then + [ -d "${current}" ] && [ ! -L "${current}" ] + else + run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0755 "${current}" + if [ "${record_created}" = true ]; then + CREATED_DIRS+=("${current}") + fi + fi + done +} + +ensure_dir_tree "${RECEIPT_ROOT}" false +[ ! -e "${RECEIPT_DIR}" ] && [ ! -L "${RECEIPT_DIR}" ] +run_root install -d -o "${REMOTE_UID}" -g "${REMOTE_GID}" -m 0700 "${RECEIPT_DIR}" +install -d -m 0700 "${PRIOR_DIR}" +: > "${RECEIPT_LOG}" + +receipt() { + local phase="$1" result="$2" detail="$3" line + line="$(printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-native-backup-toolchain","phase":"%s","result":"%s","detail":"%s"}' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${phase}" "${result}" "${detail}")" + printf '%s\n' "${line}" + printf '%s\n' "${line}" >> "${RECEIPT_LOG}" +} + +listener_up() { + local port="$1" + ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' +} + +runtime_snapshot() { + local container value running health port api_code + for container in "${CONTAINERS[@]}"; do + value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" + IFS='|' read -r _ _ _ running health <<<"${value}" + [ "${running}" = true ] + case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac + printf 'container\t%s\t%s\n' "${container}" "${value}" + done + for port in 4317 4318 8080; do + listener_up "${port}" + printf 'listener\t%s\tup\n' "${port}" + done + api_code="$(curl -sS --max-time 5 -o /dev/null -w '%{http_code}' "${API_URL}")" + [ "${api_code}" = 200 ] + printf 'api\thealth\t%s\n' "${api_code}" +} + +validate_xml_pair() { + python3 - "$1" "$2" <<'PY' +from pathlib import Path +import sys +import xml.etree.ElementTree as ET + +backup_path = Path(sys.argv[1]) +cluster_path = Path(sys.argv[2]) +backup = ET.parse(backup_path).getroot() +cluster = ET.parse(cluster_path).getroot() +assert backup.findtext("./storage_configuration/disks/backups/type") == "local" +assert backup.findtext("./storage_configuration/disks/backups/path") == "/backups/" +assert backup.findtext("./backups/allowed_disk") == "backups" +assert backup.findtext("./backups/allowed_path") == "/backups/" +assert backup.findtext("./backups/allow_concurrent_backups") == "false" +assert backup.findtext("./backups/allow_concurrent_restores") == "false" +assert cluster.findtext("./zookeeper/node/host") == "restore-zookeeper" +assert cluster.findtext("./remote_servers/cluster/shard/replica/host") == "restore-clickhouse" +assert cluster.findtext("./macros/shard") == "01" +replica = cluster.find("./macros/replica") +assert replica is not None +assert replica.attrib == {"from_env": "CLICKHOUSE_RESTORE_REPLICA"} +PY +} + +cleanup_transients() { + local index rc=0 + for index in "${!TARGETS[@]}"; do + run_root rm -f \ + "${TARGETS[index]}.candidate.${RUN_ID}" \ + "${TARGETS[index]}.rollback.${RUN_ID}" || rc=1 + done + if [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ]; then + rm -rf "${STAGE_DIR}/pycache" "${STAGE_DIR}/candidate-pycache" || rc=1 + rm -f "${STAGED[@]}" || rc=1 + rmdir "${STAGE_DIR}" 2>/dev/null || rc=1 + else + rc=1 + fi + return "${rc}" +} + +verify_residue_zero() { + local index + [ ! -e "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1 + for index in "${!TARGETS[@]}"; do + [ ! -e "${TARGETS[index]}.candidate.${RUN_ID}" ] \ + && [ ! -L "${TARGETS[index]}.candidate.${RUN_ID}" ] || return 1 + [ ! -e "${TARGETS[index]}.rollback.${RUN_ID}" ] \ + && [ ! -L "${TARGETS[index]}.rollback.${RUN_ID}" ] || return 1 + done +} + +rollback_targets() ( + set -euo pipefail + local index target rollback_candidate actual_mode actual_hash + for index in "${!TARGETS[@]}"; do + target="${TARGETS[index]}" + if [ "${PRIOR_PRESENT[index]}" = 1 ]; then + rollback_candidate="${target}.rollback.${RUN_ID}" + run_root install \ + -o "${PRIOR_UIDS[index]}" -g "${PRIOR_GIDS[index]}" \ + -m "${PRIOR_MODES[index]}" \ + "${PRIOR_DIR}/${LABELS[index]}" "${rollback_candidate}" + [ "$(root_hash "${rollback_candidate}")" = "${PRIOR_HASHES[index]}" ] + run_root mv -f "${rollback_candidate}" "${target}" + actual_mode="$(stat -c '%a' "${target}")" + actual_hash="$(root_hash "${target}")" + [ "${actual_mode}" = "${PRIOR_MODES[index]}" ] + [ "${actual_hash}" = "${PRIOR_HASHES[index]}" ] + else + run_root rm -f "${target}" + [ ! -e "${target}" ] && [ ! -L "${target}" ] + fi + done + + for ((index=${#CREATED_DIRS[@]} - 1; index >= 0; index--)); do + run_root rmdir "${CREATED_DIRS[index]}" + done + + if [ "${RUNTIME_BASELINE_CAPTURED}" -eq 1 ]; then + runtime_snapshot > "${RECEIPT_DIR}/runtime-rollback.tsv" + cmp -s "${RUNTIME_BEFORE}" "${RECEIPT_DIR}/runtime-rollback.tsv" + fi + receipt rollback pass "all_prior_files_modes_hashes_owners_and_runtime_restored" +) + +on_exit() { + local rc=$? rollback_rc=0 cleanup_rc=0 residue_rc=0 terminal=failed + trap - EXIT HUP INT TERM + set +e + if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then + rollback_targets + rollback_rc=$? + if [ "${rollback_rc}" -ne 0 ]; then + receipt rollback failed "prior_state_restore_or_runtime_verifier_failed" + rc=91 + fi + elif [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 0 ]; then + receipt rollback no_write "bounded_execution_not_started_targets_unchanged" + fi + cleanup_transients + cleanup_rc=$? + verify_residue_zero + residue_rc=$? + if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then + rc=92 + fi + if [ "${LOCK_HELD}" -eq 1 ]; then + flock -u 9 >/dev/null 2>&1 + exec 9>&- + rm -f "${LOCK_FILE}" + fi + if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then + terminal=pass + fi + receipt terminal "${terminal}" "exit_${rc}_stage_candidate_residue_0_receipt_ref_${RECEIPT_LOG}" + exit "${rc}" +} + +trap on_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +exec 9>>"${LOCK_FILE}" +flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; } +LOCK_HELD=1 + +if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then + receipt check failed active_signoz_backup_toolchain_process + exit 75 +fi +active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \ + "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" +[ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; } + +runtime_snapshot > "${RUNTIME_BEFORE}" +RUNTIME_BASELINE_CAPTURED=1 + +[ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] +for index in "${!STAGED[@]}"; do + [ -f "${STAGED[index]}" ] && [ ! -L "${STAGED[index]}" ] + [ "$(sha256sum "${STAGED[index]}" | awk '{print $1}')" = "${EXPECTED_HASHES[index]}" ] +done +for index in 0 1 2 4; do bash -n "${STAGED[index]}"; done +PYTHONPYCACHEPREFIX="${STAGE_DIR}/pycache" python3 -m py_compile "${STAGED[3]}" +validate_xml_pair "${STAGED[5]}" "${STAGED[6]}" + +: > "${PRIOR_MANIFEST}" +for index in "${!TARGETS[@]}"; do + target="${TARGETS[index]}" + [ ! -e "${target}.candidate.${RUN_ID}" ] && [ ! -L "${target}.candidate.${RUN_ID}" ] + [ ! -e "${target}.rollback.${RUN_ID}" ] && [ ! -L "${target}.rollback.${RUN_ID}" ] + if [ -e "${target}" ] || [ -L "${target}" ]; then + [ -f "${target}" ] && [ ! -L "${target}" ] + PRIOR_PRESENT[index]=1 + PRIOR_MODES[index]="$(stat -c '%a' "${target}")" + PRIOR_UIDS[index]="$(stat -c '%u' "${target}")" + PRIOR_GIDS[index]="$(stat -c '%g' "${target}")" + PRIOR_HASHES[index]="$(root_hash "${target}")" + run_root cp -p "${target}" "${PRIOR_DIR}/${LABELS[index]}" + [ "$(root_hash "${PRIOR_DIR}/${LABELS[index]}")" = "${PRIOR_HASHES[index]}" ] + else + PRIOR_PRESENT[index]=0 + PRIOR_MODES[index]=absent + PRIOR_UIDS[index]=absent + PRIOR_GIDS[index]=absent + PRIOR_HASHES[index]=absent + fi + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${LABELS[index]}" "${target}" "${PRIOR_PRESENT[index]}" \ + "${PRIOR_MODES[index]}" "${PRIOR_UIDS[index]}" "${PRIOR_GIDS[index]}" \ + "${PRIOR_HASHES[index]}" >> "${PRIOR_MANIFEST}" +done + +receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources" +receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2" +receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}" +receipt ai_decision candidate "atomic_install_native_backup_toolchain_only" +receipt risk_policy pass "risk_medium_bounded_files_only_no_runtime_execution_no_container_change_full_rollback" +receipt check pass "runtime_baseline_active_operations_0_hash_bash_xml_py_compile" + +APPLY_STARTED=1 +for directory in "${TARGET_DIRS[@]}"; do + ensure_dir_tree "${directory}" true +done + +for index in "${!TARGETS[@]}"; do + target="${TARGETS[index]}" + if [ "${PRIOR_PRESENT[index]}" = 1 ]; then + target_uid="${PRIOR_UIDS[index]}" + target_gid="${PRIOR_GIDS[index]}" + else + target_uid="${REMOTE_UID}" + target_gid="${REMOTE_GID}" + fi + run_root install -o "${target_uid}" -g "${target_gid}" -m "${MODES[index]}" \ + "${STAGED[index]}" "${target}.candidate.${RUN_ID}" + [ "$(root_hash "${target}.candidate.${RUN_ID}")" = "${EXPECTED_HASHES[index]}" ] + [ "0$(stat -c '%a' "${target}.candidate.${RUN_ID}")" = "${MODES[index]}" ] +done + +for index in "${!TARGETS[@]}"; do + run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}" +done + +for index in "${!TARGETS[@]}"; do + [ -f "${TARGETS[index]}" ] && [ ! -L "${TARGETS[index]}" ] + [ "$(root_hash "${TARGETS[index]}")" = "${EXPECTED_HASHES[index]}" ] + [ "0$(stat -c '%a' "${TARGETS[index]}")" = "${MODES[index]}" ] +done +for index in 0 1 2 4; do bash -n "${TARGETS[index]}"; done +PYTHONPYCACHEPREFIX="${STAGE_DIR}/candidate-pycache" python3 -m py_compile "${TARGETS[3]}" +validate_xml_pair "${TARGETS[5]}" "${TARGETS[6]}" + +: > "${DEPLOYED_MANIFEST}" +for index in "${!TARGETS[@]}"; do + printf '%s\t%s\t%s\t%s\n' \ + "${LABELS[index]}" "${TARGETS[index]}" "${MODES[index]}" \ + "${EXPECTED_HASHES[index]}" >> "${DEPLOYED_MANIFEST}" +done + +runtime_snapshot > "${RUNTIME_AFTER}" +cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}" +receipt execution pass "seven_candidates_atomically_replaced_expected_hashes_and_modes" +receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged" +receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull" +DEPLOY_VERIFIED=1 +REMOTE_APPLY + +trap - EXIT +cleanup_stage +local_receipt terminal pass "remote_controlled_apply_postverify_and_stage_candidate_residue_0" diff --git a/scripts/ops/deploy-signoz-prometheus-route.sh b/scripts/ops/deploy-signoz-prometheus-route.sh new file mode 100755 index 000000000..570b79190 --- /dev/null +++ b/scripts/ops/deploy-signoz-prometheus-route.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# Deploy only the canonical SigNoz blackbox route to the host110 Prometheus. +# The full Prometheus file intentionally remains untouched because production +# contains unrelated, independently governed scrape targets. + +set -euo pipefail + +TRACE_ID="${TRACE_ID:?TRACE_ID is required}" +RUN_ID="${RUN_ID:?RUN_ID is required}" +WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}" +TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}" +PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}" +PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}" +REMOTE_CONFIG="${REMOTE_CONFIG:-/home/wooo/monitoring/prometheus.yml}" +MODE="${1:-apply}" + +case "${MODE}" in + apply|--dry-run) ;; + *) printf 'usage: %s [apply|--dry-run]\n' "$0" >&2; exit 64 ;; +esac + +for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do + case "${value}" in + *[!A-Za-z0-9._-]*|'') + printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2 + exit 64 + ;; + esac +done + +ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \ + bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \ + "${PROMETHEUS_URL}" "${PROMETHEUS_CONTAINER}" "${REMOTE_CONFIG}" <<'REMOTE' +set -euo pipefail + +TRACE_ID="$1" +RUN_ID="$2" +WORK_ITEM_ID="$3" +MODE="$4" +PROMETHEUS_URL="$5" +PROMETHEUS_CONTAINER="$6" +REMOTE_CONFIG="$7" +REMOTE_CANDIDATE="/tmp/awoooi-prometheus-signoz.${RUN_ID}.yml" +RUNTIME_CANDIDATE="/etc/prometheus/.awoooi-prometheus-signoz.${RUN_ID}.yml" +BACKUP_CONFIG="${REMOTE_CONFIG}.bak.${RUN_ID}" +APPLY_STARTED=0 +DEPLOY_VERIFIED=0 + +cleanup_candidate() { + rm -f "${REMOTE_CANDIDATE}" + docker exec -u 0 "${PROMETHEUS_CONTAINER}" \ + rm -f "${RUNTIME_CANDIDATE}" >/dev/null 2>&1 || true +} + +rollback_deploy() { + cp "${BACKUP_CONFIG}" "${REMOTE_CONFIG}" + curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null + curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null + printf 'rollback_verifier trace_id=%s run_id=%s work_item_id=%s terminal=rolled_back ready=true backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_CONFIG}" +} + +on_exit() { + local rc=$? + trap - EXIT + set +e + cleanup_candidate + if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] \ + && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then + rollback_deploy + fi + exit "${rc}" +} +trap on_exit EXIT + +test -f "${REMOTE_CONFIG}" +test "$(docker inspect --format '{{.State.Running}}' "${PROMETHEUS_CONTAINER}")" = true +curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null + +python3 - "${REMOTE_CONFIG}" "${REMOTE_CANDIDATE}" <<'PY' +from pathlib import Path +import sys + +source = Path(sys.argv[1]) +candidate = Path(sys.argv[2]) +text = source.read_text(encoding="utf-8") + +canonical = " - http://192.168.0.110:8080/api/v1/health\n" +legacy = " - 192.168.0.188:3301\n" +anchor = " - http://192.168.0.110:3001\n" + + +def block_bounds(payload: str, job: str) -> tuple[int, int]: + marker = f' - job_name: "{job}"\n' + start = payload.find(marker) + if start < 0 or payload.find(marker, start + 1) >= 0: + raise SystemExit(f"expected exactly one {job} job") + end = payload.find("\n - job_name:", start + len(marker)) + return start, len(payload) if end < 0 else end + 1 + + +http_start, http_end = block_bounds(text, "blackbox-http") +http_block = text[http_start:http_end] +if http_block.count(canonical) > 1 or http_block.count(anchor) != 1: + raise SystemExit("canonical HTTP target or insertion anchor is ambiguous") +if canonical not in http_block: + http_block = http_block.replace(anchor, anchor + canonical, 1) +text = text[:http_start] + http_block + text[http_end:] + +tcp_start, tcp_end = block_bounds(text, "blackbox-tcp") +tcp_block = text[tcp_start:tcp_end] +if tcp_block.count(legacy) > 1: + raise SystemExit("legacy TCP target is ambiguous") +tcp_block = tcp_block.replace(legacy, "", 1) +text = text[:tcp_start] + tcp_block + text[tcp_end:] + +http_start, http_end = block_bounds(text, "blackbox-http") +tcp_start, tcp_end = block_bounds(text, "blackbox-tcp") +if text[http_start:http_end].count(canonical) != 1: + raise SystemExit("canonical HTTP target was not normalized exactly once") +if legacy in text[tcp_start:tcp_end]: + raise SystemExit("legacy TCP target remains after normalization") + +candidate.write_text(text, encoding="utf-8") +PY + +docker cp "${REMOTE_CANDIDATE}" \ + "${PROMETHEUS_CONTAINER}:${RUNTIME_CANDIDATE}" >/dev/null +docker exec "${PROMETHEUS_CONTAINER}" promtool check config "${RUNTIME_CANDIDATE}" + +SOURCE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" +CANDIDATE_SHA="$(sha256sum "${REMOTE_CANDIDATE}" | awk '{print $1}')" +printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s source_sha=%s candidate_sha=%s promtool=pass\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \ + "${SOURCE_SHA}" "${CANDIDATE_SHA}" + +if [ "${MODE}" = "--dry-run" ]; then + printf 'terminal trace_id=%s run_id=%s status=check_pass_no_write\n' \ + "${TRACE_ID}" "${RUN_ID}" + exit 0 +fi + +cp -p "${REMOTE_CONFIG}" "${BACKUP_CONFIG}" +APPLY_STARTED=1 +cp "${REMOTE_CANDIDATE}" "${REMOTE_CONFIG}" + +ACTIVE_SHA="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" +RUNTIME_SHA="$(docker exec "${PROMETHEUS_CONTAINER}" \ + sha256sum /etc/prometheus/prometheus.yml | awk '{print $1}')" +test "${ACTIVE_SHA}" = "${CANDIDATE_SHA}" +test "${RUNTIME_SHA}" = "${CANDIDATE_SHA}" +curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null +curl -fsS "${PROMETHEUS_URL}/-/ready" >/dev/null +printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=target_first_config_apply active_sha=%s runtime_sha=%s ready=true backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${ACTIVE_SHA}" \ + "${RUNTIME_SHA}" "${BACKUP_CONFIG}" + +target_state() { + curl -fsS "${PROMETHEUS_URL}/api/v1/targets?state=active" | python3 -c ' +import json, sys +payload = json.load(sys.stdin) +targets = payload["data"]["activeTargets"] +canonical = [t for t in targets if t.get("labels", {}).get("job") == "blackbox-http" and t.get("labels", {}).get("instance") == "http://192.168.0.110:8080/api/v1/health"] +legacy = [t for t in targets if t.get("labels", {}).get("instance") == "192.168.0.188:3301"] +if len(canonical) == 1: + target = canonical[0] + print("{}|{}|{}|{}".format( + 1, + target.get("health", "unknown"), + target.get("lastScrape", "never"), + len(legacy), + )) +else: + print(f"{len(canonical)}|missing|never|{len(legacy)}") +' +} + +FIRST_SCRAPE="" +for attempt in $(seq 1 9); do + IFS='|' read -r count health last_scrape legacy_count <<<"$(target_state)" + printf 'target_probe trace_id=%s run_id=%s attempt=%s count=%s health=%s last_scrape=%s legacy_count=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${attempt}" "${count}" "${health}" \ + "${last_scrape}" "${legacy_count}" + if [ "${count}" = 1 ] && [ "${health}" = up ] \ + && [ "${legacy_count}" = 0 ] && [ "${last_scrape}" != never ]; then + FIRST_SCRAPE="${last_scrape}" + break + fi + sleep 10 +done +test -n "${FIRST_SCRAPE}" + +sleep 16 +IFS='|' read -r count health SECOND_SCRAPE legacy_count <<<"$(target_state)" +printf 'target_probe trace_id=%s run_id=%s sample=post_cooldown count=%s health=%s last_scrape=%s legacy_count=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${count}" "${health}" \ + "${SECOND_SCRAPE}" "${legacy_count}" +test "${count}" = 1 +test "${health}" = up +test "${legacy_count}" = 0 +test "${SECOND_SCRAPE}" != "${FIRST_SCRAPE}" + +PROBE_VALUE="$(curl -fsS --get \ + --data-urlencode 'query=probe_success{job="blackbox-http",instance="http://192.168.0.110:8080/api/v1/health"}' \ + "${PROMETHEUS_URL}/api/v1/query" | python3 -c ' +import json, sys +result = json.load(sys.stdin)["data"]["result"] +print(result[0]["value"][1] if len(result) == 1 else "invalid") +')" +test "${PROBE_VALUE}" = 1 + +DEPLOY_VERIFIED=1 +printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s terminal=target_up_two_scrapes first=%s second=%s probe_success=%s legacy_target_count=0\n' \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${FIRST_SCRAPE}" \ + "${SECOND_SCRAPE}" "${PROBE_VALUE}" +printf 'terminal trace_id=%s run_id=%s status=completed_target_first backup=%s\n' \ + "${TRACE_ID}" "${RUN_ID}" "${BACKUP_CONFIG}" +REMOTE diff --git a/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py new file mode 100644 index 000000000..504e2ca11 --- /dev/null +++ b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py @@ -0,0 +1,231 @@ +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parents[3] +CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml" +OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml" +DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh" + + +def executable_text() -> str: + return "\n".join( + line + for line in DEPLOYER.read_text(encoding="utf-8").splitlines() + if not line.lstrip().startswith("#") + ) + + +def test_clickhouse_config_declares_one_allowed_local_backup_disk() -> None: + root = ET.parse(CONFIG).getroot() + assert root.tag == "clickhouse" + + disks = root.findall("./storage_configuration/disks/*") + assert [disk.tag for disk in disks] == ["backups"] + assert disks[0].findtext("type") == "local" + assert disks[0].findtext("path") == "/backups/" + + backups = root.find("./backups") + assert backups is not None + assert backups.findtext("allowed_disk") == "backups" + assert backups.findtext("allowed_path") == "/backups/" + assert backups.findtext("allow_concurrent_backups") == "false" + assert backups.findtext("allow_concurrent_restores") == "false" + assert CONFIG.read_text(encoding="utf-8").count("") == 1 + + +def test_compose_override_is_additive_clickhouse_only_and_uses_dedicated_mounts() -> ( + None +): + text = OVERRIDE.read_text(encoding="utf-8") + assert text.count(" clickhouse:") == 1 + assert "otel-collector:" not in text + assert "signoz:" not in text + assert "zookeeper:" not in text + assert "source: /backup/staging/signoz-clickhouse" in text + assert "target: /backups" in text + assert ( + "source: /home/wooo/signoz/deploy/docker/awoooi-clickhouse-backup-disk.xml" + ) in text + assert ("target: /etc/clickhouse-server/config.d/awoooi-backup-disk.xml") in text + assert text.count("read_only: true") == 1 + assert text.count("read_only: false") == 1 + for forbidden in ("image:", "build:", "command:", "environment:"): + assert forbidden not in text + + +def test_deployer_shell_is_valid_and_help_is_no_write() -> None: + syntax = subprocess.run( + ["bash", "-n", str(DEPLOYER)], + check=False, + capture_output=True, + text=True, + ) + assert syntax.returncode == 0, syntax.stderr + + help_result = subprocess.run( + ["bash", str(DEPLOYER), "--help"], + check=False, + capture_output=True, + text=True, + env={ + key: value + for key, value in os.environ.items() + if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"} + }, + ) + assert help_result.returncode == 0 + assert "--check" in help_result.stdout + assert "--apply" in help_result.stdout + + +def test_deployer_requires_all_three_controlled_apply_identifiers_before_ssh() -> None: + env = { + key: value + for key, value in os.environ.items() + if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"} + } + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 64 + assert "must be path-safe" in result.stderr + assert "ssh:" not in result.stderr + + +def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + check_terminal = text.index('if [ "${MODE}" = "--check" ]') + stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.') + first_scp = text.index("scp -q") + + assert check_terminal < stage_assignment < first_scp + assert "-f '${REMOTE_BASE}' -f - config -q" in text + assert '< "${LOCAL_OVERRIDE}"' in text + assert "check_pass_no_write" in text + + +def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + executable = executable_text() + + assert 'REMOTE_SERVICE="clickhouse"' in text + assert "--env-file /dev/null" in text + assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text + assert "timeout --signal=TERM --kill-after=30 300" in text + assert "up -d --no-deps --pull never --force-recreate" in text + assert '"${REMOTE_SERVICE}"' in text + assert "docker compose down" not in executable + assert "docker pull" not in executable + assert "docker volume" not in executable + assert "docker system prune" not in executable + assert "github.com" not in text.lower() + assert "ghcr.io" not in text.lower() + + +def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + assert "active_signoz_backup_detected" in text + assert "active_native_backup_or_restore" in text + assert "partial_managed_file_drift" in text + assert "another_deploy_holds_lock" in text + assert "backup-signoz[.]sh" in text + assert "CREATING_BACKUP" in text + assert "RESTORING" in text + + +def test_apply_preserves_image_data_volume_and_non_target_containers() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + assert "BEFORE_IMAGE_ID" in text + assert "AFTER_IMAGE_ID" in text + assert "BEFORE_DATA_VOLUME" in text + assert "AFTER_DATA_VOLUME" in text + assert 'eq .Destination "/var/lib/clickhouse"' in text + for prefix in ("COLLECTOR", "SIGNOZ", "ZOOKEEPER"): + assert f"BEFORE_{prefix}_ID" in text + assert f"BEFORE_{prefix}_RESTARTS" in text + assert "image_and_data_volume_unchanged" in text + + +def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + assert "system.disks WHERE name='backups'" in text + assert "Local|0|0|0|1" in text + assert "EXISTS TABLE system.backups" in text + assert "SHOW GRANTS" in text + assert "BACKUP([,[:space:]]|$)" in text + assert 'eq .Destination "/backups"' in text + assert "101:101:750" in text + assert "server_uid" in text + assert "server_gid" in text + assert "docker exec -u 101:101" in text + assert "test -w /backups" in text + assert "awk '/^Uid:/{print $2}' /proc/1/status" in text + assert "awk '/^Gid:/{print $2}' /proc/1/status" in text + identity_start = text.index('server_uid="') + identity_block = text[ + identity_start : text.index("docker exec -u 101:101", identity_start) + ] + assert "sh -c" not in identity_block + assert "concurrency_disabled" in text + for port in (4317, 4318, 8080): + assert f"listener_up {port}" in text + assert "api_code" in text + assert "disk_backups_local_rw_healthy" in text + + +def test_failed_apply_restores_prior_managed_compose_state() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + assert "restore_managed_files" in text + assert "rollback_deploy" in text + assert "PRIOR_OVERRIDE_PRESENT" in text + assert "PRIOR_CONFIG_PRESENT" in text + assert "prior-override.yaml" in text + assert "prior-backup-disk.xml" in text + assert "prior_compose_state_restored" in text + assert "retained_nonempty_no_delete" in text + assert "rc=91" in text + assert "trap on_exit EXIT" in text + + +def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None: + executable = executable_text() + lowered = executable.lower() + assert ".config.env" not in lowered + assert "config.env" not in lowered + assert ".env" not in lowered + assert "docker inspect --format '{{json .config.env}}'" not in lowered + assert " backup all" not in lowered + assert " restore all" not in lowered + assert "clickhouse-client --query 'backup" not in lowered + assert "clickhouse-client --query 'restore" not in lowered + assert "no_backup_or_restore_executed" in executable + + +def test_receipts_cover_controlled_apply_and_durable_terminal() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + for phase in ( + "sensor_source", + "normalized_asset_identity", + "source_of_truth_diff", + "ai_decision", + "risk_policy", + "check", + "execution", + "post_verifier", + "rollback", + "closure_writeback", + "terminal", + ): + assert phase in text + assert "receipts.jsonl" in text + assert "override_sha_" in text + assert "config_sha_" in text diff --git a/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py new file mode 100644 index 000000000..9d105ae82 --- /dev/null +++ b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py @@ -0,0 +1,339 @@ +from __future__ import annotations + +import os +from pathlib import Path +import subprocess + + +ROOT = Path(__file__).resolve().parents[3] +DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh" + + +def deployer_text() -> str: + return DEPLOYER.read_text(encoding="utf-8") + + +def executable_text() -> str: + return "\n".join( + line + for line in deployer_text().splitlines() + if not line.lstrip().startswith("#") + ) + + +def base_env() -> dict[str, str]: + env = os.environ.copy() + env.update( + { + "TRACE_ID": "P0-OBS-002", + "RUN_ID": "toolchain-test-run", + "WORK_ITEM_ID": "P0-OBS-002", + } + ) + return env + + +def test_deployer_shell_is_valid_and_help_is_no_write() -> None: + syntax = subprocess.run( + ["bash", "-n", str(DEPLOYER)], + check=False, + capture_output=True, + text=True, + ) + assert syntax.returncode == 0, syntax.stderr + + help_result = subprocess.run( + ["bash", str(DEPLOYER), "--help"], + check=False, + capture_output=True, + text=True, + env={ + key: value + for key, value in os.environ.items() + if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"} + }, + ) + assert help_result.returncode == 0 + assert "--check" in help_result.stdout + assert "--apply" in help_result.stdout + assert "CLICKHOUSE_RESTORE_INVENTORY_HELPER" in help_result.stdout + assert "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" in help_result.stdout + assert "CLICKHOUSE_RESTORE_CLUSTER_CONFIG" in help_result.stdout + + +def test_deployer_requires_controlled_apply_ids_before_network_access() -> None: + env = { + key: value + for key, value in os.environ.items() + if key not in {"TRACE_ID", "RUN_ID", "WORK_ITEM_ID"} + } + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 64 + assert "must be path-safe" in result.stderr + assert "ssh:" not in result.stderr + + +def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> None: + env = base_env() + env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/scripts/../escape" + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 64 + assert "safe absolute paths" in result.stderr + assert "ssh:" not in result.stderr + + env = base_env() + env["SIGNOZ_TOOLCHAIN_RECEIPT_ROOT"] = "/var/tmp/receipts" + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 64 + assert "receipt root must stay under /backup" in result.stderr + + env = base_env() + env["SIGNOZ_TOOLCHAIN_STAGE_ROOT"] = "/backup/deploy-staging" + env["SIGNOZ_TOOLCHAIN_SCRIPT_DIR"] = "/backup/deploy-staging/scripts" + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 64 + assert "must not overlap" in result.stderr + + +def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None: + text = deployer_text() + assert 'TARGET_HOST="wooo@192.168.0.110"' in text + assert "TARGET_HOST:-" not in text + assert "TARGET_HOST=" not in text.replace('TARGET_HOST="wooo@192.168.0.110"', "", 1) + + +def test_exact_seven_artifacts_and_target_modes_are_declared() -> None: + text = deployer_text() + for source in ( + "scripts/backup/backup-signoz.sh", + "scripts/backup/clickhouse-native-backup.sh", + "scripts/backup/clickhouse-native-restore-drill.sh", + "scripts/backup/clickhouse-restore-inventory.py", + "scripts/backup/run-signoz-backup-canary.sh", + "ops/signoz/clickhouse/config.d/backup_disk.xml", + "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml", + ): + assert source in text + + for target in ( + "${REMOTE_SCRIPT_DIR}/backup-signoz.sh", + "${REMOTE_SCRIPT_DIR}/clickhouse-native-backup.sh", + "${REMOTE_SCRIPT_DIR}/clickhouse-native-restore-drill.sh", + "${REMOTE_SCRIPT_DIR}/clickhouse-restore-inventory.py", + "${REMOTE_SCRIPT_DIR}/run-signoz-backup-canary.sh", + "${REMOTE_CONFIG_ROOT}/config.d/backup_disk.xml", + "${REMOTE_CONFIG_ROOT}/restore-drill/config.d/isolated-cluster.xml", + ): + assert target in text + assert "MODES=(0755 0755 0755 0755 0755 0644 0644)" in text + assert 'REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}"' in text + assert ( + 'REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-' + '/backup/config/signoz/clickhouse}"' + ) in text + + +def test_local_and_remote_stage_validation_cover_hash_syntax_python_and_xml() -> None: + text = deployer_text() + assert "validate_local_sources" in text + assert "SOURCE_HASHES" in text + assert "bash -n" in text + assert "python3 -m py_compile" in text + assert "validate_xml_pair" in text + for expected in ( + '"/backups/"', + '"restore-zookeeper"', + '"restore-clickhouse"', + '"CLICKHOUSE_RESTORE_REPLICA"', + ): + assert expected in text + + +def test_check_mode_exits_before_remote_stage_or_scp() -> None: + text = deployer_text() + check_exit = text.index('if [ "${MODE}" = --check ]') + stage_create = text.index("STAGE_CREATED=0") + first_scp = text.index("scp -q") + assert check_exit < stage_create < first_scp + assert "check_pass_no_write" in text + assert "remote_target_unchanged_no_stage_no_candidate" in text + + remote_check = text[ + text.index("remote_read_only_check() {") : text.index( + "local_receipt sensor_source", text.index("remote_read_only_check() {") + ) + ] + for forbidden in ("mkdir ", "install ", "mv ", "rm ", "scp "): + assert forbidden not in remote_check + + +def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + log = tmp_path / "calls.log" + ssh = fake_bin / "ssh" + ssh.write_text( + "#!/bin/sh\n" + 'printf "ssh %s\\n" "$*" >> "$FAKE_CALL_LOG"\n' + "cat >/dev/null\n" + 'printf \'{"phase":"source_of_truth_diff","result":"pass"}\\n\'\n', + encoding="utf-8", + ) + scp = fake_bin / "scp" + scp.write_text( + '#!/bin/sh\nprintf "scp %s\\n" "$*" >> "$FAKE_CALL_LOG"\nexit 99\n', + encoding="utf-8", + ) + ssh.chmod(0o755) + scp.chmod(0o755) + + env = base_env() + env["PATH"] = f"{fake_bin}:{env['PATH']}" + env["FAKE_CALL_LOG"] = str(log) + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 0, result.stderr + calls = log.read_text(encoding="utf-8").splitlines() + assert len(calls) == 1 + assert calls[0].startswith("ssh ") + assert "wooo@192.168.0.110" in calls[0] + assert "check_pass_no_write" in result.stdout + + +def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None: + text = deployer_text() + for token in ( + "prior-manifest.tsv", + "PRIOR_PRESENT", + "PRIOR_MODES", + "PRIOR_UIDS", + "PRIOR_GIDS", + "PRIOR_HASHES", + "cp -p", + "rollback_targets", + "all_prior_files_modes_hashes_owners_and_runtime_restored", + ): + assert token in text + assert "trap on_exit EXIT" in text + assert "rc=91" in text + assert 'run_root rm -f "${target}"' in text + assert 'run_root rmdir "${CREATED_DIRS[index]}"' in text + assert "bounded_execution_not_started_targets_unchanged" in text + + +def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None: + text = deployer_text() + assert '"${target}.candidate.${RUN_ID}"' in text + assert ( + 'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"' + ) in text + assert "deployed-manifest.tsv" in text + assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text + assert "stage_candidate_residue_0" in text + assert "verify_residue_zero" in text + + +def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None: + text = deployer_text() + for container in ( + "signoz-clickhouse", + "signoz-otel-collector", + "signoz", + "signoz-zookeeper-1", + ): + assert container in text + for field in ( + ".Id", + ".State.StartedAt", + ".RestartCount", + ".State.Running", + 'index .State "Health"', + ): + assert field in text + assert "runtime-before.tsv" in text + assert "runtime-after.tsv" in text + assert 'cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}"' in text + for port in (4317, 4318, 8080): + assert "for port in 4317 4318 8080" in text + assert str(port) in text + assert "api_200_unchanged" in text + + +def test_apply_fails_closed_while_backup_or_restore_is_active() -> None: + text = deployer_text() + assert "active_signoz_backup_toolchain_process" in text + assert "active_native_backup_or_restore" in text + assert "CREATING_BACKUP" in text + assert "RESTORING" in text + assert "another_toolchain_deploy_holds_lock" in text + + +def test_deployer_never_executes_runtime_work_or_uses_forbidden_supply_chain() -> None: + executable = executable_text().lower() + for forbidden in ( + "docker restart", + "docker compose", + "docker pull", + "docker run", + "backup all", + "restore all", + "ansible-playbook", + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "codeload.github.com", + "ghcr.io", + " gh ", + ): + assert forbidden not in executable + assert "no_backup_restore_restart_or_pull" in executable + + +def test_receipts_cover_complete_controlled_apply_contract() -> None: + text = deployer_text() + for phase in ( + "sensor_source", + "normalized_asset_identity", + "source_of_truth_diff", + "ai_decision", + "risk_policy", + "check", + "execution", + "post_verifier", + "rollback", + "closure_writeback", + "terminal", + ): + assert phase in text + assert "receipts.jsonl" in text + assert '"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"' in text + assert "durable_manifests_and_receipts_ack" in text diff --git a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh index ecdcc095b..9b0527662 100755 --- a/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh +++ b/scripts/reboot-recovery/reboot-recovery-readiness-audit.sh @@ -265,6 +265,12 @@ require_file scripts/backup/backup-awoooi-frequent.sh "AWOOOI high-frequency DB require_file scripts/backup/backup-langfuse.sh "Langfuse backup script" require_file scripts/backup/backup-monitoring.sh "Monitoring backup script" require_file scripts/backup/backup-signoz.sh "SignOz backup script" +require_file scripts/backup/clickhouse-native-backup.sh "ClickHouse native backup adapter" +require_file scripts/backup/clickhouse-native-restore-drill.sh "ClickHouse isolated native restore drill" +require_file scripts/backup/clickhouse-restore-inventory.py "ClickHouse restore inventory verifier" +require_file scripts/backup/run-signoz-backup-canary.sh "Guarded SignOz backup canary wrapper" +require_file ops/signoz/clickhouse/config.d/backup_disk.xml "ClickHouse backup disk config" +require_file ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml "ClickHouse isolated restore config" require_file scripts/backup/backup-open-webui.sh "Open-WebUI backup script" require_file scripts/backup/backup-clawbot.sh "ClawBot backup script" require_file scripts/backup/backup-configs.sh "Host/service config backup" diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index 3e6010ca9..a24b7e37f 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -2,8 +2,8 @@ """No-write source preflight for P0-OBS-002 Wave A. The checker intentionally does not access credentials, databases, runtime -volumes, Docker, Kubernetes, or remote hosts. Runtime evidence belongs to a -separate verifier receipt. +volumes, Docker, Kubernetes, or remote hosts. It validates runtime evidence +only from the committed post-closure receipt. """ from __future__ import annotations @@ -16,17 +16,24 @@ from typing import Any import yaml SCHEMA = "awoooi_signoz_canonical_route_preflight_v1" +POST_CLOSURE_SCHEMA = "awoooi_signoz_post_closure_regression_v2" +POST_CLOSURE_SNAPSHOT = "ops/signoz/p0-obs-002-post-closure-regression.yaml" EXPECTED_BLOCKERS = [ "organization_not_initialized", "direct_ingress_policy_not_closed", "http_bridge_not_supervised", "filelog_normalization_degraded", - "api_runtime_cooldown_pending", - "cd_probe_safety_fix_not_deployed", - "backup_collector_restore_guard_not_deployed", - "signoz_control_plane_health_route_not_deployed", "wave_b_not_authorized", ] +EXPECTED_POST_CLOSURE_BLOCKERS = [ + "signoz_sqlite_application_consistent_backup_pending", + "signoz_backup_offsite_dr_pending", + "clickhouse_native_failed_artifact_retention_pending", + "clickhouse_native_resume_submitted_inventory_pending", + "service_registry_runtime_mirror_reconciliation_pending", + "github_freeze_legacy_asset_retirement_pending", + "monitoring_generator_duplicate_awoooi_api_identity_pending", +] def _load_yaml(path: Path) -> Any: @@ -40,6 +47,16 @@ def _read(root: Path, relative: str) -> str: def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: contract = _load_yaml(contract_path) + post_closure = contract.get("runtime_observations", {}).get( + "post_closure_regression", {} + ) + snapshot_path = str(post_closure.get("snapshot", "")) + regression_path = root / POST_CLOSURE_SNAPSHOT + regression = ( + _load_yaml(regression_path) + if snapshot_path == POST_CLOSURE_SNAPSHOT and regression_path.is_file() + else {} + ) checks: dict[str, bool] = {} checks["contract_schema"] = ( @@ -66,8 +83,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: config = _read(root, "apps/api/src/core/config.py") checks["dual_route_application_allow"] = ( 'default=["192.168.0.188", "192.168.0.110"]' in config - and 'default=["192.168.0.112", "192.168.0.120", "192.168.0.121"]' - in config + and 'default=["192.168.0.112", "192.168.0.120", "192.168.0.121"]' in config ) network_documents = list( @@ -82,18 +98,14 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: ) canonical_ports: set[int] = set() for rule in egress_policy.get("spec", {}).get("egress", []): - cidrs = { - item.get("ipBlock", {}).get("cidr") for item in rule.get("to", []) - } + cidrs = {item.get("ipBlock", {}).get("cidr") for item in rule.get("to", [])} if "192.168.0.110/32" in cidrs: canonical_ports.update( int(item["port"]) for item in rule.get("ports", []) if isinstance(item.get("port"), int) ) - checks["dual_route_networkpolicy_allow"] = {4317, 4318}.issubset( - canonical_ports - ) + checks["dual_route_networkpolicy_allow"] = {4317, 4318}.issubset(canonical_ports) unit = _read(root, "ops/systemd/user/awoooi-signoz-otlp-bridge.service") checks["bridge_unit_hardened"] = all( @@ -112,14 +124,15 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: root, "scripts/reboot-recovery/verify-signoz-otel-signals.sh" ) checks["route_specific_verifier"] = ( - "OTLP_ENDPOINT" in verifier and "ROUTE_ID" in verifier + "OTLP_ENDPOINT" in verifier + and "ROUTE_ID" in verifier and all( value in signal_verifier for value in ( - 'send_signal logs', - 'send_signal metrics', - 'send_signal traces', - '/v1/$signal', + "send_signal logs", + "send_signal metrics", + "send_signal traces", + "/v1/$signal", "signoz_logs.logs_v2", "signoz_metrics.time_series_v4", "signoz_metrics.samples_v4", @@ -127,57 +140,54 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: ) ) ) - grpc_bridge = _read( - root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh" - ) - grpc_unit = _read( - root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service" - ) + grpc_bridge = _read(root, "scripts/reboot-recovery/signoz-188-otlp-grpc-bridge.sh") + grpc_unit = _read(root, "ops/systemd/user/awoooi-signoz-otlp-grpc-bridge.service") grpc_verifier = _read( root, "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh" ) _, grpc_apply_marker, grpc_after_apply = grpc_bridge.partition(" --apply)") - grpc_apply_block, grpc_status_marker, _ = grpc_after_apply.partition( - " --status)" - ) - checks["grpc_transport_recovery_source_ready"] = all( - value in grpc_bridge - for value in ( - "LISTEN_PORT:-24317", - "UPSTREAM_PORT:-4317", - "SOURCE_RANGE:-192.168.0.120/31", - "systemctl --user enable --now", - "--rollback", + grpc_apply_block, grpc_status_marker, _ = grpc_after_apply.partition(" --status)") + checks["grpc_transport_recovery_source_ready"] = ( + all( + value in grpc_bridge + for value in ( + "LISTEN_PORT:-24317", + "UPSTREAM_PORT:-4317", + "SOURCE_RANGE:-192.168.0.120/31", + "systemctl --user enable --now", + "--rollback", + ) ) - ) and all( - value in grpc_unit - for value in ( - "LISTEN_PORT=24317", - "UPSTREAM_PORT=4317", - "SOURCE_RANGE=192.168.0.120/31", - "Restart=on-failure", - "NoNewPrivileges=true", + and all( + value in grpc_unit + for value in ( + "LISTEN_PORT=24317", + "UPSTREAM_PORT=4317", + "SOURCE_RANGE=192.168.0.120/31", + "Restart=on-failure", + "NoNewPrivileges=true", + ) ) - ) and all( - value in grpc_verifier - for value in ( - "WINDOW_SECONDS:-600", - "serviceName={service:String}", - "signoz_metrics.samples_v4", - "Failed to export traces", - "restart_delta_0", + and all( + value in grpc_verifier + for value in ( + "WINDOW_SECONDS:-600", + "serviceName={service:String}", + "signoz_metrics.samples_v4", + "Failed to export traces", + "restart_delta_0", + ) ) ) checks["grpc_apply_waits_for_independent_post_verifier"] = ( bool(grpc_apply_marker) and bool(grpc_status_marker) and ( - 'receipt learning partial ' - '"application_trace_metric_10m_post_verifier_pending"' - in grpc_apply_block + "receipt learning partial " + '"application_trace_metric_10m_post_verifier_pending"' in grpc_apply_block ) and ( - 'receipt terminal partial ' + "receipt terminal partial " '"grpc_transport_restored_independent_600s_post_verifier_pending"' in grpc_apply_block ) @@ -185,7 +195,7 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and grpc_apply_block.index("receipt learning partial") < grpc_apply_block.index("receipt terminal partial") and ( - 'receipt terminal pass ' + "receipt terminal pass " '"real_application_otlp_grpc_trace_metric_freshness_closed"' in grpc_verifier ) @@ -202,14 +212,11 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and contract.get("runtime_observations", {}).get("grpc_bridge_supervisor") == "user_systemd_enabled_active" and grpc_runtime.get("runtime_status") == "restored_verified" - and grpc_source_fix.get("runtime_apply_status") - == "applied_post_verifier_pass" + and grpc_source_fix.get("runtime_apply_status") == "applied_post_verifier_pass" and grpc_source_fix.get("runtime_verifier_terminal") == "pass" and int(grpc_source_fix.get("runtime_verifier_window_seconds") or 0) >= 600 ) - static_recovery = _read( - root, "scripts/reboot-recovery/signoz-110-static-ingest.sh" - ) + static_recovery = _read(root, "scripts/reboot-recovery/signoz-110-static-ingest.sh") checks["managed_promotion_fail_closed"] = all( value in static_recovery for value in ( @@ -219,6 +226,453 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: ) ) + checks["post_closure_snapshot_path"] = snapshot_path == POST_CLOSURE_SNAPSHOT + checks["post_closure_schema_v2"] = ( + regression.get("schema") == POST_CLOSURE_SCHEMA + and post_closure.get("snapshot_schema") == POST_CLOSURE_SCHEMA + and regression.get("work_item_id") == "P0-OBS-002" + and regression.get("trace_id") == "P0-OBS-002" + ) + + asset_reconciliation = regression.get("asset_reconciliation", {}) + source_registry = _load_yaml(root / "ops/config/service-registry.yaml") + runtime_configmap = _load_yaml( + root / "k8s/awoooi-prod/15-service-registry-configmap.yaml" + ) + runtime_registry = yaml.safe_load( + runtime_configmap.get("data", {}).get("service-registry.yaml", "") + ) + source_services = { + item["name"]: item for item in source_registry.get("services", []) + } + runtime_services = { + item["name"]: item for item in runtime_registry.get("services", []) + } + shared_services = set(source_services) & set(runtime_services) + exact_shared_services = { + name + for name in shared_services + if source_services[name] == runtime_services[name] + } + source_only_services = sorted(set(source_services) - set(runtime_services)) + checks["service_registry_runtime_mirror_drift_explicit"] = ( + asset_reconciliation.get("drift_work_item_id") == "P0-OBS-002-ASSET-DRIFT-001" + and asset_reconciliation.get("source_service_count") + == len(source_services) + == 27 + and asset_reconciliation.get("runtime_mirror_service_count") + == len(runtime_services) + == 24 + and asset_reconciliation.get("exact_shared_service_count") + == len(exact_shared_services) + == 24 + and asset_reconciliation.get("shared_service_drift_count") == 0 + and asset_reconciliation.get("source_only_services") + == source_only_services + == ["bitan-app", "sentry", "signoz"] + and source_services.get("signoz-clickhouse") + == runtime_services.get("signoz-clickhouse") + and asset_reconciliation.get("signoz_clickhouse_exact_match") is True + and asset_reconciliation.get("live_signoz_query_host") == "192.168.0.110" + and asset_reconciliation.get("source_signoz_host") == "192.168.0.188" + and asset_reconciliation.get("terminal") == "partial_degraded" + ) + github_legacy_drift = asset_reconciliation.get( + "github_freeze_legacy_asset_drift", {} + ) + checks["github_freeze_legacy_asset_drift_explicit"] = ( + github_legacy_drift.get("drift_work_item_id") == "P0-OBS-002-ASSET-DRIFT-002" + and github_legacy_drift.get("superseded_by") == "global_product_governance_v2" + and github_legacy_drift.get("status") == "pending_controlled_retirement" + and github_legacy_drift.get("active_github_use_in_this_run") is False + and len(github_legacy_drift.get("source_references", [])) == 3 + and github_legacy_drift.get("replacement") + == "gitea_runner_readiness_and_queue_receipts" + and github_legacy_drift.get("p0_p1_github_recovery_scheduled") is False + ) + duplicate_identity_drift = asset_reconciliation.get( + "monitoring_generator_duplicate_identity_drift", {} + ) + checks["monitoring_generator_duplicate_identity_drift_explicit"] = ( + duplicate_identity_drift.get("drift_work_item_id") + == "P0-OBS-002-ASSET-DRIFT-003" + and duplicate_identity_drift.get("status") == "pending_source_deduplication" + and duplicate_identity_drift.get("registry_service_count") == 26 + and duplicate_identity_drift.get("awoooi_api_registry_entry_count") == 2 + and duplicate_identity_drift.get("generated_scrape_config_count") == 24 + and duplicate_identity_drift.get("generated_awoooi_api_job_count") == 2 + and duplicate_identity_drift.get("generated_blackbox_target_count") == 20 + and duplicate_identity_drift.get("runtime_apply_performed") is False + ) + + cd = regression.get("release_receipts", {}).get("gitea_cd_5140", {}) + cd_probe = cd.get("cd_probe", {}) + checks["cd_5140_receipt_consistent"] = ( + cd.get("gitea_run_id") == 5140 + and cd.get("source_sha") == "e4da6570a645e504cc93647d396f45f167b3172a" + and cd.get("workflow_terminal") == "success" + and cd.get("unit_tests", {}).get("passed") == 4817 + and cd.get("integration_tests_passed") == 5 + and cd.get("production_readback", {}).get("build_tag_matched") is True + and cd.get("production_readback", {}).get("desired_tag_matched") is True + and cd_probe.get("receipt_provenance") == "gitea_run_5140_job_log" + and cd_probe.get("passed") is True + and cd_probe.get("workers") == 4 + and cd_probe.get("safety_reserve_connections") == 4 + and cd_probe.get("restart_free") is True + ) + + applies = regression.get("controlled_applies", {}) + config_route = applies.get("config_route_patch", {}) + checks["config_route_patch_receipt_consistent"] = ( + config_route.get("run_id") + == "P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1" + and config_route.get("source_commit") + == "9a9d1464a586ba172d1613db5bc285def43eb3e8" + and config_route.get("included_in_release_sha") == cd.get("source_sha") + and config_route.get("normalized_asset", {}).get("internal_url") + == "http://192.168.0.110:8080" + and config_route.get("source_truth_diff", {}).get("otlp_producer_route_changed") + is False + and config_route.get("execution_receipt", {}).get("deployment_status") + == "deployed_verified" + and config_route.get("terminal") == "applied_post_verifier_pass" + ) + + prometheus = applies.get("prometheus_target_route", {}) + prometheus_check = prometheus.get("check_receipt", {}) + prometheus_execution = prometheus.get("execution_receipt", {}) + prometheus_post = prometheus.get("post_verifier", {}) + prometheus_independent = prometheus.get("independent_verifier", {}) + checks["prometheus_target_first_receipt_consistent"] = ( + prometheus.get("run_id") + == "P0-OBS-002-prometheus-route-20260714T215106Z-apply1" + and prometheus_check.get("promtool") == "pass" + and prometheus_check.get("candidate_sha") + == prometheus_execution.get("active_sha") + == prometheus_execution.get("runtime_sha") + and prometheus_execution.get("action") == "target_first_config_apply" + and prometheus_post.get("terminal") == "target_up_two_scrapes" + and prometheus_post.get("target_count") == 1 + and prometheus_post.get("target_health") == "up" + and prometheus_post.get("probe_success") == 1 + and prometheus_post.get("legacy_target_count") == 0 + and prometheus_independent.get("active_sha_matches_runtime") is True + and prometheus_independent.get("target_count") == 1 + and prometheus_independent.get("target_health") == "up" + and prometheus_independent.get("legacy_target_count") == 0 + and prometheus_independent.get("terminal") == "pass" + and prometheus.get("terminal") == "completed_target_first" + ) + + alert_rules = applies.get("alert_rules", {}) + alert_check = alert_rules.get("check_receipt", {}) + alert_execution = alert_rules.get("execution_receipt", {}) + alert_post = alert_rules.get("post_verifier", {}) + checks["alert_rules_receipt_consistent"] = ( + alert_rules.get("run_id") == "P0-OBS-002-alert-rules-20260714T215328Z-apply1" + and alert_check.get("promtool") == "pass" + and alert_check.get("source_sha") + == alert_execution.get("remote_sha") + == alert_execution.get("canonical_sha") + == alert_execution.get("runtime_sha") + and alert_execution.get("hashes_match") is True + and alert_post.get("window_seconds") == 130 + and alert_post.get("terminal") == "pass" + and alert_post.get("signoz_rule_count") == 1 + and alert_post.get("signoz_rule_health") == "ok" + and alert_rules.get("terminal") == "completed_verified" + ) + + native_disk = applies.get("clickhouse_native_backup_disk", {}) + native_disk_attempts = native_disk.get("attempts", []) + disk_apply1 = native_disk_attempts[0] if len(native_disk_attempts) == 2 else {} + disk_retry2 = native_disk_attempts[1] if len(native_disk_attempts) == 2 else {} + checks["clickhouse_native_backup_disk_receipt_consistent"] = ( + native_disk.get("source", {}).get("override_sha256") + == "cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488" + and native_disk.get("source", {}).get("config_sha256") + == "c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9" + and native_disk.get("check_receipt", {}).get("terminal") + == "check_pass_no_write" + and disk_apply1.get("terminal") == "failed_rolled_back" + and disk_apply1.get("exit_code") == 2 + and disk_apply1.get("rollback", {}).get("terminal") == "pass" + and disk_apply1.get("rollback", {}).get("original_data_volume_preserved") + is True + and disk_retry2.get("run_id") + == "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2" + and disk_retry2.get("terminal") == "pass" + and disk_retry2.get("execution", {}).get("image_digest_unchanged") is True + and disk_retry2.get("execution", {}).get("data_volume_unchanged") is True + and disk_retry2.get("post_verifier", {}).get("independent") is True + and disk_retry2.get("post_verifier", {}).get("server_write_access") is True + and disk_retry2.get("post_verifier", {}).get("dependent_restart_count_delta") + == 0 + and disk_retry2.get("post_verifier", {}).get("api_http_status") == 200 + and disk_retry2.get("post_verifier", {}).get("active_backup_or_restore_count") + == 0 + and native_disk.get("terminal") == "deployed_verified" + ) + + native_canary = applies.get("clickhouse_native_backup_restore_canary", {}) + native_attempts = native_canary.get("attempts", []) + native_apply2 = native_attempts[1] if len(native_attempts) == 4 else {} + native_apply3 = native_attempts[2] if len(native_attempts) == 4 else {} + native_apply4 = native_attempts[3] if len(native_attempts) == 4 else {} + native_backup = native_apply4.get("backup", {}) + native_restore = native_apply4.get("isolated_restore", {}) + native_restic = native_apply4.get("restic", {}) + native_post = native_apply4.get("independent_post_verifier", {}) + toolchain_apply = native_canary.get("toolchain_controlled_apply", {}) + checks["native_backup_restore_canary_closed"] = ( + native_canary.get("completion_scope") == "clickhouse_native_only" + and native_canary.get("source", {}).get("native_backup_sha256") + == "754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a" + and native_canary.get("source", {}).get("isolated_restore_sha256") + == "57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7" + and native_canary.get("source", {}).get("post_canary_focused_tests_passed") + == 94 + and native_canary.get("source", {}).get("post_format_focused_tests_passed") + == 94 + and native_canary.get("source", {}).get("ruff_format") == "pass" + and toolchain_apply.get("apply_run_id") + == "P0-OBS-002-native-toolchain-20260715T001252Z-apply4" + and toolchain_apply.get("apply_terminal") == "pass" + and toolchain_apply.get("postcheck_terminal") == "check_pass_no_write" + and toolchain_apply.get("postcheck_diff") + == "matching_7_drift_0_absent_0_active_operations_0" + and native_apply2.get("artifact_retention") == "preserved_failed_run_evidence" + and native_apply2.get("cleanup_verified") is True + and native_apply3.get("artifact_retention") == "preserved_failed_run_evidence" + and native_apply3.get("cleanup_verified") is True + and native_apply4.get("run_id") + == "P0-OBS-002-native-canary-20260715T001329Z-apply4" + and native_apply4.get("check_terminal") == "check_pass_no_write" + and native_apply4.get("terminal") == "pass" + and native_backup.get("terminal") == "BACKUP_CREATED" + and native_backup.get("error") == "" + and native_backup.get("artifact_sha256") + == "4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc" + and native_restore.get("terminal") == "verified" + and native_restore.get("clickhouse_restore_terminal") == "RESTORED" + and native_restore.get("production_network_attached") is False + and native_restore.get("production_restore_performed") is False + and native_restore.get("source_staged_clickhouse_artifact_sha_match") is True + and native_restore.get("database_count") + == native_restore.get("restored_database_count") + == 6 + and native_restore.get("table_count") + == native_restore.get("restored_table_count") + == 100 + and native_restore.get("table_engine_schema_manifest_parity") is True + and native_restore.get("check_table_count") + == native_restore.get("check_table_pass_count") + == 44 + and native_restore.get("critical_nonzero_count") == 4 + and native_restore.get("aggregate_counter_exact_parity_gate") is False + and native_restic.get("snapshot_id") == "574e2a1a" + and native_restic.get("snapshot_readback") == "pass" + and native_restic.get("repository_scope") == "local_same_failure_domain" + and native_restic.get("offsite_verified") is False + and native_post.get("production_restart_count_delta") == 0 + and native_post.get("api_http_status") == 200 + and native_post.get("new_server_artifact_removed") is True + and native_post.get("apply2_apply3_preserved_artifact_hashes_unchanged") is True + and native_post.get( + "exact_ephemeral_container_volume_network_process_tmp_residue_count" + ) + == 0 + and native_post.get("cleanup_verified") is True + and native_post.get("secret_values_collected") is False + and native_canary.get("terminal") == "production_closed_clickhouse_native_scope" + ) + + backup = applies.get("backup_canary", {}) + backup_attempts = backup.get("attempts", []) + backup_attempt = backup_attempts[0] if len(backup_attempts) == 2 else {} + retry2 = backup_attempts[1] if len(backup_attempts) == 2 else {} + backup_rollback = backup_attempt.get("rollback", {}) + retry2_rollback = retry2.get("rollback", {}) + checks["backup_canary_failure_and_rollback_consistent"] = ( + backup.get("guard_source", {}).get("deployment_status") == "deployed_verified" + and backup_attempt.get("run_id") + == "P0-OBS-002-backup-canary-20260714T213210Z-retry1" + and backup_attempt.get("terminal") == "failed_rolled_back" + and backup_attempt.get("classification") + == "failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention" + and backup_attempt.get("archive_container_exit_code") == 137 + and backup_attempt.get("root_cause") + == "docker_health_monitor_restarted_collector_during_intentional_backup_stop" + and backup_attempt.get("clickhouse_archive_committed") is False + and backup_attempt.get("sqlite_stage_reached") is False + and backup_attempt.get("restic_stage_reached") is False + and backup_attempt.get("retention_stage_reached") is False + and backup_attempt.get("success_stage_reached") is False + and backup_rollback.get("collector_running") is True + and backup_rollback.get("collector_restart_count") == 0 + and backup_rollback.get("grpc_4317_listening") is True + and backup_rollback.get("http_4318_listening") is True + and backup_rollback.get("current_temp_dir_absent") is True + and backup_rollback.get("partial_artifacts_absent") is True + and retry2.get("run_id") == "P0-OBS-002-backup-canary-20260714T221230Z-retry2" + and retry2.get("terminal") == "failed_rolled_back" + and retry2.get("backup_script_exit_code") == 1 + and retry2.get("archive_container_exit_code") == "unknown_not_durably_captured" + and retry2.get("root_cause") + == "raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract" + and retry2.get("exact_tar_error_proven") is False + and retry2.get("clickhouse_archive_committed") is False + and retry2.get("sqlite_stage_reached") is False + and retry2.get("restic_stage_reached") is False + and retry2.get("retention_stage_reached") is False + and retry2.get("success_stage_reached") is False + and retry2.get("monitor_window", {}).get("auto_repair_count") == 0 + and retry2_rollback.get("cooldown_lease_restored") is True + and retry2_rollback.get("collector_running") is True + and retry2_rollback.get("collector_restart_count") == 0 + and retry2_rollback.get("grpc_4317_listening") is True + and retry2_rollback.get("http_4318_listening") is True + and retry2_rollback.get("current_temp_dir_absent") is True + and retry2_rollback.get("partial_artifacts_absent") is True + ) + + post_release = regression.get("post_release_runtime_verifier", {}) + checks["post_release_600s_verifier_consistent"] = ( + post_release.get("run_id") == "P0-OBS-002-post-release-20260714T215500Z" + and post_release.get("window_seconds") == 600 + and post_release.get("terminal") == "pass" + and post_release.get("restart_delta") == 0 + and post_release.get("exporter_trace_errors") == 0 + and post_release.get("exporter_metric_errors") == 0 + and post_release.get("trace_count") == 4026 + and post_release.get("metric_sample_count") == 76369 + and post_release.get("api_runtime_cooldown_closed") is True + and regression.get("regressions", {}) + .get("api_runtime", {}) + .get("last_observed_cooldown_closed") + is True + ) + + native_migration = regression.get("completed_verifiers", {}).get( + "clickhouse_native_backup_migration", {} + ) + pending_verifiers = regression.get("pending_verifiers", {}) + sqlite_backup = pending_verifiers.get( + "signoz_sqlite_application_consistent_backup", {} + ) + offsite_backup = pending_verifiers.get("signoz_backup_offsite_dr", {}) + failed_artifacts = pending_verifiers.get( + "clickhouse_native_failed_artifact_retention", {} + ) + resume_inventory = pending_verifiers.get( + "clickhouse_native_resume_submitted_inventory", {} + ) + checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = ( + native_migration.get("prior_run_id") == retry2.get("run_id") + and native_migration.get("run_id") == native_apply4.get("run_id") + and native_migration.get("status") + == "production_closed_clickhouse_native_scope" + and native_migration.get("native_backup_terminal") == "BACKUP_CREATED" + and native_migration.get("isolated_restore_terminal") == "verified" + and native_migration.get("restic_snapshot_id") == "574e2a1a" + and native_migration.get("independent_verifier") == "pass" + and native_migration.get("exact_cleanup") == "pass" + and sqlite_backup.get("status") + == "pending_supported_non_raw_export_or_coordinated_snapshot" + and sqlite_backup.get("raw_sqlite_read_or_sync_allowed") is False + and sqlite_backup.get("sqlite_cli_present_in_runtime") is False + and offsite_backup.get("status") + == "pending_offsite_snapshot_and_restore_verifier" + and offsite_backup.get("current_repository_scope") + == "local_same_failure_domain" + and offsite_backup.get("offsite_verified") is False + and failed_artifacts.get("status") == "pending_bounded_retention_decision" + and failed_artifacts.get("destructive_cleanup_authorized") is False + and failed_artifacts.get("active_backup_or_restore_count") == 0 + and len(failed_artifacts.get("preserved_artifacts", [])) == 2 + and resume_inventory.get("status") + == "pending_durable_submitted_inventory_contract" + and resume_inventory.get("same_run_resume_safe") is False + and len(resume_inventory.get("required_preconditions", [])) == 4 + ) + + checks["post_closure_contract_mirror_consistent"] = ( + post_closure.get("gitea_cd_run_id") == cd.get("gitea_run_id") + and post_closure.get("release_source_sha") == cd.get("source_sha") + and post_closure.get("api_runtime_cooldown_closed") is True + and post_closure.get("cd_probe_safety_fix_deployed") is True + and post_closure.get("config_route_patch_deployed") is True + and post_closure.get("signoz_control_plane_health_route_deployed") is True + and post_closure.get("backup_collector_restore_guard_deployed") is True + and post_closure.get("backup_canary_scope") == "clickhouse_native_only" + and post_closure.get("backup_canary_verified") is True + and post_closure.get("backup_canary_last_run_id") == native_apply4.get("run_id") + and post_closure.get("backup_canary_last_terminal") == "pass" + and post_closure.get("backup_canary_cooldown_retry_status") + == "superseded_by_online_native_no_stop_contract" + and post_closure.get("clickhouse_native_backup_migration_status") + == "production_closed" + and post_closure.get("clickhouse_native_backup_disk_run_id") + == disk_retry2.get("run_id") + and post_closure.get("clickhouse_native_backup_disk_status") + == "deployed_verified" + and post_closure.get("clickhouse_native_backup_toolchain_run_id") + == toolchain_apply.get("apply_run_id") + and post_closure.get("clickhouse_native_backup_toolchain_postcheck_run_id") + == toolchain_apply.get("postcheck_run_id") + and post_closure.get("clickhouse_native_restore_terminal") == "verified" + and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a" + and post_closure.get("clickhouse_native_offsite_status") == "pending" + and post_closure.get("clickhouse_native_failed_artifact_retention_status") + == "pending_bounded_retention_decision" + and post_closure.get("clickhouse_native_resume_inventory_status") + == "pending_durable_submitted_inventory_contract" + and post_closure.get("signoz_sqlite_application_consistent_backup_status") + == "pending" + and post_closure.get("service_registry_runtime_mirror_status") + == "partial_degraded" + and post_closure.get("service_registry_runtime_mirror_work_item_id") + == "P0-OBS-002-ASSET-DRIFT-001" + and post_closure.get("github_freeze_legacy_asset_status") + == "pending_controlled_retirement" + and post_closure.get("github_freeze_legacy_asset_work_item_id") + == "P0-OBS-002-ASSET-DRIFT-002" + and post_closure.get("monitoring_generator_duplicate_identity_status") + == "pending_source_deduplication" + and post_closure.get("monitoring_generator_duplicate_identity_work_item_id") + == "P0-OBS-002-ASSET-DRIFT-003" + and post_closure.get("post_release_otlp_grpc_600s_verifier_status") == "pass" + ) + + regression_terminal = regression.get("terminal", {}) + checks["post_closure_terminal_blocker_consistent"] = ( + regression_terminal.get("status") == "partial_degraded" + and list(regression_terminal.get("blockers", [])) + == EXPECTED_POST_CLOSURE_BLOCKERS + ) + + regression_text = _read(root, POST_CLOSURE_SNAPSHOT).lower() + checks["post_closure_no_secret_or_github_dependency"] = ( + regression.get("scope_boundaries", {}).get("secret_read") is False + and regression.get("scope_boundaries", {}).get("github_used") is False + and all( + forbidden not in regression_text + for forbidden in ( + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "codeload.github.com", + "ghcr.io", + "authorization:", + "password:", + "private_key:", + "cookie:", + "session:", + ) + ) + ) + terminal = contract.get("terminal", {}) blockers = list(terminal.get("blockers", [])) checks["expected_blockers_explicit"] = blockers == EXPECTED_BLOCKERS @@ -240,7 +694,9 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument( + "--root", type=Path, default=Path(__file__).resolve().parents[2] + ) parser.add_argument( "--contract", type=Path, diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index ef0400221..831e82f3d 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -27,7 +27,9 @@ POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression. def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None: payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8")) - assert payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1" + assert ( + payload["schema"] == "awoooi_signoz_organization_canonical_route_migration_v1" + ) assert payload["work_item_id"] == "P0-OBS-002" assert payload["phase"] == "wave_a_dual_allow" assert payload["routes"]["active_route"] == "incumbent" @@ -43,14 +45,33 @@ def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> ) -def test_contract_contains_no_credential_values_or_destructive_db_plan() -> None: - text = CONTRACT.read_text(encoding="utf-8").lower() - assert "signoz_user_root_password:" not in text - assert "signoz-api-key:" not in text - assert "authorization:" not in text - assert "raw_relational_database_update" in text - assert "stateful_volume_restore" in text - assert "apply_allowed_in_wave_a: false" in text +def test_contract_and_receipt_contain_no_secret_values_or_github_dependency() -> None: + contract_text = CONTRACT.read_text(encoding="utf-8").lower() + regression_text = POST_CLOSURE_REGRESSION.read_text(encoding="utf-8").lower() + forbidden = ( + "signoz_user_root_password:", + "signoz-api-key:", + "authorization:", + "password:", + "private_key:", + "cookie:", + "session:", + "github.com", + "api.github.com", + "raw.githubusercontent.com", + "codeload.github.com", + "ghcr.io", + ) + for text in (contract_text, regression_text): + assert all(value not in text for value in forbidden) + + assert "raw_relational_database_update" in contract_text + assert "stateful_volume_restore" in contract_text + assert "apply_allowed_in_wave_a: false" in contract_text + + regression = yaml.safe_load(regression_text) + assert regression["scope_boundaries"]["secret_read"] is False + assert regression["scope_boundaries"]["github_used"] is False def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: @@ -69,9 +90,31 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: assert payload["checks"] and all(payload["checks"].values()) assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True + assert payload["checks"]["post_closure_schema_v2"] is True + assert payload["checks"]["post_release_600s_verifier_consistent"] is True + assert ( + payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"] + is True + ) + assert payload["checks"]["service_registry_runtime_mirror_drift_explicit"] + assert payload["checks"]["github_freeze_legacy_asset_drift_explicit"] + assert payload["checks"]["monitoring_generator_duplicate_identity_drift_explicit"] assert "organization_not_initialized" in payload["blockers"] - assert "signoz_control_plane_health_route_not_deployed" in payload["blockers"] assert "wave_b_not_authorized" in payload["blockers"] + assert payload["blockers"] == [ + "organization_not_initialized", + "direct_ingress_policy_not_closed", + "http_bridge_not_supervised", + "filelog_normalization_degraded", + "wave_b_not_authorized", + ] + for closed_blocker in ( + "api_runtime_cooldown_pending", + "cd_probe_safety_fix_not_deployed", + "backup_collector_restore_guard_not_deployed", + "signoz_control_plane_health_route_not_deployed", + ): + assert closed_blocker not in payload["blockers"] def test_bridge_unit_is_user_scoped_restarting_and_hardened() -> None: @@ -129,7 +172,10 @@ def test_three_signal_snapshot_keeps_synthetic_parity_separate_from_promotion() assert snapshot["trace_id"] == parity["trace_id"] assert snapshot["route_parity_terminal"] == "pass" assert snapshot["program_terminal"] == "partial_degraded" - assert snapshot["controlled_apply_contract"]["learning_writeback"]["status"] == "partial" + assert ( + snapshot["controlled_apply_contract"]["learning_writeback"]["status"] + == "partial" + ) assert ( snapshot["controlled_apply_contract"]["learning_writeback"][ "durable_gitea_feature_acknowledgement" @@ -151,13 +197,13 @@ def test_filelog_fallback_preserves_body_and_exposes_normalization_coverage() -> text = OTEL_DAEMONSET.read_text(encoding="utf-8") assert "id: cri_parser" in text assert "on_error: send_quiet" not in text - assert 'if: \'body matches "^[^ ]+ (stdout|stderr) [^ ]* .*$"\'' in text + assert "if: 'body matches \"^[^ ]+ (stdout|stderr) [^ ]* .*$\"'" in text assert "id: mark_cri_parsed" in text - assert 'value: parsed_cri' in text + assert "value: parsed_cri" in text assert "id: move_cri_body" in text assert "if: attributes.log != nil" in text assert "id: mark_cri_fallback" in text - assert 'value: fallback_unparsed' in text + assert "value: fallback_unparsed" in text assert 'if: attributes["awoooi.log.parse_status"] == nil' in text contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8")) @@ -231,11 +277,10 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier() apply_block = bridge.split(" --apply)", 1)[1].split(" --status)", 1)[0] learning = ( - 'receipt learning partial ' - '"application_trace_metric_10m_post_verifier_pending"' + 'receipt learning partial "application_trace_metric_10m_post_verifier_pending"' ) terminal = ( - 'receipt terminal partial ' + "receipt terminal partial " '"grpc_transport_restored_independent_600s_post_verifier_pending"' ) assert learning in apply_block @@ -243,9 +288,8 @@ def test_grpc_apply_terminal_stays_partial_until_independent_runtime_verifier() assert apply_block.index(learning) < apply_block.index(terminal) assert "receipt terminal pass" not in apply_block assert ( - 'receipt terminal pass ' - '"real_application_otlp_grpc_trace_metric_freshness_closed"' - in verifier + "receipt terminal pass " + '"real_application_otlp_grpc_trace_metric_freshness_closed"' in verifier ) @@ -266,7 +310,7 @@ def test_contract_grpc_route_state_matches_verified_runtime_evidence() -> None: assert grpc["source_fix"]["runtime_verifier_window_seconds"] >= 600 -def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate() -> None: +def test_grpc_runtime_verifier_uses_real_aggregates_and_ten_minute_gate() -> None: text = GRPC_VERIFIER.read_text(encoding="utf-8") for expected in ( 'WINDOW_SECONDS="${WINDOW_SECONDS:-600}"', @@ -293,10 +337,7 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate( assert grpc["recent_awoooi_api_metric_sample_count"] == 60180 assert grpc["http_only_verifier_blind_spot"] is True assert grpc["source_fix"]["check_terminal"] == "pass" - assert ( - grpc["source_fix"]["runtime_apply_status"] - == "applied_post_verifier_pass" - ) + assert grpc["source_fix"]["runtime_apply_status"] == "applied_post_verifier_pass" assert grpc["source_fix"]["runtime_verifier_terminal"] == "pass" receipt = yaml.safe_load(GRPC_RUNTIME_RECEIPT.read_text(encoding="utf-8")) @@ -312,13 +353,431 @@ def test_grpc_runtime_verifier_uses_real_service_aggregates_and_ten_minute_gate( regression = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) assert regression["baseline"]["verifier_terminal"] == "pass" - assert regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is False + assert ( + regression["regressions"]["api_runtime"]["last_observed_cooldown_closed"] + is True + ) assert regression["controlled_recovery"]["stateful_services_touched"] == 0 assert regression["controlled_recovery"]["grpc_4317_listening"] is True assert regression["scope_boundaries"]["grpc_bridge_rollback_indicated"] is False assert regression["terminal"]["status"] == "partial_degraded" +def test_post_closure_v2_records_cd_5140_and_config_route_patch() -> None: + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + assert receipt["schema"] == "awoooi_signoz_post_closure_regression_v2" + + cd = receipt["release_receipts"]["gitea_cd_5140"] + assert cd["gitea_run_id"] == 5140 + assert cd["source_sha"] == "e4da6570a645e504cc93647d396f45f167b3172a" + assert cd["workflow_terminal"] == "success" + assert cd["focused_tests_passed"] == 6 + assert cd["unit_tests"] == {"passed": 4817, "skipped": 23, "warnings": 69} + assert cd["integration_tests_passed"] == 5 + assert cd["production_readback"]["build_tag_matched"] is True + assert cd["production_readback"]["desired_tag_matched"] is True + assert cd["production_readback"]["rollout_terminal"] == "success" + assert cd["postdeploy"]["alert_chain_terminal"] == "pass" + assert cd["postdeploy"]["playwright_passed"] == 5 + assert cd["cd_probe"]["receipt_provenance"] == "gitea_run_5140_job_log" + assert cd["cd_probe"]["restart_free"] is True + assert cd["cd_probe"]["workers"] == 4 + assert cd["cd_probe"]["safety_reserve_connections"] == 4 + + route = receipt["controlled_applies"]["config_route_patch"] + assert route["run_id"] == ("P0-OBS-002-signoz-config-drift-20260714T205605Z-retry1") + assert route["source_commit"] == "9a9d1464a586ba172d1613db5bc285def43eb3e8" + assert route["included_in_release_sha"] == cd["source_sha"] + assert route["normalized_asset"]["internal_url"] == "http://192.168.0.110:8080" + assert route["normalized_asset"]["public_url"] == "https://signoz.wooo.work" + assert route["source_truth_diff"]["otlp_producer_route_changed"] is False + assert route["execution_receipt"]["deployment_status"] == "deployed_verified" + assert route["terminal"] == "applied_post_verifier_pass" + + +def test_prometheus_target_first_receipt_has_independent_verifier() -> None: + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + route = receipt["controlled_applies"]["prometheus_target_route"] + check = route["check_receipt"] + execution = route["execution_receipt"] + post = route["post_verifier"] + independent = route["independent_verifier"] + + assert route["run_id"] == ("P0-OBS-002-prometheus-route-20260714T215106Z-apply1") + assert check["promtool"] == "pass" + assert check["candidate_sha"] == execution["active_sha"] == execution["runtime_sha"] + assert execution["action"] == "target_first_config_apply" + assert execution["backup_sha"] == check["source_sha"] + assert post["terminal"] == "target_up_two_scrapes" + assert post["first_scrape_at"] == "2026-07-15T05:52:38.146935853+08:00" + assert post["second_scrape_at"] == "2026-07-15T05:52:53.146935853+08:00" + assert post["first_scrape_at"] < post["second_scrape_at"] + assert post["target_count"] == 1 + assert post["target_health"] == "up" + assert post["probe_success"] == 1 + assert post["legacy_target_count"] == 0 + assert independent["mode"] == "independent_readback" + assert independent["active_sha_matches_runtime"] is True + assert independent["target_count"] == 1 + assert independent["target_health"] == "up" + assert independent["probe_success"] == 1 + assert independent["legacy_target_count"] == 0 + assert independent["terminal"] == "pass" + assert route["rollback"] == {"required": False, "available": True} + assert route["terminal"] == "completed_target_first" + + +def test_alert_rules_receipt_hashes_and_runtime_rule_match() -> None: + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + alert = receipt["controlled_applies"]["alert_rules"] + source_sha = alert["check_receipt"]["source_sha"] + execution = alert["execution_receipt"] + verifier = alert["post_verifier"] + + assert alert["run_id"] == "P0-OBS-002-alert-rules-20260714T215328Z-apply1" + assert alert["check_receipt"]["promtool"] == "pass" + assert source_sha == execution["remote_sha"] + assert source_sha == execution["canonical_sha"] + assert source_sha == execution["runtime_sha"] + assert execution["hashes_match"] is True + assert execution["prometheus_ready"] is True + assert verifier["window_seconds"] == 130 + assert verifier["terminal"] == "pass" + assert verifier["signoz_rule_count"] == 1 + assert verifier["signoz_rule_health"] == "ok" + assert verifier["signoz_rule_state"] == "inactive" + assert verifier["exact_query"] == ( + '(probe_success{instance="http://192.168.0.110:8080/api/v1/health",' + 'job="blackbox-http"} == 0) or absent(probe_success{instance="http://192.168.0.110:8080/api/v1/health",' + 'job="blackbox-http"})' + ) + assert alert["terminal"] == "completed_verified" + + +def test_failed_backup_canary_preserves_failure_and_verified_rollback() -> None: + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + backup = receipt["controlled_applies"]["backup_canary"] + assert backup["guard_source"]["deployment_status"] == "deployed_verified" + assert len(backup["attempts"]) == 2 + + attempt = backup["attempts"][0] + assert attempt["run_id"] == ("P0-OBS-002-backup-canary-20260714T213210Z-retry1") + assert attempt["terminal"] == "failed_rolled_back" + assert attempt["classification"] == ( + "failed_before_clickhouse_archive_commit_rollback_verified_no_restic_no_retention" + ) + assert attempt["failed_stage"] == "clickhouse_archive_create" + assert attempt["archive_container_exit_code"] == 137 + assert attempt["root_cause"] == ( + "docker_health_monitor_restarted_collector_during_intentional_backup_stop" + ) + for false_field in ( + "consistent_archive_valid", + "clickhouse_archive_committed", + "sqlite_stage_reached", + "restic_stage_reached", + "retention_stage_reached", + "success_stage_reached", + ): + assert attempt[false_field] is False + + rollback = attempt["rollback"] + assert rollback["collector_running"] is True + assert rollback["collector_restart_count"] == 0 + assert rollback["grpc_4317_listening"] is True + assert rollback["http_4318_listening"] is True + assert rollback["canary_processes_absent"] is True + assert rollback["canary_container_absent"] is True + assert rollback["current_temp_dir_absent"] is True + assert rollback["partial_artifacts_absent"] is True + assert not attempt["terminal"].startswith("completed") + assert "learning_recorded" not in attempt + + retry2 = backup["attempts"][1] + assert retry2["run_id"] == ("P0-OBS-002-backup-canary-20260714T221230Z-retry2") + assert retry2["terminal"] == "failed_rolled_back" + assert retry2["backup_script_exit_code"] == 1 + assert retry2["archive_container_exit_code"] == "unknown_not_durably_captured" + assert retry2["archive_stderr_receipt"] == "suppressed_by_legacy_pipeline" + assert retry2["root_cause"] == ( + "raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract" + ) + assert retry2["exact_tar_error_proven"] is False + assert retry2["monitor_window"]["cron_intervals_crossed"] == 2 + assert retry2["monitor_window"]["auto_repair_count"] == 0 + for false_field in ( + "consistent_archive_valid", + "clickhouse_archive_committed", + "sqlite_stage_reached", + "restic_stage_reached", + "retention_stage_reached", + "success_stage_reached", + ): + assert retry2[false_field] is False + retry2_rollback = retry2["rollback"] + assert retry2_rollback["cooldown_lease_restored"] is True + assert retry2_rollback["collector_running"] is True + assert retry2_rollback["collector_restart_count"] == 0 + assert retry2_rollback["grpc_4317_listening"] is True + assert retry2_rollback["http_4318_listening"] is True + assert retry2_rollback["current_temp_dir_absent"] is True + assert retry2_rollback["partial_artifacts_absent"] is True + assert retry2["residual_cleanup_debt"] == [] + + +def test_clickhouse_native_backup_disk_preserves_failed_attempt_and_closes_retry() -> ( + None +): + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + native_disk = receipt["controlled_applies"]["clickhouse_native_backup_disk"] + + assert native_disk["source"]["override_sha256"] == ( + "cba7f4428ebf54f2890f3836d883d611d964b226210f228279206c84596a6488" + ) + assert native_disk["source"]["config_sha256"] == ( + "c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9" + ) + assert native_disk["check_receipt"]["terminal"] == "check_pass_no_write" + assert len(native_disk["attempts"]) == 2 + + apply1, retry2 = native_disk["attempts"] + assert apply1["terminal"] == "failed_rolled_back" + assert apply1["exit_code"] == 2 + assert apply1["rollback"]["terminal"] == "pass" + assert apply1["rollback"]["original_data_volume_preserved"] is True + assert apply1["rollback"]["residue_count"] == 0 + + assert retry2["run_id"] == ( + "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2" + ) + assert retry2["terminal"] == "pass" + assert retry2["execution"]["image_digest_unchanged"] is True + assert retry2["execution"]["data_volume_unchanged"] is True + assert retry2["post_verifier"]["independent"] is True + assert retry2["post_verifier"]["server_write_access"] is True + assert retry2["post_verifier"]["dependent_restart_count_delta"] == 0 + assert retry2["post_verifier"]["api_http_status"] == 200 + assert retry2["post_verifier"]["active_backup_or_restore_count"] == 0 + assert retry2["post_verifier"]["stage_candidate_process_residue_count"] == 0 + assert native_disk["terminal"] == "deployed_verified" + + +def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> None: + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"] + assert canary["completion_scope"] == "clickhouse_native_only" + assert canary["source"]["post_canary_focused_tests_passed"] == 94 + assert canary["source"]["post_format_focused_tests_passed"] == 94 + assert canary["source"]["ruff_format"] == "pass" + assert canary["toolchain_controlled_apply"]["apply_terminal"] == "pass" + assert canary["toolchain_controlled_apply"]["postcheck_diff"] == ( + "matching_7_drift_0_absent_0_active_operations_0" + ) + assert len(canary["attempts"]) == 4 + _, apply2, apply3, apply4 = canary["attempts"] + assert apply2["artifact_retention"] == "preserved_failed_run_evidence" + assert apply2["cleanup_verified"] is True + assert apply3["artifact_retention"] == "preserved_failed_run_evidence" + assert apply3["cleanup_verified"] is True + + assert apply4["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4") + assert apply4["check_terminal"] == "check_pass_no_write" + assert apply4["terminal"] == "pass" + assert apply4["backup"]["terminal"] == "BACKUP_CREATED" + assert apply4["backup"]["error"] == "" + assert apply4["backup"]["artifact_sha256"] == ( + "4fecd3ed99bfdc219b909cc028f3c1f31b8fdd15c1fdccdef18d710365113bdc" + ) + + restore = apply4["isolated_restore"] + assert restore["terminal"] == "verified" + assert restore["clickhouse_restore_terminal"] == "RESTORED" + assert restore["production_network_attached"] is False + assert restore["production_restore_performed"] is False + assert restore["source_staged_clickhouse_artifact_sha_match"] is True + assert restore["database_count"] == restore["restored_database_count"] == 6 + assert restore["table_count"] == restore["restored_table_count"] == 100 + assert restore["table_engine_schema_manifest_parity"] is True + assert restore["check_table_count"] == restore["check_table_pass_count"] == 44 + assert restore["critical_nonzero_count"] == 4 + assert restore["aggregate_counter_exact_parity_gate"] is False + + restic = apply4["restic"] + assert restic["snapshot_id"] == "574e2a1a" + assert restic["snapshot_readback"] == "pass" + assert restic["repository_scope"] == "local_same_failure_domain" + assert restic["offsite_verified"] is False + + post = apply4["independent_post_verifier"] + assert post["production_restart_count_delta"] == 0 + assert post["api_http_status"] == 200 + assert post["new_server_artifact_removed"] is True + assert post["apply2_apply3_preserved_artifact_hashes_unchanged"] is True + assert ( + post["exact_ephemeral_container_volume_network_process_tmp_residue_count"] == 0 + ) + assert post["cleanup_verified"] is True + assert post["secret_values_collected"] is False + assert canary["terminal"] == "production_closed_clickhouse_native_scope" + + +def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None: + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + verifier = receipt["post_release_runtime_verifier"] + assert verifier["run_id"] == "P0-OBS-002-post-release-20260714T215500Z" + assert verifier["window_seconds"] == 600 + assert verifier["terminal"] == "pass" + assert verifier["restart_delta"] == 0 + assert verifier["exporter_trace_errors"] == 0 + assert verifier["exporter_metric_errors"] == 0 + assert verifier["trace_count"] == 4026 + assert verifier["metric_sample_count"] == 76369 + assert verifier["api_runtime_cooldown_closed"] is True + assert ( + receipt["regressions"]["api_runtime"]["last_observed_cooldown_closed"] is True + ) + + closed = receipt["completed_verifiers"]["clickhouse_native_backup_migration"] + assert closed["prior_run_id"] == ( + "P0-OBS-002-backup-canary-20260714T221230Z-retry2" + ) + assert closed["run_id"] == ("P0-OBS-002-native-canary-20260715T001329Z-apply4") + assert closed["status"] == "production_closed_clickhouse_native_scope" + assert closed["native_backup_terminal"] == "BACKUP_CREATED" + assert closed["isolated_restore_terminal"] == "verified" + assert closed["restic_snapshot_id"] == "574e2a1a" + assert closed["independent_verifier"] == "pass" + assert closed["exact_cleanup"] == "pass" + sqlite_pending = receipt["pending_verifiers"][ + "signoz_sqlite_application_consistent_backup" + ] + assert sqlite_pending["raw_sqlite_read_or_sync_allowed"] is False + assert sqlite_pending["sqlite_cli_present_in_runtime"] is False + offsite = receipt["pending_verifiers"]["signoz_backup_offsite_dr"] + assert offsite["current_repository_scope"] == "local_same_failure_domain" + assert offsite["offsite_verified"] is False + artifacts = receipt["pending_verifiers"][ + "clickhouse_native_failed_artifact_retention" + ] + assert artifacts["destructive_cleanup_authorized"] is False + assert artifacts["active_backup_or_restore_count"] == 0 + assert len(artifacts["preserved_artifacts"]) == 2 + resume = receipt["pending_verifiers"][ + "clickhouse_native_resume_submitted_inventory" + ] + assert resume["same_run_resume_safe"] is False + assert len(resume["required_preconditions"]) == 4 + assets = receipt["asset_reconciliation"] + assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001" + assert assets["source_service_count"] == 27 + assert assets["runtime_mirror_service_count"] == 24 + assert assets["exact_shared_service_count"] == 24 + assert assets["shared_service_drift_count"] == 0 + assert assets["source_only_services"] == ["bitan-app", "sentry", "signoz"] + assert assets["signoz_clickhouse_exact_match"] is True + assert assets["live_signoz_query_host"] == "192.168.0.110" + assert assets["source_signoz_host"] == "192.168.0.188" + github_drift = assets["github_freeze_legacy_asset_drift"] + assert github_drift["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-002" + assert github_drift["superseded_by"] == "global_product_governance_v2" + assert github_drift["active_github_use_in_this_run"] is False + assert len(github_drift["source_references"]) == 3 + assert github_drift["p0_p1_github_recovery_scheduled"] is False + duplicate_identity = assets["monitoring_generator_duplicate_identity_drift"] + assert duplicate_identity["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-003" + assert duplicate_identity["status"] == "pending_source_deduplication" + assert duplicate_identity["awoooi_api_registry_entry_count"] == 2 + assert duplicate_identity["generated_scrape_config_count"] == 24 + assert duplicate_identity["generated_awoooi_api_job_count"] == 2 + assert duplicate_identity["generated_blackbox_target_count"] == 20 + assert duplicate_identity["runtime_apply_performed"] is False + assert receipt["terminal"]["status"] == "partial_degraded" + assert receipt["terminal"]["blockers"] == [ + "signoz_sqlite_application_consistent_backup_pending", + "signoz_backup_offsite_dr_pending", + "clickhouse_native_failed_artifact_retention_pending", + "clickhouse_native_resume_submitted_inventory_pending", + "service_registry_runtime_mirror_reconciliation_pending", + "github_freeze_legacy_asset_retirement_pending", + "monitoring_generator_duplicate_awoooi_api_identity_pending", + ] + + +def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> None: + contract = yaml.safe_load(CONTRACT.read_text(encoding="utf-8")) + post_closure = contract["runtime_observations"]["post_closure_regression"] + assert post_closure["snapshot_schema"] == ( + "awoooi_signoz_post_closure_regression_v2" + ) + assert post_closure["gitea_cd_run_id"] == 5140 + assert post_closure["api_runtime_cooldown_closed"] is True + assert post_closure["cd_probe_safety_fix_deployed"] is True + assert post_closure["config_route_patch_deployed"] is True + assert post_closure["signoz_control_plane_health_route_deployed"] is True + assert post_closure["backup_collector_restore_guard_deployed"] is True + assert post_closure["backup_canary_scope"] == "clickhouse_native_only" + assert post_closure["backup_canary_verified"] is True + assert post_closure["backup_canary_last_terminal"] == "pass" + assert post_closure["backup_canary_last_run_id"] == ( + "P0-OBS-002-native-canary-20260715T001329Z-apply4" + ) + assert post_closure["backup_canary_cooldown_retry_status"] == ( + "superseded_by_online_native_no_stop_contract" + ) + assert post_closure["clickhouse_native_backup_migration_status"] == ( + "production_closed" + ) + assert post_closure["clickhouse_native_backup_disk_run_id"] == ( + "P0-OBS-002-clickhouse-backup-disk-20260715T064900P0800-retry2" + ) + assert post_closure["clickhouse_native_backup_disk_status"] == "deployed_verified" + assert post_closure["clickhouse_native_backup_toolchain_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T001252Z-apply4" + ) + assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4" + ) + assert post_closure["clickhouse_native_restore_terminal"] == "verified" + assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a" + assert post_closure["clickhouse_native_offsite_status"] == "pending" + assert post_closure["clickhouse_native_failed_artifact_retention_status"] == ( + "pending_bounded_retention_decision" + ) + assert post_closure["clickhouse_native_resume_inventory_status"] == ( + "pending_durable_submitted_inventory_contract" + ) + assert ( + post_closure["signoz_sqlite_application_consistent_backup_status"] == "pending" + ) + assert post_closure["service_registry_runtime_mirror_status"] == ( + "partial_degraded" + ) + assert post_closure["service_registry_runtime_mirror_work_item_id"] == ( + "P0-OBS-002-ASSET-DRIFT-001" + ) + assert post_closure["github_freeze_legacy_asset_status"] == ( + "pending_controlled_retirement" + ) + assert post_closure["github_freeze_legacy_asset_work_item_id"] == ( + "P0-OBS-002-ASSET-DRIFT-002" + ) + assert post_closure["monitoring_generator_duplicate_identity_status"] == ( + "pending_source_deduplication" + ) + assert post_closure["monitoring_generator_duplicate_identity_work_item_id"] == ( + "P0-OBS-002-ASSET-DRIFT-003" + ) + assert post_closure["post_release_otlp_grpc_600s_verifier_status"] == "pass" + assert contract["terminal"]["status"] == "partial_degraded" + assert contract["terminal"]["blockers"] == [ + "organization_not_initialized", + "direct_ingress_policy_not_closed", + "http_bridge_not_supervised", + "filelog_normalization_degraded", + "wave_b_not_authorized", + ] + + def test_managed_promotion_and_bridge_retirement_fail_closed_without_receipts() -> None: env = os.environ.copy() env.pop("MANAGED_PROMOTION_RECEIPT", None) From 032be5f40012e12ad6681866278b0eacd44ce20e Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 09:43:40 +0800 Subject: [PATCH 2/9] fix(observability): harden native backup controlled apply --- ...rganization-canonical-route-migration.yaml | 2 + .../p0-obs-002-post-closure-regression.yaml | 42 ++ scripts/backup/backup-signoz.sh | 98 +++- scripts/backup/run-signoz-backup-canary.sh | 364 ++++++++++++++- .../test_backup_signoz_collector_restore.py | 77 +++- .../test_signoz_backup_canary_wrapper.py | 303 ++++++++++++- .../deploy-signoz-clickhouse-backup-disk.sh | 426 +++++++++++++----- .../deploy-signoz-native-backup-toolchain.sh | 149 +++++- ...st_signoz_clickhouse_backup_disk_deploy.py | 188 +++++++- ...t_signoz_native_backup_toolchain_deploy.py | 140 +++++- .../signoz-canonical-route-preflight.py | 145 +++++- .../test_signoz_canonical_route_migration.py | 105 +++++ 12 files changed, 1849 insertions(+), 190 deletions(-) diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index 3a94d61c3..afda9b611 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -181,6 +181,8 @@ runtime_observations: clickhouse_native_backup_disk_status: deployed_verified clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4 clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4 + clickhouse_native_current_source_status: pending_toolchain_deploy_and_runtime_canary + clickhouse_native_current_source_deployed: false clickhouse_native_restore_terminal: verified clickhouse_native_restic_snapshot_id: 574e2a1a clickhouse_native_offsite_status: pending diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 0bb28a860..73f015236 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -347,6 +347,8 @@ controlled_applies: risk: high completion_scope: clickhouse_native_only source: + evidence_scope: historical_production_receipt + deployment_status: deployed_verified backup_orchestrator_sha256: 2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520 native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7 @@ -361,6 +363,36 @@ controlled_applies: ruff: pass ruff_format: pass yaml_xml_parse: pass + current_source: + observed_at: "2026-07-15T09:40:50+08:00" + evidence_scope: committed_current_source + status: pending_toolchain_deploy_and_runtime_canary + deployment_status: pending + current_source_deployed: false + runtime_verifier_terminal: pending + historical_production_receipt_preserved: true + expected_file_count: 7 + hashes: + backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2 + native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a + isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7 + inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863 + canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886 + backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9 + isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b + drift_from_historical_production_receipt: + - backup_orchestrator_sha256 + - canary_wrapper_sha256 + - inventory_verifier_sha256 + source_validation: + backup_restore_deployer_tests_passed: 92 + canonical_preflight_tests_passed: 23 + bash_syntax: pass + ruff: pass + ruff_format: pass + yaml_parse: pass + diff_check: pass + independent_reaudit: no_commit_blocker toolchain_controlled_apply: check_run_id: P0-OBS-002-native-toolchain-20260715T001124Z-check5 check_terminal: pass @@ -609,6 +641,15 @@ pending_verifiers: - bind_inventory_hash_operation_id_artifact_hash_and_run_identity - resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered - regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits + clickhouse_native_current_source_toolchain: + status: pending_toolchain_deploy_and_runtime_canary + current_source_deployed: false + runtime_verifier_terminal: pending + historical_production_receipt_preserved: true + required_preconditions: + - controlled_toolchain_apply_with_current_source_hash_readback + - same_run_native_backup_and_isolated_restore_canary + - independent_runtime_and_cleanup_verifier scope_boundaries: grpc_bridge_rollback_indicated: false @@ -629,6 +670,7 @@ terminal: - signoz_backup_offsite_dr_pending - clickhouse_native_failed_artifact_retention_pending - clickhouse_native_resume_submitted_inventory_pending + - clickhouse_native_current_source_toolchain_pending - service_registry_runtime_mirror_reconciliation_pending - github_freeze_legacy_asset_retirement_pending - monitoring_generator_duplicate_awoooi_api_identity_pending diff --git a/scripts/backup/backup-signoz.sh b/scripts/backup/backup-signoz.sh index 406a9ceab..6b4127ef9 100755 --- a/scripts/backup/backup-signoz.sh +++ b/scripts/backup/backup-signoz.sh @@ -15,14 +15,19 @@ source "$(dirname "$0")/common.sh" SERVICE="signoz" LOCAL_REPO="${BACKUP_BASE}/signoz" -DUMP_DIR="/tmp/signoz-backup-$$" +TMP_ROOT="${SIGNOZ_BACKUP_TMP_ROOT:-/tmp}" +DUMP_DIR="" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" CLICKHOUSE_NATIVE_BACKUP_SCRIPT="${CLICKHOUSE_NATIVE_BACKUP_SCRIPT:-${SCRIPT_DIR}/clickhouse-native-backup.sh}" COLLECTOR_NAME="${SIGNOZ_COLLECTOR_NAME:-signoz-otel-collector}" COLLECTOR_DOCKER_TIMEOUT_SECONDS="${COLLECTOR_DOCKER_TIMEOUT_SECONDS:-10}" COLLECTOR_CONTINUITY_POLL_SECONDS="${COLLECTOR_CONTINUITY_POLL_SECONDS:-1}" TIMEOUT_KILL_AFTER_SECONDS="${TIMEOUT_KILL_AFTER_SECONDS:-30}" -BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-0}" +# Destructive Restic forget/prune is a separate critical break-glass workflow. +# The regular and canary backup paths are backup-only and fail closed if a +# caller attempts to re-enable inline retention cleanup. +BACKUP_SKIP_RETENTION_CLEANUP="${BACKUP_SKIP_RETENTION_CLEANUP:-1}" +OPERATION_LOCK_FILE="${SIGNOZ_BACKUP_OPERATION_LOCK_FILE:-/tmp/awoooi-signoz-backup-operation.lock}" RESTIC_RECEIPT_ROOT="${SIGNOZ_BACKUP_RECEIPT_ROOT:-/backup/logs/signoz-backup}" RESTIC_INIT_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_INIT_TIMEOUT_SECONDS:-300}" RESTIC_BACKUP_TIMEOUT_SECONDS="${SIGNOZ_RESTIC_BACKUP_TIMEOUT_SECONDS:-1800}" @@ -40,7 +45,8 @@ COLLECTOR_STARTED_AT_BEFORE="" COLLECTOR_RESTARTS_BEFORE="" COLLECTOR_HEALTH_BEFORE="" COLLECTOR_OBSERVER_PID="" -COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv" +COLLECTOR_OBSERVATIONS="" +WORKSPACE_CREATED=0 CLEANUP_COMPLETE=0 FAILURE_NOTIFIED=0 @@ -126,13 +132,24 @@ cleanup() { local exit_code=$? if [ "${CLEANUP_COMPLETE}" -eq 1 ]; then - return "${exit_code}" + exit "${exit_code}" fi CLEANUP_COMPLETE=1 - trap '' HUP INT TERM + trap - EXIT HUP INT TERM + set +e stop_collector_observer - rm -rf -- "${DUMP_DIR}" - return "${exit_code}" + if [ "${WORKSPACE_CREATED}" -eq 1 ]; then + if [ -n "${DUMP_DIR}" ] && [ -d "${DUMP_DIR}" ] \ + && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \ + && [[ "${DUMP_DIR}" == "${TMP_ROOT%/}"/signoz-backup.* ]]; then + rm -rf -- "${DUMP_DIR}" || exit_code=90 + [ ! -e "${DUMP_DIR}" ] || exit_code=90 + else + log_error "拒絕清理不符合本 run identity 的 SigNoz workspace" + exit_code=90 + fi + fi + exit "${exit_code}" } on_signal() { @@ -140,6 +157,28 @@ on_signal() { exit "$((128 + signal_number))" } +create_private_workspace() { + [[ "${TMP_ROOT}" == /* ]] \ + && [[ ! "${TMP_ROOT}" =~ (^|/)\.\.?(/|$) ]] \ + && [[ "${TMP_ROOT}" != *$'\n'* ]] \ + && [ -d "${TMP_ROOT}" ] && [ -w "${TMP_ROOT}" ] \ + && [ ! -L "${TMP_ROOT}" ] || { + log_error "SigNoz workspace root 必須是安全、可寫、非 symlink 的絕對目錄" + return 1 + } + + DUMP_DIR="$(mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX")" || return 1 + WORKSPACE_CREATED=1 + [ -d "${DUMP_DIR}" ] && [ ! -L "${DUMP_DIR}" ] && [ -O "${DUMP_DIR}" ] \ + && [ -r "${DUMP_DIR}" ] && [ -w "${DUMP_DIR}" ] && [ -x "${DUMP_DIR}" ] || { + log_error "mktemp 未建立本 run 擁有的 private SigNoz workspace" + return 1 + } + chmod 700 "${DUMP_DIR}" || return 1 + COLLECTOR_OBSERVATIONS="${DUMP_DIR}/collector-continuity.tsv" + log_info "建立 SigNoz private workspace: ${DUMP_DIR}" +} + run_native_clickhouse_backup() { local mode="$1" @@ -184,6 +223,32 @@ write_restic_receipt() { mv "${receipt_tmp}" "${receipt_file}" } +acquire_operation_lock() { + case "${OPERATION_LOCK_FILE}" in + /*) ;; + *) + log_error "SigNoz operation lock path 必須是絕對路徑" + return 1 + ;; + esac + [ ! -L "${OPERATION_LOCK_FILE}" ] || { + log_error "SigNoz operation lock 不得是 symlink" + return 1 + } + if [ -e "${OPERATION_LOCK_FILE}" ]; then + [ -f "${OPERATION_LOCK_FILE}" ] || { + log_error "SigNoz operation lock 必須是 regular file" + return 1 + } + fi + exec 8>>"${OPERATION_LOCK_FILE}" + flock -n 8 || { + log_error "另一個 SigNoz backup/deploy operation 正在執行" + return 1 + } + log_info "取得 SigNoz backup/deploy 共用 operation lock" +} + main() { local start_time end_time duration snapshot_id snapshot_json tags local initial_observation initial_running final_id final_running @@ -210,7 +275,18 @@ main() { exit 1 } done - mkdir -p "${DUMP_DIR}" + [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ] || { + notify_failure_once "inline Restic forget/prune 已停用；僅允許獨立 critical break-glass workflow" + exit 1 + } + acquire_operation_lock || { + notify_failure_once "SigNoz backup/deploy operation lock 取得失敗" + exit 1 + } + create_private_workspace || { + notify_failure_once "建立 private SigNoz backup workspace 失敗" + exit 1 + } # Check-mode must pass before the backup window. It writes only durable # receipts and does not submit BACKUP or create a server artifact. @@ -305,11 +381,7 @@ main() { } log_success "Restic 備份完成: ${snapshot_id}" - if [ "${BACKUP_SKIP_RETENTION_CLEANUP}" = "1" ]; then - log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)" - else - cleanup_old_backups "${LOCAL_REPO}" - fi + log_info "略過 SignOz retention cleanup (BACKUP_SKIP_RETENTION_CLEANUP=1)" # Final readback is separate from the continuity observer receipt. initial_observation="$(collector_identity_state)" || { diff --git a/scripts/backup/run-signoz-backup-canary.sh b/scripts/backup/run-signoz-backup-canary.sh index c6bdddfe4..15ac97425 100755 --- a/scripts/backup/run-signoz-backup-canary.sh +++ b/scripts/backup/run-signoz-backup-canary.sh @@ -12,6 +12,12 @@ set -euo pipefail COLLECTOR_NAME="${SIGNOZ_CANARY_COLLECTOR_NAME:-signoz-otel-collector}" BACKUP_SCRIPT="${SIGNOZ_CANARY_BACKUP_SCRIPT:-/backup/scripts/backup-signoz.sh}" +CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-/backup/scripts/clickhouse-native-restore-drill.sh}" +CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-/backup/scripts/clickhouse-restore-inventory.py}" +CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-/backup/config/signoz/clickhouse/config.d/backup_disk.xml}" +CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml}" +CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-/backup/logs/clickhouse-native}" +CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-/backup/logs/clickhouse-native-restore-drill}" MONITOR_SCRIPT="${SIGNOZ_CANARY_MONITOR_SCRIPT:-/home/wooo/awoooi-ops/docker-health-monitor.sh}" MONITOR_LOG="${SIGNOZ_CANARY_MONITOR_LOG:-/home/wooo/awoooi-ops/monitor.log}" MONITOR_COOLDOWN_DIR="${SIGNOZ_CANARY_COOLDOWN_DIR:-/tmp/docker-health-monitor-cooldown}" @@ -21,6 +27,7 @@ LOCK_FILE="${SIGNOZ_CANARY_LOCK_FILE:-/tmp/awoooi-signoz-backup-canary.lock}" CANARY_TIMEOUT_SECONDS="${SIGNOZ_CANARY_TIMEOUT_SECONDS:-12000}" TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_CANARY_KILL_AFTER_SECONDS:-60}" +DOCKER_TIMEOUT_SECONDS="${SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS:-15}" LEASE_MARGIN_SECONDS="${SIGNOZ_CANARY_LEASE_MARGIN_SECONDS:-60}" MONITOR_CRON_INTERVAL_SECONDS="${SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS:-300}" STATE_POLL_SECONDS="${SIGNOZ_CANARY_STATE_POLL_SECONDS:-1}" @@ -94,6 +101,307 @@ require_command() { command -v "$1" >/dev/null 2>&1 || fail_closed "required_command_missing_${1}" } +run_collector_docker() { + timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${DOCKER_TIMEOUT_SECONDS}" docker "$@" +} + +require_native_restore_contract() { + local label path + + for label in \ + CLICKHOUSE_NATIVE_POST_VERIFY_HOOK \ + CLICKHOUSE_RESTORE_INVENTORY_HELPER \ + CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG \ + CLICKHOUSE_RESTORE_CLUSTER_CONFIG; do + path="${!label}" + case "${path}" in + /*) ;; + *) fail_closed "native_restore_contract_${label}_path_not_absolute" ;; + esac + [ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \ + || fail_closed "native_restore_contract_${label}_missing_unreadable_or_symlink" + done + + [ -x "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" ] \ + || fail_closed "native_restore_contract_post_verify_hook_not_executable" + [ -x "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" ] \ + || fail_closed "native_restore_contract_inventory_helper_not_executable" +} + +acquire_canary_lock() { + [ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe" + if [ -e "${LOCK_FILE}" ]; then + [ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file" + fi + exec 9>>"${LOCK_FILE}" + flock -n 9 || fail_closed "another_signoz_canary_holds_lock" +} + +require_native_receipt_contract() { + local label path run_dir + + for label in CLICKHOUSE_NATIVE_RECEIPT_ROOT CLICKHOUSE_RESTORE_RECEIPT_ROOT; do + path="${!label}" + case "${path}" in + /*) ;; + *) fail_closed "native_receipt_contract_${label}_path_not_absolute" ;; + esac + [[ "${path}" != *$'\n'* ]] \ + || fail_closed "native_receipt_contract_${label}_path_contains_newline" + [ ! -L "${path}" ] \ + || fail_closed "native_receipt_contract_${label}_root_symlink_unsafe" + if [ -e "${path}" ]; then + [ -d "${path}" ] && [ -r "${path}" ] && [ -w "${path}" ] \ + || fail_closed "native_receipt_contract_${label}_root_not_readable_writable_directory" + fi + run_dir="${path}/${RUN_ID}" + [ ! -e "${run_dir}" ] && [ ! -L "${run_dir}" ] \ + || fail_closed "native_receipt_contract_${label}_stale_same_run_path_present" + done + + [ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" != "${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" ] \ + || fail_closed "native_and_restore_receipt_roots_must_be_distinct" +} + +verify_same_run_native_restore_receipts() { + local native_receipts restore_receipts restore_status + + native_receipts="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl" + restore_receipts="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/receipts.jsonl" + restore_status="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}/${RUN_ID}/status.json" + + for path in "${native_receipts}" "${restore_receipts}" "${restore_status}"; do + [ -f "${path}" ] && [ -r "${path}" ] && [ ! -L "${path}" ] \ + || fail_closed "same_run_native_restore_receipt_missing_unreadable_or_symlink" + done + + python3 - "${native_receipts}" "${restore_receipts}" "${restore_status}" \ + "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" <<'PY' \ + || fail_closed "same_run_native_restore_receipt_validation_failed" +import json +import os +import sys + + +def reject(message: str) -> None: + print(f"receipt_verifier:{message}", file=sys.stderr) + raise SystemExit(1) + + +native_path, restore_path, status_path, trace_id, run_id, work_item_id = sys.argv[1:] +identity = { + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": work_item_id, +} + + +def read_jsonl(path: str, label: str) -> list[dict[str, object]]: + if os.path.islink(path) or not os.path.isfile(path): + reject(f"{label}_unsafe") + records: list[dict[str, object]] = [] + try: + with open(path, encoding="utf-8") as source: + for line_number, raw_line in enumerate(source, start=1): + if not raw_line.strip(): + continue + value = json.loads(raw_line) + if not isinstance(value, dict): + reject(f"{label}_line_{line_number}_not_object") + for key, expected in identity.items(): + if value.get(key) != expected: + reject(f"{label}_line_{line_number}_{key}_mismatch") + invocation_id = value.get("invocation_id") + if not isinstance(invocation_id, str) or not invocation_id: + reject(f"{label}_line_{line_number}_invocation_id_invalid") + records.append(value) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + reject(f"{label}_malformed_{type(exc).__name__}") + if not records: + reject(f"{label}_empty") + return records + + +def mode_records( + records: list[dict[str, object]], mode: str, label: str +) -> list[dict[str, object]]: + selected = [ + record + for record in records + if str(record.get("invocation_id", "")).startswith(f"{mode}-") + ] + if not selected: + reject(f"{label}_{mode}_invocation_missing") + return selected + + +def require_exact( + records: list[dict[str, object]], + *, + label: str, + stage: str, + terminal: str, + detail_prefix: str | None = None, +) -> None: + matches = [ + record + for record in records + if record.get("stage") == stage + and record.get("terminal") == terminal + and ( + detail_prefix is None + or str(record.get("detail", "")).startswith(detail_prefix) + ) + ] + if len(matches) != 1: + reject(f"{label}_{stage}_{terminal}_count_{len(matches)}") + + +native_records = read_jsonl(native_path, "native_receipts") +native_check = mode_records(native_records, "check", "native") +native_apply = mode_records(native_records, "apply", "native") +require_exact( + native_check, + label="native_check", + stage="terminal", + terminal="check_pass_no_write", +) +require_exact( + native_apply, + label="native_apply", + stage="terminal", + terminal="pass", +) +require_exact( + native_apply, + label="native_apply", + stage="independent_verifier", + terminal="pass", +) +require_exact( + native_apply, + label="native_apply_hook_check", + stage="command", + terminal="pass", + detail_prefix="step_post_verify_hook_check_exit_0_", +) +require_exact( + native_apply, + label="native_apply_hook", + stage="command", + terminal="pass", + detail_prefix="step_post_verify_hook_exit_0_", +) + +restore_records = read_jsonl(restore_path, "restore_receipts") +restore_check = mode_records(restore_records, "check", "restore") +restore_apply = mode_records(restore_records, "apply", "restore") +require_exact( + restore_check, + label="restore_check", + stage="terminal", + terminal="check_pass_no_runtime_write", +) +require_exact( + restore_apply, + label="restore_apply", + stage="terminal", + terminal="verified", +) +require_exact( + restore_apply, + label="restore_apply", + stage="independent_post_verifier", + terminal="pass", +) +require_exact( + restore_apply, + label="restore_apply", + stage="cleanup", + terminal="pass", +) + +if os.path.islink(status_path) or not os.path.isfile(status_path): + reject("restore_status_unsafe") +try: + with open(status_path, encoding="utf-8") as source: + status = json.load(source) +except (OSError, UnicodeError, json.JSONDecodeError) as exc: + reject(f"restore_status_malformed_{type(exc).__name__}") +if not isinstance(status, dict): + reject("restore_status_not_object") +for key, expected in identity.items(): + if status.get(key) != expected: + reject(f"restore_status_{key}_mismatch") + +expected_strings = { + "schema_version": "clickhouse_native_restore_drill_status_v1", + "mode": "apply", + "terminal": "verified", + "restore_terminal": "RESTORED", + "network_mode": "docker_internal_only", + "staging_network_mode": "none", +} +for key, expected in expected_strings.items(): + if not isinstance(status.get(key), str) or status.get(key) != expected: + reject(f"restore_status_{key}_invalid") +for key, expected in (("exit_code", 0), ("manifest_parity", 1)): + if type(status.get(key)) is not int or status.get(key) != expected: + reject(f"restore_status_{key}_invalid") +for key in ( + "check_pass", + "apply_pass", + "cleanup_verified", + "artifact_staging_verified", + "clickhouse_artifact_readback_verified", + "ephemeral_runtime_created", + "isolated_keeper", +): + if status.get(key) is not True: + reject(f"restore_status_{key}_not_true") +for key in ( + "production_network_attached", + "production_restore_performed", + "allow_non_empty_tables", +): + if status.get(key) is not False: + reject(f"restore_status_{key}_not_false") + + +def positive_integer(key: str) -> int: + value = status.get(key) + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + reject(f"restore_status_{key}_not_positive_integer") + return value + + +source_database_count = positive_integer("source_database_count") +restored_database_count = positive_integer("restored_database_count") +if source_database_count != restored_database_count: + reject("restore_status_database_count_mismatch") +source_table_count = positive_integer("source_table_count") +restored_table_count = positive_integer("restored_table_count") +if source_table_count != restored_table_count: + reject("restore_status_table_count_mismatch") +positive_integer("critical_nonzero_count") +check_count = status.get("check_table_count") +check_pass_count = status.get("check_table_pass_count") +if ( + isinstance(check_count, bool) + or not isinstance(check_count, int) + or isinstance(check_pass_count, bool) + or not isinstance(check_pass_count, int) + or check_count <= 0 + or check_count != check_pass_count +): + reject("restore_status_check_table_counts_invalid") +PY + + emit_receipt "native_restore_receipt_verifier" "pass" \ + "same_run_machine_readable_native_check_apply_hook_and_isolated_restore_status_verified" +} + require_monitor_fragment() { local fragment="$1" local count @@ -138,7 +446,7 @@ require_monitor_function_fragment() { collector_running_state() { local running - if ! running="$(docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then + if ! running="$(run_collector_docker inspect --format '{{.State.Running}}' "${COLLECTOR_NAME}" 2>/dev/null)"; then return 1 fi case "${running}" in @@ -149,7 +457,7 @@ collector_running_state() { } collector_runtime_metadata() { - docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \ + run_collector_docker inspect --format $'{{.Id}}\t{{.State.Running}}\t{{.State.StartedAt}}\t{{.RestartCount}}\t{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}none{{end}}' \ "${COLLECTOR_NAME}" 2>/dev/null } @@ -247,13 +555,15 @@ cleanup() { if [ "${exit_code}" -eq 0 ] && [ "${MODE}" = "check" ] && [ "${CHECK_VERIFIED}" -eq 1 ]; then overall_terminal="check_pass_no_write" elif [ "${exit_code}" -eq 0 ] && [ "${CANARY_VERIFIED}" -eq 1 ] && [ "${lease_terminal}" != "restore_failed" ]; then - overall_terminal="pass" + overall_terminal="partial_degraded" fi emit_receipt "rollback" "${lease_terminal}" "cooldown_lease_${lease_terminal}" - if [ "${overall_terminal}" = "pass" ]; then - emit_receipt "closure_writeback" "pass" \ - "no_incident_notification_or_learning_delta_local_durable_receipt_ack" + if [ "${overall_terminal}" = "partial_degraded" ]; then + emit_receipt "runtime_operation" "pass" \ + "clickhouse_native_backup_and_isolated_restore_runtime_verifier_pass" + emit_receipt "closure_writeback" "partial_degraded" \ + "local_durable_receipt_ack_incident_not_applicable_telegram_not_sent_km_rag_writeback_pending_offsite_false_sqlite_application_consistent_export_pending" elif [ "${overall_terminal}" = "check_pass_no_write" ]; then emit_receipt "closure_writeback" "not_applicable" \ "check_mode_no_runtime_change_local_durable_receipt_ack" @@ -261,7 +571,8 @@ cleanup() { emit_receipt "closure_writeback" "deferred" \ "failed_terminal_requires_followup_no_completion_claim" fi - emit_receipt "terminal" "${overall_terminal}" "backup_canary_${overall_terminal}" + emit_receipt "terminal" "${overall_terminal}" \ + "backup_canary_${overall_terminal}_clickhouse_native_scope_only_offsite_false_sqlite_pending" printf 'SIGNOZ_BACKUP_CANARY_TERMINAL trace_id=%s run_id=%s work_item_id=%s terminal=%s lease=%s exit_code=%s\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${overall_terminal}" \ "${lease_terminal}" "${exit_code}" @@ -521,7 +832,7 @@ main() { local container_health_before collector_metadata_before local monitor_log_identity="" monitor_log_size="" local source_diff_detail ai_decision_detail - local backup_status + local backup_status native_restore_contract_detail="not_applicable" setup_receipts @@ -532,11 +843,17 @@ main() { *) fail_closed "unsupported_mode_use_check_or_apply" ;; esac - for command_name in awk date docker env flock grep kill pgrep sha256sum sleep ss timeout tr; do + # Hold the canary lock before inspecting or hashing any mutable backup or + # restore toolchain path. This closes the check/apply substitution window. + require_command flock + acquire_canary_lock + + for command_name in awk date docker env grep kill pgrep python3 sha256sum sleep ss timeout tr; do require_command "${command_name}" done valid_positive_integer "${CANARY_TIMEOUT_SECONDS}" || fail_closed "invalid_canary_timeout" valid_positive_integer "${TIMEOUT_KILL_AFTER_SECONDS}" || fail_closed "invalid_timeout_kill_after" + valid_positive_integer "${DOCKER_TIMEOUT_SECONDS}" || fail_closed "invalid_docker_timeout" valid_poll_interval "${STATE_POLL_SECONDS}" || fail_closed "invalid_state_poll_interval" case "${EXPECT_COLLECTOR_STOP}" in 0|1) ;; @@ -545,7 +862,11 @@ main() { [ -f "${BACKUP_SCRIPT}" ] && [ -x "${BACKUP_SCRIPT}" ] && [ ! -L "${BACKUP_SCRIPT}" ] \ || fail_closed "backup_script_missing_unexecutable_or_symlink" - if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then + if [ "${EXPECT_COLLECTOR_STOP}" = "0" ]; then + require_native_restore_contract + require_native_receipt_contract + native_restore_contract_detail="hook_$(sha256sum "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" | awk '{print $1}')_inventory_$(sha256sum "${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" | awk '{print $1}')_disk_config_$(sha256sum "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" | awk '{print $1}')_cluster_config_$(sha256sum "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" | awk '{print $1}')" + else for command_name in cp crontab id mktemp mv rm stat tail; do require_command "${command_name}" done @@ -555,12 +876,6 @@ main() { verify_monitor_contract fi - [ ! -L "${LOCK_FILE}" ] || fail_closed "canary_lock_file_symlink_unsafe" - if [ -e "${LOCK_FILE}" ]; then - [ -f "${LOCK_FILE}" ] || fail_closed "canary_lock_path_not_regular_file" - fi - exec 9>>"${LOCK_FILE}" - flock -n 9 || fail_closed "another_signoz_canary_holds_lock" if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then fail_closed "another_signoz_backup_is_running" fi @@ -591,18 +906,18 @@ main() { PRIOR_LEASE_PRESENT=0 PRIOR_LEASE_HASH="not_applicable" emit_receipt "sensor_source" "pass" \ - "online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}" - source_diff_detail="online_native_backup_monitor_and_cooldown_contract_not_applicable" - ai_decision_detail="run_online_native_backup_without_collector_or_monitor_mutation" + "online_backup_hash_$(sha256sum "${BACKUP_SCRIPT}" | awk '{print $1}')_${native_restore_contract_detail}_collector_${container_id_before}_started_${container_started_at_before}_restarts_${container_restart_count_before}_health_${container_health_before}" + source_diff_detail="online_native_backup_isolated_restore_contract_bound_monitor_and_cooldown_contract_not_applicable" + ai_decision_detail="run_online_native_backup_with_mandatory_isolated_restore_verifier_without_collector_or_monitor_mutation" fi emit_receipt "normalized_asset_identity" "pass" \ "asset_container_${COLLECTOR_NAME}_runtime_id_${container_id_before}_cluster_host_docker" emit_receipt "source_of_truth_diff" "pass" "${source_diff_detail}" emit_receipt "ai_decision" "candidate" "${ai_decision_detail}" emit_receipt "risk_policy" "pass" \ - "risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_independent_verifier" + "risk_medium_bounded_timeout_no_retention_cleanup_runtime_metadata_mandatory_isolated_restore_independent_verifier" emit_receipt "check" "pass" \ - "mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent" + "mode_${MODE}_expect_stop_${EXPECT_COLLECTOR_STOP}_${native_restore_contract_detail}_collector_same_identity_started_restart_health_running_listeners_lock_clear_backup_absent" if [ "${MODE}" = "check" ]; then CHECK_VERIFIED=1 @@ -622,6 +937,12 @@ main() { set +e timeout --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" "${CANARY_TIMEOUT_SECONDS}" \ env TRACE_ID="${TRACE_ID}" RUN_ID="${RUN_ID}" WORK_ITEM_ID="${WORK_ITEM_ID}" \ + CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}" \ + CLICKHOUSE_RESTORE_INVENTORY_HELPER="${CLICKHOUSE_RESTORE_INVENTORY_HELPER}" \ + CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG="${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG}" \ + CLICKHOUSE_RESTORE_CLUSTER_CONFIG="${CLICKHOUSE_RESTORE_CLUSTER_CONFIG}" \ + CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}" \ + CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}" \ BACKUP_SKIP_RETENTION_CLEANUP=1 "${BACKUP_SCRIPT}" \ > "${BACKUP_OUTPUT}" 2>&1 & BACKUP_PID=$! @@ -645,6 +966,7 @@ main() { else grep -a -F -q '========== SigNoz ClickHouse native 備份完成 (' "${BACKUP_OUTPUT}" \ || fail_closed "native_backup_success_terminal_missing" + verify_same_run_native_restore_receipts fi if [ "${EXPECT_COLLECTOR_STOP}" = "1" ]; then diff --git a/scripts/backup/tests/test_backup_signoz_collector_restore.py b/scripts/backup/tests/test_backup_signoz_collector_restore.py index 71113b96c..52a7b64f5 100644 --- a/scripts/backup/tests/test_backup_signoz_collector_restore.py +++ b/scripts/backup/tests/test_backup_signoz_collector_restore.py @@ -25,7 +25,9 @@ def _run_backup( native_apply_status: int = 0, continuity_mode: str = "stable", restic_backup_status: int = 0, - skip_retention: bool = False, + skip_retention: bool = True, + operation_lock_status: int = 0, + workspace_root_symlink: bool = False, ) -> tuple[subprocess.CompletedProcess[str], list[str], Path]: harness = tmp_path / "backup" fake_bin = tmp_path / "bin" @@ -33,6 +35,7 @@ def _run_backup( event_log = tmp_path / "events.log" collector_id = tmp_path / "collector.id" collector_state = tmp_path / "collector.state" + workspace_root = tmp_path / "workspace-root" harness.mkdir() fake_bin.mkdir() (backup_base / "signoz" / "data").mkdir(parents=True) @@ -43,6 +46,12 @@ def _run_backup( "true\n" if initial_running else "false\n", encoding="utf-8", ) + if workspace_root_symlink: + workspace_target = tmp_path / "workspace-target" + workspace_target.mkdir() + workspace_root.symlink_to(workspace_target, target_is_directory=True) + else: + workspace_root.mkdir() (harness / "common.sh").write_text( """\ @@ -139,6 +148,15 @@ case " $* " in printf '[{"short_id":"native123"}]\\n' ;; esac +""", + ) + _write_executable( + fake_bin / "flock", + """\ +#!/bin/bash +set -eu +printf 'flock %s\n' "$*" >> "${TEST_EVENT_LOG:?}" +exit "${FAKE_OPERATION_LOCK_STATUS:-0}" """, ) @@ -161,6 +179,9 @@ esac "COLLECTOR_DOCKER_TIMEOUT_SECONDS": "1", "TIMEOUT_KILL_AFTER_SECONDS": "1", "BACKUP_SKIP_RETENTION_CLEANUP": "1" if skip_retention else "0", + "SIGNOZ_BACKUP_OPERATION_LOCK_FILE": str(tmp_path / "operation.lock"), + "FAKE_OPERATION_LOCK_STATUS": str(operation_lock_status), + "SIGNOZ_BACKUP_TMP_ROOT": str(workspace_root), } process = subprocess.Popen( ["bash", str(harness / SCRIPT.name)], @@ -177,7 +198,13 @@ esac stderr, ) events = event_log.read_text(encoding="utf-8").splitlines() - return result, events, Path(f"/tmp/signoz-backup-{process.pid}") + workspace_events = _events(events, "info 建立 SigNoz private workspace: ") + dump_dir = ( + Path(workspace_events[-1].split(": ", 1)[1]) + if workspace_events + else tmp_path / "workspace-not-created" + ) + return result, events, dump_dir def _events(events: list[str], prefix: str) -> list[str]: @@ -201,6 +228,12 @@ def test_source_sunsets_raw_volumes_and_collector_mutation() -> None: assert "--format '{{.Id}}\\t{{.State.Running}}'" not in source assert 'notify_clawbot "success"' not in source assert "no-downtime continuity verifier" in source + assert "awoooi-signoz-backup-operation.lock" in source + assert "flock -n 8" in source + assert "cleanup_old_backups" not in source + assert 'mktemp -d "${TMP_ROOT%/}/signoz-backup.XXXXXXXX"' in source + assert "/tmp/signoz-backup-$$" not in source + assert "WORKSPACE_CREATED=1" in source def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime( @@ -233,6 +266,32 @@ def test_success_is_clickhouse_scoped_with_metadata_gap_and_no_downtime( assert not dump_dir.exists() +def test_operation_lock_contention_fails_before_native_or_restic( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup(tmp_path, operation_lock_status=1) + + assert result.returncode != 0 + assert _events(events, "flock -n 8") + assert not _events(events, "native ") + assert not _events(events, "restic ") + assert len(_events(events, "notify failed signoz ")) == 1 + assert not dump_dir.exists() + + +def test_symlink_workspace_root_fails_before_native_or_restic( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup(tmp_path, workspace_root_symlink=True) + + assert result.returncode != 0 + assert _events(events, "flock -n 8") + assert not _events(events, "native ") + assert not _events(events, "restic ") + assert any("workspace root" in event for event in events) + assert not dump_dir.exists() + + def test_native_check_failure_occurs_before_apply_or_restic(tmp_path: Path) -> None: result, events, dump_dir = _run_backup(tmp_path, native_check_status=19) @@ -307,6 +366,20 @@ def test_canary_retention_skip_is_explicit(tmp_path: Path) -> None: assert not _events(events, "retention ") +def test_inline_destructive_retention_request_fails_before_backup( + tmp_path: Path, +) -> None: + result, events, dump_dir = _run_backup(tmp_path, skip_retention=False) + + assert result.returncode != 0 + assert not _events(events, "flock ") + assert not _events(events, "native ") + assert not _events(events, "restic ") + assert not _events(events, "retention ") + assert any("critical break-glass workflow" in event for event in events) + assert not dump_dir.exists() + + def test_backup_jobs_deploys_native_adapter_and_orchestrator() -> None: playbook = DEPLOYMENT_PLAYBOOK.read_text(encoding="utf-8") diff --git a/scripts/backup/tests/test_signoz_backup_canary_wrapper.py b/scripts/backup/tests/test_signoz_backup_canary_wrapper.py index f2ba902af..c7629e394 100644 --- a/scripts/backup/tests/test_signoz_backup_canary_wrapper.py +++ b/scripts/backup/tests/test_signoz_backup_canary_wrapper.py @@ -7,6 +7,8 @@ import subprocess import time from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[3] WRAPPER = ROOT / "scripts" / "backup" / "run-signoz-backup-canary.sh" @@ -115,6 +117,9 @@ while [[ "${1:-}" == --kill-after=* ]]; do done [ "$#" -gt 1 ] shift +if [ "${FAKE_DOCKER_TIMEOUT:-0}" = "1" ] && [ "${1:-}" = "docker" ]; then + exit 124 +fi exec "$@" """, ) @@ -122,6 +127,9 @@ exec "$@" fake_bin / "flock", """\ #!/bin/bash +if [ -n "${TEST_FLOCK_MARKER:-}" ]; then + printf 'held\n' > "${TEST_FLOCK_MARKER}" +fi exit 0 """, ) @@ -171,14 +179,103 @@ print(f"{digest} {sys.argv[1]}") ) +def _restore_hook_source() -> str: + return """\ +#!/bin/bash +set -eu +[ "$#" -eq 1 ] +case "$1" in + --check) mode="check" ;; + --apply) mode="apply" ;; + *) exit 64 ;; +esac +receipt_dir="${CLICKHOUSE_RESTORE_RECEIPT_ROOT:?}/${RUN_ID:?}" +mkdir -p "$receipt_dir" +receipt_log="$receipt_dir/receipts.jsonl" +identity_run="$RUN_ID" +if [ "${FAKE_RECEIPT_FAULT:-valid}" = "forged_identity" ]; then + identity_run="stale-$RUN_ID" +fi +invocation_id="${mode}-fake-restore-$$" +emit() { + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"%s","stage":"%s","terminal":"%s","detail":"fake"}\n' \ + "$TRACE_ID" "$identity_run" "$WORK_ITEM_ID" "$invocation_id" "$1" "$2" \ + >> "$receipt_log" +} +if [ "$mode" = "check" ]; then + emit check check_pass_no_runtime_write + emit cleanup pass + emit terminal check_pass_no_runtime_write + exit 0 +fi +emit check pass +emit independent_post_verifier pass +emit cleanup pass +emit terminal verified +if [ "${FAKE_RECEIPT_FAULT:-valid}" != "missing_status" ]; then + cat > "$receipt_dir/status.json" <> "$receipt_log" +fi +""" + + def _backup_source() -> str: return """\ #!/bin/bash set -eu [ "${BACKUP_SKIP_RETENTION_CLEANUP:-}" = "1" ] || exit 80 [ -n "${TRACE_ID:-}" ] && [ -n "${RUN_ID:-}" ] && [ -n "${WORK_ITEM_ID:-}" ] +[ "${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK:-}" = "${TEST_RESTORE_HOOK:?}" ] || exit 82 +[ "${CLICKHOUSE_RESTORE_INVENTORY_HELPER:-}" = "${TEST_INVENTORY_HELPER:?}" ] || exit 83 +[ "${CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG:-}" = "${TEST_BACKUP_DISK_CONFIG:?}" ] || exit 84 +[ "${CLICKHOUSE_RESTORE_CLUSTER_CONFIG:-}" = "${TEST_CLUSTER_CONFIG:?}" ] || exit 85 +[ "${CLICKHOUSE_NATIVE_RECEIPT_ROOT:-}" = "${TEST_NATIVE_RECEIPT_ROOT:?}" ] || exit 86 +[ "${CLICKHOUSE_RESTORE_RECEIPT_ROOT:-}" = "${TEST_RESTORE_RECEIPT_ROOT:?}" ] || exit 87 printf '%s\t%s\t%s\n' "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" \ > "${TEST_BACKUP_IDENTITIES:?}" +printf '%s\t%s\t%s\t%s\n' \ + "$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" \ + "$CLICKHOUSE_RESTORE_INVENTORY_HELPER" \ + "$CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG" \ + "$CLICKHOUSE_RESTORE_CLUSTER_CONFIG" \ + > "${TEST_RESTORE_CONTRACT:?}" if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then cat "${TEST_COOLDOWN_FILE:?}" > "${TEST_LEASE_OBSERVED:?}" printf 'false\n' > "${TEST_COLLECTOR_STATE:?}" @@ -204,6 +301,23 @@ if [ "${FAKE_BACKUP_STOPS_COLLECTOR:-1}" = "1" ]; then printf 'true\n' > "${TEST_COLLECTOR_STATE:?}" else sleep "${FAKE_BACKUP_SLEEP_SECONDS:-0.2}" + native_dir="$CLICKHOUSE_NATIVE_RECEIPT_ROOT/$RUN_ID" + mkdir -p "$native_dir" + native_receipts="$native_dir/receipts.jsonl" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"check-fake-native","stage":"terminal","terminal":"check_pass_no_write","detail":"fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" + if [ "${FAKE_RECEIPT_FAULT:-valid}" != "no_hook" ]; then + "$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --check + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_check_exit_0_stdout_fake_stderr_fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" + "$CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" --apply + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"command","terminal":"pass","detail":"step_post_verify_hook_exit_0_stdout_fake_stderr_fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" + fi + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"independent_verifier","terminal":"pass","detail":"fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" + printf '{"trace_id":"%s","run_id":"%s","work_item_id":"%s","invocation_id":"apply-fake-native","stage":"terminal","terminal":"pass","detail":"fake"}\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" >> "$native_receipts" fi if [ "${FAKE_DROP_LISTENER:-0}" = "1" ]; then printf '4317\n' > "${TEST_LISTENERS:?}" @@ -229,13 +343,30 @@ def _run_canary( duplicate_cron: bool = False, drop_listener: bool = False, expect_collector_stop: bool = True, + native_restore_contract: str = "valid", + native_receipt_state: str = "valid", + docker_timeout: bool = False, ) -> tuple[subprocess.CompletedProcess[str], dict[str, Path], str]: + if native_receipt_state not in { + "valid", + "no_hook", + "forged_identity", + "missing_status", + "symlink_status", + "malformed_restore_receipt", + }: + raise ValueError(f"unsupported native receipt state: {native_receipt_state}") + fake_bin = tmp_path / "bin" cooldown_dir = tmp_path / "cooldown" receipt_root = tmp_path / "receipts" + native_receipt_root = tmp_path / "native-receipts" + restore_receipt_root = tmp_path / "restore-receipts" fake_bin.mkdir() cooldown_dir.mkdir() receipt_root.mkdir() + native_receipt_root.mkdir() + restore_receipt_root.mkdir() monitor_script = tmp_path / "docker-health-monitor.sh" monitor_log = tmp_path / "monitor.log" @@ -244,6 +375,12 @@ def _run_canary( listeners = tmp_path / "listeners" identities = tmp_path / "backup-identities.tsv" lease_observed = tmp_path / "lease-observed.txt" + restore_contract_observed = tmp_path / "restore-contract.tsv" + flock_marker = tmp_path / "canary-lock-held.txt" + restore_hook = tmp_path / "clickhouse-native-restore-drill.sh" + inventory_helper = tmp_path / "clickhouse-restore-inventory.py" + backup_disk_config = tmp_path / "backup_disk.xml" + cluster_config = tmp_path / "isolated-cluster.xml" cooldown_file = cooldown_dir / "signoz-otel-collector.cooldown" run_id = f"canary-{tmp_path.name}" @@ -252,6 +389,19 @@ def _run_canary( _monitor_source(cooldown_dir, omit_contract=omit_monitor_contract), ) _write_executable(backup_script, _backup_source()) + if native_restore_contract == "symlink": + restore_hook_target = tmp_path / "restore-hook-target.sh" + _write_executable(restore_hook_target, _restore_hook_source()) + restore_hook.symlink_to(restore_hook_target) + elif native_restore_contract == "valid": + _write_executable(restore_hook, _restore_hook_source()) + elif native_restore_contract != "missing": + raise ValueError( + f"unsupported native restore contract: {native_restore_contract}" + ) + _write_executable(inventory_helper, "#!/bin/sh\nexit 0\n") + backup_disk_config.write_text("\n", encoding="utf-8") + cluster_config.write_text("\n", encoding="utf-8") _install_fake_commands(fake_bin) monitor_log.write_text("[test] monitor seed\n", encoding="utf-8") collector_state.write_text("true\n", encoding="utf-8") @@ -274,10 +424,17 @@ def _run_canary( "SIGNOZ_CANARY_LOCK_FILE": str(tmp_path / "canary.lock"), "SIGNOZ_CANARY_TIMEOUT_SECONDS": "5", "SIGNOZ_CANARY_KILL_AFTER_SECONDS": "2", + "SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS": "2", "SIGNOZ_CANARY_LEASE_MARGIN_SECONDS": "2", "SIGNOZ_CANARY_MONITOR_CRON_INTERVAL_SECONDS": str(monitor_interval_seconds), "SIGNOZ_CANARY_STATE_POLL_SECONDS": "0.05", "SIGNOZ_CANARY_EXPECT_COLLECTOR_STOP": ("1" if expect_collector_stop else "0"), + "CLICKHOUSE_NATIVE_POST_VERIFY_HOOK": str(restore_hook), + "CLICKHOUSE_RESTORE_INVENTORY_HELPER": str(inventory_helper), + "CLICKHOUSE_RESTORE_BACKUP_DISK_CONFIG": str(backup_disk_config), + "CLICKHOUSE_RESTORE_CLUSTER_CONFIG": str(cluster_config), + "CLICKHOUSE_NATIVE_RECEIPT_ROOT": str(native_receipt_root), + "CLICKHOUSE_RESTORE_RECEIPT_ROOT": str(restore_receipt_root), "TEST_MONITOR_SCRIPT": str(monitor_script), "TEST_MONITOR_LOG": str(monitor_log), "TEST_COLLECTOR_STATE": str(collector_state), @@ -285,11 +442,21 @@ def _run_canary( "TEST_BACKUP_IDENTITIES": str(identities), "TEST_COOLDOWN_FILE": str(cooldown_file), "TEST_LEASE_OBSERVED": str(lease_observed), + "TEST_RESTORE_CONTRACT": str(restore_contract_observed), + "TEST_RESTORE_HOOK": str(restore_hook), + "TEST_INVENTORY_HELPER": str(inventory_helper), + "TEST_BACKUP_DISK_CONFIG": str(backup_disk_config), + "TEST_CLUSTER_CONFIG": str(cluster_config), + "TEST_NATIVE_RECEIPT_ROOT": str(native_receipt_root), + "TEST_RESTORE_RECEIPT_ROOT": str(restore_receipt_root), + "TEST_FLOCK_MARKER": str(flock_marker), "FAKE_MONITOR_MODE": monitor_mode, "FAKE_BACKUP_SLEEP_SECONDS": str(backup_sleep_seconds), "FAKE_DUPLICATE_CRON": "1" if duplicate_cron else "0", "FAKE_DROP_LISTENER": "1" if drop_listener else "0", "FAKE_BACKUP_STOPS_COLLECTOR": "1" if expect_collector_stop else "0", + "FAKE_RECEIPT_FAULT": native_receipt_state, + "FAKE_DOCKER_TIMEOUT": "1" if docker_timeout else "0", } started_at = int(time.time()) result = subprocess.run( @@ -308,6 +475,13 @@ def _run_canary( "lease_observed": lease_observed, "collector_state": collector_state, "listeners": listeners, + "restore_contract": restore_contract_observed, + "restore_hook": restore_hook, + "flock_marker": flock_marker, + "native_receipts": native_receipt_root / run_id / "receipts.jsonl", + "restore_receipts": restore_receipt_root / run_id / "receipts.jsonl", + "restore_status": restore_receipt_root / run_id / "status.json", + "backup_output": receipt_root / run_id / "backup-output.log", } return result, paths, str(started_at) @@ -337,7 +511,9 @@ def test_existing_lease_is_atomically_replaced_and_exactly_restored( assert identities[2] == "P0-OBS-002\n" receipts = _receipts(paths["receipts"]) assert _terminal(receipts, "rollback") == "restored" - assert _terminal(receipts, "terminal") == "pass" + assert _terminal(receipts, "runtime_operation") == "pass" + assert _terminal(receipts, "closure_writeback") == "partial_degraded" + assert _terminal(receipts, "terminal") == "partial_degraded" metadata = json.loads(paths["rollback_metadata"].read_text(encoding="utf-8")) assert metadata["trace_id"] == "trace-P0-OBS-002" assert metadata["prior_present"] == 1 @@ -368,7 +544,7 @@ def test_absent_lease_is_removed_after_success(tmp_path: Path) -> None: assert not paths["cooldown"].exists() receipts = _receipts(paths["receipts"]) assert _terminal(receipts, "rollback") == "restored" - assert _terminal(receipts, "terminal") == "pass" + assert _terminal(receipts, "terminal") == "partial_degraded" def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease( @@ -380,13 +556,30 @@ def test_online_native_canary_keeps_collector_running_and_does_not_arm_lease( assert paths["cooldown"].read_text(encoding="utf-8") == "123\n" assert not paths["lease_observed"].exists() assert paths["collector_state"].read_text(encoding="utf-8") == "true\n" + assert paths["restore_contract"].read_text(encoding="utf-8").split("\t") == [ + str(paths["restore_hook"]), + str(tmp_path / "clickhouse-restore-inventory.py"), + str(tmp_path / "backup_disk.xml"), + f"{tmp_path / 'isolated-cluster.xml'}\n", + ] receipts = _receipts(paths["receipts"]) + assert paths["native_receipts"].is_file() + assert paths["restore_receipts"].is_file() + restore_status = json.loads(paths["restore_status"].read_text(encoding="utf-8")) + assert restore_status["terminal"] == "verified" + assert restore_status["check_table_count"] == 44 + assert restore_status["check_table_pass_count"] == 44 + assert any( + item["stage"] == "native_restore_receipt_verifier" + and item["terminal"] == "pass" + for item in receipts + ) assert any( item["stage"] == "lease" and item["terminal"] == "not_required" for item in receipts ) assert _terminal(receipts, "rollback") == "no_write" - assert _terminal(receipts, "terminal") == "pass" + assert _terminal(receipts, "terminal") == "partial_degraded" def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract( @@ -406,7 +599,7 @@ def test_online_native_canary_is_not_coupled_to_legacy_monitor_contract( item for item in receipts if item["stage"] == "source_of_truth_diff" ][-1] assert "monitor_and_cooldown_contract_not_applicable" in source_diff["detail"] - assert _terminal(receipts, "terminal") == "pass" + assert _terminal(receipts, "terminal") == "partial_degraded" def test_auto_repair_contamination_fails_and_restores_lease(tmp_path: Path) -> None: @@ -496,6 +689,34 @@ def test_wrapper_deployment_and_static_contracts_are_owned() -> None: assert 'WORK_ITEM_ID="${WORK_ITEM_ID:-}"' in wrapper assert "monitor_auto_repair_contaminated_canary" in wrapper assert "collector_post_backup_runtime_failed" in wrapper + assert "/backup/scripts/clickhouse-native-restore-drill.sh" in wrapper + assert "/backup/scripts/clickhouse-restore-inventory.py" in wrapper + assert "/backup/config/signoz/clickhouse/config.d/backup_disk.xml" in wrapper + assert ( + "/backup/config/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + in wrapper + ) + assert ( + 'CLICKHOUSE_NATIVE_POST_VERIFY_HOOK="${CLICKHOUSE_NATIVE_POST_VERIFY_HOOK}"' + in wrapper + ) + assert ( + 'CLICKHOUSE_NATIVE_RECEIPT_ROOT="${CLICKHOUSE_NATIVE_RECEIPT_ROOT}"' in wrapper + ) + assert ( + 'CLICKHOUSE_RESTORE_RECEIPT_ROOT="${CLICKHOUSE_RESTORE_RECEIPT_ROOT}"' + in wrapper + ) + main_source = wrapper[wrapper.index("main() {") :] + assert main_source.index("acquire_canary_lock") < main_source.index( + "require_native_restore_contract" + ) + assert "same_run_native_restore_receipt_validation_failed" in wrapper + assert '"check_table_count"' in wrapper + assert "mandatory_isolated_restore_verifier" in wrapper + assert "SIGNOZ_CANARY_DOCKER_TIMEOUT_SECONDS" in wrapper + assert 'emit_receipt "runtime_operation" "pass"' in wrapper + assert 'emit_receipt "closure_writeback" "partial_degraded"' in wrapper assert WRAPPER.name in PLAYBOOK.read_text(encoding="utf-8") assert str(WRAPPER.relative_to(ROOT)) in ANSIBLE_VALIDATE.read_text( encoding="utf-8" @@ -518,6 +739,80 @@ def test_check_mode_proves_contract_without_lease_or_backup_write( assert _terminal(check_receipts, "terminal") == "check_pass_no_write" +@pytest.mark.parametrize("contract_state", ["missing", "symlink"]) +def test_native_check_fails_closed_for_unsafe_restore_contract( + tmp_path: Path, + contract_state: str, +) -> None: + result, paths, _ = _run_canary( + tmp_path, + mode="check", + expect_collector_stop=False, + native_restore_contract=contract_state, + ) + + assert result.returncode != 0 + assert "native_restore_contract_CLICKHOUSE_NATIVE_POST_VERIFY_HOOK" in result.stderr + assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n" + assert not paths["identities"].exists() + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "terminal") == "failed" + + +@pytest.mark.parametrize( + "receipt_state", + [ + "no_hook", + "forged_identity", + "missing_status", + "symlink_status", + "malformed_restore_receipt", + ], +) +def test_native_apply_rejects_missing_or_forged_machine_receipts( + tmp_path: Path, + receipt_state: str, +) -> None: + result, paths, _ = _run_canary( + tmp_path, + expect_collector_stop=False, + native_receipt_state=receipt_state, + ) + + assert result.returncode != 0 + assert "========== SigNoz ClickHouse native 備份完成 (1s) ==========" in paths[ + "backup_output" + ].read_text(encoding="utf-8") + assert ( + "same_run_native_restore_receipt_validation_failed" in result.stderr + or "same_run_native_restore_receipt_missing_unreadable_or_symlink" + in result.stderr + ) + assert paths["flock_marker"].read_text(encoding="utf-8") == "held\n" + receipts = _receipts(paths["receipts"]) + assert not any( + item["stage"] == "native_restore_receipt_verifier" + and item["terminal"] == "pass" + for item in receipts + ) + assert _terminal(receipts, "terminal") == "failed" + + +def test_docker_metadata_timeout_fails_before_backup(tmp_path: Path) -> None: + result, paths, _ = _run_canary( + tmp_path, + mode="check", + expect_collector_stop=False, + docker_timeout=True, + ) + + assert result.returncode != 0 + assert "collector_runtime_metadata_unreadable" in result.stderr + assert not paths["identities"].exists() + receipts = _receipts(paths["receipts"]) + assert _terminal(receipts, "terminal") == "failed" + + def test_wrapper_is_valid_bash() -> None: result = subprocess.run( ["bash", "-n", str(WRAPPER)], diff --git a/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh index 22a79e705..da4e27be0 100755 --- a/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh +++ b/scripts/ops/deploy-signoz-clickhouse-backup-disk.sh @@ -11,7 +11,7 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" LOCAL_OVERRIDE="${ROOT_DIR}/ops/signoz/docker-compose.clickhouse-backup.override.yaml" LOCAL_CONFIG="${ROOT_DIR}/ops/signoz/clickhouse/config.d/backup_disk.xml" -TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}" +TARGET_HOST="wooo@192.168.0.110" REMOTE_DIR="/home/wooo/signoz/deploy/docker" REMOTE_BASE="${REMOTE_DIR}/docker-compose.yaml" REMOTE_OVERRIDE="${REMOTE_DIR}/docker-compose.awoooi-clickhouse-backup.yaml" @@ -31,6 +31,12 @@ RECEIPT_ROOT="/backup/deploy-receipts/signoz-clickhouse-backup-disk" API_URL="http://127.0.0.1:8080/api/v1/health" SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" SAFE_HOME="/home/wooo" +LOCAL_KILL_AFTER_SECONDS=15 +REMOTE_CHECK_TIMEOUT_SECONDS=180 +REMOTE_STAGE_TIMEOUT_SECONDS=60 +REMOTE_TRANSFER_TIMEOUT_SECONDS=60 +REMOTE_APPLY_TIMEOUT_SECONDS=3600 +REMOTE_CLEANUP_TIMEOUT_SECONDS=60 SSH_OPTS=( -o BatchMode=yes @@ -79,7 +85,7 @@ for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do } done -for command_name in python3 scp ssh; do +for command_name in python3 scp ssh timeout; do command -v "${command_name}" >/dev/null 2>&1 || { printf 'required local command missing: %s\n' "${command_name}" >&2 exit 69 @@ -94,6 +100,18 @@ hash_file() { fi } +bounded_ssh() { + local timeout_seconds="$1" + shift + timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \ + "${timeout_seconds}" ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" "$@" +} + +bounded_scp() { + timeout --signal=TERM --kill-after="${LOCAL_KILL_AFTER_SECONDS}" \ + "${REMOTE_TRANSFER_TIMEOUT_SECONDS}" scp -q "${SSH_OPTS[@]}" "$@" +} + validate_local_sources() { [ -f "${LOCAL_OVERRIDE}" ] && [ ! -L "${LOCAL_OVERRIDE}" ] [ -f "${LOCAL_CONFIG}" ] && [ ! -L "${LOCAL_CONFIG}" ] @@ -136,13 +154,13 @@ local_receipt() { } remote_candidate_compose_check() { - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" \ - "env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \ + bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" \ + "timeout --signal=TERM --kill-after=15 120 env -i PATH='${SAFE_PATH}' HOME='${SAFE_HOME}' docker compose --env-file /dev/null -p '${REMOTE_PROJECT}' -f '${REMOTE_BASE}' -f - config -q >/dev/null" \ < "${LOCAL_OVERRIDE}" } remote_read_only_check() { - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ + bounded_ssh "${REMOTE_CHECK_TIMEOUT_SECONDS}" bash -s -- \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ "${OVERRIDE_SHA}" "${CONFIG_SHA}" <<'REMOTE_CHECK' set -euo pipefail @@ -159,6 +177,24 @@ REMOTE_CONFIG="${REMOTE_DIR}/awoooi-clickhouse-backup-disk.xml" HOST_BACKUP_DIR="/backup/staging/signoz-clickhouse" SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" SAFE_HOME="/home/wooo" +COMMAND_TIMEOUT_SECONDS=30 +COMPOSE_TIMEOUT_SECONDS=120 +KILL_AFTER_SECONDS=15 + +for command_name in docker timeout; do + command -v "${command_name}" >/dev/null 2>&1 +done + +docker_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker "$@" +} + +compose_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMPOSE_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p signoz "$@" +} file_hash_or_absent() { if [ -f "$1" ] && [ ! -L "$1" ]; then @@ -172,14 +208,10 @@ file_hash_or_absent() { [ -f "${REMOTE_BASE}" ] && [ ! -L "${REMOTE_BASE}" ] for container in signoz-clickhouse signoz-otel-collector signoz signoz-zookeeper-1; do - docker inspect "${container}" >/dev/null + docker_cmd inspect "${container}" >/dev/null done -env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \ - config -q >/dev/null -env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p signoz -f "${REMOTE_BASE}" \ - config --services | grep -qx clickhouse +compose_cmd -f "${REMOTE_BASE}" config -q >/dev/null +compose_cmd -f "${REMOTE_BASE}" config --services | grep -qx clickhouse override_hash="$(file_hash_or_absent "${REMOTE_OVERRIDE}")" config_hash="$(file_hash_or_absent "${REMOTE_CONFIG}")" @@ -195,13 +227,13 @@ elif [ -e "${HOST_BACKUP_DIR}" ]; then host_dir=unsafe fi -disk_count="$(docker exec signoz-clickhouse clickhouse-client --query \ +disk_count="$(docker_cmd exec signoz-clickhouse clickhouse-client --query \ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" -mount_count="$(docker inspect --format \ +mount_count="$(docker_cmd inspect --format \ '{{range .Mounts}}{{if eq .Destination "/backups"}}1{{end}}{{end}}' \ signoz-clickhouse | tr -cd '1' | wc -c | xargs)" -collector_running="$(docker inspect --format '{{.State.Running}}' signoz-otel-collector)" -collector_restarts="$(docker inspect --format '{{.RestartCount}}' signoz-otel-collector)" +collector_running="$(docker_cmd inspect --format '{{.State.Running}}' signoz-otel-collector)" +collector_restarts="$(docker_cmd inspect --format '{{.RestartCount}}' signoz-otel-collector)" printf '{"schema":"awoooi_controlled_apply_receipt_v1","trace_id":"%s","run_id":"%s","work_item_id":"%s","asset_id":"host110-signoz-clickhouse-backup-disk","phase":"source_of_truth_diff","result":"pass","detail":"override_match_%s_config_match_%s_host_dir_%s_live_disk_count_%s_live_mount_count_%s_collector_running_%s_collector_restarts_%s"}\n' \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${override_match}" \ @@ -223,20 +255,58 @@ fi STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.${RUN_ID}" STAGE_CREATED=0 +LOCAL_APPLY_VERIFIED=0 cleanup_stage() { - if [ "${STAGE_CREATED}" -eq 1 ]; then - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true -set -u + local cleanup_rc + [ "${STAGE_CREATED}" -eq 1 ] || return 0 + bounded_ssh "${REMOTE_CLEANUP_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 +set -euo pipefail stage_dir="$1" +if [ ! -e "${stage_dir}" ]; then + exit 0 +fi +[ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ] rm -f "${stage_dir}/override.yaml" "${stage_dir}/backup_disk.xml" -rmdir "${stage_dir}" 2>/dev/null || true +rmdir "${stage_dir}" +[ ! -e "${stage_dir}" ] REMOTE_CLEANUP - fi + cleanup_rc=$? + [ "${cleanup_rc}" -eq 0 ] || return "${cleanup_rc}" + STAGE_CREATED=0 } -trap cleanup_stage EXIT -ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' +on_local_exit() { + local rc=$? + local cleanup_rc=0 cleanup_terminal=not_created terminal=failed + trap - EXIT HUP INT TERM + set +e + if [ "${STAGE_CREATED}" -eq 1 ]; then + cleanup_stage + cleanup_rc=$? + if [ "${cleanup_rc}" -eq 0 ]; then + cleanup_terminal=pass_absence_verified + else + cleanup_terminal=failed_or_residue_present + rc=92 + fi + fi + local_receipt cleanup "$([ "${cleanup_rc}" -eq 0 ] && printf pass || printf failed)" \ + "remote_stage_${cleanup_terminal}" + if [ "${rc}" -eq 0 ] && [ "${LOCAL_APPLY_VERIFIED}" -eq 1 ]; then + terminal=pass + fi + local_receipt terminal "${terminal}" \ + "remote_controlled_apply_${terminal}_stage_${cleanup_terminal}_exit_${rc}" + exit "${rc}" +} + +trap on_local_exit EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM + +bounded_ssh "${REMOTE_STAGE_TIMEOUT_SECONDS}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' set -euo pipefail stage_dir="$1" [ ! -e "${stage_dir}" ] @@ -245,12 +315,12 @@ mkdir -m 0700 "${stage_dir}" REMOTE_STAGE STAGE_CREATED=1 -scp -q "${SSH_OPTS[@]}" "${LOCAL_OVERRIDE}" \ +bounded_scp "${LOCAL_OVERRIDE}" \ "${TARGET_HOST}:${STAGE_DIR}/override.yaml" -scp -q "${SSH_OPTS[@]}" "${LOCAL_CONFIG}" \ +bounded_scp "${LOCAL_CONFIG}" \ "${TARGET_HOST}:${STAGE_DIR}/backup_disk.xml" -ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ +bounded_ssh "${REMOTE_APPLY_TIMEOUT_SECONDS}" bash -s -- \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ "${OVERRIDE_SHA}" "${CONFIG_SHA}" "${STAGE_DIR}" <<'REMOTE_APPLY' set -euo pipefail @@ -282,12 +352,26 @@ API_URL="http://127.0.0.1:8080/api/v1/health" SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" SAFE_HOME="/home/wooo" LOCK_FILE="/tmp/awoooi-signoz-clickhouse-backup-disk.lock" +BACKUP_OPERATION_LOCK_FILE="/tmp/awoooi-signoz-backup-operation.lock" +CANARY_LOCK_FILE="/tmp/awoooi-signoz-backup-canary.lock" +COMMAND_TIMEOUT_SECONDS=30 +COMPOSE_CONFIG_TIMEOUT_SECONDS=120 +COMPOSE_APPLY_TIMEOUT_SECONDS=300 +KILL_AFTER_SECONDS=15 APPLY_STARTED=0 DEPLOY_VERIFIED=0 HOST_DIR_CREATED=0 PRIOR_OVERRIDE_PRESENT=0 PRIOR_CONFIG_PRESENT=0 +PRIOR_OVERRIDE_HASH="absent" +PRIOR_OVERRIDE_MODE="absent" +PRIOR_OVERRIDE_UID="absent" +PRIOR_OVERRIDE_GID="absent" +PRIOR_CONFIG_HASH="absent" +PRIOR_CONFIG_MODE="absent" +PRIOR_CONFIG_UID="absent" +PRIOR_CONFIG_GID="absent" PRIOR_DISK_COUNT=0 BEFORE_IMAGE_ID="" BEFORE_DATA_VOLUME="" @@ -307,38 +391,60 @@ receipt() { printf '%s\n' "${line}" >> "${RECEIPT_LOG}" } +docker_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" docker "$@" +} + +bounded_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" "$@" +} + +compose_config_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMPOSE_CONFIG_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@" +} + +compose_apply_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMPOSE_APPLY_TIMEOUT_SECONDS}" env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ + docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" "$@" +} + +pgrep_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" pgrep "$@" +} + +ss_cmd() { + timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}" \ + "${COMMAND_TIMEOUT_SECONDS}" ss "$@" +} + compose_base_config() { - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" config -q >/dev/null 2>&1 + compose_config_cmd -f "${REMOTE_BASE}" config -q >/dev/null 2>&1 } compose_managed_config() { - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" config -q >/dev/null 2>&1 + compose_config_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \ + config -q >/dev/null 2>&1 } compose_stage_config() { - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config -q >/dev/null 2>&1 + compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" \ + config -q >/dev/null 2>&1 } recreate_managed() { - timeout --signal=TERM --kill-after=30 300 \ - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \ + compose_apply_cmd -f "${REMOTE_BASE}" -f "${REMOTE_OVERRIDE}" \ up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \ >/dev/null 2>&1 } recreate_base() { - timeout --signal=TERM --kill-after=30 300 \ - env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" \ + compose_apply_cmd -f "${REMOTE_BASE}" \ up -d --no-deps --pull never --force-recreate "${REMOTE_SERVICE}" \ >/dev/null 2>&1 } @@ -346,11 +452,11 @@ recreate_base() { wait_clickhouse() { local attempt running health for attempt in $(seq 1 36); do - running="$(docker inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)" - health="$(docker inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \ + running="$(docker_cmd inspect --format '{{.State.Running}}' "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)" + health="$(docker_cmd inspect --format '{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' \ "${CLICKHOUSE_CONTAINER}" 2>/dev/null || true)" if [ "${running}" = true ] && [ "${health}" = healthy ] \ - && timeout --kill-after=5 10 docker exec "${CLICKHOUSE_CONTAINER}" \ + && docker_cmd exec "${CLICKHOUSE_CONTAINER}" \ clickhouse-client --query 'SELECT 1' >/dev/null 2>&1; then return 0 fi @@ -361,7 +467,7 @@ wait_clickhouse() { listener_up() { local port="$1" - ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' + ss_cmd -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' } wait_api_and_listeners() { @@ -376,25 +482,72 @@ wait_api_and_listeners() { return 1 } +file_matches_metadata() { + local path="$1" expected_hash="$2" expected_mode="$3" expected_uid="$4" expected_gid="$5" + [ -f "${path}" ] && [ ! -L "${path}" ] \ + && [ "$(sha256sum "${path}" | awk '{print $1}')" = "${expected_hash}" ] \ + && [ "$(stat -c '%a' "${path}")" = "${expected_mode}" ] \ + && [ "$(stat -c '%u' "${path}")" = "${expected_uid}" ] \ + && [ "$(stat -c '%g' "${path}")" = "${expected_gid}" ] +} + +restore_exact_file() { + local source="$1" target="$2" expected_hash="$3" expected_mode="$4" + local expected_uid="$5" expected_gid="$6" tmp="$7" + local current_uid current_gid + + bounded_cmd install -m "${expected_mode}" "${source}" "${tmp}" || return 1 + current_uid="$(stat -c '%u' "${tmp}")" || return 1 + current_gid="$(stat -c '%g' "${tmp}")" || return 1 + if [ "${current_uid}" != "${expected_uid}" ] || [ "${current_gid}" != "${expected_gid}" ]; then + bounded_cmd sudo -n chown "${expected_uid}:${expected_gid}" "${tmp}" || return 1 + fi + bounded_cmd mv -f "${tmp}" "${target}" || return 1 + file_matches_metadata "${target}" "${expected_hash}" "${expected_mode}" \ + "${expected_uid}" "${expected_gid}" +} + restore_managed_files() { local tmp if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then tmp="${REMOTE_OVERRIDE}.rollback.${RUN_ID}" - install -m 0644 "${RECEIPT_DIR}/prior-override.yaml" "${tmp}" - mv -f "${tmp}" "${REMOTE_OVERRIDE}" + restore_exact_file "${RECEIPT_DIR}/prior-override.yaml" "${REMOTE_OVERRIDE}" \ + "${PRIOR_OVERRIDE_HASH}" "${PRIOR_OVERRIDE_MODE}" \ + "${PRIOR_OVERRIDE_UID}" "${PRIOR_OVERRIDE_GID}" "${tmp}" || return 1 else - rm -f "${REMOTE_OVERRIDE}" + bounded_cmd rm -f "${REMOTE_OVERRIDE}" || return 1 + [ ! -e "${REMOTE_OVERRIDE}" ] || return 1 fi if [ "${PRIOR_CONFIG_PRESENT}" -eq 1 ]; then tmp="${REMOTE_CONFIG}.rollback.${RUN_ID}" - install -m 0644 "${RECEIPT_DIR}/prior-backup-disk.xml" "${tmp}" - mv -f "${tmp}" "${REMOTE_CONFIG}" + restore_exact_file "${RECEIPT_DIR}/prior-backup-disk.xml" "${REMOTE_CONFIG}" \ + "${PRIOR_CONFIG_HASH}" "${PRIOR_CONFIG_MODE}" \ + "${PRIOR_CONFIG_UID}" "${PRIOR_CONFIG_GID}" "${tmp}" || return 1 else - rm -f "${REMOTE_CONFIG}" + bounded_cmd rm -f "${REMOTE_CONFIG}" || return 1 + [ ! -e "${REMOTE_CONFIG}" ] || return 1 fi } +cleanup_owned_residue() { + local path + for path in \ + "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" \ + "${REMOTE_CONFIG}.candidate.${RUN_ID}" \ + "${REMOTE_OVERRIDE}.rollback.${RUN_ID}" \ + "${REMOTE_CONFIG}.rollback.${RUN_ID}"; do + bounded_cmd rm -f "${path}" || return 1 + [ ! -e "${path}" ] || return 1 + done + if [ -e "${STAGE_DIR}" ]; then + [ -d "${STAGE_DIR}" ] && [ ! -L "${STAGE_DIR}" ] || return 1 + bounded_cmd rm -f "${STAGE_OVERRIDE}" "${STAGE_CONFIG}" || return 1 + bounded_cmd rmdir "${STAGE_DIR}" || return 1 + fi + [ ! -e "${STAGE_DIR}" ] || return 1 +} + rollback_deploy() { local disk_count rollback_dir_terminal=retained restore_managed_files || return 1 @@ -409,26 +562,27 @@ rollback_deploy() { wait_clickhouse || return 1 wait_api_and_listeners || return 1 - disk_count="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + disk_count="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" || return 1 [ "${disk_count}" = "${PRIOR_DISK_COUNT}" ] || return 1 - [ "$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1 - [ "$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1 - [ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1 - [ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1 + [ "$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_IMAGE_ID}" ] || return 1 + [ "$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" = "${BEFORE_DATA_VOLUME}" ] || return 1 + [ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] || return 1 + [ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] || return 1 if [ "${HOST_DIR_CREATED}" -eq 1 ]; then - if sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then + if bounded_cmd sudo -n rmdir "${HOST_BACKUP_DIR}" 2>/dev/null; then rollback_dir_terminal=removed_empty else rollback_dir_terminal=retained_nonempty_no_delete fi fi - receipt rollback pass "prior_compose_state_restored_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}" + receipt rollback pass \ + "prior_compose_state_restored_exact_hash_mode_uid_gid_disk_count_${disk_count}_host_dir_${rollback_dir_terminal}" } on_exit() { - local rc=$? rollback_rc=0 terminal=failed + local rc=$? rollback_rc=0 cleanup_rc=0 terminal=failed trap - EXIT HUP INT TERM set +e if [ "${rc}" -ne 0 ] && [ "${APPLY_STARTED}" -eq 1 ] && [ "${DEPLOY_VERIFIED}" -eq 0 ]; then @@ -439,7 +593,15 @@ on_exit() { rc=91 fi fi - if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ]; then + cleanup_owned_residue + cleanup_rc=$? + if [ "${cleanup_rc}" -ne 0 ]; then + receipt cleanup failed "exact_stage_candidate_or_rollback_residue_cleanup_failed" + rc=92 + else + receipt cleanup pass "exact_stage_candidate_and_rollback_residue_absence_verified" + fi + if [ "${rc}" -eq 0 ] && [ "${DEPLOY_VERIFIED}" -eq 1 ] && [ "${cleanup_rc}" -eq 0 ]; then terminal=pass fi receipt terminal "${terminal}" "exit_${rc}_receipt_ref_${RECEIPT_LOG}" @@ -455,10 +617,22 @@ trap 'exit 129' HUP trap 'exit 130' INT trap 'exit 143' TERM +for lock_path in "${LOCK_FILE}" "${BACKUP_OPERATION_LOCK_FILE}" "${CANARY_LOCK_FILE}"; do + [ ! -L "${lock_path}" ] \ + || { receipt check failed operation_lock_symlink_unsafe; exit 75; } + if [ -e "${lock_path}" ]; then + [ -f "${lock_path}" ] \ + || { receipt check failed operation_lock_not_regular; exit 75; } + fi +done exec 9>>"${LOCK_FILE}" flock -n 9 || { receipt check failed another_deploy_holds_lock; exit 75; } +exec 7>>"${CANARY_LOCK_FILE}" +flock -n 7 || { receipt check failed backup_canary_lock_held; exit 75; } +exec 8>>"${BACKUP_OPERATION_LOCK_FILE}" +flock -n 8 || { receipt check failed backup_operation_lock_held; exit 75; } -for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss sudo timeout; do +for command_name in awk curl docker flock install mv pgrep python3 seq sha256sum ss stat sudo timeout; do command -v "${command_name}" >/dev/null 2>&1 \ || { receipt check failed "required_command_missing_${command_name}"; exit 69; } done @@ -474,21 +648,39 @@ done compose_base_config compose_stage_config -env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}" \ - docker compose --env-file /dev/null -p "${REMOTE_PROJECT}" \ - -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \ +compose_config_cmd -f "${REMOTE_BASE}" -f "${STAGE_OVERRIDE}" config --services \ | grep -qx "${REMOTE_SERVICE}" -if pgrep -af '(^|/|[[:space:]])backup-signoz[.]sh([[:space:]]|$)' >/dev/null 2>&1; then - receipt check failed active_signoz_backup_detected - exit 75 -fi -active_native_backups="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ - "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" -[ "${active_native_backups}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; } +require_backup_idle() { + local phase="$1" active_native_backups process_rc + set +e + pgrep_cmd -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' \ + >/dev/null 2>&1 + process_rc=$? + set -e + case "${process_rc}" in + 0) + receipt check failed "active_signoz_backup_or_restore_process_detected_${phase}" + return 75 + ;; + 1) ;; + *) + receipt check failed "signoz_backup_process_probe_failed_${phase}_exit_${process_rc}" + return 69 + ;; + esac + active_native_backups="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ + "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" \ + || return 1 + [ "${active_native_backups}" = 0 ] \ + || { receipt check failed "active_native_backup_or_restore_${phase}"; return 75; } + receipt check pass "backup_idle_${phase}_shared_operation_and_canary_locks_held" +} + +require_backup_idle preflight for container in "${CLICKHOUSE_CONTAINER}" "${COLLECTOR_CONTAINER}" "${SIGNOZ_CONTAINER}" "${ZOOKEEPER_CONTAINER}"; do - [ "$(docker inspect --format '{{.State.Running}}' "${container}")" = true ] + [ "$(docker_cmd inspect --format '{{.State.Running}}' "${container}")" = true ] done wait_clickhouse wait_api_and_listeners @@ -505,42 +697,56 @@ fi || { receipt check failed partial_managed_file_drift; exit 78; } if [ "${PRIOR_OVERRIDE_PRESENT}" -eq 1 ]; then compose_managed_config - cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml" - cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml" + PRIOR_OVERRIDE_HASH="$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" + PRIOR_OVERRIDE_MODE="$(stat -c '%a' "${REMOTE_OVERRIDE}")" + PRIOR_OVERRIDE_UID="$(stat -c '%u' "${REMOTE_OVERRIDE}")" + PRIOR_OVERRIDE_GID="$(stat -c '%g' "${REMOTE_OVERRIDE}")" + PRIOR_CONFIG_HASH="$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" + PRIOR_CONFIG_MODE="$(stat -c '%a' "${REMOTE_CONFIG}")" + PRIOR_CONFIG_UID="$(stat -c '%u' "${REMOTE_CONFIG}")" + PRIOR_CONFIG_GID="$(stat -c '%g' "${REMOTE_CONFIG}")" + bounded_cmd cp -p "${REMOTE_OVERRIDE}" "${RECEIPT_DIR}/prior-override.yaml" + bounded_cmd cp -p "${REMOTE_CONFIG}" "${RECEIPT_DIR}/prior-backup-disk.xml" + [ "$(sha256sum "${RECEIPT_DIR}/prior-override.yaml" | awk '{print $1}')" = "${PRIOR_OVERRIDE_HASH}" ] + [ "$(sha256sum "${RECEIPT_DIR}/prior-backup-disk.xml" | awk '{print $1}')" = "${PRIOR_CONFIG_HASH}" ] fi -PRIOR_DISK_COUNT="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ +PRIOR_DISK_COUNT="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ "SELECT count() FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" -BEFORE_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" -BEFORE_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +BEFORE_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" +BEFORE_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" [ -n "${BEFORE_DATA_VOLUME}" ] -BEFORE_COLLECTOR_ID="$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" -BEFORE_COLLECTOR_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" -BEFORE_SIGNOZ_ID="$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" -BEFORE_SIGNOZ_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" -BEFORE_ZOOKEEPER_ID="$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" -BEFORE_ZOOKEEPER_RESTARTS="$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" +BEFORE_COLLECTOR_ID="$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" +BEFORE_COLLECTOR_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" +BEFORE_SIGNOZ_ID="$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" +BEFORE_SIGNOZ_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" +BEFORE_ZOOKEEPER_ID="$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" +BEFORE_ZOOKEEPER_RESTARTS="$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" receipt sensor_source pass "override_sha_${EXPECTED_OVERRIDE_SHA}_config_sha_${EXPECTED_CONFIG_SHA}_base_compose_valid" receipt normalized_asset_identity pass "host_110_project_signoz_service_clickhouse_data_volume_${BEFORE_DATA_VOLUME}" -receipt source_of_truth_diff pass "prior_override_${PRIOR_OVERRIDE_PRESENT}_prior_config_${PRIOR_CONFIG_PRESENT}_prior_disk_count_${PRIOR_DISK_COUNT}" +receipt source_of_truth_diff pass \ + "prior_override_${PRIOR_OVERRIDE_PRESENT}_hash_${PRIOR_OVERRIDE_HASH}_mode_${PRIOR_OVERRIDE_MODE}_uid_${PRIOR_OVERRIDE_UID}_gid_${PRIOR_OVERRIDE_GID}_prior_config_${PRIOR_CONFIG_PRESENT}_hash_${PRIOR_CONFIG_HASH}_mode_${PRIOR_CONFIG_MODE}_uid_${PRIOR_CONFIG_UID}_gid_${PRIOR_CONFIG_GID}_prior_disk_count_${PRIOR_DISK_COUNT}" receipt ai_decision candidate "install_dedicated_backups_disk_recreate_clickhouse_only" receipt risk_policy pass "risk_high_canary_single_service_no_pull_bounded_timeout_rollback" -receipt check pass "compose_config_q_stage_hashes_runtime_preflight_backup_idle" +receipt check pass \ + "compose_config_q_stage_hashes_runtime_preflight_dual_locks_backup_idle" + +require_backup_idle pre_mutation APPLY_STARTED=1 if [ -e "${HOST_BACKUP_DIR}" ]; then [ -d "${HOST_BACKUP_DIR}" ] && [ ! -L "${HOST_BACKUP_DIR}" ] [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ] else - sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}" + bounded_cmd sudo -n install -d -o 101 -g 101 -m 0750 "${HOST_BACKUP_DIR}" HOST_DIR_CREATED=1 fi -install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" -install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}" -mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}" -mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}" +bounded_cmd install -m 0644 "${STAGE_OVERRIDE}" "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" +bounded_cmd install -m 0644 "${STAGE_CONFIG}" "${REMOTE_CONFIG}.candidate.${RUN_ID}" +bounded_cmd mv -f "${REMOTE_OVERRIDE}.candidate.${RUN_ID}" "${REMOTE_OVERRIDE}" +bounded_cmd mv -f "${REMOTE_CONFIG}.candidate.${RUN_ID}" "${REMOTE_CONFIG}" [ "$(sha256sum "${REMOTE_OVERRIDE}" | awk '{print $1}')" = "${EXPECTED_OVERRIDE_SHA}" ] [ "$(sha256sum "${REMOTE_CONFIG}" | awk '{print $1}')" = "${EXPECTED_CONFIG_SHA}" ] compose_managed_config @@ -548,39 +754,39 @@ compose_managed_config recreate_managed wait_clickhouse -AFTER_IMAGE_ID="$(docker inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" -AFTER_DATA_VOLUME="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +AFTER_IMAGE_ID="$(docker_cmd inspect --format '{{.Image}}' "${CLICKHOUSE_CONTAINER}")" +AFTER_DATA_VOLUME="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/var/lib/clickhouse"}}{{.Name}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" [ "${AFTER_IMAGE_ID}" = "${BEFORE_IMAGE_ID}" ] [ "${AFTER_DATA_VOLUME}" = "${BEFORE_DATA_VOLUME}" ] -backup_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" -config_mount="$(docker inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +backup_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/backups"}}{{.Source}}|{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" +config_mount="$(docker_cmd inspect --format '{{range .Mounts}}{{if eq .Destination "/etc/clickhouse-server/config.d/awoooi-backup-disk.xml"}}{{.RW}}{{end}}{{end}}' "${CLICKHOUSE_CONTAINER}")" [ "${backup_mount}" = "${HOST_BACKUP_DIR}|true" ] [ "${config_mount}" = false ] [ "$(stat -c '%u:%g:%a' "${HOST_BACKUP_DIR}")" = "101:101:750" ] -server_uid="$(docker exec "${CLICKHOUSE_CONTAINER}" \ +server_uid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \ awk '/^Uid:/{print $2}' /proc/1/status 2>/dev/null)" -server_gid="$(docker exec "${CLICKHOUSE_CONTAINER}" \ +server_gid="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" \ awk '/^Gid:/{print $2}' /proc/1/status 2>/dev/null)" [ "${server_uid}" = 101 ] [ "${server_gid}" = 101 ] -docker exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \ +docker_cmd exec -u 101:101 "${CLICKHOUSE_CONTAINER}" sh -c \ 'test -d /backups && test -w /backups' -disk_state="$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ +disk_state="$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ "SELECT concat(type,'|',toString(is_read_only),'|',toString(is_remote),'|',toString(is_broken),'|',toString(free_space>0)) FROM system.disks WHERE name='backups' FORMAT TSVRaw" 2>/dev/null)" [ "${disk_state}" = "Local|0|0|0|1" ] -[ "$(docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ +[ "$(docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query \ "EXISTS TABLE system.backups FORMAT TSVRaw" 2>/dev/null)" = 1 ] -docker exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \ +docker_cmd exec "${CLICKHOUSE_CONTAINER}" clickhouse-client --query 'SHOW GRANTS' 2>/dev/null \ | grep -Eqi '(^|[,[:space:]])BACKUP([,[:space:]]|$)' -[ "$(docker inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] -[ "$(docker inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] -[ "$(docker inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ] -[ "$(docker inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ] -[ "$(docker inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ] -[ "$(docker inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ] +[ "$(docker_cmd inspect --format '{{.Id}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_ID}" ] +[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${COLLECTOR_CONTAINER}")" = "${BEFORE_COLLECTOR_RESTARTS}" ] +[ "$(docker_cmd inspect --format '{{.Id}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_ID}" ] +[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${SIGNOZ_CONTAINER}")" = "${BEFORE_SIGNOZ_RESTARTS}" ] +[ "$(docker_cmd inspect --format '{{.Id}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_ID}" ] +[ "$(docker_cmd inspect --format '{{.RestartCount}}' "${ZOOKEEPER_CONTAINER}")" = "${BEFORE_ZOOKEEPER_RESTARTS}" ] wait_api_and_listeners receipt execution pass "clickhouse_recreated_no_pull_image_and_data_volume_unchanged" @@ -589,6 +795,4 @@ receipt closure_writeback pass "durable_receipt_ack_no_backup_or_restore_execute DEPLOY_VERIFIED=1 REMOTE_APPLY -trap - EXIT -cleanup_stage -local_receipt terminal pass "remote_controlled_apply_and_postverify_complete" +LOCAL_APPLY_VERIFIED=1 diff --git a/scripts/ops/deploy-signoz-native-backup-toolchain.sh b/scripts/ops/deploy-signoz-native-backup-toolchain.sh index 02ae66a68..6d4efb305 100755 --- a/scripts/ops/deploy-signoz-native-backup-toolchain.sh +++ b/scripts/ops/deploy-signoz-native-backup-toolchain.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# P0-OBS-002 - atomically deploy the host110 SigNoz native backup toolchain. +# P0-OBS-002 - bounded controlled deploy of the host110 SigNoz native backup toolchain. # # This deployer owns seven repo-sourced files only. It does not run a backup or # restore, restart a container, pull an image, or inspect a secret-bearing @@ -12,6 +12,13 @@ TARGET_HOST="wooo@192.168.0.110" SAFE_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" SAFE_HOME="/home/wooo" API_URL="http://127.0.0.1:8080/api/v1/health" +CANARY_LOCK="/tmp/awoooi-signoz-backup-canary.lock" +OPERATION_LOCK="/tmp/awoooi-signoz-backup-operation.lock" + +SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}" +SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}" +DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}" +TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}" REMOTE_SCRIPT_DIR="${SIGNOZ_TOOLCHAIN_SCRIPT_DIR:-/backup/scripts}" REMOTE_CONFIG_ROOT="${SIGNOZ_TOOLCHAIN_CONFIG_ROOT:-/backup/config/signoz/clickhouse}" @@ -61,10 +68,18 @@ Optional safe absolute path environment: SIGNOZ_TOOLCHAIN_RECEIPT_ROOT SIGNOZ_TOOLCHAIN_STAGE_ROOT +Optional bounded-command timeout environment (positive integer seconds): + SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS + SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS + SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS + SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS + --check validates local sources and performs a remote read-only target diff and runtime readiness check. --apply stages all seven sources, validates hashes and -syntax, atomically replaces exact targets, and rolls every target back to its -prior hash/mode/owner if any bounded verifier fails. +syntax, uses a same-filesystem atomic replacement for each exact target, and +rolls every target back to its prior hash/mode/owner if any bounded verifier +fails. SSH, SCP, and Docker calls are bounded, and apply holds both the SigNoz +canary lock and backup operation lock through post-verification or rollback. Flat restore-driver invocation must set: CLICKHOUSE_RESTORE_INVENTORY_HELPER=/clickhouse-restore-inventory.py @@ -89,6 +104,10 @@ valid_identity() { [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$ ]] } +valid_positive_integer() { + [[ "$1" =~ ^[1-9][0-9]*$ ]] +} + safe_absolute_path() { local path="$1" [[ "${path}" == /* ]] || return 1 @@ -111,6 +130,24 @@ for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do } done +for value in \ + "${SSH_COMMAND_TIMEOUT_SECONDS}" \ + "${SCP_COMMAND_TIMEOUT_SECONDS}" \ + "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \ + "${TIMEOUT_KILL_AFTER_SECONDS}"; do + valid_positive_integer "${value}" || { + printf 'toolchain command timeouts must be positive integer seconds\n' >&2 + exit 64 + } +done +[ "${SSH_COMMAND_TIMEOUT_SECONDS}" -le 1200 ] \ + && [ "${SCP_COMMAND_TIMEOUT_SECONDS}" -le 600 ] \ + && [ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" -le 300 ] \ + && [ "${TIMEOUT_KILL_AFTER_SECONDS}" -le 60 ] || { + printf 'toolchain command timeout exceeds bounded maximum\n' >&2 + exit 64 +} + for path in "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}"; do safe_absolute_path "${path}" || { printf 'toolchain paths must be safe absolute paths: %s\n' "${path}" >&2 @@ -133,13 +170,25 @@ if paths_overlap "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" \ exit 64 fi -for command_name in bash python3 scp ssh; do +for command_name in bash python3 scp ssh timeout; do command -v "${command_name}" >/dev/null 2>&1 || { printf 'required local command missing: %s\n' "${command_name}" >&2 exit 69 } done + +bounded_ssh() { + timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${SSH_COMMAND_TIMEOUT_SECONDS}" ssh "$@" +} + + +bounded_scp() { + timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${SCP_COMMAND_TIMEOUT_SECONDS}" scp "$@" +} + hash_file() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print $1}' @@ -241,9 +290,10 @@ local_receipt() { } remote_read_only_check() { - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ + bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_ROOT}" \ + "${DOCKER_COMMAND_TIMEOUT_SECONDS}" "${TIMEOUT_KILL_AFTER_SECONDS}" \ "${SOURCE_HASHES[@]}" <<'REMOTE_CHECK' set -euo pipefail @@ -254,7 +304,9 @@ REMOTE_SCRIPT_DIR="$4" REMOTE_CONFIG_ROOT="$5" RECEIPT_ROOT="$6" STAGE_ROOT="$7" -shift 7 +DOCKER_COMMAND_TIMEOUT_SECONDS="$8" +TIMEOUT_KILL_AFTER_SECONDS="$9" +shift 9 EXPECTED_HASHES=("$@") LABELS=( backup-signoz.sh @@ -279,10 +331,18 @@ API_URL="http://127.0.0.1:8080/api/v1/health" [ "${#EXPECTED_HASHES[@]}" -eq 7 ] [ -d /backup ] && [ ! -L /backup ] -for command_name in awk curl docker pgrep sha256sum ss stat; do +for command_name in awk curl docker pgrep sha256sum ss stat timeout; do command -v "${command_name}" >/dev/null 2>&1 done +[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]] +[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]] + +bounded_docker() { + timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@" +} + listener_up() { local port="$1" ss -H -lnt | awk -v port="${port}" '$4 ~ (":" port "$") { found = 1 } END { exit(found ? 0 : 1) }' @@ -305,7 +365,7 @@ validate_existing_ancestors() { runtime_line() { local container="$1" value running health - value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" + value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" IFS='|' read -r _ _ _ running health <<<"${value}" [ "${running}" = true ] case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac @@ -322,7 +382,7 @@ validate_existing_ancestors "${REMOTE_CONFIG_ROOT}/restore-drill/config.d" validate_existing_ancestors "${RECEIPT_ROOT}" validate_existing_ancestors "${STAGE_ROOT}" [ -d "${STAGE_ROOT}" ] && [ ! -L "${STAGE_ROOT}" ] && [ -w "${STAGE_ROOT}" ] -active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \ +active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \ "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" [ "${active_operations}" = 0 ] @@ -373,7 +433,7 @@ fi STAGE_CREATED=0 cleanup_stage() { if [ "${STAGE_CREATED}" -eq 1 ]; then - ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true + bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_CLEANUP' >/dev/null 2>&1 || true set -u stage_dir="$1" if [ -d "${stage_dir}" ] && [ ! -L "${stage_dir}" ]; then @@ -393,7 +453,7 @@ REMOTE_CLEANUP } trap cleanup_stage EXIT -ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' +bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- "${STAGE_DIR}" <<'REMOTE_STAGE' set -euo pipefail stage_dir="$1" [ ! -e "${stage_dir}" ] && [ ! -L "${stage_dir}" ] @@ -405,13 +465,15 @@ REMOTE_STAGE STAGE_CREATED=1 for index in "${!LOCAL_SOURCES[@]}"; do - scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \ + bounded_scp -q "${SSH_OPTS[@]}" "${LOCAL_SOURCES[index]}" \ "${TARGET_HOST}:${STAGE_DIR}/${LABELS[index]}" done -ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ +bounded_ssh "${SSH_OPTS[@]}" "${TARGET_HOST}" bash -s -- \ "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" \ "${REMOTE_SCRIPT_DIR}" "${REMOTE_CONFIG_ROOT}" "${RECEIPT_ROOT}" "${STAGE_DIR}" \ + "${CANARY_LOCK}" "${OPERATION_LOCK}" "${DOCKER_COMMAND_TIMEOUT_SECONDS}" \ + "${TIMEOUT_KILL_AFTER_SECONDS}" \ "${SOURCE_HASHES[@]}" <<'REMOTE_APPLY' set -euo pipefail @@ -422,7 +484,11 @@ REMOTE_SCRIPT_DIR="$4" REMOTE_CONFIG_ROOT="$5" RECEIPT_ROOT="$6" STAGE_DIR="$7" -shift 7 +CANARY_LOCK="$8" +OPERATION_LOCK="$9" +DOCKER_COMMAND_TIMEOUT_SECONDS="${10}" +TIMEOUT_KILL_AFTER_SECONDS="${11}" +shift 11 EXPECTED_HASHES=("$@") LABELS=( backup-signoz.sh @@ -472,6 +538,8 @@ APPLY_STARTED=0 DEPLOY_VERIFIED=0 RUNTIME_BASELINE_CAPTURED=0 LOCK_HELD=0 +CANARY_LOCK_HELD=0 +OPERATION_LOCK_HELD=0 REMOTE_UID="$(id -u)" REMOTE_GID="$(id -g)" CREATED_DIRS=() @@ -481,7 +549,7 @@ PRIOR_UIDS=() PRIOR_GIDS=() PRIOR_HASHES=() -for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat; do +for command_name in awk bash cmp cp curl docker flock id install mv pgrep python3 rm rmdir sha256sum ss stat timeout; do command -v "${command_name}" >/dev/null 2>&1 || { printf 'required remote command missing: %s\n' "${command_name}" >&2 exit 69 @@ -493,6 +561,10 @@ if [ "${REMOTE_UID}" -ne 0 ]; then fi [ "${#EXPECTED_HASHES[@]}" -eq 7 ] [ -d /backup ] && [ ! -L /backup ] +[[ "${DOCKER_COMMAND_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]] +[[ "${TIMEOUT_KILL_AFTER_SECONDS}" =~ ^[1-9][0-9]*$ ]] +[ "${CANARY_LOCK}" = "/tmp/awoooi-signoz-backup-canary.lock" ] +[ "${OPERATION_LOCK}" = "/tmp/awoooi-signoz-backup-operation.lock" ] run_root() { if [ "${REMOTE_UID}" -eq 0 ]; then @@ -502,6 +574,11 @@ run_root() { fi } +bounded_docker() { + timeout --signal=TERM --kill-after="${TIMEOUT_KILL_AFTER_SECONDS}" \ + "${DOCKER_COMMAND_TIMEOUT_SECONDS}" docker "$@" +} + root_hash() { if [ "${REMOTE_UID}" -eq 0 ]; then sha256sum "$1" | awk '{print $1}' @@ -550,7 +627,7 @@ listener_up() { runtime_snapshot() { local container value running health port api_code for container in "${CONTAINERS[@]}"; do - value="$(docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" + value="$(bounded_docker inspect --format '{{.Id}}|{{.State.StartedAt}}|{{.RestartCount}}|{{.State.Running}}|{{if (index .State "Health")}}{{(index .State "Health").Status}}{{else}}not_configured{{end}}' "${container}")" IFS='|' read -r _ _ _ running health <<<"${value}" [ "${running}" = true ] case "${health}" in healthy|not_configured) ;; *) return 1 ;; esac @@ -673,6 +750,14 @@ on_exit() { if [ "${cleanup_rc}" -ne 0 ] || [ "${residue_rc}" -ne 0 ]; then rc=92 fi + if [ "${OPERATION_LOCK_HELD}" -eq 1 ]; then + flock -u 7 >/dev/null 2>&1 + exec 7>&- + fi + if [ "${CANARY_LOCK_HELD}" -eq 1 ]; then + flock -u 8 >/dev/null 2>&1 + exec 8>&- + fi if [ "${LOCK_HELD}" -eq 1 ]; then flock -u 9 >/dev/null 2>&1 exec 9>&- @@ -694,11 +779,33 @@ exec 9>>"${LOCK_FILE}" flock -n 9 || { receipt check failed another_toolchain_deploy_holds_lock; exit 75; } LOCK_HELD=1 +[ ! -L "${CANARY_LOCK}" ] \ + || { receipt check failed canary_lock_symlink_unsafe; exit 75; } +if [ -e "${CANARY_LOCK}" ]; then + [ -f "${CANARY_LOCK}" ] \ + || { receipt check failed canary_lock_not_regular_file; exit 75; } +fi +exec 8>>"${CANARY_LOCK}" +flock -n 8 \ + || { receipt check failed another_signoz_canary_holds_canary_lock; exit 75; } +CANARY_LOCK_HELD=1 + +[ ! -L "${OPERATION_LOCK}" ] \ + || { receipt check failed backup_operation_lock_symlink_unsafe; exit 75; } +if [ -e "${OPERATION_LOCK}" ]; then + [ -f "${OPERATION_LOCK}" ] \ + || { receipt check failed backup_operation_lock_not_regular_file; exit 75; } +fi +exec 7>>"${OPERATION_LOCK}" +flock -n 7 \ + || { receipt check failed another_signoz_backup_holds_operation_lock; exit 75; } +OPERATION_LOCK_HELD=1 + if pgrep -af '(^|/|[[:space:]])(backup-signoz|clickhouse-native-backup|clickhouse-native-restore-drill|run-signoz-backup-canary)[.]sh([[:space:]]|$)' >/dev/null 2>&1; then receipt check failed active_signoz_backup_toolchain_process exit 75 fi -active_operations="$(docker exec signoz-clickhouse clickhouse-client --query \ +active_operations="$(bounded_docker exec signoz-clickhouse clickhouse-client --query \ "SELECT count() FROM system.backups WHERE status IN ('CREATING_BACKUP','RESTORING') FORMAT TSVRaw" 2>/dev/null)" [ "${active_operations}" = 0 ] || { receipt check failed active_native_backup_or_restore; exit 75; } @@ -744,9 +851,9 @@ done receipt sensor_source pass "seven_stage_hashes_and_syntax_match_repo_sources" receipt normalized_asset_identity pass "host_110_backup_scripts_5_clickhouse_configs_2" receipt source_of_truth_diff pass "prior_manifest_${PRIOR_MANIFEST}" -receipt ai_decision candidate "atomic_install_native_backup_toolchain_only" -receipt risk_policy pass "risk_medium_bounded_files_only_no_runtime_execution_no_container_change_full_rollback" -receipt check pass "runtime_baseline_active_operations_0_hash_bash_xml_py_compile" +receipt ai_decision candidate "same_filesystem_candidate_replace_native_backup_toolchain_only" +receipt risk_policy pass "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_backup_operation_locks_no_runtime_execution_no_container_change_full_rollback" +receipt check pass "dual_signoz_canary_and_backup_operation_locks_held_runtime_baseline_active_operations_0_hash_bash_xml_py_compile" APPLY_STARTED=1 for directory in "${TARGET_DIRS[@]}"; do @@ -790,7 +897,7 @@ done runtime_snapshot > "${RUNTIME_AFTER}" cmp -s "${RUNTIME_BEFORE}" "${RUNTIME_AFTER}" -receipt execution pass "seven_candidates_atomically_replaced_expected_hashes_and_modes" +receipt execution pass "seven_same_filesystem_candidates_replaced_with_full_rollback_expected_hashes_and_modes" receipt post_verifier pass "container_id_started_at_restart_count_health_ports_4317_4318_8080_api_200_unchanged" receipt closure_writeback pass "durable_manifests_and_receipts_ack_no_backup_restore_restart_or_pull" DEPLOY_VERIFIED=1 diff --git a/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py index 504e2ca11..23d7597d2 100644 --- a/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py +++ b/scripts/ops/tests/test_signoz_clickhouse_backup_disk_deploy.py @@ -2,6 +2,7 @@ from __future__ import annotations import os from pathlib import Path +import re import subprocess import xml.etree.ElementTree as ET @@ -10,6 +11,12 @@ ROOT = Path(__file__).resolve().parents[3] CONFIG = ROOT / "ops/signoz/clickhouse/config.d/backup_disk.xml" OVERRIDE = ROOT / "ops/signoz/docker-compose.clickhouse-backup.override.yaml" DEPLOYER = ROOT / "scripts/ops/deploy-signoz-clickhouse-backup-disk.sh" +BACKUP_SCRIPT = ROOT / "scripts/backup/backup-signoz.sh" + + +def write_executable(path: Path, text: str) -> None: + path.write_text(text, encoding="utf-8") + path.chmod(0o755) def executable_text() -> str: @@ -105,9 +112,9 @@ def test_check_mode_validates_candidate_via_stdin_without_remote_staging() -> No text = DEPLOYER.read_text(encoding="utf-8") check_terminal = text.index('if [ "${MODE}" = "--check" ]') stage_assignment = text.index('STAGE_DIR="/tmp/awoooi-signoz-clickhouse-backup.') - first_scp = text.index("scp -q") + first_transfer = text.index('bounded_scp "${LOCAL_OVERRIDE}"') - assert check_terminal < stage_assignment < first_scp + assert check_terminal < stage_assignment < first_transfer assert "-f '${REMOTE_BASE}' -f - config -q" in text assert '< "${LOCAL_OVERRIDE}"' in text assert "check_pass_no_write" in text @@ -120,7 +127,12 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None: assert 'REMOTE_SERVICE="clickhouse"' in text assert "--env-file /dev/null" in text assert 'env -i PATH="${SAFE_PATH}" HOME="${SAFE_HOME}"' in text - assert "timeout --signal=TERM --kill-after=30 300" in text + assert "bounded_ssh" in text + assert "bounded_scp" in text + assert "docker_cmd" in text + assert "compose_config_cmd" in text + assert "compose_apply_cmd" in text + assert "REMOTE_APPLY_TIMEOUT_SECONDS=3600" in text assert "up -d --no-deps --pull never --force-recreate" in text assert '"${REMOTE_SERVICE}"' in text assert "docker compose down" not in executable @@ -133,15 +145,135 @@ def test_apply_is_single_service_bounded_no_pull_and_env_isolated() -> None: def test_apply_fails_closed_when_backup_is_active_or_managed_files_drift() -> None: text = DEPLOYER.read_text(encoding="utf-8") - assert "active_signoz_backup_detected" in text + assert "active_signoz_backup_or_restore_process_detected" in text assert "active_native_backup_or_restore" in text + assert "signoz_backup_process_probe_failed" in text assert "partial_managed_file_drift" in text assert "another_deploy_holds_lock" in text - assert "backup-signoz[.]sh" in text + for process in ( + "backup-signoz", + "clickhouse-native-backup", + "clickhouse-native-restore-drill", + "run-signoz-backup-canary", + ): + assert process in text + probe_block = text[ + text.index("require_backup_idle()") : text.index( + "require_backup_idle preflight" + ) + ] + assert "process_rc=$?" in probe_block + assert 'case "${process_rc}" in' in probe_block + assert "1) ;;" in probe_block + assert "return 69" in probe_block assert "CREATING_BACKUP" in text assert "RESTORING" in text +def test_target_host_is_fixed_and_ignores_ambient_override(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + ssh_log = tmp_path / "ssh.log" + write_executable( + fake_bin / "ssh", + """#!/bin/bash +printf '%s\\n' "$*" >> "${FAKE_SSH_LOG:?}" +cat >/dev/null +""", + ) + write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n") + env = os.environ.copy() + env.update( + { + "PATH": f"{fake_bin}:{env['PATH']}", + "TRACE_ID": "P0-OBS-002-test", + "RUN_ID": "fixed-host-test", + "WORK_ITEM_ID": "P0-OBS-002", + "TARGET_HOST": "attacker@example.invalid", + "FAKE_SSH_LOG": str(ssh_log), + } + ) + + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 0, result.stderr + invocations = ssh_log.read_text(encoding="utf-8").splitlines() + assert len(invocations) == 2 + assert all("wooo@192.168.0.110" in line for line in invocations) + assert all("attacker@example.invalid" not in line for line in invocations) + + +def test_all_transport_and_remote_docker_operations_are_bounded() -> None: + text = DEPLOYER.read_text(encoding="utf-8") + executable = executable_text() + + assert executable.count(' ssh "${SSH_OPTS[@]}"') == 1 + assert executable.count(' scp -q "${SSH_OPTS[@]}"') == 1 + assert not re.search(r"\bdocker (inspect|exec)\b", executable) + assert 'timeout --signal=TERM --kill-after="${KILL_AFTER_SECONDS}"' in text + assert 'docker_cmd exec "${CLICKHOUSE_CONTAINER}"' in text + assert "ss_cmd -H -lnt" in text + for mutation in ( + "bounded_cmd cp -p", + "bounded_cmd install -m", + "bounded_cmd mv -f", + "bounded_cmd sudo -n install", + "bounded_cmd rm -f", + ): + assert mutation in text + + +def test_stage_cleanup_failure_changes_apply_terminal_to_failed(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + ssh_count = tmp_path / "ssh-count" + write_executable( + fake_bin / "ssh", + """#!/bin/bash +count=0 +if [ -f "${FAKE_SSH_COUNT:?}" ]; then + count="$(cat "${FAKE_SSH_COUNT}")" +fi +count=$((count + 1)) +printf '%s\\n' "${count}" > "${FAKE_SSH_COUNT}" +cat >/dev/null +if [ "${count}" -eq 5 ]; then + exit 1 +fi +""", + ) + write_executable(fake_bin / "scp", "#!/bin/bash\nexit 0\n") + env = os.environ.copy() + env.update( + { + "PATH": f"{fake_bin}:{env['PATH']}", + "TRACE_ID": "P0-OBS-002-test", + "RUN_ID": "cleanup-failure-test", + "WORK_ITEM_ID": "P0-OBS-002", + "FAKE_SSH_COUNT": str(ssh_count), + } + ) + + result = subprocess.run( + ["bash", str(DEPLOYER), "--apply"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + assert result.returncode == 92 + assert '"phase":"cleanup","result":"failed"' in result.stdout + assert '"phase":"terminal","result":"failed"' in result.stdout + assert "failed_or_residue_present" in result.stdout + + def test_apply_preserves_image_data_volume_and_non_target_containers() -> None: text = DEPLOYER.read_text(encoding="utf-8") assert "BEFORE_IMAGE_ID" in text @@ -166,13 +298,13 @@ def test_post_verifier_checks_disk_mount_health_listeners_and_api() -> None: assert "101:101:750" in text assert "server_uid" in text assert "server_gid" in text - assert "docker exec -u 101:101" in text + assert "docker_cmd exec -u 101:101" in text assert "test -w /backups" in text assert "awk '/^Uid:/{print $2}' /proc/1/status" in text assert "awk '/^Gid:/{print $2}' /proc/1/status" in text identity_start = text.index('server_uid="') identity_block = text[ - identity_start : text.index("docker exec -u 101:101", identity_start) + identity_start : text.index("docker_cmd exec -u 101:101", identity_start) ] assert "sh -c" not in identity_block assert "concurrency_disabled" in text @@ -194,6 +326,48 @@ def test_failed_apply_restores_prior_managed_compose_state() -> None: assert "retained_nonempty_no_delete" in text assert "rc=91" in text assert "trap on_exit EXIT" in text + for token in ( + "PRIOR_OVERRIDE_HASH", + "PRIOR_OVERRIDE_MODE", + "PRIOR_OVERRIDE_UID", + "PRIOR_OVERRIDE_GID", + "PRIOR_CONFIG_HASH", + "PRIOR_CONFIG_MODE", + "PRIOR_CONFIG_UID", + "PRIOR_CONFIG_GID", + "file_matches_metadata", + "restore_exact_file", + "prior_compose_state_restored_exact_hash_mode_uid_gid", + ): + assert token in text + assert "exact_stage_candidate_and_rollback_residue_absence_verified" in text + + +def test_deployer_and_backup_execution_share_operation_lock_contract() -> None: + deployer = DEPLOYER.read_text(encoding="utf-8") + backup = BACKUP_SCRIPT.read_text(encoding="utf-8") + operation_lock = "/tmp/awoooi-signoz-backup-operation.lock" + canary_lock = "/tmp/awoooi-signoz-backup-canary.lock" + + assert operation_lock in deployer + assert operation_lock in backup + assert canary_lock in deployer + assert 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' in deployer + assert 'exec 7>>"${CANARY_LOCK_FILE}"' in deployer + assert "flock -n 8 || { receipt check failed backup_operation_lock_held" in deployer + assert "flock -n 7 || { receipt check failed backup_canary_lock_held" in deployer + assert 'exec 8>>"${OPERATION_LOCK_FILE}"' in backup + assert "require_backup_idle pre_mutation" in deployer + assert deployer.index('exec 7>>"${CANARY_LOCK_FILE}"') < deployer.index( + 'exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"' + ) + assert deployer.index('exec 8>>"${BACKUP_OPERATION_LOCK_FILE}"') < deployer.index( + "require_backup_idle pre_mutation" + ) + remote_cleanup_call = deployer.rindex(" cleanup_owned_residue") + assert remote_cleanup_call < deployer.index( + 'receipt terminal "${terminal}"', remote_cleanup_call + ) def test_deployer_never_reads_runtime_environment_or_executes_backup() -> None: diff --git a/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py index 9d105ae82..0abf7e6a1 100644 --- a/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py +++ b/scripts/ops/tests/test_signoz_native_backup_toolchain_deploy.py @@ -3,10 +3,13 @@ from __future__ import annotations import os from pathlib import Path import subprocess +import time ROOT = Path(__file__).resolve().parents[3] DEPLOYER = ROOT / "scripts/ops/deploy-signoz-native-backup-toolchain.sh" +BACKUP = ROOT / "scripts/backup/backup-signoz.sh" +CANARY = ROOT / "scripts/backup/run-signoz-backup-canary.sh" def deployer_text() -> str: @@ -119,6 +122,33 @@ def test_safe_absolute_path_overrides_fail_closed_before_network_access() -> Non assert "must not overlap" in result.stderr +def test_timeout_overrides_fail_closed_before_network_access() -> None: + for key, value, expected in ( + ( + "SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS", + "0", + "positive integer seconds", + ), + ( + "SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS", + "301", + "exceeds bounded maximum", + ), + ): + env = base_env() + env[key] = value + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + ) + assert result.returncode == 64 + assert expected in result.stderr + assert "ssh:" not in result.stderr + + def test_target_host_is_fixed_to_host110_and_not_environment_overridable() -> None: text = deployer_text() assert 'TARGET_HOST="wooo@192.168.0.110"' in text @@ -229,6 +259,70 @@ def test_check_mode_uses_only_ssh_and_never_scp(tmp_path: Path) -> None: assert "check_pass_no_write" in result.stdout +def test_hung_ssh_is_bounded_and_cannot_claim_check_pass(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + ssh = fake_bin / "ssh" + ssh.write_text("#!/bin/sh\nsleep 5\n", encoding="utf-8") + scp = fake_bin / "scp" + scp.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8") + ssh.chmod(0o755) + scp.chmod(0o755) + + env = base_env() + env["PATH"] = f"{fake_bin}:{env['PATH']}" + env["SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS"] = "1" + env["SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS"] = "1" + started = time.monotonic() + result = subprocess.run( + ["bash", str(DEPLOYER), "--check"], + check=False, + capture_output=True, + text=True, + env=env, + timeout=5, + ) + elapsed = time.monotonic() - started + + assert result.returncode == 124 + assert elapsed < 4 + assert "check_pass_no_write" not in result.stdout + + +def test_all_ssh_scp_and_docker_calls_use_bounded_wrappers() -> None: + text = deployer_text() + executable = executable_text() + + assert ( + 'SSH_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SSH_TIMEOUT_SECONDS:-300}"' + in text + ) + assert ( + 'SCP_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_SCP_TIMEOUT_SECONDS:-120}"' + in text + ) + assert ( + 'DOCKER_COMMAND_TIMEOUT_SECONDS="${SIGNOZ_TOOLCHAIN_DOCKER_TIMEOUT_SECONDS:-30}"' + in text + ) + assert ( + 'TIMEOUT_KILL_AFTER_SECONDS="${SIGNOZ_TOOLCHAIN_TIMEOUT_KILL_AFTER_SECONDS:-10}"' + in text + ) + assert "bounded_ssh()" in text + assert "bounded_scp()" in text + assert text.count('ssh "$@"') == 1 + assert text.count('scp "$@"') == 1 + assert executable.count("bounded_docker()") == 2 + assert "$(docker inspect" not in executable + assert "$(docker exec" not in executable + assert "\n docker inspect" not in executable + assert "\n docker exec" not in executable + assert executable.count("bounded_docker inspect") == 2 + assert executable.count("bounded_docker exec") == 2 + assert "timeout --signal=TERM" in executable + + def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None: text = deployer_text() for token in ( @@ -250,18 +344,60 @@ def test_apply_persists_exact_prior_metadata_and_has_full_rollback() -> None: assert "bounded_execution_not_started_targets_unchanged" in text -def test_apply_uses_same_filesystem_candidates_and_atomic_replace() -> None: +def test_apply_uses_same_filesystem_candidates_and_accurate_replace_receipt() -> None: text = deployer_text() assert '"${target}.candidate.${RUN_ID}"' in text assert ( 'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}" "${TARGETS[index]}"' ) in text assert "deployed-manifest.tsv" in text - assert "seven_candidates_atomically_replaced_expected_hashes_and_modes" in text + assert ( + "seven_same_filesystem_candidates_replaced_with_full_rollback_" + "expected_hashes_and_modes" + ) in text + assert "seven_candidates_atomically_replaced" not in text + assert "atomically replaces exact targets" not in text assert "stage_candidate_residue_0" in text assert "verify_residue_zero" in text +def test_apply_holds_dual_backup_locks_through_postverify_or_rollback() -> None: + text = deployer_text() + backup_text = BACKUP.read_text(encoding="utf-8") + canary_text = CANARY.read_text(encoding="utf-8") + canary_lock = "/tmp/awoooi-signoz-backup-canary.lock" + operation_lock = "/tmp/awoooi-signoz-backup-operation.lock" + + assert f'CANARY_LOCK="{canary_lock}"' in text + assert f'OPERATION_LOCK="{operation_lock}"' in text + assert canary_lock in canary_text + assert operation_lock in backup_text + canary_acquire = text.index('exec 8>>"${CANARY_LOCK}"') + operation_acquire = text.index('exec 7>>"${OPERATION_LOCK}"') + active_check = text.index("if pgrep -af", operation_acquire) + apply_started = text.index("APPLY_STARTED=1", operation_acquire) + target_replace = text.index( + 'run_root mv -f "${TARGETS[index]}.candidate.${RUN_ID}"', apply_started + ) + assert canary_acquire < operation_acquire < active_check < apply_started + assert apply_started < target_replace + + on_exit = text[text.index("on_exit() {") : text.index("trap on_exit EXIT")] + rollback = on_exit.index("rollback_targets") + release_operation = on_exit.index("flock -u 7") + release_canary = on_exit.index("flock -u 8") + assert rollback < release_operation < release_canary + assert 'rm -f "${OPERATION_LOCK}"' not in text + assert 'rm -f "${CANARY_LOCK}"' not in text + assert "another_signoz_backup_holds_operation_lock" in text + assert "another_signoz_canary_holds_canary_lock" in text + assert ( + "risk_medium_bounded_ssh_scp_docker_dual_signoz_canary_and_" + "backup_operation_locks_" + "no_runtime_execution_no_container_change_full_rollback" + ) in text + + def test_runtime_post_verifier_requires_exact_identity_and_lifecycle_parity() -> None: text = deployer_text() for container in ( diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index a24b7e37f..e3d857987 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -9,6 +9,7 @@ only from the committed post-closure receipt. from __future__ import annotations import argparse +import hashlib import json from pathlib import Path from typing import Any @@ -30,10 +31,45 @@ EXPECTED_POST_CLOSURE_BLOCKERS = [ "signoz_backup_offsite_dr_pending", "clickhouse_native_failed_artifact_retention_pending", "clickhouse_native_resume_submitted_inventory_pending", + "clickhouse_native_current_source_toolchain_pending", "service_registry_runtime_mirror_reconciliation_pending", "github_freeze_legacy_asset_retirement_pending", "monitoring_generator_duplicate_awoooi_api_identity_pending", ] +TOOLCHAIN_SOURCE_PATHS = { + "backup_orchestrator_sha256": "scripts/backup/backup-signoz.sh", + "native_backup_sha256": "scripts/backup/clickhouse-native-backup.sh", + "isolated_restore_sha256": "scripts/backup/clickhouse-native-restore-drill.sh", + "inventory_verifier_sha256": "scripts/backup/clickhouse-restore-inventory.py", + "canary_wrapper_sha256": "scripts/backup/run-signoz-backup-canary.sh", + "backup_disk_config_sha256": "ops/signoz/clickhouse/config.d/backup_disk.xml", + "isolated_cluster_config_sha256": ( + "ops/signoz/clickhouse/restore-drill/config.d/isolated-cluster.xml" + ), +} +HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES = { + "backup_orchestrator_sha256": ( + "2fff0ded4ece9104b910f9e2db6deb4eebdc173d12c2be2bad2a59596e0d7520" + ), + "native_backup_sha256": ( + "754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a" + ), + "isolated_restore_sha256": ( + "57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7" + ), + "inventory_verifier_sha256": ( + "79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c" + ), + "canary_wrapper_sha256": ( + "64d23d9f7a64d1e4e1347b98d45e50e26eadea1741d0e28a5e2e5d74b22322d8" + ), + "backup_disk_config_sha256": ( + "c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9" + ), + "isolated_cluster_config_sha256": ( + "e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b" + ), +} def _load_yaml(path: Path) -> Any: @@ -45,6 +81,27 @@ def _read(root: Path, relative: str) -> str: return (root / relative).read_text(encoding="utf-8") +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _current_toolchain_source_hashes(root: Path) -> dict[str, str]: + hashes: dict[str, str] = {} + for field, relative in TOOLCHAIN_SOURCE_PATHS.items(): + source = root / relative + if not source.is_file() or source.is_symlink(): + return {} + try: + hashes[field] = _sha256_file(source) + except OSError: + return {} + return hashes + + def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: contract = _load_yaml(contract_path) post_closure = contract.get("runtime_observations", {}).get( @@ -425,17 +482,61 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: native_restic = native_apply4.get("restic", {}) native_post = native_apply4.get("independent_post_verifier", {}) toolchain_apply = native_canary.get("toolchain_controlled_apply", {}) + historical_native_source = native_canary.get("source", {}) + current_native_source = native_canary.get("current_source", {}) + historical_source_hashes = { + field: historical_native_source.get(field) for field in TOOLCHAIN_SOURCE_PATHS + } + declared_current_hashes = current_native_source.get("hashes", {}) + actual_current_hashes = _current_toolchain_source_hashes(root) + actual_historical_drift = sorted( + field + for field in TOOLCHAIN_SOURCE_PATHS + if actual_current_hashes.get(field) != historical_source_hashes.get(field) + ) + declared_historical_drift = current_native_source.get( + "drift_from_historical_production_receipt", [] + ) + checks["native_backup_historical_source_receipt_preserved"] = ( + historical_native_source.get("evidence_scope") + == "historical_production_receipt" + and historical_native_source.get("deployment_status") == "deployed_verified" + and historical_source_hashes == HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES + ) + checks["native_backup_current_source_hashes_bound"] = ( + current_native_source.get("evidence_scope") == "committed_current_source" + and current_native_source.get("expected_file_count") + == len(TOOLCHAIN_SOURCE_PATHS) + and set(declared_current_hashes) == set(TOOLCHAIN_SOURCE_PATHS) + and declared_current_hashes == actual_current_hashes + and declared_historical_drift == actual_historical_drift + ) + pending_source_state = ( + current_native_source.get("status") + == "pending_toolchain_deploy_and_runtime_canary" + and current_native_source.get("deployment_status") == "pending" + and current_native_source.get("current_source_deployed") is False + and current_native_source.get("runtime_verifier_terminal") == "pending" + ) + deployed_source_state = ( + current_native_source.get("status") == "deployed_verified" + and current_native_source.get("deployment_status") == "deployed" + and current_native_source.get("current_source_deployed") is True + and current_native_source.get("runtime_verifier_terminal") == "pass" + and current_native_source.get("deployed_source_hashes") == actual_current_hashes + and bool(current_native_source.get("toolchain_apply_run_id")) + and bool(current_native_source.get("runtime_canary_run_id")) + ) + checks["native_backup_current_source_deployment_state_explicit"] = ( + current_native_source.get("historical_production_receipt_preserved") is True + and (pending_source_state or deployed_source_state) + ) checks["native_backup_restore_canary_closed"] = ( native_canary.get("completion_scope") == "clickhouse_native_only" - and native_canary.get("source", {}).get("native_backup_sha256") - == "754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a" - and native_canary.get("source", {}).get("isolated_restore_sha256") - == "57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7" - and native_canary.get("source", {}).get("post_canary_focused_tests_passed") - == 94 - and native_canary.get("source", {}).get("post_format_focused_tests_passed") - == 94 - and native_canary.get("source", {}).get("ruff_format") == "pass" + and checks["native_backup_historical_source_receipt_preserved"] + and historical_native_source.get("post_canary_focused_tests_passed") == 94 + and historical_native_source.get("post_format_focused_tests_passed") == 94 + and historical_native_source.get("ruff_format") == "pass" and toolchain_apply.get("apply_run_id") == "P0-OBS-002-native-toolchain-20260715T001252Z-apply4" and toolchain_apply.get("apply_terminal") == "pass" @@ -568,6 +669,9 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: resume_inventory = pending_verifiers.get( "clickhouse_native_resume_submitted_inventory", {} ) + current_source_toolchain = pending_verifiers.get( + "clickhouse_native_current_source_toolchain", {} + ) checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = ( native_migration.get("prior_run_id") == retry2.get("run_id") and native_migration.get("run_id") == native_apply4.get("run_id") @@ -595,6 +699,14 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == "pending_durable_submitted_inventory_contract" and resume_inventory.get("same_run_resume_safe") is False and len(resume_inventory.get("required_preconditions", [])) == 4 + and current_source_toolchain.get("status") + == current_native_source.get("status") + == "pending_toolchain_deploy_and_runtime_canary" + and current_source_toolchain.get("current_source_deployed") is False + and current_source_toolchain.get("runtime_verifier_terminal") == "pending" + and current_source_toolchain.get("historical_production_receipt_preserved") + is True + and len(current_source_toolchain.get("required_preconditions", [])) == 3 ) checks["post_closure_contract_mirror_consistent"] = ( @@ -621,6 +733,10 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == toolchain_apply.get("apply_run_id") and post_closure.get("clickhouse_native_backup_toolchain_postcheck_run_id") == toolchain_apply.get("postcheck_run_id") + and post_closure.get("clickhouse_native_current_source_status") + == current_native_source.get("status") + and post_closure.get("clickhouse_native_current_source_deployed") + == current_native_source.get("current_source_deployed") and post_closure.get("clickhouse_native_restore_terminal") == "verified" and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a" and post_closure.get("clickhouse_native_offsite_status") == "pending" @@ -684,10 +800,21 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: "phase": "wave_a_dual_allow", "source_ready": source_ready, "runtime_status": terminal.get("status", "unknown"), + "backup_toolchain_current_source_status": current_native_source.get( + "status", "unknown" + ), + "backup_toolchain_current_source_deployed": current_native_source.get( + "current_source_deployed", False + ), + "backup_toolchain_runtime_verifier_terminal": current_native_source.get( + "runtime_verifier_terminal", "unknown" + ), + "backup_toolchain_source_drift_fields": actual_historical_drift, "active_route": "incumbent", "writes_performed": 0, "checks": checks, "blockers": blockers, + "post_closure_blockers": list(regression_terminal.get("blockers", [])), "terminal": "pass" if source_ready else "failed", } diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index 831e82f3d..463fc9496 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import importlib.util import json import os import subprocess @@ -25,6 +27,16 @@ GRPC_RUNTIME_RECEIPT = ROOT / "ops/signoz/p0-obs-002-grpc-runtime-receipt.yaml" POST_CLOSURE_REGRESSION = ROOT / "ops/signoz/p0-obs-002-post-closure-regression.yaml" +def _load_preflight_module(): + spec = importlib.util.spec_from_file_location( + "signoz_canonical_route_preflight_under_test", PREFLIGHT + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_wave_a_contract_keeps_active_route_and_all_producers_on_incumbent() -> None: payload = yaml.safe_load(CONTRACT.read_text(encoding="utf-8")) assert ( @@ -86,11 +98,28 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: assert payload["writes_performed"] == 0 assert payload["terminal"] == "pass" assert payload["runtime_status"] == "partial_degraded" + assert ( + payload["backup_toolchain_current_source_status"] + == "pending_toolchain_deploy_and_runtime_canary" + ) + assert payload["backup_toolchain_current_source_deployed"] is False + assert payload["backup_toolchain_runtime_verifier_terminal"] == "pending" + assert payload["backup_toolchain_source_drift_fields"] == [ + "backup_orchestrator_sha256", + "canary_wrapper_sha256", + "inventory_verifier_sha256", + ] assert payload["active_route"] == "incumbent" assert payload["checks"] and all(payload["checks"].values()) assert payload["checks"]["grpc_apply_waits_for_independent_post_verifier"] is True assert payload["checks"]["grpc_contract_runtime_state_consistent"] is True assert payload["checks"]["post_closure_schema_v2"] is True + assert payload["checks"]["native_backup_current_source_hashes_bound"] is True + assert ( + payload["checks"]["native_backup_current_source_deployment_state_explicit"] + is True + ) + assert payload["checks"]["native_backup_historical_source_receipt_preserved"] assert payload["checks"]["post_release_600s_verifier_consistent"] is True assert ( payload["checks"]["native_backup_migration_closed_and_remaining_gaps_explicit"] @@ -621,6 +650,69 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No assert canary["terminal"] == "production_closed_clickhouse_native_scope" +def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt() -> ( + None +): + module = _load_preflight_module() + receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) + canary = receipt["controlled_applies"]["clickhouse_native_backup_restore_canary"] + historical = canary["source"] + current = canary["current_source"] + + historical_hashes = { + field: historical[field] for field in module.TOOLCHAIN_SOURCE_PATHS + } + assert historical["evidence_scope"] == "historical_production_receipt" + assert historical["deployment_status"] == "deployed_verified" + assert historical_hashes == module.HISTORICAL_PRODUCTION_TOOLCHAIN_HASHES + assert historical["inventory_verifier_sha256"] == ( + "79cf5fcbb9e31ee994dda03bf0043eee6b6cf412eb6febaeca4ce82ea5b7df3c" + ) + + current_hashes = { + field: hashlib.sha256((ROOT / relative).read_bytes()).hexdigest() + for field, relative in module.TOOLCHAIN_SOURCE_PATHS.items() + } + assert current["hashes"] == current_hashes + assert current["hashes"]["inventory_verifier_sha256"] == ( + "c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863" + ) + assert current["status"] == "pending_toolchain_deploy_and_runtime_canary" + assert current["deployment_status"] == "pending" + assert current["current_source_deployed"] is False + assert current["runtime_verifier_terminal"] == "pending" + assert current["historical_production_receipt_preserved"] is True + assert current["drift_from_historical_production_receipt"] == [ + "backup_orchestrator_sha256", + "canary_wrapper_sha256", + "inventory_verifier_sha256", + ] + + +def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale( + monkeypatch, +) -> None: + module = _load_preflight_module() + real_sha256_file = module._sha256_file + + def stale_inventory_hash(path: Path) -> str: + if path.name == "clickhouse-restore-inventory.py": + return "0" * 64 + return real_sha256_file(path) + + monkeypatch.setattr(module, "_sha256_file", stale_inventory_hash) + payload = module.evaluate(ROOT, CONTRACT) + + assert payload["source_ready"] is False + assert payload["terminal"] == "failed" + assert payload["checks"]["native_backup_current_source_hashes_bound"] is False + assert payload["checks"]["native_backup_historical_source_receipt_preserved"] + assert ( + payload["backup_toolchain_current_source_status"] + == "pending_toolchain_deploy_and_runtime_canary" + ) + + def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None: receipt = yaml.safe_load(POST_CLOSURE_REGRESSION.read_text(encoding="utf-8")) verifier = receipt["post_release_runtime_verifier"] @@ -667,6 +759,14 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> ] assert resume["same_run_resume_safe"] is False assert len(resume["required_preconditions"]) == 4 + current_source = receipt["pending_verifiers"][ + "clickhouse_native_current_source_toolchain" + ] + assert current_source["status"] == "pending_toolchain_deploy_and_runtime_canary" + assert current_source["current_source_deployed"] is False + assert current_source["runtime_verifier_terminal"] == "pending" + assert current_source["historical_production_receipt_preserved"] is True + assert len(current_source["required_preconditions"]) == 3 assets = receipt["asset_reconciliation"] assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001" assert assets["source_service_count"] == 27 @@ -697,6 +797,7 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> "signoz_backup_offsite_dr_pending", "clickhouse_native_failed_artifact_retention_pending", "clickhouse_native_resume_submitted_inventory_pending", + "clickhouse_native_current_source_toolchain_pending", "service_registry_runtime_mirror_reconciliation_pending", "github_freeze_legacy_asset_retirement_pending", "monitoring_generator_duplicate_awoooi_api_identity_pending", @@ -737,6 +838,10 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == ( "P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4" ) + assert post_closure["clickhouse_native_current_source_status"] == ( + "pending_toolchain_deploy_and_runtime_canary" + ) + assert post_closure["clickhouse_native_current_source_deployed"] is False assert post_closure["clickhouse_native_restore_terminal"] == "verified" assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a" assert post_closure["clickhouse_native_offsite_status"] == "pending" From ca8403ce79ce363fa3ea7730d2001698def96d29 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 10:17:00 +0800 Subject: [PATCH 3/9] chore(observability): record native backup production verifier --- ...rganization-canonical-route-migration.yaml | 23 +- .../p0-obs-002-post-closure-regression.yaml | 181 +++++++++-- .../signoz-canonical-route-preflight.py | 280 ++++++++++++++++-- .../test_signoz_canonical_route_migration.py | 204 +++++++++++-- 4 files changed, 611 insertions(+), 77 deletions(-) diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index afda9b611..4810622a4 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -172,8 +172,10 @@ runtime_observations: backup_collector_restore_guard_deployed: true backup_canary_scope: clickhouse_native_only backup_canary_verified: true - backup_canary_last_run_id: P0-OBS-002-native-canary-20260715T001329Z-apply4 - backup_canary_last_terminal: pass + backup_canary_last_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + backup_canary_last_terminal: partial_degraded + backup_canary_runtime_operation_terminal: pass + backup_canary_machine_receipt_verifier_terminal: pass backup_canary_prior_root_cause: raw_tar_of_active_clickhouse_rw_volume_has_no_consistent_snapshot_contract backup_canary_cooldown_retry_status: superseded_by_online_native_no_stop_contract clickhouse_native_backup_migration_status: production_closed @@ -181,10 +183,21 @@ runtime_observations: clickhouse_native_backup_disk_status: deployed_verified clickhouse_native_backup_toolchain_run_id: P0-OBS-002-native-toolchain-20260715T001252Z-apply4 clickhouse_native_backup_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4 - clickhouse_native_current_source_status: pending_toolchain_deploy_and_runtime_canary - clickhouse_native_current_source_deployed: false + clickhouse_native_current_source_status: deployed_verified + clickhouse_native_current_source_deployed: true + clickhouse_native_current_source_runtime_verifier_terminal: pass + clickhouse_native_current_source_toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6 + clickhouse_native_current_source_toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5 + clickhouse_native_current_source_toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5 + clickhouse_native_current_source_toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5 + clickhouse_native_current_source_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5 + clickhouse_native_current_source_canary_apply_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + clickhouse_native_current_source_wrapper_terminal: partial_degraded + clickhouse_native_current_source_runtime_operation_terminal: pass + clickhouse_native_current_source_machine_receipt_verifier_terminal: pass + clickhouse_native_current_source_artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83 clickhouse_native_restore_terminal: verified - clickhouse_native_restic_snapshot_id: 574e2a1a + clickhouse_native_restic_snapshot_id: 135485ab clickhouse_native_offsite_status: pending clickhouse_native_failed_artifact_retention_status: pending_bounded_retention_decision clickhouse_native_resume_inventory_status: pending_durable_submitted_inventory_contract diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index 73f015236..48f1f5f8c 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -364,12 +364,12 @@ controlled_applies: ruff_format: pass yaml_xml_parse: pass current_source: - observed_at: "2026-07-15T09:40:50+08:00" + observed_at: "2026-07-15T09:53:46+08:00" evidence_scope: committed_current_source - status: pending_toolchain_deploy_and_runtime_canary - deployment_status: pending - current_source_deployed: false - runtime_verifier_terminal: pending + status: deployed_verified + deployment_status: deployed + current_source_deployed: true + runtime_verifier_terminal: pass historical_production_receipt_preserved: true expected_file_count: 7 hashes: @@ -380,13 +380,38 @@ controlled_applies: canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886 backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9 isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b + deployed_source_hashes: + backup_orchestrator_sha256: d1950c6ada4be4696f38dabf904bea3167c89293d7ca300365e62c2831cf4df2 + native_backup_sha256: 754b76aac707aa65f1e3b9cd5e3cbb1c1413c224a22a76ff524c6dbd3ba8dd2a + isolated_restore_sha256: 57554d46251bcc532a41be4fb8240f423e54e179ed692565282d2afda7c591c7 + inventory_verifier_sha256: c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863 + canary_wrapper_sha256: e6707c1398e066cbeec5e43a936533682b881b6e2763e21085f85b3de33c5886 + backup_disk_config_sha256: c1e6267940be5381ce83d759be55ac4172c5c34fcf319be1f320aa5a366d15f9 + isolated_cluster_config_sha256: e78a5cedd8b211c5cc30777d2c9e52cbc22f37b396eef78d0f39c7842fe7700b drift_from_historical_production_receipt: - backup_orchestrator_sha256 - canary_wrapper_sha256 - inventory_verifier_sha256 + toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6 + toolchain_check_terminal: check_pass_no_write + toolchain_check_diff: matching_4_drift_3_absent_0_active_operations_0 + toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5 + toolchain_apply_terminal: pass + toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5 + toolchain_postcheck_terminal: check_pass_no_write + toolchain_postcheck_diff: matching_7_drift_0_absent_0_active_operations_0 + toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5 + toolchain_postcanary_terminal: check_pass_no_write + toolchain_postcanary_diff: matching_7_drift_0_absent_0_active_operations_0 + default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5 + default_canary_check_terminal: check_pass_no_write + runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + wrapper_terminal: partial_degraded + runtime_operation_terminal: pass + nested_machine_receipt_verifier_terminal: pass source_validation: backup_restore_deployer_tests_passed: 92 - canonical_preflight_tests_passed: 23 + canonical_preflight_tests_passed: 24 bash_syntax: pass ruff: pass ruff_format: pass @@ -503,6 +528,109 @@ controlled_applies: cleanup_verified: true secret_values_collected: false closure_writeback: durable_local_receipts_acknowledged + - run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5 + check_terminal: check_pass_no_write + wrapper_terminal: partial_degraded + terminal: partial_degraded + runtime_operation: + terminal: pass + scope: clickhouse_native_backup_and_isolated_restore + nested_machine_receipt_verifier: + stage: native_restore_receipt_verifier + terminal: pass + backup: + terminal: BACKUP_CREATED + terminal_sequence: + - check_pass_no_write + - pass + independent_verifier_terminal: pass + operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1 + operation_status: "BACKUP_CREATED|4885|108006575|99461897|" + file_count: 4885 + total_size_bytes: 108006575 + compressed_size_bytes: 99461897 + artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83 + active_operation_count_after: 0 + server_artifact_absent_after: true + isolated_restore: + terminal: verified + terminal_sequence: + - check_pass_no_runtime_write + - verified + clickhouse_restore_terminal: RESTORED + independent_post_verifier_terminal: pass + cleanup_terminal: pass + network_scope: internal_only + internal_network_attached: true + staging_network_attached: false + production_network_attached: false + production_restore_performed: false + cleanup_verified: true + database_count: 6 + restored_database_count: 6 + table_count: 100 + restored_table_count: 100 + table_engine_schema_manifest_parity: true + check_table_count: 44 + check_table_pass_count: 44 + critical_nonzero_count: 4 + restic: + snapshot_id: 135485ab + snapshot_readback: pass + repository_scope: local_same_failure_domain + offsite_verified: false + sqlite: + terminal: pending_application_consistent_export + independent_post_verifier: + terminal: pass + independent_monitor: pass + deployed_source_hash_drift_count: 0 + runtime_drift_count: 0 + production_container_count: 4 + production_container_identity_unchanged: true + production_container_restart_count_delta: 0 + production_containers: + clickhouse: + started_at: "2026-07-14T22:49:13.665350281Z" + restart_count: 0 + health: healthy + collector: + started_at: "2026-07-14T22:20:53.725583861Z" + restart_count: 0 + health: not_configured + signoz: + started_at: "2026-07-11T08:45:17.179074963Z" + restart_count: 0 + health: healthy + zookeeper: + started_at: "2026-07-11T08:45:17.055893064Z" + restart_count: 0 + health: healthy + api_http_status: 200 + listener_counts: + "4317": 2 + "4318": 2 + "8080": 2 + active_backup_operation_count: 0 + restore_container_count: 0 + restore_volume_count: 0 + restore_network_count: 0 + toolchain_stage_residue_count: 0 + toolchain_candidate_residue_count: 0 + toolchain_rollback_residue_count: 0 + backup_process_count: 0 + lock_holders: + canary: none + operation: none + toolchain: none + server_artifact_absent: true + exact_ephemeral_container_volume_network_process_tmp_residue_count: 0 + cleanup_verified: true + closure_writeback: + terminal: partial_degraded + current_source_runtime_closed: true + overall_program_partial_degraded: true terminal: production_closed_clickhouse_native_scope backup_canary: @@ -599,6 +727,34 @@ completed_verifiers: restic_snapshot_id: 574e2a1a independent_verifier: pass exact_cleanup: pass + clickhouse_native_current_source_toolchain: + status: deployed_verified + current_source_deployed: true + runtime_verifier_terminal: pass + historical_production_receipt_preserved: true + toolchain_check_run_id: P0-OBS-002-native-toolchain-20260715T015040Z-check6 + toolchain_apply_run_id: P0-OBS-002-native-toolchain-20260715T015059Z-apply5 + toolchain_postcheck_run_id: P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5 + toolchain_postcanary_run_id: P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5 + default_canary_check_run_id: P0-OBS-002-native-canary-default-20260715T015131Z-check5 + runtime_canary_run_id: P0-OBS-002-native-canary-default-20260715T015149Z-apply5 + wrapper_terminal: partial_degraded + runtime_operation_terminal: pass + nested_machine_receipt_verifier_terminal: pass + operation_identity_hash: d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1 + operation_status: "BACKUP_CREATED|4885|108006575|99461897|" + restic_snapshot_id: 135485ab + artifact_sha256: 08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83 + offsite_verified: false + sqlite_terminal: pending_application_consistent_export + independent_monitor: pass + api_http_status: 200 + production_container_identity_unchanged: true + production_container_restart_count_delta: 0 + active_backup_operation_count: 0 + runtime_drift_count: 0 + residue_count: 0 + exact_cleanup: pass pending_verifiers: signoz_sqlite_application_consistent_backup: @@ -611,7 +767,7 @@ pending_verifiers: signoz_backup_offsite_dr: status: pending_offsite_snapshot_and_restore_verifier current_repository_scope: local_same_failure_domain - latest_local_snapshot_id: 574e2a1a + latest_local_snapshot_id: 135485ab offsite_verified: false required_preconditions: - immutable_or_versioned_offsite_target @@ -641,16 +797,6 @@ pending_verifiers: - bind_inventory_hash_operation_id_artifact_hash_and_run_identity - resume_reuses_original_inventory_and_fails_closed_if_missing_or_tampered - regression_test_with_live_counter_drift_and_zero_duplicate_backup_submits - clickhouse_native_current_source_toolchain: - status: pending_toolchain_deploy_and_runtime_canary - current_source_deployed: false - runtime_verifier_terminal: pending - historical_production_receipt_preserved: true - required_preconditions: - - controlled_toolchain_apply_with_current_source_hash_readback - - same_run_native_backup_and_isolated_restore_canary - - independent_runtime_and_cleanup_verifier - scope_boundaries: grpc_bridge_rollback_indicated: false api_manual_rollback_performed: false @@ -670,7 +816,6 @@ terminal: - signoz_backup_offsite_dr_pending - clickhouse_native_failed_artifact_retention_pending - clickhouse_native_resume_submitted_inventory_pending - - clickhouse_native_current_source_toolchain_pending - service_registry_runtime_mirror_reconciliation_pending - github_freeze_legacy_asset_retirement_pending - monitoring_generator_duplicate_awoooi_api_identity_pending diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index e3d857987..6694cda2a 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -31,7 +31,6 @@ EXPECTED_POST_CLOSURE_BLOCKERS = [ "signoz_backup_offsite_dr_pending", "clickhouse_native_failed_artifact_retention_pending", "clickhouse_native_resume_submitted_inventory_pending", - "clickhouse_native_current_source_toolchain_pending", "service_registry_runtime_mirror_reconciliation_pending", "github_freeze_legacy_asset_retirement_pending", "monitoring_generator_duplicate_awoooi_api_identity_pending", @@ -474,16 +473,35 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: native_canary = applies.get("clickhouse_native_backup_restore_canary", {}) native_attempts = native_canary.get("attempts", []) - native_apply2 = native_attempts[1] if len(native_attempts) == 4 else {} - native_apply3 = native_attempts[2] if len(native_attempts) == 4 else {} - native_apply4 = native_attempts[3] if len(native_attempts) == 4 else {} + native_attempts_by_run_id = { + attempt.get("run_id"): attempt + for attempt in native_attempts + if isinstance(attempt, dict) and attempt.get("run_id") + } + native_apply2 = native_attempts_by_run_id.get( + "P0-OBS-002-native-canary-20260714T234436Z-apply2", {} + ) + native_apply3 = native_attempts_by_run_id.get( + "P0-OBS-002-native-canary-20260715T000638Z-apply3", {} + ) + native_apply4 = native_attempts_by_run_id.get( + "P0-OBS-002-native-canary-20260715T001329Z-apply4", {} + ) + native_apply5 = native_attempts_by_run_id.get( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5", {} + ) native_backup = native_apply4.get("backup", {}) native_restore = native_apply4.get("isolated_restore", {}) native_restic = native_apply4.get("restic", {}) native_post = native_apply4.get("independent_post_verifier", {}) + current_backup = native_apply5.get("backup", {}) + current_restore = native_apply5.get("isolated_restore", {}) + current_restic = native_apply5.get("restic", {}) + current_post = native_apply5.get("independent_post_verifier", {}) toolchain_apply = native_canary.get("toolchain_controlled_apply", {}) historical_native_source = native_canary.get("source", {}) current_native_source = native_canary.get("current_source", {}) + current_source_validation = current_native_source.get("source_validation", {}) historical_source_hashes = { field: historical_native_source.get(field) for field in TOOLCHAIN_SOURCE_PATHS } @@ -511,25 +529,166 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and declared_current_hashes == actual_current_hashes and declared_historical_drift == actual_historical_drift ) - pending_source_state = ( - current_native_source.get("status") - == "pending_toolchain_deploy_and_runtime_canary" - and current_native_source.get("deployment_status") == "pending" - and current_native_source.get("current_source_deployed") is False - and current_native_source.get("runtime_verifier_terminal") == "pending" - ) deployed_source_state = ( current_native_source.get("status") == "deployed_verified" and current_native_source.get("deployment_status") == "deployed" and current_native_source.get("current_source_deployed") is True and current_native_source.get("runtime_verifier_terminal") == "pass" and current_native_source.get("deployed_source_hashes") == actual_current_hashes - and bool(current_native_source.get("toolchain_apply_run_id")) - and bool(current_native_source.get("runtime_canary_run_id")) + and current_native_source.get("toolchain_check_run_id") + == "P0-OBS-002-native-toolchain-20260715T015040Z-check6" + and current_native_source.get("toolchain_check_terminal") + == "check_pass_no_write" + and current_native_source.get("toolchain_check_diff") + == "matching_4_drift_3_absent_0_active_operations_0" + and current_native_source.get("toolchain_apply_run_id") + == "P0-OBS-002-native-toolchain-20260715T015059Z-apply5" + and current_native_source.get("toolchain_apply_terminal") == "pass" + and current_native_source.get("toolchain_postcheck_run_id") + == "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5" + and current_native_source.get("toolchain_postcheck_terminal") + == "check_pass_no_write" + and current_native_source.get("toolchain_postcheck_diff") + == "matching_7_drift_0_absent_0_active_operations_0" + and current_native_source.get("toolchain_postcanary_run_id") + == "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5" + and current_native_source.get("toolchain_postcanary_terminal") + == "check_pass_no_write" + and current_native_source.get("toolchain_postcanary_diff") + == "matching_7_drift_0_absent_0_active_operations_0" + and current_native_source.get("default_canary_check_run_id") + == "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + and current_native_source.get("default_canary_check_terminal") + == "check_pass_no_write" + and current_native_source.get("runtime_canary_run_id") + == "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + and current_native_source.get("wrapper_terminal") == "partial_degraded" + and current_native_source.get("runtime_operation_terminal") == "pass" + and current_native_source.get("nested_machine_receipt_verifier_terminal") + == "pass" + and current_source_validation.get("backup_restore_deployer_tests_passed") == 92 + and current_source_validation.get("canonical_preflight_tests_passed") == 24 + and current_source_validation.get("bash_syntax") == "pass" + and current_source_validation.get("ruff") == "pass" + and current_source_validation.get("ruff_format") == "pass" + and current_source_validation.get("yaml_parse") == "pass" + and current_source_validation.get("diff_check") == "pass" + and current_source_validation.get("independent_reaudit") == "no_commit_blocker" ) checks["native_backup_current_source_deployment_state_explicit"] = ( current_native_source.get("historical_production_receipt_preserved") is True - and (pending_source_state or deployed_source_state) + and deployed_source_state + ) + checks["native_backup_current_source_runtime_closed"] = ( + native_apply5.get("check_run_id") + == "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + and native_apply5.get("check_terminal") == "check_pass_no_write" + and native_apply5.get("wrapper_terminal") == "partial_degraded" + and native_apply5.get("terminal") == "partial_degraded" + and native_apply5.get("runtime_operation", {}).get("terminal") == "pass" + and native_apply5.get("nested_machine_receipt_verifier", {}).get("stage") + == "native_restore_receipt_verifier" + and native_apply5.get("nested_machine_receipt_verifier", {}).get("terminal") + == "pass" + and current_backup.get("terminal") == "BACKUP_CREATED" + and current_backup.get("terminal_sequence") == ["check_pass_no_write", "pass"] + and current_backup.get("independent_verifier_terminal") == "pass" + and current_backup.get("operation_identity_hash") + == "d60d3b618798bc1f60eeff4bfcf693ca14c5a1da70150043af90b3fa5e459ec1" + and current_backup.get("operation_status") + == "BACKUP_CREATED|4885|108006575|99461897|" + and current_backup.get("file_count") == 4885 + and current_backup.get("total_size_bytes") == 108006575 + and current_backup.get("compressed_size_bytes") == 99461897 + and current_backup.get("artifact_sha256") + == "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" + and current_backup.get("active_operation_count_after") == 0 + and current_backup.get("server_artifact_absent_after") is True + and current_restore.get("terminal") == "verified" + and current_restore.get("terminal_sequence") + == ["check_pass_no_runtime_write", "verified"] + and current_restore.get("clickhouse_restore_terminal") == "RESTORED" + and current_restore.get("independent_post_verifier_terminal") == "pass" + and current_restore.get("cleanup_terminal") == "pass" + and current_restore.get("network_scope") == "internal_only" + and current_restore.get("internal_network_attached") is True + and current_restore.get("staging_network_attached") is False + and current_restore.get("production_network_attached") is False + and current_restore.get("production_restore_performed") is False + and current_restore.get("cleanup_verified") is True + and current_restore.get("database_count") + == current_restore.get("restored_database_count") + == 6 + and current_restore.get("table_count") + == current_restore.get("restored_table_count") + == 100 + and current_restore.get("table_engine_schema_manifest_parity") is True + and current_restore.get("check_table_count") + == current_restore.get("check_table_pass_count") + == 44 + and current_restore.get("critical_nonzero_count") == 4 + and current_restic.get("snapshot_id") == "135485ab" + and current_restic.get("snapshot_readback") == "pass" + and current_restic.get("repository_scope") == "local_same_failure_domain" + and current_restic.get("offsite_verified") is False + and native_apply5.get("sqlite", {}).get("terminal") + == "pending_application_consistent_export" + and current_post.get("terminal") == "pass" + and current_post.get("independent_monitor") == "pass" + and current_post.get("deployed_source_hash_drift_count") == 0 + and current_post.get("runtime_drift_count") == 0 + and current_post.get("production_container_count") == 4 + and current_post.get("production_container_identity_unchanged") is True + and current_post.get("production_container_restart_count_delta") == 0 + and current_post.get("production_containers") + == { + "clickhouse": { + "started_at": "2026-07-14T22:49:13.665350281Z", + "restart_count": 0, + "health": "healthy", + }, + "collector": { + "started_at": "2026-07-14T22:20:53.725583861Z", + "restart_count": 0, + "health": "not_configured", + }, + "signoz": { + "started_at": "2026-07-11T08:45:17.179074963Z", + "restart_count": 0, + "health": "healthy", + }, + "zookeeper": { + "started_at": "2026-07-11T08:45:17.055893064Z", + "restart_count": 0, + "health": "healthy", + }, + } + and current_post.get("api_http_status") == 200 + and current_post.get("listener_counts") == {"4317": 2, "4318": 2, "8080": 2} + and current_post.get("active_backup_operation_count") == 0 + and current_post.get("restore_container_count") == 0 + and current_post.get("restore_volume_count") == 0 + and current_post.get("restore_network_count") == 0 + and current_post.get("toolchain_stage_residue_count") == 0 + and current_post.get("toolchain_candidate_residue_count") == 0 + and current_post.get("toolchain_rollback_residue_count") == 0 + and current_post.get("backup_process_count") == 0 + and current_post.get("lock_holders") + == {"canary": "none", "operation": "none", "toolchain": "none"} + and current_post.get("server_artifact_absent") is True + and current_post.get( + "exact_ephemeral_container_volume_network_process_tmp_residue_count" + ) + == 0 + and current_post.get("cleanup_verified") is True + and native_apply5.get("closure_writeback", {}).get( + "current_source_runtime_closed" + ) + is True + and native_apply5.get("closure_writeback", {}).get( + "overall_program_partial_degraded" + ) + is True ) checks["native_backup_restore_canary_closed"] = ( native_canary.get("completion_scope") == "clickhouse_native_only" @@ -655,8 +814,10 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: is True ) - native_migration = regression.get("completed_verifiers", {}).get( - "clickhouse_native_backup_migration", {} + completed_verifiers = regression.get("completed_verifiers", {}) + native_migration = completed_verifiers.get("clickhouse_native_backup_migration", {}) + current_source_toolchain = completed_verifiers.get( + "clickhouse_native_current_source_toolchain", {} ) pending_verifiers = regression.get("pending_verifiers", {}) sqlite_backup = pending_verifiers.get( @@ -669,9 +830,6 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: resume_inventory = pending_verifiers.get( "clickhouse_native_resume_submitted_inventory", {} ) - current_source_toolchain = pending_verifiers.get( - "clickhouse_native_current_source_toolchain", {} - ) checks["native_backup_migration_closed_and_remaining_gaps_explicit"] = ( native_migration.get("prior_run_id") == retry2.get("run_id") and native_migration.get("run_id") == native_apply4.get("run_id") @@ -701,12 +859,47 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and len(resume_inventory.get("required_preconditions", [])) == 4 and current_source_toolchain.get("status") == current_native_source.get("status") - == "pending_toolchain_deploy_and_runtime_canary" - and current_source_toolchain.get("current_source_deployed") is False - and current_source_toolchain.get("runtime_verifier_terminal") == "pending" + == "deployed_verified" + and current_source_toolchain.get("current_source_deployed") is True + and current_source_toolchain.get("runtime_verifier_terminal") == "pass" and current_source_toolchain.get("historical_production_receipt_preserved") is True - and len(current_source_toolchain.get("required_preconditions", [])) == 3 + and current_source_toolchain.get("toolchain_check_run_id") + == current_native_source.get("toolchain_check_run_id") + and current_source_toolchain.get("toolchain_apply_run_id") + == current_native_source.get("toolchain_apply_run_id") + and current_source_toolchain.get("toolchain_postcheck_run_id") + == current_native_source.get("toolchain_postcheck_run_id") + and current_source_toolchain.get("toolchain_postcanary_run_id") + == current_native_source.get("toolchain_postcanary_run_id") + and current_source_toolchain.get("default_canary_check_run_id") + == current_native_source.get("default_canary_check_run_id") + and current_source_toolchain.get("runtime_canary_run_id") + == current_native_source.get("runtime_canary_run_id") + and current_source_toolchain.get("wrapper_terminal") == "partial_degraded" + and current_source_toolchain.get("runtime_operation_terminal") == "pass" + and current_source_toolchain.get("nested_machine_receipt_verifier_terminal") + == "pass" + and current_source_toolchain.get("operation_identity_hash") + == current_backup.get("operation_identity_hash") + and current_source_toolchain.get("operation_status") + == current_backup.get("operation_status") + and current_source_toolchain.get("restic_snapshot_id") == "135485ab" + and current_source_toolchain.get("artifact_sha256") + == "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" + and current_source_toolchain.get("offsite_verified") is False + and current_source_toolchain.get("sqlite_terminal") + == "pending_application_consistent_export" + and current_source_toolchain.get("independent_monitor") == "pass" + and current_source_toolchain.get("api_http_status") == 200 + and current_source_toolchain.get("production_container_identity_unchanged") + is True + and current_source_toolchain.get("production_container_restart_count_delta") + == 0 + and current_source_toolchain.get("active_backup_operation_count") == 0 + and current_source_toolchain.get("runtime_drift_count") == 0 + and current_source_toolchain.get("residue_count") == 0 + and current_source_toolchain.get("exact_cleanup") == "pass" ) checks["post_closure_contract_mirror_consistent"] = ( @@ -719,8 +912,11 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: and post_closure.get("backup_collector_restore_guard_deployed") is True and post_closure.get("backup_canary_scope") == "clickhouse_native_only" and post_closure.get("backup_canary_verified") is True - and post_closure.get("backup_canary_last_run_id") == native_apply4.get("run_id") - and post_closure.get("backup_canary_last_terminal") == "pass" + and post_closure.get("backup_canary_last_run_id") == native_apply5.get("run_id") + and post_closure.get("backup_canary_last_terminal") == "partial_degraded" + and post_closure.get("backup_canary_runtime_operation_terminal") == "pass" + and post_closure.get("backup_canary_machine_receipt_verifier_terminal") + == "pass" and post_closure.get("backup_canary_cooldown_retry_status") == "superseded_by_online_native_no_stop_contract" and post_closure.get("clickhouse_native_backup_migration_status") @@ -737,8 +933,40 @@ def evaluate(root: Path, contract_path: Path) -> dict[str, Any]: == current_native_source.get("status") and post_closure.get("clickhouse_native_current_source_deployed") == current_native_source.get("current_source_deployed") + and post_closure.get( + "clickhouse_native_current_source_runtime_verifier_terminal" + ) + == current_native_source.get("runtime_verifier_terminal") + and post_closure.get("clickhouse_native_current_source_toolchain_check_run_id") + == current_native_source.get("toolchain_check_run_id") + and post_closure.get("clickhouse_native_current_source_toolchain_apply_run_id") + == current_native_source.get("toolchain_apply_run_id") + and post_closure.get( + "clickhouse_native_current_source_toolchain_postcheck_run_id" + ) + == current_native_source.get("toolchain_postcheck_run_id") + and post_closure.get( + "clickhouse_native_current_source_toolchain_postcanary_run_id" + ) + == current_native_source.get("toolchain_postcanary_run_id") + and post_closure.get("clickhouse_native_current_source_canary_check_run_id") + == current_native_source.get("default_canary_check_run_id") + and post_closure.get("clickhouse_native_current_source_canary_apply_run_id") + == current_native_source.get("runtime_canary_run_id") + and post_closure.get("clickhouse_native_current_source_wrapper_terminal") + == "partial_degraded" + and post_closure.get( + "clickhouse_native_current_source_runtime_operation_terminal" + ) + == "pass" + and post_closure.get( + "clickhouse_native_current_source_machine_receipt_verifier_terminal" + ) + == "pass" + and post_closure.get("clickhouse_native_current_source_artifact_sha256") + == "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" and post_closure.get("clickhouse_native_restore_terminal") == "verified" - and post_closure.get("clickhouse_native_restic_snapshot_id") == "574e2a1a" + and post_closure.get("clickhouse_native_restic_snapshot_id") == "135485ab" and post_closure.get("clickhouse_native_offsite_status") == "pending" and post_closure.get("clickhouse_native_failed_artifact_retention_status") == "pending_bounded_retention_decision" diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index 463fc9496..410f5e68e 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -98,12 +98,9 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: assert payload["writes_performed"] == 0 assert payload["terminal"] == "pass" assert payload["runtime_status"] == "partial_degraded" - assert ( - payload["backup_toolchain_current_source_status"] - == "pending_toolchain_deploy_and_runtime_canary" - ) - assert payload["backup_toolchain_current_source_deployed"] is False - assert payload["backup_toolchain_runtime_verifier_terminal"] == "pending" + assert payload["backup_toolchain_current_source_status"] == "deployed_verified" + assert payload["backup_toolchain_current_source_deployed"] is True + assert payload["backup_toolchain_runtime_verifier_terminal"] == "pass" assert payload["backup_toolchain_source_drift_fields"] == [ "backup_orchestrator_sha256", "canary_wrapper_sha256", @@ -119,6 +116,7 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: payload["checks"]["native_backup_current_source_deployment_state_explicit"] is True ) + assert payload["checks"]["native_backup_current_source_runtime_closed"] is True assert payload["checks"]["native_backup_historical_source_receipt_preserved"] assert payload["checks"]["post_release_600s_verifier_consistent"] is True assert ( @@ -602,8 +600,8 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No assert canary["toolchain_controlled_apply"]["postcheck_diff"] == ( "matching_7_drift_0_absent_0_active_operations_0" ) - assert len(canary["attempts"]) == 4 - _, apply2, apply3, apply4 = canary["attempts"] + assert len(canary["attempts"]) == 5 + _, apply2, apply3, apply4, apply5 = canary["attempts"] assert apply2["artifact_retention"] == "preserved_failed_run_evidence" assert apply2["cleanup_verified"] is True assert apply3["artifact_retention"] == "preserved_failed_run_evidence" @@ -647,6 +645,54 @@ def test_clickhouse_native_backup_restore_canary_closes_production_scope() -> No ) assert post["cleanup_verified"] is True assert post["secret_values_collected"] is False + + assert apply5["run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + ) + assert apply5["check_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + ) + assert apply5["wrapper_terminal"] == "partial_degraded" + assert apply5["terminal"] == "partial_degraded" + assert apply5["runtime_operation"]["terminal"] == "pass" + assert apply5["nested_machine_receipt_verifier"] == { + "stage": "native_restore_receipt_verifier", + "terminal": "pass", + } + assert apply5["backup"]["terminal"] == "BACKUP_CREATED" + assert apply5["backup"]["file_count"] == 4885 + assert apply5["backup"]["total_size_bytes"] == 108006575 + assert apply5["backup"]["compressed_size_bytes"] == 99461897 + assert apply5["backup"]["artifact_sha256"] == ( + "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" + ) + current_restore = apply5["isolated_restore"] + assert ( + current_restore["database_count"] + == current_restore["restored_database_count"] + == 6 + ) + assert ( + current_restore["table_count"] == current_restore["restored_table_count"] == 100 + ) + assert ( + current_restore["check_table_count"] + == current_restore["check_table_pass_count"] + == 44 + ) + assert current_restore["critical_nonzero_count"] == 4 + assert apply5["restic"]["snapshot_id"] == "135485ab" + current_post = apply5["independent_post_verifier"] + assert current_post["independent_monitor"] == "pass" + assert current_post["deployed_source_hash_drift_count"] == 0 + assert current_post["runtime_drift_count"] == 0 + assert ( + current_post[ + "exact_ephemeral_container_volume_network_process_tmp_residue_count" + ] + == 0 + ) + assert current_post["cleanup_verified"] is True assert canary["terminal"] == "production_closed_clickhouse_native_scope" @@ -677,16 +723,38 @@ def test_current_toolchain_hashes_are_bound_without_rewriting_production_receipt assert current["hashes"]["inventory_verifier_sha256"] == ( "c2d2159eafc0b06139c52fd6992f75f75966075ba27cc9711208a4f4750ea863" ) - assert current["status"] == "pending_toolchain_deploy_and_runtime_canary" - assert current["deployment_status"] == "pending" - assert current["current_source_deployed"] is False - assert current["runtime_verifier_terminal"] == "pending" + assert current["deployed_source_hashes"] == current_hashes + assert current["status"] == "deployed_verified" + assert current["deployment_status"] == "deployed" + assert current["current_source_deployed"] is True + assert current["runtime_verifier_terminal"] == "pass" assert current["historical_production_receipt_preserved"] is True assert current["drift_from_historical_production_receipt"] == [ "backup_orchestrator_sha256", "canary_wrapper_sha256", "inventory_verifier_sha256", ] + assert current["toolchain_check_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015040Z-check6" + ) + assert current["toolchain_apply_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015059Z-apply5" + ) + assert current["toolchain_postcheck_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5" + ) + assert current["toolchain_postcanary_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5" + ) + assert current["default_canary_check_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + ) + assert current["runtime_canary_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + ) + assert current["wrapper_terminal"] == "partial_degraded" + assert current["runtime_operation_terminal"] == "pass" + assert current["nested_machine_receipt_verifier_terminal"] == "pass" def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale( @@ -707,10 +775,31 @@ def test_preflight_fails_closed_when_current_toolchain_source_hash_is_stale( assert payload["terminal"] == "failed" assert payload["checks"]["native_backup_current_source_hashes_bound"] is False assert payload["checks"]["native_backup_historical_source_receipt_preserved"] - assert ( - payload["backup_toolchain_current_source_status"] - == "pending_toolchain_deploy_and_runtime_canary" - ) + assert payload["backup_toolchain_current_source_status"] == "deployed_verified" + + +def test_preflight_fails_closed_when_current_runtime_receipt_is_incomplete( + monkeypatch, +) -> None: + module = _load_preflight_module() + real_load_yaml = module._load_yaml + + def load_without_operation_identity(path: Path): + payload = real_load_yaml(path) + if path.name == POST_CLOSURE_REGRESSION.name: + attempts = payload["controlled_applies"][ + "clickhouse_native_backup_restore_canary" + ]["attempts"] + attempts[-1]["backup"].pop("operation_identity_hash") + return payload + + monkeypatch.setattr(module, "_load_yaml", load_without_operation_identity) + payload = module.evaluate(ROOT, CONTRACT) + + assert payload["source_ready"] is False + assert payload["terminal"] == "failed" + assert payload["checks"]["native_backup_current_source_runtime_closed"] is False + assert payload["runtime_status"] == "partial_degraded" def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> None: @@ -759,14 +848,33 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> ] assert resume["same_run_resume_safe"] is False assert len(resume["required_preconditions"]) == 4 - current_source = receipt["pending_verifiers"][ + current_source = receipt["completed_verifiers"][ "clickhouse_native_current_source_toolchain" ] - assert current_source["status"] == "pending_toolchain_deploy_and_runtime_canary" - assert current_source["current_source_deployed"] is False - assert current_source["runtime_verifier_terminal"] == "pending" + assert current_source["status"] == "deployed_verified" + assert current_source["current_source_deployed"] is True + assert current_source["runtime_verifier_terminal"] == "pass" assert current_source["historical_production_receipt_preserved"] is True - assert len(current_source["required_preconditions"]) == 3 + assert current_source["toolchain_apply_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015059Z-apply5" + ) + assert current_source["runtime_canary_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + ) + assert current_source["wrapper_terminal"] == "partial_degraded" + assert current_source["runtime_operation_terminal"] == "pass" + assert current_source["nested_machine_receipt_verifier_terminal"] == "pass" + assert current_source["restic_snapshot_id"] == "135485ab" + assert current_source["artifact_sha256"] == ( + "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" + ) + assert current_source["independent_monitor"] == "pass" + assert current_source["runtime_drift_count"] == 0 + assert current_source["residue_count"] == 0 + assert current_source["exact_cleanup"] == "pass" + assert ( + "clickhouse_native_current_source_toolchain" not in receipt["pending_verifiers"] + ) assets = receipt["asset_reconciliation"] assert assets["drift_work_item_id"] == "P0-OBS-002-ASSET-DRIFT-001" assert assets["source_service_count"] == 27 @@ -797,7 +905,6 @@ def test_post_release_verifier_and_native_backup_close_with_remaining_gaps() -> "signoz_backup_offsite_dr_pending", "clickhouse_native_failed_artifact_retention_pending", "clickhouse_native_resume_submitted_inventory_pending", - "clickhouse_native_current_source_toolchain_pending", "service_registry_runtime_mirror_reconciliation_pending", "github_freeze_legacy_asset_retirement_pending", "monitoring_generator_duplicate_awoooi_api_identity_pending", @@ -818,10 +925,12 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N assert post_closure["backup_collector_restore_guard_deployed"] is True assert post_closure["backup_canary_scope"] == "clickhouse_native_only" assert post_closure["backup_canary_verified"] is True - assert post_closure["backup_canary_last_terminal"] == "pass" + assert post_closure["backup_canary_last_terminal"] == "partial_degraded" assert post_closure["backup_canary_last_run_id"] == ( - "P0-OBS-002-native-canary-20260715T001329Z-apply4" + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" ) + assert post_closure["backup_canary_runtime_operation_terminal"] == "pass" + assert post_closure["backup_canary_machine_receipt_verifier_terminal"] == "pass" assert post_closure["backup_canary_cooldown_retry_status"] == ( "superseded_by_online_native_no_stop_contract" ) @@ -838,12 +947,51 @@ def test_post_closure_contract_mirror_and_program_blockers_are_consistent() -> N assert post_closure["clickhouse_native_backup_toolchain_postcheck_run_id"] == ( "P0-OBS-002-native-toolchain-20260715T001305Z-postcheck4" ) - assert post_closure["clickhouse_native_current_source_status"] == ( - "pending_toolchain_deploy_and_runtime_canary" + assert ( + post_closure["clickhouse_native_current_source_status"] == "deployed_verified" + ) + assert post_closure["clickhouse_native_current_source_deployed"] is True + assert ( + post_closure["clickhouse_native_current_source_runtime_verifier_terminal"] + == "pass" + ) + assert post_closure["clickhouse_native_current_source_toolchain_check_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015040Z-check6" + ) + assert post_closure["clickhouse_native_current_source_toolchain_apply_run_id"] == ( + "P0-OBS-002-native-toolchain-20260715T015059Z-apply5" + ) + assert ( + post_closure["clickhouse_native_current_source_toolchain_postcheck_run_id"] + == "P0-OBS-002-native-toolchain-20260715T015116Z-postcheck5" + ) + assert ( + post_closure["clickhouse_native_current_source_toolchain_postcanary_run_id"] + == "P0-OBS-002-native-toolchain-20260715T015346Z-postcanary5" + ) + assert post_closure["clickhouse_native_current_source_canary_check_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015131Z-check5" + ) + assert post_closure["clickhouse_native_current_source_canary_apply_run_id"] == ( + "P0-OBS-002-native-canary-default-20260715T015149Z-apply5" + ) + assert post_closure["clickhouse_native_current_source_wrapper_terminal"] == ( + "partial_degraded" + ) + assert post_closure[ + "clickhouse_native_current_source_runtime_operation_terminal" + ] == ("pass") + assert ( + post_closure[ + "clickhouse_native_current_source_machine_receipt_verifier_terminal" + ] + == "pass" + ) + assert post_closure["clickhouse_native_current_source_artifact_sha256"] == ( + "08d86054110e9faf769976be6483f70ab742974f14fc9880cc5d1515d2788e83" ) - assert post_closure["clickhouse_native_current_source_deployed"] is False assert post_closure["clickhouse_native_restore_terminal"] == "verified" - assert post_closure["clickhouse_native_restic_snapshot_id"] == "574e2a1a" + assert post_closure["clickhouse_native_restic_snapshot_id"] == "135485ab" assert post_closure["clickhouse_native_offsite_status"] == "pending" assert post_closure["clickhouse_native_failed_artifact_retention_status"] == ( "pending_bounded_retention_decision" From e2cd70501d86b981600b12fd0ede718c865736b0 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 11:07:00 +0800 Subject: [PATCH 4/9] fix(mcp): enforce exact replay sandbox contract --- docs/LOGBOOK.md | 2 +- .../external_mcp_replay_controller.py | 76 ++++++++++++++----- .../test_external_mcp_replay_controller.py | 25 +++++- ...test_verify_external_mcp_replay_receipt.py | 20 +++++ .../verify_external_mcp_replay_receipt.py | 41 +++++++++- 5 files changed, 140 insertions(+), 24 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index b90f03160..98bd573b1 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -10,7 +10,7 @@ **source/test evidence**： - controller/verifier 本機真實雙 replay 皆通過：MCP `Playwright@1.62.0-alpha-1783623505000`、protocol `2025-06-18`、24 tools、tool-name hash `12d23fea9d8d1a1de3f44863dc00cf393f0a2165323838cf93fed30a6a4cc237`、schema hash `c3737ced21147d0512b0400df5885c95a545bc915892b5c8f928cee78428ac99`，兩次均 clean exit/container removed。 -- 新 replay tests `32 passed`；artifact + replay supply-chain tests與 MCP control-plane/federation/version lifecycle/runner-health 聚焦回歸合計 `84 passed`。Ruff、py_compile、JSON/YAML parse、source-control owner guard與 `git diff --check` 通過。 +- 新 replay tests `37 passed`；artifact + replay supply-chain tests與 MCP control-plane/federation/version lifecycle/runner-health 聚焦回歸合計 `89 passed`。Ruff、py_compile、JSON/YAML parse、source-control owner guard與 `git diff --check` 通過。 - global security mirror guard另抓到 latest main 既有 `apps/web/src/app/[locale]/iwooos/page.tsx` 的 `raw_blocked_waiting_state` 敏感字樣；本 work item未修改該檔，保留為獨立 drift blocker，不以 replay source 綠燈覆蓋。 **尚未完成 / 不誤報**： diff --git a/scripts/security/external_mcp_replay_controller.py b/scripts/security/external_mcp_replay_controller.py index 319a32222..7b76e7f0f 100644 --- a/scripts/security/external_mcp_replay_controller.py +++ b/scripts/security/external_mcp_replay_controller.py @@ -52,6 +52,27 @@ EXPECTED_PACKAGE_KEYS = { "playwright@1.62.0-alpha-1783623505000", "playwright-core@1.62.0-alpha-1783623505000", } +EXPECTED_WORK_ITEM_ID = "MCP-REPLAY-PLAYWRIGHT-001" +EXPECTED_CANDIDATE_ID = "external.playwright-verifier" +EXPECTED_ADAPTER_ID = "playwright_public_accessibility_verifier_v1" +EXPECTED_OWNER_LANE = "awoooi_mcp_supply_chain" +EXPECTED_ARTIFACT_REPOSITORY = ( + "registry.wooo.work/awoooi/mcp-artifacts/playwright-mcp@" +) +EXPECTED_ARTIFACT_RUNTIME_REPOSITORY = ( + "192.168.0.110:5000/awoooi/mcp-artifacts/playwright-mcp@" +) +EXPECTED_RUNTIME_REPOSITORY = "registry.wooo.work/awoooi/ci-runner@" +EXPECTED_RUNTIME_ARGUMENTS = ( + "--headless", + "--isolated", + "--allowed-origins", + "https://awoooi.wooo.work", + "--output-dir", + "/tmp/mcp-output", + "--executable-path", + "/nonexistent/browser", +) class ReplayError(RuntimeError): @@ -203,6 +224,13 @@ def load_policy(path: Path) -> ReplayPolicy: raise ReplayError("replay_scope_invalid") if payload.get("risk_class") != "high": raise ReplayError("replay_risk_class_invalid") + if ( + payload.get("work_item_id") != EXPECTED_WORK_ITEM_ID + or payload.get("candidate_id") != EXPECTED_CANDIDATE_ID + or payload.get("adapter_id") != EXPECTED_ADAPTER_ID + or payload.get("owner_lane") != EXPECTED_OWNER_LANE + ): + raise ReplayError("replay_asset_identity_invalid") expiry_text = _require_text(payload.get("expires_at"), "replay_expiry_invalid") try: @@ -220,6 +248,8 @@ def load_policy(path: Path) -> ReplayPolicy: _require_text(payload.get(field), f"replay_{field}_invalid") artifact = _require_dict(payload.get("artifact_source"), "artifact_source_invalid") + if artifact.get("audit_branch") != "mcp-artifact-receipts": + raise ReplayError("audit_branch_invalid") audit_commit = _require_text(artifact.get("audit_commit"), "audit_commit_invalid") if not GIT_SHA_PATTERN.fullmatch(audit_commit): raise ReplayError("audit_commit_invalid") @@ -235,32 +265,42 @@ def load_policy(path: Path) -> ReplayPolicy: raise ReplayError("package_integrity_invalid") normalized_integrities[key] = digest + artifact_ref = _validate_oci_ref(artifact.get("artifact_ref"), "artifact_ref_invalid") + artifact_runtime_ref = _validate_oci_ref( + artifact.get("artifact_runtime_ref"), "artifact_runtime_ref_invalid" + ) + if not artifact_ref.startswith(EXPECTED_ARTIFACT_REPOSITORY): + raise ReplayError("artifact_repository_not_allowlisted") + if not artifact_runtime_ref.startswith(EXPECTED_ARTIFACT_RUNTIME_REPOSITORY): + raise ReplayError("artifact_runtime_repository_not_allowlisted") + runtime = _require_dict(payload.get("runtime"), "runtime_invalid") + runtime_image_ref = _validate_oci_ref( + runtime.get("image_ref"), "runtime_image_ref_invalid" + ) + if not runtime_image_ref.startswith(EXPECTED_RUNTIME_REPOSITORY): + raise ReplayError("runtime_repository_not_allowlisted") if ( runtime.get("network_mode") != "none" or runtime.get("read_only_root") is not True or runtime.get("no_new_privileges") is not True or runtime.get("cap_drop") != ["ALL"] + or runtime.get("platform") != "linux/amd64" + or runtime.get("user") != "65534:65534" + or runtime.get("pids_limit") != 64 + or runtime.get("memory_limit") != "384m" + or runtime.get("cpu_limit") != "0.5" + or runtime.get("tmpfs") != "/tmp:rw,noexec,nosuid,size=32m" + or runtime.get("startup_timeout_seconds") != 20 + or runtime.get("shutdown_timeout_seconds") != 5 ): raise ReplayError("runtime_isolation_invalid") entrypoint = _require_text_list(runtime.get("entrypoint"), "entrypoint_invalid") arguments = _require_text_list(runtime.get("arguments"), "arguments_invalid") if entrypoint != ("node", "node_modules/@playwright/mcp/cli.js"): raise ReplayError("entrypoint_not_allowlisted") - required_arguments = { - "--headless", - "--isolated", - "--allowed-origins", - "https://awoooi.wooo.work", - "--output-dir", - "/tmp/mcp-output", - "--executable-path", - "/nonexistent/browser", - } - if set(arguments) != required_arguments or len(arguments) != 8: + if arguments != EXPECTED_RUNTIME_ARGUMENTS: raise ReplayError("arguments_not_allowlisted") - if runtime.get("user") != "65534:65534": - raise ReplayError("runtime_user_invalid") protocol = _require_dict( payload.get("protocol_contract"), "protocol_contract_invalid" @@ -332,10 +372,8 @@ def load_policy(path: Path) -> ReplayPolicy: artifact.get("verifier_receipt_sha256"), "verifier_receipt_sha256_invalid", ), - artifact_ref=_validate_oci_ref(artifact.get("artifact_ref"), "artifact_ref_invalid"), - artifact_runtime_ref=_validate_oci_ref( - artifact.get("artifact_runtime_ref"), "artifact_runtime_ref_invalid" - ), + artifact_ref=artifact_ref, + artifact_runtime_ref=artifact_runtime_ref, bundle_manifest_sha256=_validate_sha256( artifact.get("bundle_manifest_sha256"), "bundle_manifest_sha256_invalid" ), @@ -343,9 +381,7 @@ def load_policy(path: Path) -> ReplayPolicy: artifact.get("cyclonedx_sbom_sha256"), "cyclonedx_sbom_sha256_invalid" ), package_integrities=normalized_integrities, - runtime_image_ref=_validate_oci_ref( - runtime.get("image_ref"), "runtime_image_ref_invalid" - ), + runtime_image_ref=runtime_image_ref, runtime_platform=_require_text( runtime.get("platform"), "runtime_platform_invalid" ), diff --git a/scripts/security/tests/test_external_mcp_replay_controller.py b/scripts/security/tests/test_external_mcp_replay_controller.py index 0a9063d2a..c267a0550 100644 --- a/scripts/security/tests/test_external_mcp_replay_controller.py +++ b/scripts/security/tests/test_external_mcp_replay_controller.py @@ -157,7 +157,23 @@ class ReplayPolicyTests(unittest.TestCase): path = self._mutated_policy( lambda payload: payload["runtime"].__setitem__("user", "0:0") ) - with self.assertRaisesRegex(controller.ReplayError, "runtime_user_invalid"): + with self.assertRaisesRegex(controller.ReplayError, "runtime_isolation_invalid"): + controller.load_policy(path) + + def test_runtime_memory_limit_cannot_be_weakened(self) -> None: + path = self._mutated_policy( + lambda payload: payload["runtime"].__setitem__("memory_limit", "2g") + ) + with self.assertRaisesRegex(controller.ReplayError, "runtime_isolation_invalid"): + controller.load_policy(path) + + def test_runtime_argument_order_is_part_of_allowlist(self) -> None: + def mutate(payload: dict) -> None: + arguments = payload["runtime"]["arguments"] + arguments[2], arguments[4] = arguments[4], arguments[2] + + path = self._mutated_policy(mutate) + with self.assertRaisesRegex(controller.ReplayError, "arguments_not_allowlisted"): controller.load_policy(path) def test_floating_runtime_reference_is_rejected(self) -> None: @@ -178,6 +194,13 @@ class ReplayPolicyTests(unittest.TestCase): with self.assertRaisesRegex(controller.ReplayError, "execution_boundary_invalid"): controller.load_policy(path) + def test_candidate_identity_cannot_be_retargeted(self) -> None: + path = self._mutated_policy( + lambda payload: payload.__setitem__("candidate_id", "external.other") + ) + with self.assertRaisesRegex(controller.ReplayError, "replay_asset_identity_invalid"): + controller.load_policy(path) + def test_dangerous_tool_cannot_leave_denied_partition(self) -> None: def mutate(payload: dict) -> None: payload["protocol_contract"]["adapter_denied_tools"].remove( diff --git a/scripts/security/tests/test_verify_external_mcp_replay_receipt.py b/scripts/security/tests/test_verify_external_mcp_replay_receipt.py index 27c4ae9b3..633eebce0 100644 --- a/scripts/security/tests/test_verify_external_mcp_replay_receipt.py +++ b/scripts/security/tests/test_verify_external_mcp_replay_receipt.py @@ -154,6 +154,26 @@ class StandalonePolicyTests(unittest.TestCase): with self.assertRaisesRegex(verifier.VerifierError, "artifact_ref_invalid"): verifier.load_and_validate_policy(path) + def test_runtime_resource_limits_cannot_be_weakened(self) -> None: + with tempfile.TemporaryDirectory() as temp: + payload = _policy_payload() + payload["runtime"]["pids_limit"] = 512 + path = Path(temp) / "policy.json" + _write(path, payload) + with self.assertRaisesRegex(verifier.VerifierError, "runtime_isolation_invalid"): + verifier.load_and_validate_policy(path) + + def test_runtime_argument_order_drift_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + payload = _policy_payload() + payload["runtime"]["arguments"] = list( + reversed(payload["runtime"]["arguments"]) + ) + path = Path(temp) / "policy.json" + _write(path, payload) + with self.assertRaisesRegex(verifier.VerifierError, "runtime_isolation_invalid"): + verifier.load_and_validate_policy(path) + class ControllerReceiptTests(unittest.TestCase): def test_valid_controller_receipt_passes_standalone_validation(self) -> None: diff --git a/scripts/security/verify_external_mcp_replay_receipt.py b/scripts/security/verify_external_mcp_replay_receipt.py index 5733edc65..b643d82df 100644 --- a/scripts/security/verify_external_mcp_replay_receipt.py +++ b/scripts/security/verify_external_mcp_replay_receipt.py @@ -41,6 +41,16 @@ MAX_JSON_BYTES = 4 * 1024 * 1024 MAX_PROTOCOL_LINE_BYTES = 2 * 1024 * 1024 MAX_FILES = 4_096 MAX_UNPACKED_BYTES = 40 * 1024 * 1024 +EXPECTED_RUNTIME_ARGUMENTS = [ + "--headless", + "--isolated", + "--allowed-origins", + "https://awoooi.wooo.work", + "--output-dir", + "/tmp/mcp-output", + "--executable-path", + "/nonexistent/browser", +] class VerifierError(RuntimeError): @@ -113,6 +123,11 @@ def load_and_validate_policy(path: Path) -> tuple[dict[str, Any], str]: or policy.get("risk_class") != "high" or policy.get("scope") != "offline_protocol_and_tool_schema_compatibility_only" + or policy.get("work_item_id") != "MCP-REPLAY-PLAYWRIGHT-001" + or policy.get("candidate_id") != "external.playwright-verifier" + or policy.get("adapter_id") + != "playwright_public_accessibility_verifier_v1" + or policy.get("owner_lane") != "awoooi_mcp_supply_chain" ): raise VerifierError("policy_contract_invalid") try: @@ -134,12 +149,34 @@ def load_and_validate_policy(path: Path) -> tuple[dict[str, Any], str]: or runtime.get("cap_drop") != ["ALL"] or runtime.get("user") != "65534:65534" or runtime.get("platform") != "linux/amd64" + or runtime.get("pids_limit") != 64 + or runtime.get("memory_limit") != "384m" + or runtime.get("cpu_limit") != "0.5" + or runtime.get("tmpfs") != "/tmp:rw,noexec,nosuid,size=32m" + or runtime.get("startup_timeout_seconds") != 20 + or runtime.get("shutdown_timeout_seconds") != 5 + or runtime.get("entrypoint") + != ["node", "node_modules/@playwright/mcp/cli.js"] + or runtime.get("arguments") != EXPECTED_RUNTIME_ARGUMENTS ): raise VerifierError("runtime_isolation_invalid") artifact_ref = _oci_ref(artifact.get("artifact_ref"), "artifact_ref_invalid") runtime_ref = _oci_ref(runtime.get("image_ref"), "runtime_ref_invalid") - if artifact_ref == runtime_ref: - raise VerifierError("artifact_runtime_separation_invalid") + artifact_runtime_ref = _oci_ref( + artifact.get("artifact_runtime_ref"), "artifact_runtime_ref_invalid" + ) + if ( + artifact_ref == runtime_ref + or not artifact_ref.startswith( + "registry.wooo.work/awoooi/mcp-artifacts/playwright-mcp@" + ) + or not artifact_runtime_ref.startswith( + "192.168.0.110:5000/awoooi/mcp-artifacts/playwright-mcp@" + ) + or not runtime_ref.startswith("registry.wooo.work/awoooi/ci-runner@") + or artifact.get("audit_branch") != "mcp-artifact-receipts" + ): + raise VerifierError("artifact_runtime_repository_contract_invalid") for field in ( "controller_receipt_sha256", "verifier_receipt_sha256", From 6505e2cede2d9e25114052882a054a1c84eab882 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 10:38:32 +0800 Subject: [PATCH 5/9] fix(automation): stabilize asset reconciliation source --- .../asset_capability_reconciliation_job.py | 105 +++++++++++++- apps/api/src/jobs/asset_change_tracker_job.py | 11 +- apps/api/src/jobs/asset_scanner_job.py | 6 +- apps/api/src/jobs/rule_catalog_sync_job.py | 29 ++-- ...t_ai_automation_asset_capability_matrix.py | 114 ++++++++++++++- .../tests/test_asset_change_tracker_job.py | 54 +++++++ apps/api/tests/test_asset_scanner_job.py | 6 + apps/api/tests/test_rule_catalog_sync_job.py | 133 ++++++++++++++++++ ..._signal_worker_ansible_executor_binding.py | 1 + k8s/awoooi-prod/08-deployment-worker.yaml | 4 + 10 files changed, 442 insertions(+), 21 deletions(-) create mode 100644 apps/api/tests/test_asset_change_tracker_job.py create mode 100644 apps/api/tests/test_rule_catalog_sync_job.py diff --git a/apps/api/src/jobs/asset_capability_reconciliation_job.py b/apps/api/src/jobs/asset_capability_reconciliation_job.py index 446da976e..c4dd07b0b 100644 --- a/apps/api/src/jobs/asset_capability_reconciliation_job.py +++ b/apps/api/src/jobs/asset_capability_reconciliation_job.py @@ -13,6 +13,7 @@ import time import uuid from collections import Counter from collections.abc import Mapping, Sequence +from datetime import UTC, datetime from typing import Any import structlog @@ -29,9 +30,12 @@ from src.services.ai_automation_asset_capability_matrix import ( logger = structlog.get_logger(__name__) -_FIRST_DELAY_SECONDS = 30 +_FIRST_DELAY_SECONDS = 7 * 60 _LOOP_BACKOFF_SECONDS = 5 * 60 _RECONCILIATION_POLL_SECONDS = 5 * 60 +_SOURCE_SETTLE_SECONDS = 5 * 60 +_SOURCE_STABILITY_DELAY_SECONDS = 5 +_SOURCE_STABILITY_RETRY_SECONDS = 60 _RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS = 30.0 _WORK_ITEM_INSERT_BATCH_SIZE = 250 _MAX_RECONCILIATION_CANDIDATES = 20_000 @@ -39,6 +43,8 @@ _RETRYABLE_RESULT_STATUSES = { "blocked_safe_no_write", "degraded_cache_refresh", "degraded_no_write", + "source_snapshot_settling", + "source_snapshot_unstable", } @@ -50,6 +56,24 @@ def _canonical_json(value: Any) -> str: return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) +def _latest_discovery_age_seconds( + matrix: Mapping[str, Any], + *, + now: datetime | None = None, +) -> float | None: + ended_at = str(_dict(matrix.get("latest_discovery_run")).get("ended_at") or "") + if not ended_at: + return None + try: + parsed = datetime.fromisoformat(ended_at.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + current = (now or datetime.now(UTC)).astimezone(UTC) + return max(0.0, (current - parsed.astimezone(UTC)).total_seconds()) + + def build_reconciliation_summary_payload( matrix: Mapping[str, Any], candidates: Sequence[Mapping[str, Any]], @@ -177,11 +201,13 @@ def build_reconciliation_work_item_input( async def run_asset_capability_reconciliation_loop() -> None: - """Reconcile shortly after startup and retry each discovery fingerprint.""" + """Reconcile settled source snapshots and retry each fingerprint.""" logger.info( "asset_capability_reconciliation_loop_started", + first_delay_seconds=_FIRST_DELAY_SECONDS, poll_interval_seconds=_RECONCILIATION_POLL_SECONDS, + source_settle_seconds=_SOURCE_SETTLE_SECONDS, ) await asyncio.sleep(_FIRST_DELAY_SECONDS) triggered_by = "startup" @@ -197,13 +223,20 @@ async def run_asset_capability_reconciliation_loop() -> None: triggered_by = "retry" continue if str(result.get("status") or "") in _RETRYABLE_RESULT_STATUSES: + retry_seconds = max( + 30, + min( + _LOOP_BACKOFF_SECONDS, + int(result.get("retry_seconds") or _LOOP_BACKOFF_SECONDS), + ), + ) logger.warning( "asset_capability_reconciliation_retry_scheduled", status=result.get("status"), reason=result.get("reason"), - retry_seconds=_LOOP_BACKOFF_SECONDS, + retry_seconds=retry_seconds, ) - await asyncio.sleep(_LOOP_BACKOFF_SECONDS) + await asyncio.sleep(retry_seconds) triggered_by = "retry" continue await asyncio.sleep(_RECONCILIATION_POLL_SECONDS) @@ -235,6 +268,70 @@ async def reconcile_once( "candidate_count": 0, } + discovery_age_seconds = _latest_discovery_age_seconds(matrix) + if discovery_age_seconds is None or discovery_age_seconds < _SOURCE_SETTLE_SECONDS: + remaining_seconds = ( + _SOURCE_STABILITY_RETRY_SECONDS + if discovery_age_seconds is None + else max( + _SOURCE_STABILITY_RETRY_SECONDS, + int(_SOURCE_SETTLE_SECONDS - discovery_age_seconds) + 1, + ) + ) + logger.info( + "asset_capability_reconciliation_source_snapshot_settling", + discovery_age_seconds=discovery_age_seconds, + required_age_seconds=_SOURCE_SETTLE_SECONDS, + retry_seconds=remaining_seconds, + ) + return { + "status": "source_snapshot_settling", + "reason": "latest_discovery_run_not_settled", + "discovery_age_seconds": discovery_age_seconds, + "required_age_seconds": _SOURCE_SETTLE_SECONDS, + "retry_seconds": remaining_seconds, + "external_runtime_write_performed": False, + } + + await asyncio.sleep(_SOURCE_STABILITY_DELAY_SECONDS) + stable_matrix = await build_asset_capability_matrix_with_live_readback( + project_id=project_id, + timeout_seconds=_RECONCILIATION_LIVE_READBACK_TIMEOUT_SECONDS, + include_live_only_assets=True, + ) + if stable_matrix.get("live_readback_status") != "ready": + return { + "status": "degraded_no_write", + "reason": "stability_verifier_live_readback_degraded", + "live_readback_status": stable_matrix.get("live_readback_status"), + "candidate_count": 0, + "retry_seconds": _SOURCE_STABILITY_RETRY_SECONDS, + "external_runtime_write_performed": False, + } + + first_identity = ( + str(matrix.get("matrix_fingerprint") or ""), + str(_dict(matrix.get("latest_discovery_run")).get("run_id") or ""), + ) + stable_identity = ( + str(stable_matrix.get("matrix_fingerprint") or ""), + str(_dict(stable_matrix.get("latest_discovery_run")).get("run_id") or ""), + ) + if not all(stable_identity) or stable_identity != first_identity: + logger.warning( + "asset_capability_reconciliation_source_snapshot_unstable", + discovery_run_changed=first_identity[1] != stable_identity[1], + matrix_fingerprint_changed=first_identity[0] != stable_identity[0], + retry_seconds=_SOURCE_STABILITY_RETRY_SECONDS, + ) + return { + "status": "source_snapshot_unstable", + "reason": "matrix_changed_during_stability_verifier", + "retry_seconds": _SOURCE_STABILITY_RETRY_SECONDS, + "external_runtime_write_performed": False, + } + matrix = stable_matrix + candidates = build_reconciliation_candidates(matrix) if len(candidates) > _MAX_RECONCILIATION_CANDIDATES: logger.error( diff --git a/apps/api/src/jobs/asset_change_tracker_job.py b/apps/api/src/jobs/asset_change_tracker_job.py index e46b7875c..5b8dc0761 100644 --- a/apps/api/src/jobs/asset_change_tracker_job.py +++ b/apps/api/src/jobs/asset_change_tracker_job.py @@ -29,7 +29,6 @@ from __future__ import annotations import asyncio import json as _json import time as _time -from typing import Any import structlog @@ -38,6 +37,7 @@ logger = structlog.get_logger(__name__) _TRACK_INTERVAL_SEC = 3600 _FIRST_DELAY_SEC = 360 _LOOP_BACKOFF_SEC = 600 +_PROJECT_ID = "awoooi" async def run_asset_change_tracker_loop() -> None: @@ -92,9 +92,10 @@ async def track_once() -> dict[str, int]: async def _get_recent_runs(limit: int = 2) -> list[str]: """取最近 N 個 success 的 run_id (降序).""" from sqlalchemy import text as _sql + from src.db.base import get_db_context - async with get_db_context() as db: + async with get_db_context(_PROJECT_ID) as db: rows = await db.execute( _sql("SELECT run_id FROM asset_discovery_run WHERE status='success' ORDER BY ended_at DESC LIMIT :lim"), {"lim": limit}, @@ -113,11 +114,12 @@ async def _diff_runs(newer_run: str, older_run: str) -> dict[str, int]: - newer ∩ older 且 metadata 變 = modified """ from sqlalchemy import text as _sql + from src.db.base import get_db_context stats = {"added": 0, "removed": 0, "modified": 0} - async with get_db_context() as db: + async with get_db_context(_PROJECT_ID) as db: # 1. Added: newer run 有但 older run 沒有 result = await db.execute( _sql(""" @@ -216,10 +218,11 @@ async def _diff_runs(newer_run: str, older_run: str) -> dict[str, int]: async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) -> None: try: from sqlalchemy import text as _sql + from src.db.base import get_db_context aol_status = "failed" if error else "success" - async with get_db_context() as db: + async with get_db_context(_PROJECT_ID) as db: await db.execute( _sql(""" INSERT INTO automation_operation_log ( diff --git a/apps/api/src/jobs/asset_scanner_job.py b/apps/api/src/jobs/asset_scanner_job.py index cef3d872f..5cc49c313 100644 --- a/apps/api/src/jobs/asset_scanner_job.py +++ b/apps/api/src/jobs/asset_scanner_job.py @@ -206,7 +206,7 @@ async def run_asset_scanner_loop() -> None: while True: try: - await scan_once(triggered_by="cron") + await scan_once(triggered_by="cron", raise_on_failure=True) except Exception as e: logger.exception("asset_scanner_loop_error", error=str(e)) await asyncio.sleep(_LOOP_BACKOFF_SEC) @@ -227,6 +227,7 @@ async def scan_once( "domain_tls_inventory", ), scan_depth: str = "shallow", + raise_on_failure: bool = False, ) -> str: """ 執行一次資產盤點。 @@ -235,6 +236,7 @@ async def scan_once( triggered_by: cron / ai / human / incident scope: 本次掃描範圍標籤 (寫入 asset_discovery_run.scope) scan_depth: shallow (僅 list) / deep (含 describe) / full + raise_on_failure: durable terminal 寫入後拋錯，供排程進入 bounded retry Returns: run_id (UUID 字串)。無法建立 durable header 時會拋出例外。 @@ -324,6 +326,8 @@ async def scan_once( disappeared=disappeared_count, duration_ms=duration_ms, ) + if error_msg and raise_on_failure: + raise RuntimeError(f"asset_scan_failed:{run_id}:{error_msg}") return run_id diff --git a/apps/api/src/jobs/rule_catalog_sync_job.py b/apps/api/src/jobs/rule_catalog_sync_job.py index 30655e93b..facbf3589 100644 --- a/apps/api/src/jobs/rule_catalog_sync_job.py +++ b/apps/api/src/jobs/rule_catalog_sync_job.py @@ -48,6 +48,7 @@ _SYNC_INTERVAL_SEC = 3600 # 每 1 小時 _FIRST_DELAY_SEC = 90 # 啟動後等 90s (等 Prometheus 就緒) _HTTP_TIMEOUT_SEC = 10 _LOOP_BACKOFF_SEC = 300 +_PROJECT_ID = "awoooi" _PROM_RULES_ENDPOINT = "/api/v1/rules" @@ -65,11 +66,19 @@ async def run_rule_catalog_sync_loop() -> None: while True: try: - await sync_once() + stats = await sync_once() except Exception as e: logger.exception("rule_catalog_sync_loop_error", error=str(e)) await asyncio.sleep(_LOOP_BACKOFF_SEC) continue + if stats.get("failed", 0) > 0: + logger.warning( + "rule_catalog_sync_retry_scheduled", + failed=stats["failed"], + retry_seconds=_LOOP_BACKOFF_SEC, + ) + await asyncio.sleep(_LOOP_BACKOFF_SEC) + continue await asyncio.sleep(_SYNC_INTERVAL_SEC) @@ -78,11 +87,11 @@ async def sync_once() -> dict[str, int]: 執行一次 Prometheus → alert_rule_catalog 同步. Returns: - {"total": N, "new": M, "updated": K, "unchanged": L} - 失敗時所有值為 0,error 會寫 aol. + {"total": N, "new": M, "updated": K, "unchanged": L, "failed": F} + 失敗時 failed > 0，error 會寫 AOL 並由排程 bounded retry. """ started_ms = _time.time() - stats = {"total": 0, "new": 0, "updated": 0, "unchanged": 0} + stats = {"total": 0, "new": 0, "updated": 0, "unchanged": 0, "failed": 0} error_msg: str | None = None try: @@ -98,6 +107,8 @@ async def sync_once() -> dict[str, int]: stats["unchanged"] += 1 except Exception as e: error_msg = f"{type(e).__name__}: {e}"[:1000] + processed = stats["new"] + stats["updated"] + stats["unchanged"] + stats["failed"] = max(1, stats["total"] - processed) logger.exception("rule_catalog_sync_once_failed", error=error_msg) duration_ms = int((_time.time() - started_ms) * 1000) @@ -159,7 +170,7 @@ async def _fetch_prometheus_rules() -> list[dict[str, Any]]: def _parse_duration(d: Any) -> int: """Prometheus 的 duration 可能是 秒 (int/float) 或 '5m' 字串.""" - if isinstance(d, (int, float)): + if isinstance(d, int | float): return int(d) if isinstance(d, str) and d: try: @@ -189,10 +200,11 @@ async def _upsert_rule(rule: dict[str, Any]) -> str: 比較 expr/severity/labels/annotations,有變化 → updated,否則 unchanged. """ from sqlalchemy import text as _sql + from src.db.base import get_db_context try: - async with get_db_context() as db: + async with get_db_context(_PROJECT_ID) as db: row = await db.execute( _sql(""" INSERT INTO alert_rule_catalog ( @@ -237,7 +249,7 @@ async def _upsert_rule(rule: dict[str, Any]) -> str: return "new" if inserted else "updated" except Exception as e: logger.warning("rule_upsert_failed", rule_name=rule["rule_name"], error=str(e)) - return "unchanged" + raise async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) -> None: @@ -256,9 +268,10 @@ async def _log_aol(stats: dict[str, int], duration_ms: int, error: str | None) - try: from sqlalchemy import text as _sql + from src.db.base import get_db_context - async with get_db_context() as db: + async with get_db_context(_PROJECT_ID) as db: await db.execute( _sql(""" INSERT INTO automation_operation_log ( diff --git a/apps/api/tests/test_ai_automation_asset_capability_matrix.py b/apps/api/tests/test_ai_automation_asset_capability_matrix.py index 0e9d3185f..8235dbb42 100644 --- a/apps/api/tests/test_ai_automation_asset_capability_matrix.py +++ b/apps/api/tests/test_ai_automation_asset_capability_matrix.py @@ -454,13 +454,24 @@ def test_reconcile_once_publishes_closed_public_matrix_cache( monkeypatch: Any, ) -> None: live_readback_calls: list[dict[str, object]] = [] + settled_at = (datetime.now(UTC) - timedelta(minutes=10)).isoformat() async def _fake_live_readback(**kwargs: object) -> dict[str, object]: live_readback_calls.append(dict(kwargs)) if kwargs["include_live_only_assets"] is True: - return {"live_readback_status": "ready"} + return { + "live_readback_status": "ready", + "matrix_fingerprint": "stable-fingerprint", + "latest_discovery_run": { + "run_id": "discovery-run-1", + "ended_at": settled_at, + }, + } return {"live_readback_status": "ready", "closed": True} + async def _no_wait(_seconds: int) -> None: + return None + async def _fake_persist( _matrix: object, _candidates: object, @@ -481,6 +492,7 @@ def test_reconcile_once_publishes_closed_public_matrix_cache( lambda _matrix: [], ) monkeypatch.setattr(reconciliation_job, "_persist_reconciliation", _fake_persist) + monkeypatch.setattr(reconciliation_job.asyncio, "sleep", _no_wait) result = asyncio.run( reconciliation_job.reconcile_once( @@ -495,6 +507,11 @@ def test_reconcile_once_publishes_closed_public_matrix_cache( "timeout_seconds": 30.0, "include_live_only_assets": True, }, + { + "project_id": "awoooi", + "timeout_seconds": 30.0, + "include_live_only_assets": True, + }, { "project_id": "awoooi", "timeout_seconds": 30.0, @@ -505,6 +522,95 @@ def test_reconcile_once_publishes_closed_public_matrix_cache( assert result["public_cache_refreshed"] is True +def test_reconcile_once_waits_for_settled_discovery_before_write( + monkeypatch: Any, +) -> None: + async def _fake_live_readback(**_kwargs: object) -> dict[str, object]: + return { + "live_readback_status": "ready", + "matrix_fingerprint": "too-fresh", + "latest_discovery_run": { + "run_id": "discovery-run-1", + "ended_at": datetime.now(UTC).isoformat(), + }, + } + + async def _unexpected_persist(*_args: object, **_kwargs: object) -> None: + raise AssertionError("unsettled source must not persist reconciliation") + + monkeypatch.setattr( + reconciliation_job, + "build_asset_capability_matrix_with_live_readback", + _fake_live_readback, + ) + monkeypatch.setattr( + reconciliation_job, + "_persist_reconciliation", + _unexpected_persist, + ) + + result = asyncio.run( + reconciliation_job.reconcile_once( + triggered_by="controlled_replay", + project_id="awoooi", + ) + ) + + assert result["status"] == "source_snapshot_settling" + assert result["reason"] == "latest_discovery_run_not_settled" + assert result["retry_seconds"] >= 60 + assert result["external_runtime_write_performed"] is False + + +def test_reconcile_once_retries_when_source_changes_during_stability_check( + monkeypatch: Any, +) -> None: + fingerprints = iter(("first-fingerprint", "second-fingerprint")) + settled_at = (datetime.now(UTC) - timedelta(minutes=10)).isoformat() + + async def _fake_live_readback(**_kwargs: object) -> dict[str, object]: + return { + "live_readback_status": "ready", + "matrix_fingerprint": next(fingerprints), + "latest_discovery_run": { + "run_id": "discovery-run-1", + "ended_at": settled_at, + }, + } + + async def _no_wait(_seconds: int) -> None: + return None + + async def _unexpected_persist(*_args: object, **_kwargs: object) -> None: + raise AssertionError("unstable source must not persist reconciliation") + + monkeypatch.setattr( + reconciliation_job, + "build_asset_capability_matrix_with_live_readback", + _fake_live_readback, + ) + monkeypatch.setattr(reconciliation_job.asyncio, "sleep", _no_wait) + monkeypatch.setattr( + reconciliation_job, + "_persist_reconciliation", + _unexpected_persist, + ) + + result = asyncio.run( + reconciliation_job.reconcile_once( + triggered_by="controlled_replay", + project_id="awoooi", + ) + ) + + assert result == { + "status": "source_snapshot_unstable", + "reason": "matrix_changed_during_stability_verifier", + "retry_seconds": 60, + "external_runtime_write_performed": False, + } + + def test_reconciliation_loop_retries_degraded_result_before_poll( monkeypatch: Any, ) -> None: @@ -524,9 +630,9 @@ def test_reconciliation_loop_retries_degraded_result_before_poll( monkeypatch.setattr(reconciliation_job, "reconcile_once", _fake_reconcile_once) monkeypatch.setattr(reconciliation_job.asyncio, "sleep", _fake_sleep) - monkeypatch.setattr(reconciliation_job, "_FIRST_DELAY_SECONDS", 90) + monkeypatch.setattr(reconciliation_job, "_FIRST_DELAY_SECONDS", 420) monkeypatch.setattr(reconciliation_job, "_LOOP_BACKOFF_SECONDS", 1_800) - monkeypatch.setattr(reconciliation_job, "_RECONCILIATION_POLL_SECONDS", 1) + monkeypatch.setattr(reconciliation_job, "_RECONCILIATION_POLL_SECONDS", 3_600) try: asyncio.run(reconciliation_job.run_asset_capability_reconciliation_loop()) @@ -536,7 +642,7 @@ def test_reconciliation_loop_retries_degraded_result_before_poll( raise AssertionError("loop should stop at the test cancellation point") assert triggered_by_values == ["startup", "retry"] - assert sleep_values == [90, 1_800, 1] + assert sleep_values == [420, 1_800, 3_600] def test_priority_overlay_projects_aia_p0_006_runtime_controls() -> None: diff --git a/apps/api/tests/test_asset_change_tracker_job.py b/apps/api/tests/test_asset_change_tracker_job.py new file mode 100644 index 000000000..11f473995 --- /dev/null +++ b/apps/api/tests/test_asset_change_tracker_job.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import asyncio +from typing import Any + +import src.jobs.asset_change_tracker_job as tracker + + +class _FakeResult: + def __init__(self, rows: list[tuple[str]] | None = None) -> None: + self._rows = rows or [] + self.rowcount = 0 + + def fetchall(self) -> list[tuple[str]]: + return self._rows + + +class _FakeDb: + async def execute( + self, + statement: Any, + _params: dict[str, Any] | None = None, + ) -> _FakeResult: + if "SELECT run_id FROM asset_discovery_run" in str(statement): + return _FakeResult([("newer-run",), ("older-run",)]) + return _FakeResult() + + +class _FakeDbContext: + async def __aenter__(self) -> _FakeDb: + return _FakeDb() + + async def __aexit__(self, *_args: object) -> None: + return None + + +def test_change_tracker_scopes_every_database_path_to_awoooi(monkeypatch: Any) -> None: + project_ids: list[str] = [] + + def _get_db_context(project_id: str) -> _FakeDbContext: + project_ids.append(project_id) + return _FakeDbContext() + + monkeypatch.setattr("src.db.base.get_db_context", _get_db_context) + + assert asyncio.run(tracker._get_recent_runs()) == ["newer-run", "older-run"] + assert asyncio.run(tracker._diff_runs("newer-run", "older-run")) == { + "added": 0, + "removed": 0, + "modified": 0, + } + asyncio.run(tracker._log_aol({"added": 0, "removed": 0, "modified": 0}, 1, None)) + + assert project_ids == ["awoooi", "awoooi", "awoooi"] diff --git a/apps/api/tests/test_asset_scanner_job.py b/apps/api/tests/test_asset_scanner_job.py index 8ccb262d9..adaefb78b 100644 --- a/apps/api/tests/test_asset_scanner_job.py +++ b/apps/api/tests/test_asset_scanner_job.py @@ -4,6 +4,8 @@ import asyncio import json from pathlib import Path +import pytest + from src.jobs import asset_scanner_job @@ -1547,6 +1549,10 @@ def test_scan_terminal_is_failed_when_any_collector_is_incomplete(monkeypatch) - "RuntimeError: asset_collector_failures=prometheus_targets" ) + with pytest.raises(RuntimeError, match="asset_scan_failed"): + asyncio.run(asset_scanner_job.scan_once(raise_on_failure=True)) + assert terminal["status"] == "failed" + def test_database_catalog_collector_reads_metadata_only(monkeypatch) -> None: requested_projects: list[str | None] = [] diff --git a/apps/api/tests/test_rule_catalog_sync_job.py b/apps/api/tests/test_rule_catalog_sync_job.py new file mode 100644 index 000000000..5c8f182c7 --- /dev/null +++ b/apps/api/tests/test_rule_catalog_sync_job.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import asyncio +from typing import Any + +import src.jobs.rule_catalog_sync_job as rule_sync + + +class _FakeResult: + def one_or_none(self) -> tuple[str, bool]: + return "rule-id", True + + +class _FakeDb: + async def execute( + self, + _statement: Any, + _parameters: dict[str, Any] | None = None, + ) -> _FakeResult: + return _FakeResult() + + +class _FakeDbContext: + async def __aenter__(self) -> _FakeDb: + return _FakeDb() + + async def __aexit__(self, *_args: object) -> None: + return None + + +def _rule(name: str = "ExampleRule") -> dict[str, Any]: + return { + "rule_name": name, + "expr": "vector(1)", + "duration_seconds": 60, + "severity": "warning", + "labels": {}, + "annotations": {}, + "group_name": "example", + } + + +def test_rule_catalog_database_paths_are_tenant_scoped(monkeypatch: Any) -> None: + project_ids: list[str] = [] + + def _get_db_context(project_id: str) -> _FakeDbContext: + project_ids.append(project_id) + return _FakeDbContext() + + monkeypatch.setattr("src.db.base.get_db_context", _get_db_context) + + assert asyncio.run(rule_sync._upsert_rule(_rule())) == "new" + asyncio.run( + rule_sync._log_aol( + {"total": 1, "new": 1, "updated": 0, "unchanged": 0, "failed": 0}, + duration_ms=1, + error=None, + ) + ) + + assert project_ids == ["awoooi", "awoooi"] + + +def test_rule_catalog_failure_is_visible_and_durably_logged(monkeypatch: Any) -> None: + captured: dict[str, Any] = {} + + async def _fetch() -> list[dict[str, Any]]: + return [_rule("RuleA"), _rule("RuleB")] + + async def _fail_upsert(_item: dict[str, Any]) -> str: + raise RuntimeError("database unavailable") + + async def _capture_log( + stats: dict[str, int], + duration_ms: int, + error: str | None, + ) -> None: + captured.update(stats=stats.copy(), duration_ms=duration_ms, error=error) + + monkeypatch.setattr(rule_sync, "_fetch_prometheus_rules", _fetch) + monkeypatch.setattr(rule_sync, "_upsert_rule", _fail_upsert) + monkeypatch.setattr(rule_sync, "_log_aol", _capture_log) + + result = asyncio.run(rule_sync.sync_once()) + + assert result == { + "total": 2, + "new": 0, + "updated": 0, + "unchanged": 0, + "failed": 2, + } + assert str(captured["error"]).startswith("RuntimeError: database unavailable") + assert captured["stats"] == result + + +def test_rule_catalog_loop_retries_failed_sync_before_hourly_wait( + monkeypatch: Any, +) -> None: + sync_count = 0 + sleep_values: list[int] = [] + + async def _sync_once() -> dict[str, int]: + nonlocal sync_count + sync_count += 1 + return { + "total": 1, + "new": 0, + "updated": 0, + "unchanged": 0 if sync_count == 1 else 1, + "failed": 1 if sync_count == 1 else 0, + } + + async def _sleep(seconds: int) -> None: + sleep_values.append(seconds) + if len(sleep_values) == 3: + raise asyncio.CancelledError + + monkeypatch.setattr(rule_sync, "sync_once", _sync_once) + monkeypatch.setattr(rule_sync.asyncio, "sleep", _sleep) + monkeypatch.setattr(rule_sync, "_FIRST_DELAY_SEC", 1) + monkeypatch.setattr(rule_sync, "_LOOP_BACKOFF_SEC", 2) + monkeypatch.setattr(rule_sync, "_SYNC_INTERVAL_SEC", 3) + + try: + asyncio.run(rule_sync.run_rule_catalog_sync_loop()) + except asyncio.CancelledError: + pass + else: + raise AssertionError("loop should stop at the test cancellation point") + + assert sync_count == 2 + assert sleep_values == [1, 2, 3] diff --git a/apps/api/tests/test_signal_worker_ansible_executor_binding.py b/apps/api/tests/test_signal_worker_ansible_executor_binding.py index 7eaf00e8a..086d00987 100644 --- a/apps/api/tests/test_signal_worker_ansible_executor_binding.py +++ b/apps/api/tests/test_signal_worker_ansible_executor_binding.py @@ -93,6 +93,7 @@ def test_worker_manifest_has_no_execution_transport_or_write_loop() -> None: assert worker_env["AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WINDOW_HOURS"] == "168" assert worker_env["DATABASE_NULL_POOL"] == "true" assert worker_env["DATABASE_NULL_POOL_CONCURRENCY_LIMIT"] == "1" + assert worker_env["DATABASE_POOL_TIMEOUT_SECONDS"] == "30" def test_execution_broker_is_only_workload_with_ssh_transport() -> None: diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index c9a589790..33118d88e 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -174,6 +174,10 @@ spec: # NullPool releases each connection after use, while this cap # keeps concurrent background loops inside the worker role budget. value: "1" + - name: DATABASE_POOL_TIMEOUT_SECONDS + # Preserve the one-connection budget while allowing serialized + # background jobs to wait through a bounded long query. + value: "30" - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA From 2563b9c460e13f03973ff7f39f9f30f887efd2c8 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 11:25:00 +0800 Subject: [PATCH 6/9] chore(cd): deploy 6505e2c [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index 6dca50685..d8032c803 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7" + value: "6505e2cede2d9e25114052882a054a1c84eab882" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7" + value: "6505e2cede2d9e25114052882a054a1c84eab882" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 683b639a9..729b79f92 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7" + value: "6505e2cede2d9e25114052882a054a1c84eab882" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index 33118d88e..a56468090 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -181,7 +181,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "e01db7391e89a52958d9c3f06c3218fdc5053eb7" + value: "6505e2cede2d9e25114052882a054a1c84eab882" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index dbf720196..c638d5532 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:a06af69531087ef123b91bcae5f4170f322035156a64e5d09ef8ebdc23ea1c7a +- digest: sha256:d8304318081a6c4267d78cc1bc6b43a88902a17bfccc7e1146989609282453ad name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:9e6ada8cef17243c83230d7890acc1da154add49569b407a590e633d5a5ea70a +- digest: sha256:7891f1fb9c2e6473d3ae102efc6c80ae1be48e1fda96a4de0a0c4d8f6cad4238 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From dd32cbf9cdc7e699f19cf91dcd7782945c4ed475 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 11:27:36 +0800 Subject: [PATCH 7/9] fix(ci): make MCP audit writeback repeatable --- .../mcp-external-artifact-mirror.yaml | 48 +-- .gitea/workflows/mcp-external-replay.yaml | 50 +-- docs/LOGBOOK.md | 9 +- ...tea_workflow_runner_health_2026-06-05.json | 3 + .../tests/test_write_gitea_audit_receipts.py | 290 ++++++++++++++++++ scripts/ci/write-gitea-audit-receipts.sh | 188 ++++++++++++ .../test_external_mcp_artifact_controller.py | 11 +- .../test_external_mcp_replay_controller.py | 9 + 8 files changed, 539 insertions(+), 69 deletions(-) create mode 100644 scripts/ci/tests/test_write_gitea_audit_receipts.py create mode 100755 scripts/ci/write-gitea-audit-receipts.sh diff --git a/.gitea/workflows/mcp-external-artifact-mirror.yaml b/.gitea/workflows/mcp-external-artifact-mirror.yaml index 1fa6be8da..607db7544 100644 --- a/.gitea/workflows/mcp-external-artifact-mirror.yaml +++ b/.gitea/workflows/mcp-external-artifact-mirror.yaml @@ -22,6 +22,8 @@ on: - scripts/security/verify_external_mcp_artifact_receipt.py - scripts/security/tests/test_external_mcp_artifact_controller.py - scripts/security/tests/test_verify_external_mcp_artifact_receipt.py + - scripts/ci/write-gitea-audit-receipts.sh + - scripts/ci/tests/test_write_gitea_audit_receipts.py - .gitea/workflows/mcp-external-artifact-mirror.yaml concurrency: @@ -68,9 +70,11 @@ jobs: scripts/security/external_mcp_artifact_controller.py \ scripts/security/verify_external_mcp_artifact_receipt.py \ scripts/security/tests/test_external_mcp_artifact_controller.py \ - scripts/security/tests/test_verify_external_mcp_artifact_receipt.py + scripts/security/tests/test_verify_external_mcp_artifact_receipt.py \ + scripts/ci/tests/test_write_gitea_audit_receipts.py python3 scripts/security/tests/test_external_mcp_artifact_controller.py python3 scripts/security/tests/test_verify_external_mcp_artifact_receipt.py + python3 scripts/ci/tests/test_write_gitea_audit_receipts.py python3 - <<'PY' import json from pathlib import Path @@ -194,14 +198,11 @@ jobs: set +x test -s "${CONTROLLER_RECEIPT_PATH}" test -s "${VERIFIER_RECEIPT_PATH}" + WRITEBACK_DIR="$(mktemp -d)" + cp "${CONTROLLER_RECEIPT_PATH}" "${WRITEBACK_DIR}/controller.json" + cp "${VERIFIER_RECEIPT_PATH}" "${WRITEBACK_DIR}/verifier.json" git config user.email "mcp-artifact-controller@awoooi.internal" git config user.name "AWOOOI MCP Artifact Controller" - git add "${CONTROLLER_RECEIPT_PATH}" "${VERIFIER_RECEIPT_PATH}" - git diff --cached --quiet && { - echo "receipt_writeback=no_change" - exit 0 - } - git commit -m "chore(mcp): record verified Playwright artifact mirror [skip ci] [metadata-only]" ASKPASS_PATH="$(mktemp)" export ASKPASS_PATH @@ -223,31 +224,16 @@ jobs: PY cleanup_push_auth() { rm -f "${ASKPASS_PATH}" + rm -rf "${WRITEBACK_DIR}" } trap cleanup_push_auth EXIT export GIT_ASKPASS="${ASKPASS_PATH}" export GIT_TERMINAL_PROMPT=0 - git remote remove gitea 2>/dev/null || true - git remote add gitea "${GITEA_SOURCE_URL}" - - for attempt in 1 2 3; do - if git ls-remote --exit-code --heads gitea \ - "refs/heads/${RECEIPT_BRANCH}" >/dev/null 2>&1; then - git fetch --no-tags --depth=100 gitea "${RECEIPT_BRANCH}" - if ! git merge --no-edit FETCH_HEAD; then - git merge --abort || true - echo "BLOCKER receipt_writeback_merge_conflict" - exit 1 - fi - fi - if git push gitea "HEAD:refs/heads/${RECEIPT_BRANCH}"; then - echo "receipt_writeback=verified_audit_branch_normal_push" - echo "receipt_branch=${RECEIPT_BRANCH}" - echo "receipt_commit_sha=$(git rev-parse HEAD)" - exit 0 - fi - echo "receipt_writeback_retry=${attempt}" - sleep 5 - done - echo "BLOCKER receipt_writeback_non_fast_forward_after_bounded_retry" - exit 1 + export GITEA_AUDIT_REMOTE_URL="${GITEA_SOURCE_URL}" + export GITEA_AUDIT_BRANCH="${RECEIPT_BRANCH}" + export GITEA_AUDIT_CONTROLLER_SOURCE="${WRITEBACK_DIR}/controller.json" + export GITEA_AUDIT_VERIFIER_SOURCE="${WRITEBACK_DIR}/verifier.json" + export GITEA_AUDIT_CONTROLLER_TARGET="${CONTROLLER_RECEIPT_PATH}" + export GITEA_AUDIT_VERIFIER_TARGET="${VERIFIER_RECEIPT_PATH}" + export GITEA_AUDIT_COMMIT_MESSAGE="chore(mcp): record verified Playwright artifact mirror [skip ci] [metadata-only]" + bash scripts/ci/write-gitea-audit-receipts.sh diff --git a/.gitea/workflows/mcp-external-replay.yaml b/.gitea/workflows/mcp-external-replay.yaml index 6328f0e84..bf1eff5a7 100644 --- a/.gitea/workflows/mcp-external-replay.yaml +++ b/.gitea/workflows/mcp-external-replay.yaml @@ -22,6 +22,8 @@ on: - scripts/security/verify_external_mcp_replay_receipt.py - scripts/security/tests/test_external_mcp_replay_controller.py - scripts/security/tests/test_verify_external_mcp_replay_receipt.py + - scripts/ci/write-gitea-audit-receipts.sh + - scripts/ci/tests/test_write_gitea_audit_receipts.py - .gitea/workflows/mcp-external-replay.yaml concurrency: @@ -76,9 +78,11 @@ jobs: scripts/security/external_mcp_replay_controller.py \ scripts/security/verify_external_mcp_replay_receipt.py \ scripts/security/tests/test_external_mcp_replay_controller.py \ - scripts/security/tests/test_verify_external_mcp_replay_receipt.py + scripts/security/tests/test_verify_external_mcp_replay_receipt.py \ + scripts/ci/tests/test_write_gitea_audit_receipts.py python3 scripts/security/tests/test_external_mcp_replay_controller.py python3 scripts/security/tests/test_verify_external_mcp_replay_receipt.py + python3 scripts/ci/tests/test_write_gitea_audit_receipts.py python3 - <<'PY' import json from pathlib import Path @@ -267,16 +271,11 @@ jobs: set +x test -s "${REPLAY_CONTROLLER_RECEIPT_PATH}" test -s "${REPLAY_VERIFIER_RECEIPT_PATH}" + WRITEBACK_DIR="$(mktemp -d)" + cp "${REPLAY_CONTROLLER_RECEIPT_PATH}" "${WRITEBACK_DIR}/controller.json" + cp "${REPLAY_VERIFIER_RECEIPT_PATH}" "${WRITEBACK_DIR}/verifier.json" git config user.email "mcp-replay-controller@awoooi.internal" git config user.name "AWOOOI MCP Replay Controller" - git add \ - "${REPLAY_CONTROLLER_RECEIPT_PATH}" \ - "${REPLAY_VERIFIER_RECEIPT_PATH}" - git diff --cached --quiet && { - echo "receipt_writeback=no_change" - exit 0 - } - git commit -m "chore(mcp): record verified Playwright protocol replay [skip ci] [metadata-only]" ASKPASS_PATH="$(mktemp)" export ASKPASS_PATH @@ -298,31 +297,16 @@ jobs: PY cleanup_push_auth() { rm -f "${ASKPASS_PATH}" + rm -rf "${WRITEBACK_DIR}" } trap cleanup_push_auth EXIT export GIT_ASKPASS="${ASKPASS_PATH}" export GIT_TERMINAL_PROMPT=0 - git remote remove gitea 2>/dev/null || true - git remote add gitea "${GITEA_SOURCE_URL}" - - for attempt in 1 2 3; do - if git ls-remote --exit-code --heads gitea \ - "refs/heads/${REPLAY_RECEIPT_BRANCH}" >/dev/null 2>&1; then - git fetch --no-tags --depth=100 gitea "${REPLAY_RECEIPT_BRANCH}" - if ! git merge --no-edit FETCH_HEAD; then - git merge --abort || true - echo "BLOCKER receipt_writeback_merge_conflict" - exit 1 - fi - fi - if git push gitea "HEAD:refs/heads/${REPLAY_RECEIPT_BRANCH}"; then - echo "receipt_writeback=verified_audit_branch_normal_push" - echo "receipt_branch=${REPLAY_RECEIPT_BRANCH}" - echo "receipt_commit_sha=$(git rev-parse HEAD)" - exit 0 - fi - echo "receipt_writeback_retry=${attempt}" - sleep 5 - done - echo "BLOCKER receipt_writeback_non_fast_forward_after_bounded_retry" - exit 1 + export GITEA_AUDIT_REMOTE_URL="${GITEA_SOURCE_URL}" + export GITEA_AUDIT_BRANCH="${REPLAY_RECEIPT_BRANCH}" + export GITEA_AUDIT_CONTROLLER_SOURCE="${WRITEBACK_DIR}/controller.json" + export GITEA_AUDIT_VERIFIER_SOURCE="${WRITEBACK_DIR}/verifier.json" + export GITEA_AUDIT_CONTROLLER_TARGET="${REPLAY_CONTROLLER_RECEIPT_PATH}" + export GITEA_AUDIT_VERIFIER_TARGET="${REPLAY_VERIFIER_RECEIPT_PATH}" + export GITEA_AUDIT_COMMIT_MESSAGE="chore(mcp): record verified Playwright protocol replay [skip ci] [metadata-only]" + bash scripts/ci/write-gitea-audit-receipts.sh diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 98bd573b1..04276b79f 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -10,16 +10,19 @@ **source/test evidence**： - controller/verifier 本機真實雙 replay 皆通過：MCP `Playwright@1.62.0-alpha-1783623505000`、protocol `2025-06-18`、24 tools、tool-name hash `12d23fea9d8d1a1de3f44863dc00cf393f0a2165323838cf93fed30a6a4cc237`、schema hash `c3737ced21147d0512b0400df5885c95a545bc915892b5c8f928cee78428ac99`，兩次均 clean exit/container removed。 -- 新 replay tests `37 passed`；artifact + replay supply-chain tests與 MCP control-plane/federation/version lifecycle/runner-health 聚焦回歸合計 `89 passed`。Ruff、py_compile、JSON/YAML parse、source-control owner guard與 `git diff --check` 通過。 +- artifact/replay controller + independent verifier `61 passed`、MCP control-plane/federation/version lifecycle/runner-health API `27 passed`、shared audit writeback `4 passed`，本輪聚焦回歸合計 `92 passed`。Ruff、py_compile、JSON/YAML parse、source-control owner guard與 `git diff --check` 通過。 +- Gitea replay `#5183` 已在 source `387f841...` 完成 controller + independent verifier，run `d3b8d15e-e5c6-43c0-b657-f9f23089677c`，audit commit `281d6e0058721dfbf84235d85c2f894a004ceea1`；24 tools、offline clean exit、container removal與所有 runtime/write boundary false 均有 durable receipt。 +- 後續 replay `#5186` 在 source `5eb46f9...` 的 controller/verifier 仍成功，但 final writeback 因 shallow main 與 audit branch `unrelated histories` 失敗；scheduled artifact `#5178` 有相同 terminal。根因不是 MCP replay 或 artifact verification，而是舊 workflow 把 audit branch merge 回 depth-1 main checkout。 +- 新增共用 fail-closed audit writeback：直接 checkout audit branch tip、只 stage兩個 allowlisted receipt、同 run identity與 branch/path/schema 綁定、最多 3 次 normal fast-forward push，無 merge、無 force。真實 temporary bare-Git functional tests涵蓋連續第二/第三次線性 push與拒絕 branch/path/identity mismatch，`4 passed`。 - global security mirror guard另抓到 latest main 既有 `apps/web/src/app/[locale]/iwooos/page.tsx` 的 `raw_blocked_waiting_state` 敏感字樣；本 work item未修改該檔，保留為獨立 drift blocker，不以 replay source 綠燈覆蓋。 **尚未完成 / 不誤報**： -- 目前是 source + 本機真實 replay證據；Gitea replay workflow、audit branch receipt、main CD/deploy marker與 production runtime/UI readback尚未產生，因此 production compatibility仍不得標 completed。 +- `#5183` 證明 protocol/schema replay本身與 durable receipt可成立，但最新 source `5eb46f9...` 的 repeatability writeback修正尚未 normal push / Gitea workflow terminal；artifact/replay recurring lanes仍要各取得一筆新的 success terminal，production catalog也尚未讀取該 receipt，因此不得把 compatibility或週期升級 loop標 completed。 - Browser binary/runtime immutable supply chain、public-origin egress proxy與 private-network denial、prompt-injection replay、accessibility/latency shadow、canary、Gateway adapter registration、rollback execution與正式 KM/RAG learning acknowledgement仍是 promotion blocker。 - 未讀 secret value、`.env`、raw session、SQLite/auth；未連線或下載 GitHub/Actions/hosted artifact，未使用 `npx -y`、`@latest`、`:latest`、force push、外部 RAG write或 production mutation。 **下一步**： -- normal push Gitea main並取得 replay workflow controller + standalone verifier audit receipts；再更新 MCP production read model/UI為 compatibility verified但 promotion disabled，之後才建立 immutable browser runtime與隔離 public-origin shadow/canary/rollback lane。 +- normal push shared writeback修正到 Gitea main，要求 artifact與 replay recurring workflow各自取得線性 audit commit；再更新 MCP production read model/UI為 compatibility verified但 promotion disabled，之後才建立 immutable browser runtime與隔離 public-origin shadow/canary/rollback lane。 ## 2026-07-15 — P0 MCP Playwright immutable mirror 與獨立供應鏈 verifier diff --git a/docs/evaluations/gitea_workflow_runner_health_2026-06-05.json b/docs/evaluations/gitea_workflow_runner_health_2026-06-05.json index 9656d8c3c..a3f3b4225 100644 --- a/docs/evaluations/gitea_workflow_runner_health_2026-06-05.json +++ b/docs/evaluations/gitea_workflow_runner_health_2026-06-05.json @@ -30,6 +30,7 @@ "docs/evaluations/ai_agent_version_lifecycle_update_proposal_2026-07-10.json", "scripts/ci/check-gitea-step-env-secrets.js", "scripts/ci/cleanup-host-runner-workspace.sh", + "scripts/ci/write-gitea-audit-receipts.sh", "scripts/ci/wait-host-web-build-pressure.sh", "scripts/ci/notify-awoooi-cicd.sh", "scripts/setup-runner-watchdog.sh", @@ -367,6 +368,7 @@ "config/mcp/playwright-mcp-artifact-policy.json", "scripts/security/external_mcp_artifact_controller.py", "scripts/security/verify_external_mcp_artifact_receipt.py", + "scripts/ci/write-gitea-audit-receipts.sh", "scripts/ci/wait-host-web-build-pressure.sh" ], "next_action": "維持 bounded capacity wait、獨立 verifier 與 mcp-artifact-receipts audit branch；不得啟動 artifact、寫 RAG、切 production route 或推回 deploy main。" @@ -397,6 +399,7 @@ "config/mcp/playwright-mcp-replay-policy.json", "scripts/security/external_mcp_replay_controller.py", "scripts/security/verify_external_mcp_replay_receipt.py", + "scripts/ci/write-gitea-audit-receipts.sh", "scripts/ci/wait-host-web-build-pressure.sh" ], "next_action": "先取得 mcp-replay-receipts 獨立 verifier receipt；browser runtime、public-origin egress proxy、shadow、canary、Gateway adapter registration、RAG 與 production route 仍保持停用。" diff --git a/scripts/ci/tests/test_write_gitea_audit_receipts.py b/scripts/ci/tests/test_write_gitea_audit_receipts.py new file mode 100644 index 000000000..2299b7a8b --- /dev/null +++ b/scripts/ci/tests/test_write_gitea_audit_receipts.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""Functional tests for linear Gitea audit-branch receipt writeback.""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts/ci/write-gitea-audit-receipts.sh" +CONTROLLER_TARGET = ( + "docs/operations/external-mcp-replays/fixture-controller.snapshot.json" +) +VERIFIER_TARGET = ( + "docs/operations/external-mcp-replays/fixture-verifier.snapshot.json" +) + + +def _run(command: list[str], *, cwd: Path, env: dict[str, str] | None = None) -> str: + completed = subprocess.run( + command, + cwd=cwd, + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if completed.returncode != 0: + raise AssertionError( + f"command failed: {command}\nstdout={completed.stdout}\nstderr={completed.stderr}" + ) + return completed.stdout.strip() + + +def _receipt(schema: str, run_id: str) -> dict: + return { + "schema_version": schema, + "run_id": run_id, + "trace_id": f"mcp-replay-{run_id}", + "work_item_id": "MCP-REPLAY-PLAYWRIGHT-001", + "status": "fixture_verified", + } + + +def _write_receipts(directory: Path, run_id: str) -> tuple[Path, Path]: + controller = directory / "controller.json" + verifier = directory / "verifier.json" + controller.write_text( + json.dumps( + _receipt("awoooi_external_mcp_replay_receipt_v1", run_id), + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + verifier.write_text( + json.dumps( + _receipt("awoooi_external_mcp_replay_verifier_receipt_v1", run_id), + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + return controller, verifier + + +class AuditReceiptWritebackTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory() + self.root = Path(self.temp.name) + self.remote = self.root / "remote.git" + self.seed = self.root / "seed" + _run(["git", "init", "--bare", str(self.remote)], cwd=self.root) + self.seed.mkdir() + _run(["git", "init"], cwd=self.seed) + _run(["git", "config", "user.email", "fixture@awoooi.internal"], cwd=self.seed) + _run(["git", "config", "user.name", "Fixture"], cwd=self.seed) + (self.seed / "base.txt").write_text("base\n", encoding="utf-8") + _run(["git", "add", "base.txt"], cwd=self.seed) + _run(["git", "commit", "-m", "base"], cwd=self.seed) + _run(["git", "branch", "-M", "main"], cwd=self.seed) + _run(["git", "remote", "add", "origin", f"file://{self.remote}"], cwd=self.seed) + _run(["git", "push", "origin", "main"], cwd=self.seed) + + _run(["git", "checkout", "-b", "mcp-replay-receipts"], cwd=self.seed) + initial_controller, initial_verifier = _write_receipts( + self.seed, "00000000-0000-4000-8000-000000000000" + ) + (self.seed / CONTROLLER_TARGET).parent.mkdir(parents=True) + (self.seed / CONTROLLER_TARGET).write_bytes(initial_controller.read_bytes()) + (self.seed / VERIFIER_TARGET).write_bytes(initial_verifier.read_bytes()) + _run(["git", "add", "docs"], cwd=self.seed) + _run(["git", "commit", "-m", "initial audit receipt"], cwd=self.seed) + _run(["git", "push", "origin", "mcp-replay-receipts"], cwd=self.seed) + self.initial_audit_sha = _run(["git", "rev-parse", "HEAD"], cwd=self.seed) + + _run(["git", "checkout", "main"], cwd=self.seed) + (self.seed / "main-only.txt").write_text("new main\n", encoding="utf-8") + _run(["git", "add", "main-only.txt"], cwd=self.seed) + _run(["git", "commit", "-m", "advance main"], cwd=self.seed) + _run(["git", "push", "origin", "main"], cwd=self.seed) + + def tearDown(self) -> None: + self.temp.cleanup() + + def _fresh_main_clone(self, name: str) -> Path: + clone = self.root / name + _run( + [ + "git", + "clone", + "--depth=1", + "--branch", + "main", + f"file://{self.remote}", + str(clone), + ], + cwd=self.root, + ) + _run(["git", "config", "user.email", "fixture@awoooi.internal"], cwd=clone) + _run(["git", "config", "user.name", "Fixture"], cwd=clone) + return clone + + def _apply(self, clone: Path, run_id: str) -> str: + source_dir = self.root / f"source-{run_id}" + source_dir.mkdir() + controller, verifier = _write_receipts(source_dir, run_id) + env = { + **os.environ, + "GIT_TERMINAL_PROMPT": "0", + "GITEA_AUDIT_TEST_FILE_REMOTE": "1", + "GITEA_AUDIT_REMOTE_URL": f"file://{self.remote}", + "GITEA_AUDIT_BRANCH": "mcp-replay-receipts", + "GITEA_AUDIT_CONTROLLER_SOURCE": str(controller), + "GITEA_AUDIT_VERIFIER_SOURCE": str(verifier), + "GITEA_AUDIT_CONTROLLER_TARGET": CONTROLLER_TARGET, + "GITEA_AUDIT_VERIFIER_TARGET": VERIFIER_TARGET, + "GITEA_AUDIT_COMMIT_MESSAGE": "fixture audit receipt", + } + return _run(["bash", str(SCRIPT)], cwd=clone, env=env) + + def test_second_and_third_writebacks_are_linear_normal_pushes(self) -> None: + first_clone = self._fresh_main_clone("clone-one") + first_output = self._apply( + first_clone, "11111111-1111-4111-8111-111111111111" + ) + self.assertIn("receipt_writeback=verified_audit_branch_normal_push", first_output) + first_sha = _run( + ["git", "--git-dir", str(self.remote), "rev-parse", "mcp-replay-receipts"], + cwd=self.root, + ) + first_parent = _run( + ["git", "--git-dir", str(self.remote), "rev-parse", f"{first_sha}^"], + cwd=self.root, + ) + self.assertEqual(first_parent, self.initial_audit_sha) + + second_clone = self._fresh_main_clone("clone-two") + second_output = self._apply( + second_clone, "22222222-2222-4222-8222-222222222222" + ) + self.assertIn("receipt_writeback=verified_audit_branch_normal_push", second_output) + second_sha = _run( + ["git", "--git-dir", str(self.remote), "rev-parse", "mcp-replay-receipts"], + cwd=self.root, + ) + second_parent = _run( + ["git", "--git-dir", str(self.remote), "rev-parse", f"{second_sha}^"], + cwd=self.root, + ) + self.assertEqual(second_parent, first_sha) + latest = _run( + [ + "git", + "--git-dir", + str(self.remote), + "show", + f"mcp-replay-receipts:{CONTROLLER_TARGET}", + ], + cwd=self.root, + ) + self.assertEqual( + json.loads(latest)["run_id"], + "22222222-2222-4222-8222-222222222222", + ) + + def test_non_allowlisted_branch_is_rejected_without_push(self) -> None: + clone = self._fresh_main_clone("clone-rejected") + source_dir = self.root / "source-rejected" + source_dir.mkdir() + controller, verifier = _write_receipts( + source_dir, "33333333-3333-4333-8333-333333333333" + ) + env = { + **os.environ, + "GITEA_AUDIT_TEST_FILE_REMOTE": "1", + "GITEA_AUDIT_REMOTE_URL": f"file://{self.remote}", + "GITEA_AUDIT_BRANCH": "main", + "GITEA_AUDIT_CONTROLLER_SOURCE": str(controller), + "GITEA_AUDIT_VERIFIER_SOURCE": str(verifier), + "GITEA_AUDIT_CONTROLLER_TARGET": CONTROLLER_TARGET, + "GITEA_AUDIT_VERIFIER_TARGET": VERIFIER_TARGET, + "GITEA_AUDIT_COMMIT_MESSAGE": "must not commit", + } + completed = subprocess.run( + ["bash", str(SCRIPT)], + cwd=clone, + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("audit_writeback_branch_not_allowlisted", completed.stdout) + + def test_branch_and_receipt_directory_must_match(self) -> None: + clone = self._fresh_main_clone("clone-mismatched-target") + source_dir = self.root / "source-mismatched-target" + source_dir.mkdir() + controller, verifier = _write_receipts( + source_dir, "44444444-4444-4444-8444-444444444444" + ) + env = { + **os.environ, + "GITEA_AUDIT_TEST_FILE_REMOTE": "1", + "GITEA_AUDIT_REMOTE_URL": f"file://{self.remote}", + "GITEA_AUDIT_BRANCH": "mcp-artifact-receipts", + "GITEA_AUDIT_CONTROLLER_SOURCE": str(controller), + "GITEA_AUDIT_VERIFIER_SOURCE": str(verifier), + "GITEA_AUDIT_CONTROLLER_TARGET": CONTROLLER_TARGET, + "GITEA_AUDIT_VERIFIER_TARGET": VERIFIER_TARGET, + "GITEA_AUDIT_COMMIT_MESSAGE": "must not commit", + } + completed = subprocess.run( + ["bash", str(SCRIPT)], + cwd=clone, + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("audit_writeback_branch_target_mismatch", completed.stdout) + + def test_controller_and_verifier_identity_must_match(self) -> None: + clone = self._fresh_main_clone("clone-mismatched-identity") + source_dir = self.root / "source-mismatched-identity" + source_dir.mkdir() + controller, verifier = _write_receipts( + source_dir, "55555555-5555-4555-8555-555555555555" + ) + verifier_payload = json.loads(verifier.read_text(encoding="utf-8")) + verifier_payload["run_id"] = "66666666-6666-4666-8666-666666666666" + verifier.write_text( + json.dumps(verifier_payload, sort_keys=True) + "\n", encoding="utf-8" + ) + env = { + **os.environ, + "GITEA_AUDIT_TEST_FILE_REMOTE": "1", + "GITEA_AUDIT_REMOTE_URL": f"file://{self.remote}", + "GITEA_AUDIT_BRANCH": "mcp-replay-receipts", + "GITEA_AUDIT_CONTROLLER_SOURCE": str(controller), + "GITEA_AUDIT_VERIFIER_SOURCE": str(verifier), + "GITEA_AUDIT_CONTROLLER_TARGET": CONTROLLER_TARGET, + "GITEA_AUDIT_VERIFIER_TARGET": VERIFIER_TARGET, + "GITEA_AUDIT_COMMIT_MESSAGE": "must not commit", + } + completed = subprocess.run( + ["bash", str(SCRIPT)], + cwd=clone, + env=env, + check=False, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(completed.returncode, 2) + self.assertIn("audit_writeback_receipt_identity_mismatch", completed.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/write-gitea-audit-receipts.sh b/scripts/ci/write-gitea-audit-receipts.sh new file mode 100755 index 000000000..08c29fd26 --- /dev/null +++ b/scripts/ci/write-gitea-audit-receipts.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +# Append two verified JSON receipts to a dedicated Gitea audit branch without +# merging the shallow workflow checkout. Authentication is supplied by the +# caller through GIT_ASKPASS; this script never accepts a token argument. + +required_env=( + GITEA_AUDIT_REMOTE_URL + GITEA_AUDIT_BRANCH + GITEA_AUDIT_CONTROLLER_SOURCE + GITEA_AUDIT_VERIFIER_SOURCE + GITEA_AUDIT_CONTROLLER_TARGET + GITEA_AUDIT_VERIFIER_TARGET + GITEA_AUDIT_COMMIT_MESSAGE +) +for name in "${required_env[@]}"; do + if [ -z "${!name:-}" ]; then + echo "BLOCKER audit_writeback_missing_${name}" + exit 2 + fi +done + +if [ "${GITEA_AUDIT_REMOTE_URL}" = "http://192.168.0.110:3001/wooo/awoooi.git" ]; then + if [ -z "${GIT_ASKPASS:-}" ] || [ ! -x "${GIT_ASKPASS}" ]; then + echo "BLOCKER audit_writeback_askpass_missing_or_not_executable" + exit 2 + fi +elif [[ "${GITEA_AUDIT_REMOTE_URL}" == file://* ]] && \ + [ "${GITEA_AUDIT_TEST_FILE_REMOTE:-0}" = "1" ]; then + : +else + echo "BLOCKER audit_writeback_remote_not_allowlisted" + exit 2 +fi +if ! [[ "${GITEA_AUDIT_BRANCH}" =~ ^mcp-(artifact|replay)-receipts$ ]]; then + echo "BLOCKER audit_writeback_branch_not_allowlisted" + exit 2 +fi +if ! [[ "${GITEA_AUDIT_CONTROLLER_TARGET}" =~ ^docs/operations/external-mcp-(artifacts|replays)/[A-Za-z0-9._-]+-controller\.snapshot\.json$ ]]; then + echo "BLOCKER audit_writeback_controller_target_not_allowlisted" + exit 2 +fi +if ! [[ "${GITEA_AUDIT_VERIFIER_TARGET}" =~ ^docs/operations/external-mcp-(artifacts|replays)/[A-Za-z0-9._-]+-verifier\.snapshot\.json$ ]]; then + echo "BLOCKER audit_writeback_verifier_target_not_allowlisted" + exit 2 +fi +if [ "$(dirname "${GITEA_AUDIT_CONTROLLER_TARGET}")" != "$(dirname "${GITEA_AUDIT_VERIFIER_TARGET}")" ]; then + echo "BLOCKER audit_writeback_target_directory_mismatch" + exit 2 +fi +case "${GITEA_AUDIT_BRANCH}" in + mcp-artifact-receipts) + expected_target_directory="docs/operations/external-mcp-artifacts" + expected_controller_schema="awoooi_external_mcp_artifact_receipt_v1" + expected_verifier_schema="awoooi_external_mcp_artifact_verifier_receipt_v1" + ;; + mcp-replay-receipts) + expected_target_directory="docs/operations/external-mcp-replays" + expected_controller_schema="awoooi_external_mcp_replay_receipt_v1" + expected_verifier_schema="awoooi_external_mcp_replay_verifier_receipt_v1" + ;; +esac +if [ "$(dirname "${GITEA_AUDIT_CONTROLLER_TARGET}")" != "${expected_target_directory}" ]; then + echo "BLOCKER audit_writeback_branch_target_mismatch" + exit 2 +fi +if [ "${#GITEA_AUDIT_COMMIT_MESSAGE}" -gt 200 ] || \ + [[ "${GITEA_AUDIT_COMMIT_MESSAGE}" == *$'\n'* ]] || \ + [[ "${GITEA_AUDIT_COMMIT_MESSAGE}" == *$'\r'* ]]; then + echo "BLOCKER audit_writeback_commit_message_invalid" + exit 2 +fi + +for source_path in \ + "${GITEA_AUDIT_CONTROLLER_SOURCE}" \ + "${GITEA_AUDIT_VERIFIER_SOURCE}"; do + if [ ! -f "${source_path}" ] || [ -L "${source_path}" ]; then + echo "BLOCKER audit_writeback_source_invalid" + exit 2 + fi + source_size="$(wc -c < "${source_path}" | tr -d ' ')" + if [ "${source_size}" -le 0 ] || [ "${source_size}" -gt 4194304 ]; then + echo "BLOCKER audit_writeback_source_size_invalid" + exit 2 + fi +done + +python3 - \ + "${GITEA_AUDIT_CONTROLLER_SOURCE}" \ + "${GITEA_AUDIT_VERIFIER_SOURCE}" \ + "${expected_controller_schema}" \ + "${expected_verifier_schema}" <<'PY' +import json +import sys +from pathlib import Path + +def fail(message: str) -> None: + print(message) + raise SystemExit(2) + + +try: + receipts = [ + json.loads(Path(value).read_text(encoding="utf-8")) for value in sys.argv[1:3] + ] +except (OSError, UnicodeError, json.JSONDecodeError): + fail("BLOCKER audit_writeback_receipt_json_invalid") + +expected_schemas = sys.argv[3:5] +identity_fields = ("run_id", "trace_id", "work_item_id") +for payload, expected_schema in zip(receipts, expected_schemas): + if not isinstance(payload, dict) or payload.get("schema_version") != expected_schema: + fail("BLOCKER audit_writeback_receipt_schema_invalid") + if any(not str(payload.get(field) or "") for field in identity_fields): + fail("BLOCKER audit_writeback_receipt_identity_missing") +if any(receipts[0][field] != receipts[1][field] for field in identity_fields): + fail("BLOCKER audit_writeback_receipt_identity_mismatch") +PY + +remote_name="gitea-audit" +git remote remove "${remote_name}" >/dev/null 2>&1 || true +git remote add "${remote_name}" "${GITEA_AUDIT_REMOTE_URL}" +rm -f \ + "${GITEA_AUDIT_CONTROLLER_TARGET}" \ + "${GITEA_AUDIT_VERIFIER_TARGET}" + +for attempt in 1 2 3; do + if git ls-remote --exit-code --heads "${remote_name}" \ + "refs/heads/${GITEA_AUDIT_BRANCH}" >/dev/null 2>&1; then + git fetch --no-tags --depth=1 "${remote_name}" "${GITEA_AUDIT_BRANCH}" + git checkout --force --detach FETCH_HEAD + fi + + target_directory="$(dirname "${GITEA_AUDIT_CONTROLLER_TARGET}")" + for ancestor in docs docs/operations "${target_directory}"; do + if [ -L "${ancestor}" ]; then + echo "BLOCKER audit_writeback_target_symlink_rejected" + exit 2 + fi + done + mkdir -p "${target_directory}" + for target_path in \ + "${GITEA_AUDIT_CONTROLLER_TARGET}" \ + "${GITEA_AUDIT_VERIFIER_TARGET}"; do + if [ -L "${target_path}" ]; then + echo "BLOCKER audit_writeback_target_symlink_rejected" + exit 2 + fi + done + install -m 0644 \ + "${GITEA_AUDIT_CONTROLLER_SOURCE}" \ + "${GITEA_AUDIT_CONTROLLER_TARGET}" + install -m 0644 \ + "${GITEA_AUDIT_VERIFIER_SOURCE}" \ + "${GITEA_AUDIT_VERIFIER_TARGET}" + git add \ + "${GITEA_AUDIT_CONTROLLER_TARGET}" \ + "${GITEA_AUDIT_VERIFIER_TARGET}" + + while IFS= read -r staged_path; do + if [ "${staged_path}" != "${GITEA_AUDIT_CONTROLLER_TARGET}" ] && \ + [ "${staged_path}" != "${GITEA_AUDIT_VERIFIER_TARGET}" ]; then + echo "BLOCKER audit_writeback_unexpected_staged_path" + exit 2 + fi + done < <(git diff --cached --name-only) + + if git diff --cached --quiet; then + echo "receipt_writeback=no_change" + echo "receipt_branch=${GITEA_AUDIT_BRANCH}" + echo "receipt_commit_sha=$(git rev-parse HEAD)" + exit 0 + fi + git commit -m "${GITEA_AUDIT_COMMIT_MESSAGE}" + if git push "${remote_name}" "HEAD:refs/heads/${GITEA_AUDIT_BRANCH}"; then + echo "receipt_writeback=verified_audit_branch_normal_push" + echo "receipt_branch=${GITEA_AUDIT_BRANCH}" + echo "receipt_commit_sha=$(git rev-parse HEAD)" + exit 0 + fi + echo "receipt_writeback_retry=${attempt}" + sleep 5 +done + +echo "BLOCKER receipt_writeback_non_fast_forward_after_bounded_retry" +exit 1 diff --git a/scripts/security/tests/test_external_mcp_artifact_controller.py b/scripts/security/tests/test_external_mcp_artifact_controller.py index 273b68ef6..121f8ac37 100644 --- a/scripts/security/tests/test_external_mcp_artifact_controller.py +++ b/scripts/security/tests/test_external_mcp_artifact_controller.py @@ -17,6 +17,7 @@ ROOT = Path(__file__).resolve().parents[3] CONTROLLER_PATH = ROOT / "scripts/security/external_mcp_artifact_controller.py" POLICY_PATH = ROOT / "config/mcp/playwright-mcp-artifact-policy.json" WORKFLOW_PATH = ROOT / ".gitea/workflows/mcp-external-artifact-mirror.yaml" +AUDIT_WRITEBACK_PATH = ROOT / "scripts/ci/write-gitea-audit-receipts.sh" RUN_ID = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" TRACE_ID = f"mcp-artifact-{RUN_ID}" @@ -130,8 +131,14 @@ class ExternalMcpArtifactControllerTest(unittest.TestCase): self.assertIn("docker logout", raw) self.assertIn("verify_external_mcp_artifact_receipt.py", raw) self.assertIn("RECEIPT_BRANCH: mcp-artifact-receipts", raw) - self.assertIn('git merge --no-edit FETCH_HEAD', raw) - self.assertIn('HEAD:refs/heads/${RECEIPT_BRANCH}', raw) + self.assertIn("bash scripts/ci/write-gitea-audit-receipts.sh", raw) + writeback = AUDIT_WRITEBACK_PATH.read_text(encoding="utf-8") + self.assertIn("git checkout --force --detach FETCH_HEAD", writeback) + self.assertNotIn("git merge --no-edit FETCH_HEAD", writeback) + self.assertIn( + 'git push "${remote_name}" "HEAD:refs/heads/${GITEA_AUDIT_BRANCH}"', + writeback, + ) self.assertNotIn("git push gitea HEAD:main", raw) self.assertIn('cron: "13 2 * * 3"', raw) diff --git a/scripts/security/tests/test_external_mcp_replay_controller.py b/scripts/security/tests/test_external_mcp_replay_controller.py index c267a0550..c4d765a88 100644 --- a/scripts/security/tests/test_external_mcp_replay_controller.py +++ b/scripts/security/tests/test_external_mcp_replay_controller.py @@ -19,6 +19,7 @@ CONTROLLER_PATH = ROOT / "scripts/security/external_mcp_replay_controller.py" POLICY_PATH = ROOT / "config/mcp/playwright-mcp-replay-policy.json" ARTIFACT_POLICY_PATH = ROOT / "config/mcp/playwright-mcp-artifact-policy.json" WORKFLOW_PATH = ROOT / ".gitea/workflows/mcp-external-replay.yaml" +AUDIT_WRITEBACK_PATH = ROOT / "scripts/ci/write-gitea-audit-receipts.sh" SPEC = importlib.util.spec_from_file_location("external_mcp_replay_controller", CONTROLLER_PATH) assert SPEC and SPEC.loader @@ -389,6 +390,14 @@ class ReceiptAndWorkflowTests(unittest.TestCase): self.assertIn("wait-host-web-build-pressure.sh", raw) self.assertIn("mcp-replay-receipts", raw) self.assertIn("verify_external_mcp_replay_receipt.py", raw) + self.assertIn("bash scripts/ci/write-gitea-audit-receipts.sh", raw) + writeback = AUDIT_WRITEBACK_PATH.read_text(encoding="utf-8").lower() + self.assertIn("git checkout --force --detach fetch_head", writeback) + self.assertNotIn("git merge --no-edit fetch_head", writeback) + self.assertIn( + 'git push "${remote_name}" "head:refs/heads/${gitea_audit_branch}"', + writeback, + ) if __name__ == "__main__": From bc66cecdb19c6d8e71f17e14df5584253b7bbf0a Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 15 Jul 2026 11:47:32 +0800 Subject: [PATCH 8/9] chore(cd): deploy 7cd0f69 [skip ci] --- k8s/awoooi-prod/06-deployment-api.yaml | 4 ++-- k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 2 +- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index d8032c803..7f194d365 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -160,12 +160,12 @@ spec: - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. - value: "6505e2cede2d9e25114052882a054a1c84eab882" + value: "7cd0f69475c4828c46c249d33d327a01bc5cb941" - name: AWOOOI_DESIRED_API_IMAGE_TAG # 2026-06-30 Codex: CD rewrites this alongside AWOOOI_BUILD_COMMIT_SHA. # Production readback compares runtime image truth against this # GitOps desired tag instead of doing a slow Gitea raw fetch. - value: "6505e2cede2d9e25114052882a054a1c84eab882" + value: "7cd0f69475c4828c46c249d33d327a01bc5cb941" - name: DATABASE_POOL_SIZE # 2026-07-01 Codex: production role `awoooi` currently has a low # connection limit. Keep API pool conservative until DB role diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 729b79f92..98d0a61f1 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -66,7 +66,7 @@ spec: - name: DATABASE_NULL_POOL value: "true" - name: AWOOOI_BUILD_COMMIT_SHA - value: "6505e2cede2d9e25114052882a054a1c84eab882" + value: "7cd0f69475c4828c46c249d33d327a01bc5cb941" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "false" - name: ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index a56468090..6b3791fa7 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -181,7 +181,7 @@ spec: - name: DATABASE_BOOTSTRAP_DDL_ENABLED value: "false" - name: AWOOOI_BUILD_COMMIT_SHA - value: "6505e2cede2d9e25114052882a054a1c84eab882" + value: "7cd0f69475c4828c46c249d33d327a01bc5cb941" - name: ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER value: "true" - name: ENABLE_SECURITY_CONTROL_PLANE_MAINTENANCE_WORKER diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index c638d5532..c5b647daf 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,9 +40,9 @@ resources: # ⚠️ 重要: name 必須與 deployment YAML 中的 image 完全匹配 (含 tag) # newName + newTag 由 CI 透過 kustomize edit set image 注入 images: -- digest: sha256:d8304318081a6c4267d78cc1bc6b43a88902a17bfccc7e1146989609282453ad +- digest: sha256:6da3614f3925161eb9c6015ed4997543c89d3a1c83259935a6e28121f6587f9c name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api -- digest: sha256:7891f1fb9c2e6473d3ae102efc6c80ae1be48e1fda96a4de0a0c4d8f6cad4238 +- digest: sha256:5c1cbb8c330c37349c3b7a60a533672f53d893e6eb565c53112be3af125be527 name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web From 82759e7cd34c174f20a585eeb6c64bc5bb15a63b Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 11:51:30 +0800 Subject: [PATCH 9/9] fix(agent99): reduce SSH performance probe churn --- agent99-bootstrap.ps1 | 10 + agent99-control-plane.ps1 | 355 +++++++++++++++++- agent99-deploy.ps1 | 60 ++- agent99.config.99.example.json | 10 + agent99.config.example.json | 10 + ...nt99_node_exporter_performance_contract.py | 165 ++++++++ 6 files changed, 594 insertions(+), 16 deletions(-) create mode 100644 scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 1ad66b9e6..be020aeec 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -353,13 +353,23 @@ if (-not (Test-Path $configPath)) { "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [] } }, "backupHealth": { diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index e16a0a0f7..68bca2b88 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -634,6 +634,9 @@ function Format-AgentTelegramLegacyText { $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null $load = Get-AgentObjectValue $Data "loadPerCore" $null $mem = Get-AgentObjectValue $Data "memAvailablePercent" $null + $kernelPercpu = Get-AgentObjectValue $Data "kernelPercpuMemoryPercent" $null + $kernelSUnreclaim = Get-AgentObjectValue $Data "kernelSUnreclaimMemoryPercent" $null + $metricSource = Get-AgentObjectValue $Data "metricSource" "unknown" $lines += "事件 Event: 主機效能 / host performance" $lines += "主機 Host: $hostName" if ($reasons -contains "perf_readback_failed" -or ($null -eq $disk -and $null -eq $load -and $null -eq $mem)) { @@ -649,6 +652,12 @@ function Format-AgentTelegramLegacyText { } elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") { $lines += "原因 Reason: CPU load/core=$load。" $lines += "Agent99 動作 Action: 僅使用 allowlisted load reduction；不得任意停止服務。" + } elseif ($reasons -contains "kernel_percpu_memory_critical" -or $reasons -contains "kernel_sunreclaim_memory_critical") { + $lines += "原因 Reason: kernel memory pressure 已達 critical；Percpu=$kernelPercpu percent，SUnreclaim=$kernelSUnreclaim percent。" + $lines += "Agent99 動作 Action: 日常採 Node Exporter 無 SSH 監測；process 診斷限流，並保留受控 kernel remediation 與獨立 verifier。" + } elseif ($reasons -contains "kernel_percpu_memory_warning") { + $lines += "原因 Reason: kernel Percpu memory=$kernelPercpu percent，已達 warning threshold。" + $lines += "Agent99 動作 Action: 持續 Node Exporter readback，SSH 診斷依 cooldown 執行，避免監控本身增加 session churn。" } elseif ($reasons -contains "memory_available_low") { $lines += "原因 Reason: 可用記憶體為 $mem percent。" $lines += "Agent99 動作 Action: 先檢查 memory pressure，再進行 allowlisted remediation。" @@ -660,6 +669,9 @@ function Format-AgentTelegramLegacyText { if ($null -ne $load -and [string]$load -ne "") { $metricParts += "load/core=$load" } if ($null -ne $mem -and [string]$mem -ne "") { $metricParts += "memAvail=$mem percent" } if ($null -ne $disk -and [string]$disk -ne "") { $metricParts += "disk=$disk percent" } + if ($null -ne $kernelPercpu -and [string]$kernelPercpu -ne "") { $metricParts += "kernel Percpu=$kernelPercpu percent" } + if ($null -ne $kernelSUnreclaim -and [string]$kernelSUnreclaim -ne "") { $metricParts += "kernel SUnreclaim=$kernelSUnreclaim percent" } + if ($metricSource -ne "unknown") { $metricParts += "source=$metricSource" } if ($metricParts.Count -gt 0) { $lines += "目前狀態 Current: $($metricParts -join ', ')." } else { @@ -964,22 +976,44 @@ function Get-AgentIncidentCardModel { $load = Get-AgentObjectValue $Data "loadPerCore" $null $memory = Get-AgentObjectValue $Data "memAvailablePercent" $null $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null - $title = if ($reasons -contains "perf_readback_failed") { "$hostName 效能讀回失敗" } else { "$hostName 主機效能異常" } + $kernelPercpu = Get-AgentObjectValue $Data "kernelPercpuMemoryPercent" $null + $kernelSUnreclaim = Get-AgentObjectValue $Data "kernelSUnreclaimMemoryPercent" $null + $metricSource = [string](Get-AgentObjectValue $Data "metricSource" "unknown") + $kernelPressure = [bool](@($reasons | Where-Object { $_ -match '^kernel_' }).Count -gt 0) + $title = if ($reasons -contains "perf_readback_failed") { + "$hostName 效能讀回失敗" + } elseif ($kernelPressure) { + "$hostName kernel memory pressure" + } else { + "$hostName 主機效能異常" + } $target = if ($hostName) { $hostName } else { "managed host" } if ($reasons -contains "perf_readback_failed" -or ($null -eq $load -and $null -eq $memory -and $null -eq $disk)) { $impact = "CPU、記憶體與磁碟數值目前不可信，不能把空白讀回當成主機故障。" $action = "先修復監控 transport/readback，再決定是否需要降載。" $verification = "等待完整三項主機指標重新讀回。" + } elseif ($kernelPressure) { + $impact = "Kernel Percpu 或不可回收記憶體持續偏高，可能壓縮服務可用記憶體；不能誤判為單一 process RSS。" + $action = "日常改用 Node Exporter 無 SSH 監測；process 診斷依 cooldown 限流，修復走受控 kernel remediation。" + $verification = "source=$metricSource；修復後須同時確認 Percpu、SUnreclaim、可用記憶體與 session churn 回落。" } else { $impact = "資源壓力可能降低服務回應速度或可用性。" $action = "只執行 allowlisted 降載或清理，禁止任意停服務或重啟主機。" - $verification = "修復後必須重新讀回 CPU、記憶體與磁碟才可關閉。" + $verification = "source=$metricSource；修復後必須重新讀回 CPU、記憶體與磁碟才可關閉。" + } + $metrics = if ($kernelPressure) { + @( + [pscustomobject]@{ name = "Kernel Percpu"; value = $kernelPercpu; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "Kernel SUnreclaim"; value = $kernelSUnreclaim; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" } + ) + } else { + @( + [pscustomobject]@{ name = "CPU load/core"; value = $load; scale = 2.0; suffix = "" }, + [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "磁碟使用"; value = $disk; scale = 100.0; suffix = "%" } + ) } - $metrics = @( - [pscustomobject]@{ name = "CPU load/core"; value = $load; scale = 2.0; suffix = "" }, - [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" }, - [pscustomobject]@{ name = "磁碟使用"; value = $disk; scale = 100.0; suffix = "%" } - ) $visualKind = "performance" } elseif ($EventType -match "^backup_") { $age = Get-AgentObjectValue $Data "ageMinutes" $null @@ -2730,12 +2764,214 @@ function Convert-AgentDouble { } } +function Get-AgentNodeExporterSource { + param([string]$TargetHost) + + $performance = $Config.performance + if ( + -not $performance -or + -not $performance.PSObject.Properties["nodeExporterUrls"] -or + -not $performance.nodeExporterUrls.PSObject.Properties[$TargetHost] + ) { + return [pscustomobject]@{ + configured = $false + valid = $false + url = $null + timeoutSeconds = 0 + reason = "node_exporter_not_configured" + } + } + + $rawUrl = [string]$performance.nodeExporterUrls.PSObject.Properties[$TargetHost].Value + $uri = $null + $validUri = [Uri]::TryCreate($rawUrl, [UriKind]::Absolute, [ref]$uri) + $valid = [bool]( + $validUri -and + $uri.Scheme -eq "http" -and + $uri.Host -eq $TargetHost -and + $uri.Port -eq 9100 -and + $uri.AbsolutePath -eq "/metrics" -and + -not $uri.UserInfo -and + -not $uri.Query -and + -not $uri.Fragment + ) + $timeoutSeconds = 8 + if ($performance.PSObject.Properties["nodeExporterTimeoutSeconds"]) { + $timeoutSeconds = [math]::Min(15, [math]::Max(2, [int]$performance.nodeExporterTimeoutSeconds)) + } + [pscustomobject]@{ + configured = $true + valid = $valid + url = if ($valid) { $uri.AbsoluteUri } else { $null } + timeoutSeconds = $timeoutSeconds + reason = if ($valid) { "ready" } else { "node_exporter_url_policy_rejected" } + } +} + +function Invoke-AgentNodeExporterPerformanceReadback { + param([string]$TargetHost) + + $source = Get-AgentNodeExporterSource $TargetHost + if (-not $source.configured -or -not $source.valid) { + return [pscustomobject]@{ + configured = [bool]$source.configured + ok = $false + route = "node_exporter_http" + reason = $source.reason + output = "" + elapsedMs = 0 + httpStatus = $null + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + try { + $response = Invoke-WebRequest -Method Get -Uri $source.url -UseBasicParsing -TimeoutSec $source.timeoutSeconds + $stopwatch.Stop() + $httpStatus = [int]$response.StatusCode + $content = [string]$response.Content + if ($httpStatus -ne 200 -or -not $content) { + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_http_not_ready" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $load1 = $null + $memAvailableBytes = $null + $memTotalBytes = $null + $kernelPercpuBytes = $null + $kernelSUnreclaimBytes = $null + $rootAvailableBytes = $null + $rootSizeBytes = $null + $cpuIds = @{} + foreach ($line in @($content -split "`r?`n")) { + if ($null -eq $load1 -and $line -match '^node_load1\s+([^\s]+)\s*$') { + $load1 = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $memAvailableBytes -and $line -match '^node_memory_MemAvailable_bytes\s+([^\s]+)\s*$') { + $memAvailableBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $memTotalBytes -and $line -match '^node_memory_MemTotal_bytes\s+([^\s]+)\s*$') { + $memTotalBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $kernelPercpuBytes -and $line -match '^node_memory_Percpu_bytes\s+([^\s]+)\s*$') { + $kernelPercpuBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($null -eq $kernelSUnreclaimBytes -and $line -match '^node_memory_SUnreclaim_bytes\s+([^\s]+)\s*$') { + $kernelSUnreclaimBytes = Convert-AgentDouble $Matches[1] + continue + } + if ($line -match '^node_cpu_seconds_total\{([^}]*)\}\s+[^\s]+\s*$') { + $labels = $Matches[1] + if ($labels -match '(^|,)mode="idle"(,|$)' -and $labels -match '(^|,)cpu="([^"]+)"(,|$)') { + $cpuIds[$Matches[2]] = $true + } + continue + } + if ($line -match '^node_filesystem_(avail|size)_bytes\{([^}]*)\}\s+([^\s]+)\s*$') { + $metricKind = $Matches[1] + $labels = $Matches[2] + $metricValue = Convert-AgentDouble $Matches[3] + if ($labels -match '(^|,)mountpoint="/"(,|$)') { + if ($metricKind -eq "avail" -and $null -eq $rootAvailableBytes) { $rootAvailableBytes = $metricValue } + if ($metricKind -eq "size" -and $null -eq $rootSizeBytes) { $rootSizeBytes = $metricValue } + } + } + } + + $cores = $cpuIds.Count + if ( + $cores -le 0 -or + $null -eq $load1 -or + $null -eq $memAvailableBytes -or + $null -eq $memTotalBytes -or + $memTotalBytes -le 0 -or + $null -eq $rootAvailableBytes -or + $null -eq $rootSizeBytes -or + $rootSizeBytes -le 0 + ) { + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_required_metrics_missing" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } + + $memAvailablePercent = [math]::Round(($memAvailableBytes / $memTotalBytes) * 100, 2) + $kernelPercpuMemoryPercent = if ($null -ne $kernelPercpuBytes) { + [math]::Round(($kernelPercpuBytes / $memTotalBytes) * 100, 2) + } else { $null } + $kernelSUnreclaimMemoryPercent = if ($null -ne $kernelSUnreclaimBytes) { + [math]::Round(($kernelSUnreclaimBytes / $memTotalBytes) * 100, 2) + } else { $null } + $diskUsedPercent = [math]::Round((1 - ($rootAvailableBytes / $rootSizeBytes)) * 100, 0) + $diskUsedPercent = [math]::Min(100, [math]::Max(0, $diskUsedPercent)) + $canonical = "perf_schema=agent99_perf_readback_v1 os=linux cores=$cores load1=$load1 mem_available_percent=$memAvailablePercent disk_used_percent=$diskUsedPercent" + return [pscustomobject]@{ + configured = $true + ok = $true + exitCode = 0 + route = "node_exporter_http" + reason = "verified" + output = $canonical + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $httpStatus + sourceUrl = $source.url + kernelPercpuMemoryPercent = $kernelPercpuMemoryPercent + kernelSUnreclaimMemoryPercent = $kernelSUnreclaimMemoryPercent + kernelMemoryMetricsAvailable = [bool]($null -ne $kernelPercpuMemoryPercent -and $null -ne $kernelSUnreclaimMemoryPercent) + transportSerialized = $false + transportLockWaitMs = 0 + } + } catch { + $stopwatch.Stop() + return [pscustomobject]@{ + configured = $true + ok = $false + route = "node_exporter_http" + reason = "node_exporter_transport_failed" + output = "" + elapsedMs = $stopwatch.ElapsedMilliseconds + httpStatus = $null + sourceUrl = $source.url + transportSerialized = $false + transportLockWaitMs = 0 + } + } +} + function Get-PerfThresholds { $perf = $Config.performance [pscustomobject]@{ loadPerCoreWarning = if ($perf.loadPerCoreWarning) { [double]$perf.loadPerCoreWarning } else { 0.9 } loadPerCoreCritical = if ($perf.loadPerCoreCritical) { [double]$perf.loadPerCoreCritical } else { 1.5 } memoryAvailablePercentCritical = if ($perf.memoryAvailablePercentCritical) { [double]$perf.memoryAvailablePercentCritical } else { 10 } + kernelPercpuMemoryPercentWarning = if ($perf.kernelPercpuMemoryPercentWarning) { [double]$perf.kernelPercpuMemoryPercentWarning } else { 10 } + kernelPercpuMemoryPercentCritical = if ($perf.kernelPercpuMemoryPercentCritical) { [double]$perf.kernelPercpuMemoryPercentCritical } else { 20 } + kernelSUnreclaimMemoryPercentCritical = if ($perf.kernelSUnreclaimMemoryPercentCritical) { [double]$perf.kernelSUnreclaimMemoryPercentCritical } else { 25 } diskUsedPercentWarning = if ($perf.diskUsedPercentWarning) { [double]$perf.diskUsedPercentWarning } else { 90 } diskUsedPercentCritical = if ($perf.diskUsedPercentCritical) { [double]$perf.diskUsedPercentCritical } else { 95 } } @@ -2754,6 +2990,17 @@ function Get-HostPerformance { $readRetries = [int]$Config.performance.readRetries } $results = @() + $previousPerfEvidence = Get-AgentLatestEvidence "agent99-Perf-*.json" + $previousPerfRows = if ( + $previousPerfEvidence.exists -and + -not $previousPerfEvidence.parseError -and + $previousPerfEvidence.data -and + $previousPerfEvidence.data.PSObject.Properties["performance"] + ) { @($previousPerfEvidence.data.performance) } else { @() } + $diagnosticCooldownMinutes = 15 + if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuDiagnosticCooldownMinutes"]) { + $diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5, [int]$Config.performance.topCpuDiagnosticCooldownMinutes)) + } $command = @' echo __AGENT99_PERF__ echo HOST=$(hostname) @@ -2772,7 +3019,16 @@ df -P / 2>/dev/null } $slowRetryUsed = $false $slowTimeoutSeconds = $null - $readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries + $nodeExporterReadback = Invoke-AgentNodeExporterPerformanceReadback $hostIp + $metricsFallbackReason = $null + if ($nodeExporterReadback.ok) { + $readback = $nodeExporterReadback + } else { + if ($nodeExporterReadback.configured) { + $metricsFallbackReason = [string]$nodeExporterReadback.reason + } + $readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries + } if (-not $readback.ok -and $readback.exitCode -eq -2) { $slowTimeoutSeconds = 90 if ($Config.performance -and $Config.performance.PSObject.Properties["slowReadTimeoutSeconds"]) { @@ -2795,6 +3051,8 @@ df -P / 2>/dev/null $load1 = $null $memAvailablePercent = $null $diskUsedPercent = $null + $kernelPercpuMemoryPercent = $null + $kernelSUnreclaimMemoryPercent = $null $canonicalPerfMatch = [regex]::Match( $text, "(?m)^perf_schema=agent99_perf_readback_v1\s+os=(?\S+)\s+cores=(?\d+)\s+load1=(?[0-9]+(?:[\.,][0-9]+)?)\s+mem_available_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s+disk_used_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s*$" @@ -2834,8 +3092,27 @@ df -P / 2>/dev/null if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value } } } + if ($nodeExporterReadback.ok) { + if ($nodeExporterReadback.PSObject.Properties["kernelPercpuMemoryPercent"]) { + $kernelPercpuMemoryPercent = $nodeExporterReadback.kernelPercpuMemoryPercent + } + if ($nodeExporterReadback.PSObject.Properties["kernelSUnreclaimMemoryPercent"]) { + $kernelSUnreclaimMemoryPercent = $nodeExporterReadback.kernelSUnreclaimMemoryPercent + } + } $topCpu = @() + $diagnosticStatus = "not_required" + $diagnosticKind = "none" + $diagnosticLastAttemptAt = $null + $diagnosticCapturedAt = $null + $previousPerfRow = $previousPerfRows | Where-Object { [string]$_.host -eq $hostIp } | Select-Object -First 1 + if ($previousPerfRow -and $previousPerfRow.PSObject.Properties["diagnosticLastAttemptAt"]) { + $diagnosticLastAttemptAt = [string]$previousPerfRow.diagnosticLastAttemptAt + } + if ($previousPerfRow -and $previousPerfRow.PSObject.Properties["diagnosticCapturedAt"]) { + $diagnosticCapturedAt = [string]$previousPerfRow.diagnosticCapturedAt + } $loadPerCore = if ($null -ne $load1 -and $cores -gt 0) { [math]::Round($load1 / $cores, 2) } else { $null } $severity = "ok" @@ -2855,6 +3132,17 @@ df -P / 2>/dev/null $severity = "critical" $reasons += "memory_available_low" } + if ($null -ne $kernelPercpuMemoryPercent -and $kernelPercpuMemoryPercent -ge $thresholds.kernelPercpuMemoryPercentCritical) { + $severity = "critical" + $reasons += "kernel_percpu_memory_critical" + } elseif ($null -ne $kernelPercpuMemoryPercent -and $kernelPercpuMemoryPercent -ge $thresholds.kernelPercpuMemoryPercentWarning) { + if ($severity -ne "critical") { $severity = "warning" } + $reasons += "kernel_percpu_memory_warning" + } + if ($null -ne $kernelSUnreclaimMemoryPercent -and $kernelSUnreclaimMemoryPercent -ge $thresholds.kernelSUnreclaimMemoryPercentCritical) { + $severity = "critical" + $reasons += "kernel_sunreclaim_memory_critical" + } if ($null -ne $diskUsedPercent -and $diskUsedPercent -ge $thresholds.diskUsedPercentCritical) { $severity = "critical" $reasons += "disk_used_high" @@ -2871,13 +3159,38 @@ df -P / 2>/dev/null if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuTimeoutSeconds"]) { $topCpuTimeoutSeconds = [int]$Config.performance.topCpuTimeoutSeconds } - $shouldCaptureTopCpu = ($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning) + $processDiagnosticReason = [bool](@($reasons | Where-Object { $_ -match '^(load_per_core|memory_available|kernel_)' }).Count -gt 0) + $shouldCaptureTopCpu = $processDiagnosticReason -and (($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning)) if ($shouldCaptureTopCpu -and $readback.ok) { - $topReadback = Invoke-HostSshText $hostIp "ps -eo pid,ppid,comm,pcpu,pmem --sort=-pcpu" $topCpuTimeoutSeconds 1 - if ($topReadback.ok) { - $topCpu = @($topReadback.output -split "`n" | Select-Object -First 8) + $diagnosticDue = $true + if ($diagnosticLastAttemptAt) { + try { + $lastAttempt = [DateTimeOffset]::Parse($diagnosticLastAttemptAt) + $diagnosticDue = [bool](([DateTimeOffset]::Now - $lastAttempt).TotalMinutes -ge $diagnosticCooldownMinutes) + } catch { + $diagnosticDue = $true + } + } + if ($diagnosticDue) { + $diagnosticNow = [DateTimeOffset]::Now.ToString("o") + $diagnosticLastAttemptAt = $diagnosticNow + $diagnosticKind = if (@($reasons | Where-Object { $_ -match '^(memory_available|kernel_)' }).Count -gt 0) { "top_rss" } else { "top_cpu" } + $diagnosticCommand = if ($diagnosticKind -eq "top_rss") { + "ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-rss" + } else { + "ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-pcpu" + } + $topReadback = Invoke-HostSshText $hostIp $diagnosticCommand $topCpuTimeoutSeconds 1 + if ($topReadback.ok) { + $topCpu = @($topReadback.output -split "`n" | Select-Object -First 8) + $diagnosticCapturedAt = $diagnosticNow + $diagnosticStatus = "captured" + } else { + $diagnosticStatus = "readback_failed_backoff_active" + } } else { - $topCpu = @("top_cpu_readback_failed: $($topReadback.output)") + $diagnosticStatus = "cooldown_suppressed" + $diagnosticKind = if (@($reasons | Where-Object { $_ -match '^(memory_available|kernel_)' }).Count -gt 0) { "top_rss" } else { "top_cpu" } } } @@ -2892,7 +3205,19 @@ df -P / 2>/dev/null loadPerCore = $loadPerCore memAvailablePercent = $memAvailablePercent diskUsedPercent = $diskUsedPercent + kernelPercpuMemoryPercent = $kernelPercpuMemoryPercent + kernelSUnreclaimMemoryPercent = $kernelSUnreclaimMemoryPercent + kernelMemoryMetricsAvailable = [bool]($null -ne $kernelPercpuMemoryPercent -and $null -ne $kernelSUnreclaimMemoryPercent) topCpu = $topCpu + metricSource = if ($nodeExporterReadback.ok) { "node_exporter" } elseif ($nodeExporterReadback.configured) { "ssh_fallback" } else { "ssh" } + metricSourceUrl = if ($nodeExporterReadback.ok) { $nodeExporterReadback.sourceUrl } else { $null } + metricSourceHttpStatus = if ($nodeExporterReadback.ok) { $nodeExporterReadback.httpStatus } else { $null } + metricsFallbackReason = $metricsFallbackReason + diagnosticStatus = $diagnosticStatus + diagnosticKind = $diagnosticKind + diagnosticLastAttemptAt = $diagnosticLastAttemptAt + diagnosticCapturedAt = $diagnosticCapturedAt + diagnosticCooldownMinutes = $diagnosticCooldownMinutes output = $text readTimeoutSeconds = $hostTimeoutSeconds slowReadTimeoutSeconds = $slowTimeoutSeconds @@ -2901,9 +3226,9 @@ df -P / 2>/dev/null transportSerialized = if ($readback.PSObject.Properties["transportSerialized"]) { [bool]$readback.transportSerialized } else { $false } transportLockWaitMs = if ($readback.PSObject.Properties["transportLockWaitMs"]) { [long]$readback.transportLockWaitMs } else { $null } } - Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route) elapsedMs=$($readback.elapsedMs) transportSerialized=$($result.transportSerialized) transportLockWaitMs=$($result.transportLockWaitMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" + Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent kernelPercpuPct=$kernelPercpuMemoryPercent kernelSUnreclaimPct=$kernelSUnreclaimMemoryPercent route=$($readback.route) metricSource=$($result.metricSource) metricsFallback=$metricsFallbackReason diagnostic=$diagnosticStatus elapsedMs=$($readback.elapsedMs) transportSerialized=$($result.transportSerialized) transportLockWaitMs=$($result.transportLockWaitMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" if ($severity -in @("warning", "critical") -and -not $SuppressAlerts) { - Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent reasons=$($reasons -join ',')" $result -Alert + Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent kernelPercpuPct=$kernelPercpuMemoryPercent kernelSUnreclaimPct=$kernelSUnreclaimMemoryPercent reasons=$($reasons -join ',')" $result -Alert } $results += $result } diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 3998fc0f4..5402d50b8 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -515,6 +515,44 @@ try { Add-DefaultProperty $config "performance" ([pscustomobject]@{}) Add-DefaultProperty $config.performance "processCpuHostPercentWarning" 25 Add-DefaultProperty $config.performance "processCpuHostPercentCritical" 50 + Add-DefaultProperty $config.performance "nodeExporterTimeoutSeconds" ([int]$policyConfig.performance.nodeExporterTimeoutSeconds) + Add-DefaultProperty $config.performance "nodeExporterUrls" ($policyConfig.performance.nodeExporterUrls | ConvertTo-Json -Depth 5 | ConvertFrom-Json) + Add-DefaultProperty $config.performance "topCpuDiagnosticCooldownMinutes" ([int]$policyConfig.performance.topCpuDiagnosticCooldownMinutes) + Add-DefaultProperty $config.performance "kernelPercpuMemoryPercentWarning" ([int]$policyConfig.performance.kernelPercpuMemoryPercentWarning) + Add-DefaultProperty $config.performance "kernelPercpuMemoryPercentCritical" ([int]$policyConfig.performance.kernelPercpuMemoryPercentCritical) + Add-DefaultProperty $config.performance "kernelSUnreclaimMemoryPercentCritical" ([int]$policyConfig.performance.kernelSUnreclaimMemoryPercentCritical) + foreach ($nodeExporterHost in @("192.168.0.110", "192.168.0.112", "192.168.0.188")) { + $expectedNodeExporterUrl = [string]$policyConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value + $currentNodeExporterUrl = if ($config.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost]) { + [string]$config.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value + } else { "" } + if ($currentNodeExporterUrl -ne $expectedNodeExporterUrl) { + Set-AgentProperty $config.performance.nodeExporterUrls $nodeExporterHost $expectedNodeExporterUrl + $script:ConfigMigrations += [pscustomobject]@{ + name = "node_exporter_performance_route_$($nodeExporterHost.Replace('.', '_'))" + from = $currentNodeExporterUrl + to = $expectedNodeExporterUrl + } + } + } + foreach ($performancePolicyName in @( + "nodeExporterTimeoutSeconds", + "topCpuDiagnosticCooldownMinutes", + "kernelPercpuMemoryPercentWarning", + "kernelPercpuMemoryPercentCritical", + "kernelSUnreclaimMemoryPercentCritical" + )) { + $expectedPerformanceValue = [int]$policyConfig.performance.PSObject.Properties[$performancePolicyName].Value + $currentPerformanceValue = [int]$config.performance.PSObject.Properties[$performancePolicyName].Value + if ($currentPerformanceValue -ne $expectedPerformanceValue) { + Set-AgentProperty $config.performance $performancePolicyName $expectedPerformanceValue + $script:ConfigMigrations += [pscustomobject]@{ + name = "performance_policy_$performancePolicyName" + from = $currentPerformanceValue + to = $expectedPerformanceValue + } + } + } foreach ($sourceAction in @($policyConfig.performance.loadShedding.actions | Where-Object { $_.PSObject.Properties["policyVersion"] })) { $liveAction = $config.performance.loadShedding.actions | Where-Object { $_.name -eq $sourceAction.name } | Select-Object -First 1 @@ -545,6 +583,25 @@ try { $persistedHosts = @($persistedConfig.hosts | ForEach-Object { [string]$_ }) $persistedSelfHealthHosts = @($persistedConfig.selfHealth.requiredHosts | ForEach-Object { [string]$_ }) $persistedPublicUrls = @($persistedConfig.publicUrls | ForEach-Object { [string]$_ }) + $nodeExporterRoutesReady = $true + foreach ($nodeExporterHost in @("192.168.0.110", "192.168.0.112", "192.168.0.188")) { + $expectedNodeExporterUrl = "http://${nodeExporterHost}:9100/metrics" + if ( + -not $persistedConfig.performance.nodeExporterUrls -or + -not $persistedConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost] -or + [string]$persistedConfig.performance.nodeExporterUrls.PSObject.Properties[$nodeExporterHost].Value -ne $expectedNodeExporterUrl + ) { + $nodeExporterRoutesReady = $false + } + } + $nodeExporterPolicyReady = [bool]( + $nodeExporterRoutesReady -and + [int]$persistedConfig.performance.nodeExporterTimeoutSeconds -eq 8 -and + [int]$persistedConfig.performance.topCpuDiagnosticCooldownMinutes -eq 15 -and + [int]$persistedConfig.performance.kernelPercpuMemoryPercentWarning -eq 10 -and + [int]$persistedConfig.performance.kernelPercpuMemoryPercentCritical -eq 20 -and + [int]$persistedConfig.performance.kernelSUnreclaimMemoryPercentCritical -eq 25 + ) $persistedEdgeServices = if ($persistedConfig.PSObject.Properties["edgeServiceRecovery"]) { $persistedConfig.edgeServiceRecovery } else { $null } $persistedEdgeServicesReady = [bool]( $persistedEdgeServices -and @@ -571,6 +628,7 @@ try { "192.168.0.111" -notin $persistedSelfHealthHosts -or "https://n8n.wooo.work" -notin $persistedPublicUrls -or "https://ollama.wooo.work" -notin $persistedPublicUrls -or + -not $nodeExporterPolicyReady -or -not $persistedEdgeServicesReady -or -not $host111ExternalReady -or $persistedHost112Vm.Count -ne 1 -or @@ -579,7 +637,7 @@ try { $persistedHost112Vmx -ne $canonicalHost112Vmx -or -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) ) { - throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed. Host188 edge recovery config post-verifier also failed." + throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed. Node-exporter performance routes or Host188 edge recovery config post-verifier also failed." } $manifestRows = @() diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 92980ff42..4533279a3 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -233,13 +233,23 @@ "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [ diff --git a/agent99.config.example.json b/agent99.config.example.json index 744ab7088..e8e664216 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -222,13 +222,23 @@ "performance": { "enabled": true, "intervalMinutes": 1, + "nodeExporterTimeoutSeconds": 8, + "nodeExporterUrls": { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics" + }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, "memoryAvailablePercentCritical": 10, + "kernelPercpuMemoryPercentWarning": 10, + "kernelPercpuMemoryPercentCritical": 20, + "kernelSUnreclaimMemoryPercentCritical": 25, "diskUsedPercentWarning": 90, "diskUsedPercentCritical": 95, "captureTopCpuOnWarning": true, "topCpuTimeoutSeconds": 10, + "topCpuDiagnosticCooldownMinutes": 15, "loadShedding": { "enabled": true, "actions": [] diff --git a/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py b/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py new file mode 100644 index 000000000..747518b37 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +CONTROL = ROOT / "agent99-control-plane.ps1" +DEPLOY = ROOT / "agent99-deploy.ps1" +BOOTSTRAP = ROOT / "agent99-bootstrap.ps1" +CONFIGS = ( + ROOT / "agent99.config.example.json", + ROOT / "agent99.config.99.example.json", +) + + +def read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def function(source: str, name: str, next_name: str) -> str: + return source[ + source.index(f"function {name} {{") : source.index( + f"function {next_name} {{" + ) + ] + + +def test_node_exporter_routes_are_explicit_and_identical_in_policy_configs() -> None: + expected = { + "192.168.0.110": "http://192.168.0.110:9100/metrics", + "192.168.0.112": "http://192.168.0.112:9100/metrics", + "192.168.0.188": "http://192.168.0.188:9100/metrics", + } + + for path in CONFIGS: + performance = json.loads(read(path))["performance"] + assert performance["nodeExporterUrls"] == expected + assert performance["nodeExporterTimeoutSeconds"] == 8 + assert performance["topCpuDiagnosticCooldownMinutes"] == 15 + assert performance["kernelPercpuMemoryPercentWarning"] == 10 + assert performance["kernelPercpuMemoryPercentCritical"] == 20 + assert performance["kernelSUnreclaimMemoryPercentCritical"] == 25 + + bootstrap = read(BOOTSTRAP) + for host, url in expected.items(): + assert f'"{host}": "{url}"' in bootstrap + + +def test_node_exporter_transport_is_fixed_to_host_port_path_and_no_credentials() -> None: + source = read(CONTROL) + policy = function( + source, + "Get-AgentNodeExporterSource", + "Invoke-AgentNodeExporterPerformanceReadback", + ) + transport = function( + source, + "Invoke-AgentNodeExporterPerformanceReadback", + "Get-PerfThresholds", + ) + + for requirement in ( + '$uri.Scheme -eq "http"', + '$uri.Host -eq $TargetHost', + '$uri.Port -eq 9100', + '$uri.AbsolutePath -eq "/metrics"', + '-not $uri.UserInfo', + '-not $uri.Query', + '-not $uri.Fragment', + 'node_exporter_url_policy_rejected', + ): + assert requirement in policy + assert "Invoke-WebRequest -Method Get" in transport + assert "-TimeoutSec $source.timeoutSeconds" in transport + assert 'reason = "node_exporter_transport_failed"' in transport + assert "password" not in transport.lower() + assert "token" not in transport.lower() + + +def test_node_exporter_parser_requires_complete_load_memory_cpu_and_root_disk() -> None: + source = read(CONTROL) + transport = function( + source, + "Invoke-AgentNodeExporterPerformanceReadback", + "Get-PerfThresholds", + ) + + for metric in ( + "node_load1", + "node_memory_MemAvailable_bytes", + "node_memory_MemTotal_bytes", + "node_memory_Percpu_bytes", + "node_memory_SUnreclaim_bytes", + "node_cpu_seconds_total", + "node_filesystem_(avail|size)_bytes", + ): + assert metric in transport + assert 'mountpoint="/"' in transport + assert 'reason = "node_exporter_required_metrics_missing"' in transport + assert "perf_schema=agent99_perf_readback_v1" in transport + assert 'route = "node_exporter_http"' in transport + assert "kernelPercpuMemoryPercent" in transport + assert "kernelSUnreclaimMemoryPercent" in transport + assert "kernelMemoryMetricsAvailable" in transport + + +def test_perf_uses_metrics_first_and_ssh_only_as_fallback_or_cooldown_diagnostic() -> None: + source = read(CONTROL) + perf = function(source, "Get-HostPerformance", "Record-PerformanceIssuesWithoutRemediation") + + metrics_at = perf.index("Invoke-AgentNodeExporterPerformanceReadback $hostIp") + fallback_at = perf.index("Invoke-HostSshText $hostIp $command") + assert metrics_at < fallback_at + assert "if ($nodeExporterReadback.ok)" in perf + assert "$metricsFallbackReason = [string]$nodeExporterReadback.reason" in perf + assert 'metricSource = if ($nodeExporterReadback.ok) { "node_exporter" }' in perf + assert 'diagnosticStatus = "cooldown_suppressed"' in perf + assert "topCpuDiagnosticCooldownMinutes" in perf + assert "$diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5" in perf + assert "memory_available_low" in perf + assert "kernel_percpu_memory_warning" in perf + assert "kernel_percpu_memory_critical" in perf + assert "kernel_sunreclaim_memory_critical" in perf + assert "kernelPercpuMemoryPercentWarning" in perf + assert "kernelPercpuMemoryPercentCritical" in perf + assert "kernelSUnreclaimMemoryPercentCritical" in perf + assert '"ps -eo pid,ppid,comm,pcpu,pmem,rss --sort=-rss"' in perf + assert "disk_used_warning" not in perf[ + perf.index("$processDiagnosticReason") : perf.index("$result =") + ] + + +def test_deploy_migrates_and_post_verifies_canonical_metric_routes() -> None: + deploy = read(DEPLOY) + + assert 'Add-DefaultProperty $config.performance "nodeExporterUrls"' in deploy + assert 'Set-AgentProperty $config.performance.nodeExporterUrls' in deploy + assert 'name = "node_exporter_performance_route_' in deploy + assert "$nodeExporterRoutesReady = $true" in deploy + assert "$nodeExporterPolicyReady = [bool](" in deploy + assert "-not $nodeExporterPolicyReady" in deploy + assert "node-exporter performance routes" in deploy.lower() + for policy_name in ( + "kernelPercpuMemoryPercentWarning", + "kernelPercpuMemoryPercentCritical", + "kernelSUnreclaimMemoryPercentCritical", + ): + assert f'Add-DefaultProperty $config.performance "{policy_name}"' in deploy + assert policy_name in deploy[deploy.index("$nodeExporterPolicyReady") :] + + +def test_performance_card_explains_kernel_memory_and_metric_source() -> None: + source = read(CONTROL) + legacy = function(source, "Format-AgentTelegramLegacyText", "Get-AgentSha256Hex") + model = function(source, "Get-AgentIncidentCardModel", "Format-AgentTelegramText") + + for formatter in (legacy, model): + assert "kernel memory pressure" in formatter + assert "Percpu" in formatter + assert "SUnreclaim" in formatter + assert 'Get-AgentObjectValue $Data "metricSource"' in formatter + assert "Node Exporter" in formatter + assert "$metrics = if ($kernelPressure)" in model + assert model.index('name = "Kernel Percpu"') < model.index('name = "CPU load/core"')