feat(observability): close native SigNoz backup restore canary

This commit is contained in:
ogt
2026-07-15 08:55:08 +08:00
parent 6c1a12e388
commit 322e812d93
44 changed files with 9209 additions and 842 deletions

View File

@@ -716,21 +716,23 @@ groups:
description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘"
- alert: SignOzDown
expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0
expr: >-
(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"} == 0)
or absent(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"})
for: 2m
labels:
severity: warning
layer: docker-188
layer: docker-110
component: signoz
host: "188"
host: "110"
team: ops
auto_repair: "true"
mcp_provider: "ssh_host"
target_host: "192.168.0.188"
target_host: "192.168.0.110"
alert_category: "devops_tool"
annotations:
summary: "SignOz 服務離線"
description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘"
description: "SignOz (http://192.168.0.110:8080/api/v1/health) 已離線或監控序列缺失超過 2 分鐘"
# ---- 110 Docker 層 ----
- alert: SentryDown

View File

@@ -400,21 +400,23 @@ groups:
description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘"
- alert: SignOzDown
expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0
expr: >-
(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"} == 0)
or absent(probe_success{job="blackbox-http", instance="http://192.168.0.110:8080/api/v1/health"})
for: 2m
labels:
severity: warning
layer: docker-188
layer: docker-110
component: signoz
host: "188"
host: "110"
team: ops
auto_repair: "true"
mcp_provider: "ssh_host"
target_host: "192.168.0.188"
target_host: "192.168.0.110"
alert_category: "devops_tool"
annotations:
summary: "SignOz 服務離線"
description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘"
description: "SignOz (http://192.168.0.110:8080/api/v1/health) 已離線或監控序列缺失超過 2 分鐘"
# ---- 110 Docker 層 ----
- alert: SentryDown

View File

@@ -54,7 +54,10 @@ def generate_prometheus_scrape_configs(registry: dict) -> list[dict]:
# K8s 服務
for svc in registry.get("services", []):
if svc.get("monitoring", {}).get("prometheus"):
monitoring = svc.get("monitoring", {})
if monitoring.get("prometheus") and monitoring.get(
"prometheus_scrape", True
):
if svc.get("type") == "k8s-deployment":
# K8s ServiceMonitor 風格
config = {

View File

@@ -335,6 +335,8 @@ services:
health_type: grpc
monitoring:
prometheus: true
prometheus_scrape: false
external_verifier: scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh
sentry: false
alerts:
- service_down
@@ -345,11 +347,13 @@ services:
# --- SignOz UI ---
- name: signoz-ui
type: docker
host: 192.168.0.188
port: 3301
health_endpoint: /
host: 192.168.0.110
port: 8080
health_endpoint: /api/v1/health
health_type: http
monitoring:
prometheus: true
prometheus_scrape: false
sentry: false
alerts:
- service_down