feat(observability): close native SigNoz backup restore canary

2026-07-15 08:55:08 +08:00
parent 6c1a12e388
commit 322e812d93
44 changed files with 9209 additions and 842 deletions
--- a/apps/api/tests/test_signoz_canonical_health_route.py
+++ b/apps/api/tests/test_signoz_canonical_health_route.py
@@ -1,5 +1,7 @@
 from __future__ import annotations

+import subprocess
+import sys
 from datetime import UTC, datetime
 from pathlib import Path
 from types import SimpleNamespace
@@ -15,9 +17,15 @@ from src.services import signoz_client as signoz_client_module
 from src.services.host_aggregator import HOST_CONFIGS

 REPO_ROOT = Path(__file__).resolve().parents[3]
+ALERT_CHAIN_SMOKE = REPO_ROOT / "scripts/alert_chain_smoke_test.py"
+PROMETHEUS_ROUTE_DEPLOY = (
+    REPO_ROOT / "scripts/ops/deploy-signoz-prometheus-route.sh"
+)
+LEGACY_PROMETHEUS_DEPLOY = REPO_ROOT / "k8s/monitoring/deploy-prometheus-config.sh"
 INTERNAL_URL = "http://192.168.0.110:8080"
 PUBLIC_URL = "https://signoz.wooo.work"
 LEGACY_URL = "192.168.0.188:3301"
+INTERNAL_HEALTH_URL = f"{INTERNAL_URL}/api/v1/health"


 def _configmap(relative_path: str) -> dict:
@@ -81,7 +89,9 @@ def test_user_facing_deep_links_use_public_route(
 ) -> None:
    monkeypatch.setattr(deep_linking_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL)
    monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL)
-    monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL)
+    monkeypatch.setattr(
+        signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL
+    )

    trace_id = "0af7651916cd43dd8448eb211c80319c"
    assert deep_linking_module.DeepLinking.signoz_trace_url(trace_id) == (
@@ -165,9 +175,176 @@ def test_active_api_source_has_no_legacy_signoz_ui_route() -> None:
    assert offenders == []


+def test_cd_alert_chain_smoke_uses_canonical_public_signoz_route() -> None:
+    text = ALERT_CHAIN_SMOKE.read_text(encoding="utf-8")
+
+    assert '"SIGNOZ_PUBLIC_URL"' in text
+    assert PUBLIC_URL in text
+    assert LEGACY_URL not in text
+
+
 def test_host_asset_map_places_signoz_on_the_canonical_query_host() -> None:
    canonical_services = HOST_CONFIGS["192.168.0.110"]["services"]
    legacy_services = HOST_CONFIGS["192.168.0.188"]["services"]

    assert ("SigNoz", 8080, "http", "/api/v1/health") in canonical_services
    assert not any(service[0] == "SigNoz" for service in legacy_services)
+
+
+def test_monitoring_registry_splits_query_health_from_otlp_transport() -> None:
+    registry = _configmap("ops/monitoring/service-registry.yaml")
+    services = {service["name"]: service for service in registry["services"]}
+
+    query = services["signoz-ui"]
+    assert query["host"] == "192.168.0.110"
+    assert query["port"] == 8080
+    assert query["health_endpoint"] == "/api/v1/health"
+    assert query["health_type"] == "http"
+    assert query["monitoring"]["prometheus"] is True
+    assert query["monitoring"]["prometheus_scrape"] is False
+
+    collector = services["signoz-collector"]
+    assert collector["host"] == "192.168.0.188"
+    assert collector["port"] == 24317
+    assert collector["health_type"] == "grpc"
+    assert collector["monitoring"]["prometheus_scrape"] is False
+    assert collector["monitoring"]["external_verifier"] == (
+        "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
+    )
+
+
+def test_stateful_registry_places_clickhouse_on_canonical_host() -> None:
+    source_registry = _configmap("ops/config/service-registry.yaml")
+    configmap = _configmap("k8s/awoooi-prod/15-service-registry-configmap.yaml")
+    runtime_registry = yaml.safe_load(configmap["data"]["service-registry.yaml"])
+
+    for registry in (source_registry, runtime_registry):
+        services = {service["name"]: service for service in registry["services"]}
+        clickhouse = services["signoz-clickhouse"]
+        assert clickhouse["host"] == "192.168.0.110"
+        assert clickhouse["stateful_level"] == "BLOCK"
+        assert clickhouse["alert_only"] is True
+
+
+def test_active_prometheus_targets_canonical_http_health_not_legacy_tcp() -> None:
+    config = _configmap("k8s/monitoring/prometheus.yml")
+    jobs = {job["job_name"]: job for job in config["scrape_configs"]}
+    http_job = jobs["blackbox-http"]
+    http_targets = http_job["static_configs"][0]["targets"]
+    tcp_targets = jobs["blackbox-tcp"]["static_configs"][0]["targets"]
+
+    assert INTERNAL_HEALTH_URL in http_targets
+    assert LEGACY_URL not in tcp_targets
+    assert http_job["relabel_configs"] == [
+        {
+            "source_labels": ["__address__"],
+            "target_label": "__param_target",
+        },
+        {
+            "source_labels": ["__param_target"],
+            "target_label": "instance",
+        },
+        {
+            "target_label": "__address__",
+            "replacement": "192.168.0.110:9115",
+        },
+    ]
+
+
+def test_signoz_prometheus_route_deploy_is_target_first_and_rollback_safe() -> None:
+    text = PROMETHEUS_ROUTE_DEPLOY.read_text(encoding="utf-8")
+
+    assert 'MODE="${1:-apply}"' in text
+    assert "promtool check config" in text
+    assert "cp -p \"${REMOTE_CONFIG}\" \"${BACKUP_CONFIG}\"" in text
+    assert 'action=target_first_config_apply' in text
+    assert text.index("promtool check config") < text.index(
+        'action=target_first_config_apply'
+    )
+    assert text.index('action=target_first_config_apply') < text.index(
+        "terminal=target_up_two_scrapes"
+    )
+    assert "rollback_deploy" in text
+    assert INTERNAL_HEALTH_URL in text
+    assert LEGACY_URL in text
+
+
+def test_legacy_prometheus_deployer_routes_to_bounded_host110_replacement() -> None:
+    text = LEGACY_PROMETHEUS_DEPLOY.read_text(encoding="utf-8")
+
+    assert "superseded_by=global_product_governance_v2" in text
+    assert "expiry=2026-07-15" in text
+    assert "scripts/ops/deploy-signoz-prometheus-route.sh" in text
+    assert "192.168.0.188" not in text
+
+
+@pytest.mark.parametrize(
+    "relative_path",
+    [
+        "ops/monitoring/alerts-unified.yml",
+        "ops/monitoring/alerts.yml",
+    ],
+)
+def test_signoz_down_alert_is_exact_and_fails_closed_on_missing_series(
+    relative_path: str,
+) -> None:
+    payload = _configmap(relative_path)
+    rules = [
+        rule
+        for group in payload["groups"]
+        for rule in group.get("rules", [])
+        if rule.get("alert") == "SignOzDown"
+    ]
+
+    assert len(rules) == 1
+    rule = rules[0]
+    assert 'job="blackbox-http"' in rule["expr"]
+    assert INTERNAL_HEALTH_URL in rule["expr"]
+    assert "absent(" in rule["expr"]
+    assert LEGACY_URL not in rule["expr"]
+    assert rule["labels"]["host"] == "110"
+    assert rule["labels"]["target_host"] == "192.168.0.110"
+
+
+def test_legacy_188_rule_plane_does_not_duplicate_canonical_signoz_alert() -> None:
+    payload = _configmap("k8s/monitoring/k3s-alerts.yaml")
+    signoz_rules = [
+        rule
+        for group in payload["groups"]
+        for rule in group.get("rules", [])
+        if rule.get("alert") == "SignOzDown"
+    ]
+    text = (REPO_ROOT / "k8s/monitoring/k3s-alerts.yaml").read_text(encoding="utf-8")
+
+    assert signoz_rules == []
+    assert LEGACY_URL not in text
+
+
+def test_monitoring_generator_uses_blackbox_without_scraping_signoz_spa(
+    tmp_path: Path,
+) -> None:
+    subprocess.run(
+        [
+            sys.executable,
+            str(REPO_ROOT / "ops/monitoring/generate_monitoring.py"),
+            "--output-dir",
+            str(tmp_path),
+        ],
+        cwd=REPO_ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    scrape = _configmap(str(tmp_path / "prometheus-scrape-generated.yaml"))
+    blackbox = yaml.safe_load(
+        (tmp_path / "blackbox-targets-generated.yaml").read_text(encoding="utf-8")
+    )
+    generated_jobs = {job["job_name"] for job in scrape["scrape_configs"]}
+    assert "signoz-ui" not in generated_jobs
+    assert "signoz-collector" not in generated_jobs
+    assert any(
+        target["labels"].get("service") == "signoz-ui"
+        and target["url"] == INTERNAL_HEALTH_URL
+        for target in blackbox
+    )
--- a/apps/sensor/agent.py
+++ b/apps/sensor/agent.py
@@ -66,7 +66,6 @@ HOST_CONFIGS: dict[str, dict] = {
            {"name": "Redis",        "host": "127.0.0.1", "port": 6380},
            {"name": "Ollama",       "host": "127.0.0.1", "port": 11434},
            {"name": "Nginx",        "host": "127.0.0.1", "port": 80},
-            {"name": "SigNoz",       "host": "127.0.0.1", "port": 3301},
            # OpenClaw 跑在 K3s (awoooi-prod)，非 188 本機服務，不在此探測
        ],
    },
@@ -77,7 +76,9 @@ HOST_CONFIGS: dict[str, dict] = {
        "services": [
            {"name": "Harbor",       "host": "127.0.0.1", "port": 5000},
            {"name": "Gitea",        "host": "127.0.0.1", "port": 3001},
-            {"name": "GH-Runner",    "host": "127.0.0.1", "port": 8080},
+            # TCP availability for canonical machine health endpoint:
+            # http://192.168.0.110:8080/api/v1/health
+            {"name": "SigNoz",       "host": "127.0.0.1", "port": 8080},
        ],
    },
 }
--- a/apps/web/src/app/[locale]/classic/page.tsx
+++ b/apps/web/src/app/[locale]/classic/page.tsx
@@ -31,6 +31,7 @@ import { HostGrid, type HostInfo, type HostService } from '@/components/infra/ho
 import { AppLayout } from '@/components/layout'

 const API_BASE = process.env.NEXT_PUBLIC_API_URL ?? ''
+const SIGNOZ_PUBLIC_URL = 'https://signoz.wooo.work'

 // =============================================================================
 // Types
@@ -152,7 +153,7 @@ function MonitoringTools() {
        const statusText  = isUp ? (hasFiring ? `${tool.firing_count} ${tDash('monitoringStatus.firing')}` : tDash('monitoringStatus.up')) : tDash('monitoringStatus.down')
        const accentColor = TOOL_ACCENT_COLOR[tool.name] ?? '#b0ad9f'
        const icon = TOOL_ICON[tool.name] ?? <Activity size={16} />
-        const link = safeExternalMonitoringUrl(tool.url)
+        const link = tool.name === 'SigNoz' ? SIGNOZ_PUBLIC_URL : safeExternalMonitoringUrl(tool.url)
        const timeStr = (() => {
          try { return new Date(tool.checked_at).toLocaleTimeString('zh-TW', { timeZone: 'Asia/Taipei', hour: '2-digit', minute: '2-digit' }) }
          catch { return '--' }
@@ -294,6 +295,7 @@ const HOST_CATALOG: Record<string, { services: HostService[]; isK3s?: boolean; r
      { name: 'Langfuse',   healthy: false, port: 3100,  description: 'LLM Tracing' },
      { name: 'Grafana',    healthy: false, port: 3002,  description: '監控面板' },
      { name: 'Prometheus', healthy: false, port: 9090,  description: '告警規則' },
+      { name: 'SigNoz',     healthy: false, port: 8080,  description: 'APM health · /api/v1/health' },
    ],
  },
  'host:kali-readonly': {
@@ -328,7 +330,6 @@ const HOST_CATALOG: Record<string, { services: HostService[]; isK3s?: boolean; r
      { name: 'Redis',      healthy: false, port: 6380,  description: 'Cache' },
      { name: 'Ollama',     healthy: false, port: 11434, description: 'LLM' },
      { name: 'OpenClaw',   healthy: false, port: 8088,  description: 'AI Agent' },
-      { name: 'SigNoz',     healthy: false, port: 3301,  description: 'APM · OTEL' },
    ],
  },
 }
--- a/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx
+++ b/apps/web/src/app/[locale]/governance/tabs/automation-inventory-tab.tsx
@@ -150,12 +150,14 @@ const PUBLIC_TEXT_ENDPOINT_ALIASES: Record<string, string> = {
  '110:3002': 'host:public-gateway/grafana',
  '110:3100': 'host:public-gateway/langfuse',
  '110:5000': 'host:public-gateway/registry',
+  '110:8080': 'host:public-gateway/signoz-health',
  '110:9000': 'host:public-gateway/sentry',
  '112:8080': 'host:kali-readonly/scanner',
  '188:11434': 'host:observability-a/ollama',
  '188:8088': 'host:observability-a/openclaw-legacy',
  '188:8089': 'host:observability-a/openclaw',
-  '188:3301': 'host:observability-a/signoz',
+  // Historical scrub-only alias; never use this legacy route for live access.
+  '188:3301': 'host:observability-a/signoz-legacy',
  '188:5432': 'host:observability-a/postgres',
  '188:6380': 'host:observability-a/redis',
 }
--- a/apps/web/src/components/dashboard/live-dashboard.tsx
+++ b/apps/web/src/components/dashboard/live-dashboard.tsx
@@ -81,6 +81,7 @@ export function LiveDashboard({ locale: _locale }: LiveDashboardProps) {
      services: [
        { name: 'Harbor', status: 'idle', port: 5000 },
        { name: 'GH Runner', status: 'idle' },
+        { name: 'SigNoz', status: 'idle', port: 8080 },
        { name: 'Docker', status: 'idle', port: 2375 },
      ],
    },
@@ -111,7 +112,6 @@ export function LiveDashboard({ locale: _locale }: LiveDashboardProps) {
        { name: 'Redis', status: 'idle', port: 6380 },
        { name: 'Ollama', status: 'idle', port: 11434 },
        { name: 'OpenClaw', status: 'idle', port: 8089 },
-        { name: 'SigNoz', status: 'idle', port: 3301 },
      ],
    },
  }