feat(observability): close native SigNoz backup restore canary

apps/api/tests/test_signoz_canonical_health_route.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import subprocess
+import sys
 from datetime import UTC, datetime
 from pathlib import Path
 from types import SimpleNamespace
@@ -15,9 +17,15 @@ from src.services import signoz_client as signoz_client_module
 from src.services.host_aggregator import HOST_CONFIGS
 
 REPO_ROOT = Path(__file__).resolve().parents[3]
+ALERT_CHAIN_SMOKE = REPO_ROOT / "scripts/alert_chain_smoke_test.py"
+PROMETHEUS_ROUTE_DEPLOY = (
+    REPO_ROOT / "scripts/ops/deploy-signoz-prometheus-route.sh"
+)
+LEGACY_PROMETHEUS_DEPLOY = REPO_ROOT / "k8s/monitoring/deploy-prometheus-config.sh"
 INTERNAL_URL = "http://192.168.0.110:8080"
 PUBLIC_URL = "https://signoz.wooo.work"
 LEGACY_URL = "192.168.0.188:3301"
+INTERNAL_HEALTH_URL = f"{INTERNAL_URL}/api/v1/health"
 
 
 def _configmap(relative_path: str) -> dict:
@@ -81,7 +89,9 @@ def test_user_facing_deep_links_use_public_route(
 ) -> None:
     monkeypatch.setattr(deep_linking_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL)
     monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL)
-    monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL)
+    monkeypatch.setattr(
+        signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL
+    )
 
     trace_id = "0af7651916cd43dd8448eb211c80319c"
     assert deep_linking_module.DeepLinking.signoz_trace_url(trace_id) == (
@@ -165,9 +175,176 @@ def test_active_api_source_has_no_legacy_signoz_ui_route() -> None:
     assert offenders == []
 
 
+def test_cd_alert_chain_smoke_uses_canonical_public_signoz_route() -> None:
+    text = ALERT_CHAIN_SMOKE.read_text(encoding="utf-8")
+
+    assert '"SIGNOZ_PUBLIC_URL"' in text
+    assert PUBLIC_URL in text
+    assert LEGACY_URL not in text
+
+
 def test_host_asset_map_places_signoz_on_the_canonical_query_host() -> None:
     canonical_services = HOST_CONFIGS["192.168.0.110"]["services"]
     legacy_services = HOST_CONFIGS["192.168.0.188"]["services"]
 
     assert ("SigNoz", 8080, "http", "/api/v1/health") in canonical_services
     assert not any(service[0] == "SigNoz" for service in legacy_services)
+
+
+def test_monitoring_registry_splits_query_health_from_otlp_transport() -> None:
+    registry = _configmap("ops/monitoring/service-registry.yaml")
+    services = {service["name"]: service for service in registry["services"]}
+
+    query = services["signoz-ui"]
+    assert query["host"] == "192.168.0.110"
+    assert query["port"] == 8080
+    assert query["health_endpoint"] == "/api/v1/health"
+    assert query["health_type"] == "http"
+    assert query["monitoring"]["prometheus"] is True
+    assert query["monitoring"]["prometheus_scrape"] is False
+
+    collector = services["signoz-collector"]
+    assert collector["host"] == "192.168.0.188"
+    assert collector["port"] == 24317
+    assert collector["health_type"] == "grpc"
+    assert collector["monitoring"]["prometheus_scrape"] is False
+    assert collector["monitoring"]["external_verifier"] == (
+        "scripts/reboot-recovery/verify-signoz-otel-grpc-runtime.sh"
+    )
+
+
+def test_stateful_registry_places_clickhouse_on_canonical_host() -> None:
+    source_registry = _configmap("ops/config/service-registry.yaml")
+    configmap = _configmap("k8s/awoooi-prod/15-service-registry-configmap.yaml")
+    runtime_registry = yaml.safe_load(configmap["data"]["service-registry.yaml"])
+
+    for registry in (source_registry, runtime_registry):
+        services = {service["name"]: service for service in registry["services"]}
+        clickhouse = services["signoz-clickhouse"]
+        assert clickhouse["host"] == "192.168.0.110"
+        assert clickhouse["stateful_level"] == "BLOCK"
+        assert clickhouse["alert_only"] is True
+
+
+def test_active_prometheus_targets_canonical_http_health_not_legacy_tcp() -> None:
+    config = _configmap("k8s/monitoring/prometheus.yml")
+    jobs = {job["job_name"]: job for job in config["scrape_configs"]}
+    http_job = jobs["blackbox-http"]
+    http_targets = http_job["static_configs"][0]["targets"]
+    tcp_targets = jobs["blackbox-tcp"]["static_configs"][0]["targets"]
+
+    assert INTERNAL_HEALTH_URL in http_targets
+    assert LEGACY_URL not in tcp_targets
+    assert http_job["relabel_configs"] == [
+        {
+            "source_labels": ["__address__"],
+            "target_label": "__param_target",
+        },
+        {
+            "source_labels": ["__param_target"],
+            "target_label": "instance",
+        },
+        {
+            "target_label": "__address__",
+            "replacement": "192.168.0.110:9115",
+        },
+    ]
+
+
+def test_signoz_prometheus_route_deploy_is_target_first_and_rollback_safe() -> None:
+    text = PROMETHEUS_ROUTE_DEPLOY.read_text(encoding="utf-8")
+
+    assert 'MODE="${1:-apply}"' in text
+    assert "promtool check config" in text
+    assert "cp -p \"${REMOTE_CONFIG}\" \"${BACKUP_CONFIG}\"" in text
+    assert 'action=target_first_config_apply' in text
+    assert text.index("promtool check config") < text.index(
+        'action=target_first_config_apply'
+    )
+    assert text.index('action=target_first_config_apply') < text.index(
+        "terminal=target_up_two_scrapes"
+    )
+    assert "rollback_deploy" in text
+    assert INTERNAL_HEALTH_URL in text
+    assert LEGACY_URL in text
+
+
+def test_legacy_prometheus_deployer_routes_to_bounded_host110_replacement() -> None:
+    text = LEGACY_PROMETHEUS_DEPLOY.read_text(encoding="utf-8")
+
+    assert "superseded_by=global_product_governance_v2" in text
+    assert "expiry=2026-07-15" in text
+    assert "scripts/ops/deploy-signoz-prometheus-route.sh" in text
+    assert "192.168.0.188" not in text
+
+
+@pytest.mark.parametrize(
+    "relative_path",
+    [
+        "ops/monitoring/alerts-unified.yml",
+        "ops/monitoring/alerts.yml",
+    ],
+)
+def test_signoz_down_alert_is_exact_and_fails_closed_on_missing_series(
+    relative_path: str,
+) -> None:
+    payload = _configmap(relative_path)
+    rules = [
+        rule
+        for group in payload["groups"]
+        for rule in group.get("rules", [])
+        if rule.get("alert") == "SignOzDown"
+    ]
+
+    assert len(rules) == 1
+    rule = rules[0]
+    assert 'job="blackbox-http"' in rule["expr"]
+    assert INTERNAL_HEALTH_URL in rule["expr"]
+    assert "absent(" in rule["expr"]
+    assert LEGACY_URL not in rule["expr"]
+    assert rule["labels"]["host"] == "110"
+    assert rule["labels"]["target_host"] == "192.168.0.110"
+
+
+def test_legacy_188_rule_plane_does_not_duplicate_canonical_signoz_alert() -> None:
+    payload = _configmap("k8s/monitoring/k3s-alerts.yaml")
+    signoz_rules = [
+        rule
+        for group in payload["groups"]
+        for rule in group.get("rules", [])
+        if rule.get("alert") == "SignOzDown"
+    ]
+    text = (REPO_ROOT / "k8s/monitoring/k3s-alerts.yaml").read_text(encoding="utf-8")
+
+    assert signoz_rules == []
+    assert LEGACY_URL not in text
+
+
+def test_monitoring_generator_uses_blackbox_without_scraping_signoz_spa(
+    tmp_path: Path,
+) -> None:
+    subprocess.run(
+        [
+            sys.executable,
+            str(REPO_ROOT / "ops/monitoring/generate_monitoring.py"),
+            "--output-dir",
+            str(tmp_path),
+        ],
+        cwd=REPO_ROOT,
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    scrape = _configmap(str(tmp_path / "prometheus-scrape-generated.yaml"))
+    blackbox = yaml.safe_load(
+        (tmp_path / "blackbox-targets-generated.yaml").read_text(encoding="utf-8")
+    )
+    generated_jobs = {job["job_name"] for job in scrape["scrape_configs"]}
+    assert "signoz-ui" not in generated_jobs
+    assert "signoz-collector" not in generated_jobs
+    assert any(
+        target["labels"].get("service") == "signoz-ui"
+        and target["url"] == INTERNAL_HEALTH_URL
+        for target in blackbox
+    )