From 9a9d1464a586ba172d1613db5bc285def43eb3e8 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 03:03:22 +0800 Subject: [PATCH] fix(obs): split SigNoz health and public routes --- apps/api/src/api/v1/health.py | 8 +- apps/api/src/api/v1/monitoring.py | 4 +- apps/api/src/core/config.py | 10 +- apps/api/src/core/deep_linking.py | 18 +- apps/api/src/plugins/mcp/mcp_bridge.py | 6 +- apps/api/src/services/host_aggregator.py | 2 +- apps/api/src/services/openclaw.py | 3 +- apps/api/src/services/public_redaction.py | 2 +- apps/api/src/services/signoz_client.py | 8 +- apps/api/src/services/telegram_gateway.py | 5 +- apps/api/tests/test_public_redaction.py | 11 +- .../test_signoz_canonical_health_route.py | 172 ++++++++++++++++++ apps/api/tests/test_signoz_fail_closed.py | 6 +- docs/LOGBOOK.md | 3 + k8s/awoooi-dev/02-configmap.yaml | 3 +- k8s/awoooi-prod/02-network-policy.yaml | 6 +- k8s/awoooi-prod/04-configmap.yaml | 3 +- ...rganization-canonical-route-migration.yaml | 1 + .../p0-obs-002-post-closure-regression.yaml | 7 + .../signoz-canonical-route-preflight.py | 1 + .../test_signoz_canonical_route_migration.py | 1 + 21 files changed, 246 insertions(+), 34 deletions(-) create mode 100644 apps/api/tests/test_signoz_canonical_health_route.py diff --git a/apps/api/src/api/v1/health.py b/apps/api/src/api/v1/health.py index 55a5408f9..702d083d9 100644 --- a/apps/api/src/api/v1/health.py +++ b/apps/api/src/api/v1/health.py @@ -13,7 +13,7 @@ Components Checked: - Redis (192.168.0.188:6380) - Ollama ADR-110 provider pool (GCP-A -> GCP-B -> 111) - OpenClaw (192.168.0.188:8089) -- SigNoz (192.168.0.188:3301) +- SigNoz canonical internal route (192.168.0.110:8080) """ import asyncio @@ -247,7 +247,11 @@ async def check_openclaw() -> ComponentHealth: async def check_signoz() -> ComponentHealth: """Async SigNoz health check""" - return await _http_health_check("signoz", settings.SIGNOZ_URL, "/api/v1/health") + return await _http_health_check( + "signoz", + settings.SIGNOZ_INTERNAL_URL, + "/api/v1/health", + ) def _determine_overall_status( diff --git a/apps/api/src/api/v1/monitoring.py b/apps/api/src/api/v1/monitoring.py index 90457f682..4483721a5 100644 --- a/apps/api/src/api/v1/monitoring.py +++ b/apps/api/src/api/v1/monitoring.py @@ -30,7 +30,7 @@ TIMEOUT = 3.0 PUBLIC_TOOL_URLS = { "Sentry": "https://sentry.wooo.work", "Langfuse": "https://langfuse.wooo.work", - "SigNoz": "https://signoz.wooo.work", + "SigNoz": settings.SIGNOZ_PUBLIC_URL, "Gitea": "https://gitea.wooo.work", } @@ -176,7 +176,7 @@ async def _probe_langfuse(client: httpx.AsyncClient) -> dict: async def _probe_signoz(client: httpx.AsyncClient) -> dict: - base = "http://192.168.0.188:3301" + base = settings.SIGNOZ_INTERNAL_URL.rstrip("/") try: r = await client.get(f"{base}/api/v1/health", timeout=TIMEOUT) if r.status_code == 200: diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py index c99bc5650..c8de53bbe 100644 --- a/apps/api/src/core/config.py +++ b/apps/api/src/core/config.py @@ -485,9 +485,13 @@ class Settings(BaseSettings): default="http://192.168.0.112:8080", description="Kali security scanner URL", ) - SIGNOZ_URL: str = Field( - default="http://192.168.0.188:3301", - description="SigNoz observability URL", + SIGNOZ_INTERNAL_URL: str = Field( + default="http://192.168.0.110:8080", + description="Canonical internal SigNoz query and health URL", + ) + SIGNOZ_PUBLIC_URL: str = Field( + default="https://signoz.wooo.work", + description="Public SigNoz UI base URL used only for user-facing deep links", ) CLICKHOUSE_URL: str = Field( default="http://192.168.0.188:8123", diff --git a/apps/api/src/core/deep_linking.py b/apps/api/src/core/deep_linking.py index 6e2fceb0f..d74293181 100644 --- a/apps/api/src/core/deep_linking.py +++ b/apps/api/src/core/deep_linking.py @@ -13,7 +13,7 @@ Phase 15.3 (2026-03-26) └─────────┘ └─────────┘ └──────────┘ URL 格式: -- SignOz Trace: http://192.168.0.188:3301/trace/{trace_id} +- SignOz Trace: https://signoz.wooo.work/trace/{trace_id} - Langfuse Trace: http://192.168.0.110:3100/trace/{langfuse_trace_id} - Sentry Issue: http://192.168.0.110:9000/organizations/sentry/issues/{issue_id}/ @@ -36,7 +36,7 @@ class DeepLinking: 統帥鐵律 (Phase 15.3): - 所有觀測系統必須能互相跳轉 - - URL 必須使用內網 IP (非 localhost) + - 使用者可見連結必須使用受控公開網址 - 確保 trace_id 格式一致 (32 hex chars) """ @@ -44,7 +44,9 @@ class DeepLinking: # SignOz URLs (Traces/Metrics/Logs) # ========================================================================== - SIGNOZ_BASE_URL = "http://192.168.0.188:3301" + @staticmethod + def _signoz_public_base_url() -> str: + return settings.SIGNOZ_PUBLIC_URL.rstrip("/") @classmethod def signoz_trace_url(cls, trace_id: str) -> str: @@ -55,12 +57,12 @@ class DeepLinking: trace_id: 32 字元 hex 格式 (e.g., "0af7651916cd43dd8448eb211c80319c") Returns: - SignOz Trace URL (e.g., http://192.168.0.188:3301/trace/0af7651916cd43dd8448eb211c80319c) + SignOz Trace URL (e.g., https://signoz.wooo.work/trace/0af7651916cd43dd8448eb211c80319c) """ if not trace_id: return "" # SignOz v3 URL 格式 - return f"{cls.SIGNOZ_BASE_URL}/trace/{trace_id}" + return f"{cls._signoz_public_base_url()}/trace/{trace_id}" @classmethod def signoz_service_url(cls, service_name: str = "awoooi-api") -> str: @@ -73,7 +75,7 @@ class DeepLinking: Returns: SignOz Service URL """ - return f"{cls.SIGNOZ_BASE_URL}/services/{service_name}" + return f"{cls._signoz_public_base_url()}/services/{service_name}" @classmethod def signoz_logs_url(cls, trace_id: str | None = None) -> str: @@ -88,8 +90,8 @@ class DeepLinking: """ if trace_id: # SignOz v3 logs 過濾語法 - return f"{cls.SIGNOZ_BASE_URL}/logs?q=trace_id%3D{trace_id}" - return f"{cls.SIGNOZ_BASE_URL}/logs" + return f"{cls._signoz_public_base_url()}/logs?q=trace_id%3D{trace_id}" + return f"{cls._signoz_public_base_url()}/logs" # ========================================================================== # Langfuse URLs (LLMOps) diff --git a/apps/api/src/plugins/mcp/mcp_bridge.py b/apps/api/src/plugins/mcp/mcp_bridge.py index 5daf14b1d..76eb5614f 100644 --- a/apps/api/src/plugins/mcp/mcp_bridge.py +++ b/apps/api/src/plugins/mcp/mcp_bridge.py @@ -23,7 +23,9 @@ from typing import Any import httpx -from src.core.config import settings # P0-13: K8s namespace 由 settings.AWOOOI_K8S_NAMESPACE 提供 +from src.core.config import ( + settings, # P0-13: K8s namespace 由 settings.AWOOOI_K8S_NAMESPACE 提供 +) from src.services.mcp_audit_context import with_mcp_audit_context from src.utils.timezone import now_taipei @@ -215,7 +217,7 @@ class MCPBridge: self._servers["signoz"] = MCPServer( name="signoz", transport=MCPTransport.HTTP, - endpoint="http://192.168.0.188:3301", # SignOz Query Service + endpoint=settings.SIGNOZ_INTERNAL_URL.rstrip("/"), ) # Phase 13.2 #82: Filesystem Provider (安全受限) self._servers["filesystem"] = MCPServer( diff --git a/apps/api/src/services/host_aggregator.py b/apps/api/src/services/host_aggregator.py index ef1a24750..e32c85740 100644 --- a/apps/api/src/services/host_aggregator.py +++ b/apps/api/src/services/host_aggregator.py @@ -287,6 +287,7 @@ HOST_CONFIGS = { ("Gitea", 3001, "tcp", None), # CI/CD 主倉，tcp 可達 ("Prometheus", 9090, "http", "/-/healthy"), # 實測可達 ("Grafana", 3002, "http", "/api/health"), # 實測可達 + ("SigNoz", 8080, "http", "/api/v1/health"), ], }, "192.168.0.112": { @@ -319,7 +320,6 @@ HOST_CONFIGS = { ("PostgreSQL", 5432, "tcp", None), ("Redis", 6380, "tcp", None), ("OpenClaw", 8088, "http", "/health"), # 修正: 8089→8088 - ("SigNoz", 3301, "http", "/api/v1/health"), ], }, } diff --git a/apps/api/src/services/openclaw.py b/apps/api/src/services/openclaw.py index 3e0392986..3dea0b0f1 100644 --- a/apps/api/src/services/openclaw.py +++ b/apps/api/src/services/openclaw.py @@ -350,7 +350,8 @@ class OpenClawService: error=str(e), ) # 降級: 返回 None 和靜態 URL - fallback_url = f"{settings.SIGNOZ_URL}/traces?service={service_name}" + public_base = settings.SIGNOZ_PUBLIC_URL.rstrip("/") + fallback_url = f"{public_base}/traces?service={service_name}" return None, fallback_url def generate_auto_tuning_command( diff --git a/apps/api/src/services/public_redaction.py b/apps/api/src/services/public_redaction.py index 69b20d9c8..06011bd1e 100644 --- a/apps/api/src/services/public_redaction.py +++ b/apps/api/src/services/public_redaction.py @@ -15,12 +15,12 @@ _ENDPOINT_ALIASES = { "192.168.0.110:3002": "host:public-gateway/grafana", "192.168.0.110:3100": "host:public-gateway/langfuse", "192.168.0.110:5000": "host:public-gateway/registry", + "192.168.0.110:8080": "host:public-gateway/signoz", "192.168.0.110:9000": "host:public-gateway/sentry", "192.168.0.112:8080": "host:kali-readonly/scanner", "192.168.0.188:11434": "host:observability-a/ollama", "192.168.0.188:8088": "host:observability-a/openclaw-legacy", "192.168.0.188:8089": "host:observability-a/openclaw", - "192.168.0.188:3301": "host:observability-a/signoz", "192.168.0.188:5432": "host:observability-a/postgres", "192.168.0.188:6380": "host:observability-a/redis", } diff --git a/apps/api/src/services/signoz_client.py b/apps/api/src/services/signoz_client.py index bec831b36..fca8d1343 100644 --- a/apps/api/src/services/signoz_client.py +++ b/apps/api/src/services/signoz_client.py @@ -10,7 +10,8 @@ Features: - 趨勢圖表數據提取 (供 AI 分析) 架構: -- SignOz Query Service: 192.168.0.188:3301 (需認證) +- SignOz Query/Health Service: 192.168.0.110:8080 (canonical internal route) +- SignOz UI deep links: https://signoz.wooo.work (public route) - ClickHouse HTTP API: 192.168.0.188:8123 (直查) """ @@ -128,7 +129,8 @@ class SignOzClient: """ def __init__(self): - self.signoz_url = settings.SIGNOZ_URL # http://192.168.0.188:3301 + self.signoz_internal_url = settings.SIGNOZ_INTERNAL_URL.rstrip("/") + self.signoz_public_url = settings.SIGNOZ_PUBLIC_URL.rstrip("/") self.clickhouse_url = settings.CLICKHOUSE_URL # http://192.168.0.188:8123 async def close(self) -> None: @@ -384,7 +386,7 @@ class SignOzClient: alert_timestamp = datetime.now(UTC) link = SignOzTraceLink( - base_url=self.signoz_url, + base_url=self.signoz_public_url, service_name=service_name, start_time=alert_timestamp - timedelta(minutes=window_minutes), end_time=alert_timestamp + timedelta(minutes=window_minutes), diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 4d3815a79..539aaa88e 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -4145,7 +4145,10 @@ class TelegramMessage: # SignOz URL (優先使用動態 URL) - 必須 HTML 轉義防止解析錯誤 service_name = self.resource_name.split("-")[0] if "-" in self.resource_name else self.resource_name - raw_url = self.signoz_trace_url or f"http://192.168.0.188:3301/traces?service={service_name}" + public_signoz_url = settings.SIGNOZ_PUBLIC_URL.rstrip("/") + raw_url = self.signoz_trace_url or ( + f"{public_signoz_url}/traces?service={service_name}" + ) signoz_url = html.escape(raw_url, quote=True) # SignOz 指標區塊（含 Trace 連結） diff --git a/apps/api/tests/test_public_redaction.py b/apps/api/tests/test_public_redaction.py index caf5e5c0d..7df19466d 100644 --- a/apps/api/tests/test_public_redaction.py +++ b/apps/api/tests/test_public_redaction.py @@ -1,7 +1,10 @@ from __future__ import annotations from src.api.v1.monitoring import public_monitoring_tool_payload -from src.services.public_redaction import redact_public_lan_text, redact_public_lan_topology +from src.services.public_redaction import ( + redact_public_lan_text, + redact_public_lan_topology, +) def test_redact_public_lan_text_replaces_internal_endpoints_with_aliases() -> None: @@ -28,13 +31,13 @@ def test_redact_public_lan_text_replaces_internal_endpoints_with_aliases() -> No def test_redact_public_lan_topology_recurses_json_values() -> None: payload = { "safe_key": "unchanged", - "nested": [{"endpoint": "192.168.0.188:3301"}], + "nested": [{"endpoint": "192.168.0.110:8080"}], } redacted = redact_public_lan_topology(payload) assert redacted["safe_key"] == "unchanged" - assert redacted["nested"][0]["endpoint"] == "host:observability-a/signoz" + assert redacted["nested"][0]["endpoint"] == "host:public-gateway/signoz" def test_redact_public_lan_text_replaces_internal_work_context_terms() -> None: @@ -80,7 +83,7 @@ def test_public_monitoring_tool_payload_uses_public_route_when_available() -> No { "name": "SigNoz", "status": "up", - "url": "http://192.168.0.188:3301", + "url": "http://192.168.0.110:8080", } ) diff --git a/apps/api/tests/test_signoz_canonical_health_route.py b/apps/api/tests/test_signoz_canonical_health_route.py new file mode 100644 index 000000000..7e5ef50c7 --- /dev/null +++ b/apps/api/tests/test_signoz_canonical_health_route.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from src.api.v1 import health as health_module +from src.api.v1 import monitoring as monitoring_module +from src.core import deep_linking as deep_linking_module +from src.core.config import Settings +from src.services import signoz_client as signoz_client_module +from src.services.host_aggregator import HOST_CONFIGS + +REPO_ROOT = Path(__file__).resolve().parents[3] +INTERNAL_URL = "http://192.168.0.110:8080" +PUBLIC_URL = "https://signoz.wooo.work" +LEGACY_URL = "192.168.0.188:3301" + + +def _configmap(relative_path: str) -> dict: + return yaml.safe_load((REPO_ROOT / relative_path).read_text(encoding="utf-8")) + + +def test_settings_split_internal_health_from_public_deep_links() -> None: + fields = Settings.model_fields + + assert fields["SIGNOZ_INTERNAL_URL"].default == INTERNAL_URL + assert fields["SIGNOZ_PUBLIC_URL"].default == PUBLIC_URL + assert "SIGNOZ_URL" not in fields + assert fields["OTEL_EXPORTER_OTLP_ENDPOINT"].default == "192.168.0.188:24317" + + +@pytest.mark.asyncio +async def test_health_check_uses_exact_canonical_internal_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str, str]] = [] + expected = SimpleNamespace(status="up") + + async def fake_http_health_check(name: str, base_url: str, path: str): + calls.append((name, base_url, path)) + return expected + + monkeypatch.setattr(health_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL) + monkeypatch.setattr( + health_module, + "_http_health_check", + fake_http_health_check, + ) + + assert await health_module.check_signoz() is expected + assert calls == [("signoz", INTERNAL_URL, "/api/v1/health")] + + +@pytest.mark.asyncio +async def test_monitoring_probe_uses_internal_route_but_returns_replaceable_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requested_urls: list[str] = [] + + class FakeClient: + async def get(self, url: str, *, timeout: float): + requested_urls.append(url) + assert timeout == monitoring_module.TIMEOUT + return SimpleNamespace(status_code=200) + + monkeypatch.setattr(monitoring_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL) + + probe = await monitoring_module._probe_signoz(FakeClient()) + + assert requested_urls == [f"{INTERNAL_URL}/api/v1/health"] + assert probe["url"] == INTERNAL_URL + assert monitoring_module.public_monitoring_tool_payload(probe)["url"] == PUBLIC_URL + + +def test_user_facing_deep_links_use_public_route( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(deep_linking_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL) + monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_PUBLIC_URL", PUBLIC_URL) + monkeypatch.setattr(signoz_client_module.settings, "SIGNOZ_INTERNAL_URL", INTERNAL_URL) + + trace_id = "0af7651916cd43dd8448eb211c80319c" + assert deep_linking_module.DeepLinking.signoz_trace_url(trace_id) == ( + f"{PUBLIC_URL}/trace/{trace_id}" + ) + assert deep_linking_module.DeepLinking.signoz_service_url() == ( + f"{PUBLIC_URL}/services/awoooi-api" + ) + assert deep_linking_module.DeepLinking.signoz_logs_url(trace_id).startswith( + f"{PUBLIC_URL}/logs?" + ) + + client = signoz_client_module.SignOzClient() + generated = client.generate_trace_url( + "awoooi-api", + alert_timestamp=datetime(2026, 7, 15, tzinfo=UTC), + ) + assert generated.startswith(f"{PUBLIC_URL}/traces?service=awoooi-api&") + assert INTERNAL_URL not in generated + + +@pytest.mark.parametrize( + "relative_path", + [ + "k8s/awoooi-prod/04-configmap.yaml", + "k8s/awoooi-dev/02-configmap.yaml", + ], +) +def test_configmaps_publish_split_routes_and_preserve_otlp( + relative_path: str, +) -> None: + data = _configmap(relative_path)["data"] + + assert data["SIGNOZ_INTERNAL_URL"] == INTERNAL_URL + assert data["SIGNOZ_PUBLIC_URL"] == PUBLIC_URL + assert "SIGNOZ_URL" not in data + assert data["OTEL_EXPORTER_OTLP_ENDPOINT"] == "http://192.168.0.188:24317" + + +def test_prod_network_policy_allows_exact_canonical_signoz_health_route() -> None: + policy_path = REPO_ROOT / "k8s/awoooi-prod/02-network-policy.yaml" + policies = [ + document + for document in yaml.safe_load_all(policy_path.read_text(encoding="utf-8")) + if document + ] + policy = next( + document + for document in policies + if document["metadata"]["name"] == "allow-required-egress" + ) + + canonical_rules = [ + rule + for rule in policy["spec"]["egress"] + if rule.get("to") == [{"ipBlock": {"cidr": "192.168.0.110/32"}}] + ] + assert len(canonical_rules) == 1 + assert {"protocol": "TCP", "port": 8080} in canonical_rules[0]["ports"] + + legacy_rules = [ + rule + for rule in policy["spec"]["egress"] + if rule.get("to") == [{"ipBlock": {"cidr": "192.168.0.188/32"}}] + ] + assert len(legacy_rules) == 1 + assert {"protocol": "TCP", "port": 3301} not in legacy_rules[0]["ports"] + assert {"protocol": "TCP", "port": 24317} in legacy_rules[0]["ports"] + assert {"protocol": "TCP", "port": 24318} in legacy_rules[0]["ports"] + + +def test_active_api_source_has_no_legacy_signoz_ui_route() -> None: + source_root = REPO_ROOT / "apps/api/src" + offenders = [ + str(path.relative_to(REPO_ROOT)) + for path in source_root.rglob("*.py") + if LEGACY_URL in path.read_text(encoding="utf-8") + ] + + assert offenders == [] + + +def test_host_asset_map_places_signoz_on_the_canonical_query_host() -> None: + canonical_services = HOST_CONFIGS["192.168.0.110"]["services"] + legacy_services = HOST_CONFIGS["192.168.0.188"]["services"] + + assert ("SigNoz", 8080, "http", "/api/v1/health") in canonical_services + assert not any(service[0] == "SigNoz" for service in legacy_services) diff --git a/apps/api/tests/test_signoz_fail_closed.py b/apps/api/tests/test_signoz_fail_closed.py index 96fd2dea6..6a4208b41 100644 --- a/apps/api/tests/test_signoz_fail_closed.py +++ b/apps/api/tests/test_signoz_fail_closed.py @@ -52,7 +52,11 @@ async def test_openclaw_marks_signoz_unavailable_when_metrics_query_fails( get_gold_metrics=AsyncMock(side_effect=RuntimeError("clickhouse unavailable")), generate_trace_url=lambda **_kwargs: "https://unused.example/traces", ) - monkeypatch.setattr(signoz_module.settings, "SIGNOZ_URL", "https://signoz.example") + monkeypatch.setattr( + signoz_module.settings, + "SIGNOZ_PUBLIC_URL", + "https://signoz.example", + ) metrics, trace_url = await service.get_signoz_context("node-exporter-110") diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index baa19e66d..ef07354ab 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -4,16 +4,19 @@ - gRPC bridge run `P0-OBS-002-20260715T013800+0800` 先通過完整 600 秒 verifier；後續 revision `2004` 於 02:11:22 起發生 liveness/readiness timeout，最低 `0/2 Ready`，兩 pod 各累積 restart `4`。bridge 維持同 PID、NRestarts `0`，因此不執行 bridge rollback。 - API log 捕獲 `too many connections for role awoooi_api_runtime`；既有 CD verifier 在 role limit `12`、最低 headroom `8` 的條件下仍固定發出 8 個 DB-backed requests，存在把 `4 + 8` 打滿 `12/12` 的結構性風險。已完成的 #5119 receipt 本身顯示 probe 當時 active=`0`、restart `0->0`，故只把此問題列為高信心 structural amplifier，不誤報為唯一直接觸發。 - 每日 `backup-all` 02:12 執行 SignOz 備份時停止 collector；volume backup 立即失敗，總備份僅 `3/9` 成功且 collector 未恢復。`P0-OBS-002-static-recovery-20260715T021600+0800` 只恢復 stateless collector，4317/4318、OTLP HTTP `200`、`.120/.121` gRPC TCP 全數通過，stateful services touched=`0`。 +- 公開 `/api/v1/health` 的 `components.signoz=down` 不是 OTLP 故障：live API 對舊 `192.168.0.188:3301` 回 connection refused，而 188 bridge 明確只 bind `127.0.0.1:3301`。同時 collector、4317/4318、gRPC bridge、trace/metric freshness 與 exporter errors 均維持正常，根因是 Query/UI control-plane health target drift。 **source/test evidence**： - CD probe 改為先檢查 connection budget、保留 4 條安全餘量、最多 4 workers、單輪、每 request 5 秒；新增 workers/reserve/skip reason machine-readable receipt fields。 - `backup-signoz.sh` 在 stop 前先 arm restore，加入 `EXIT/HUP/INT/TERM` trap、防重入 cleanup 與首次 restart 失敗後的有界 EXIT retry。 - `110-devops.yml --tags backup_jobs` 納入 `backup-signoz.sh`，確保 source guard 有可驗證的 `/backup/scripts/backup-signoz.sh` 部署路徑，不再只停留在 repo。 - `#5128` 在 `1518 passed` 後被 asset-integrity workflow shape 測試的過時注解邊界擋下；測試改以下一個 YAML `- name:` 截取單一 step，仍保留 shell body 不得出現 `${{ secrets.* }}` 的安全斷言，不放寬 secret transport policy。 +- SigNoz 路由分離為 internal Query/health `http://192.168.0.110:8080` 與 public deep link `https://signoz.wooo.work`；prod NetworkPolicy 移除無效 `.188:3301` 並只對 `.110/32` 新增 TCP 8080。OTLP `.188:24317/.24318` 與 producer active route 不變。 - 新增 `ops/signoz/p0-obs-002-post-closure-regression.yaml`，分離 API probe starvation、collector backup interruption、controlled recovery、source candidate 與 scope boundary。 **仍維持**： - API 最近一次讀回雖恢復 `2/2`，但 restart 已是 `4/4` 且較新 Gitea CD 仍連續進行，尚未完成 10 分鐘 cooldown；program 仍是 `partial_degraded`。 +- SigNoz control-plane health route 修正尚未 deploy/post-verify；即使 `components.signoz` 恢復 `up`，overall health 仍可因 Ollama primary fallback 獨立維持 `degraded`。 - 兩個 source guard 尚未由 main/CD/runtime readback 證明；filelog normalization、organization、direct ingress policy、HTTP bridge supervision 與 producer promotion 仍未完成。 ## 2026-07-15 — P0-OBS-002 production OTLP/gRPC runtime closure diff --git a/k8s/awoooi-dev/02-configmap.yaml b/k8s/awoooi-dev/02-configmap.yaml index 71f8db4ea..bc30e3220 100644 --- a/k8s/awoooi-dev/02-configmap.yaml +++ b/k8s/awoooi-dev/02-configmap.yaml @@ -22,7 +22,8 @@ data: ALERT_AI_STRICT_PROVIDER_CHAIN: "true" OPENCLAW_URL: "http://192.168.0.188:8089" KALI_SCANNER_URL: "http://192.168.0.112:8080" - SIGNOZ_URL: "http://192.168.0.188:3301" + SIGNOZ_INTERNAL_URL: "http://192.168.0.110:8080" + SIGNOZ_PUBLIC_URL: "https://signoz.wooo.work" LANGFUSE_URL: "http://192.168.0.110:3100" OTEL_ENABLED: "true" diff --git a/k8s/awoooi-prod/02-network-policy.yaml b/k8s/awoooi-prod/02-network-policy.yaml index b7a2f629d..e96ddac98 100644 --- a/k8s/awoooi-prod/02-network-policy.yaml +++ b/k8s/awoooi-prod/02-network-policy.yaml @@ -102,9 +102,6 @@ spec: # OpenClaw (clawbot Docker) - 2026-04-01 ogt: 修正為 8088 (clawbot-v5 使用 8088) - protocol: TCP port: 8088 - # SigNoz UI (Docker) - - protocol: TCP - port: 3301 # ========== OTEL 神經連線 (P0 核心) ========== # SigNoz OTEL Collector gRPC (Host 24317 -> Container 4317) - protocol: TCP @@ -158,6 +155,9 @@ spec: # Gitea — CI/CD 主倉 probe + monitoring - protocol: TCP port: 3001 + # SigNoz canonical internal query and health route. + - protocol: TCP + port: 8080 # P0-OBS-002 transition allow only. Active producers remain on the # .188 compatibility bridge until replay/shadow/canary passes. # SigNoz canonical OTLP gRPC / HTTP on host .110. diff --git a/k8s/awoooi-prod/04-configmap.yaml b/k8s/awoooi-prod/04-configmap.yaml index da3977a0a..abf03a475 100644 --- a/k8s/awoooi-prod/04-configmap.yaml +++ b/k8s/awoooi-prod/04-configmap.yaml @@ -25,7 +25,8 @@ data: OLLAMA_FALLBACK_URL: "http://192.168.0.110:11437" OPENCLAW_URL: "http://192.168.0.188:8088" KALI_SCANNER_URL: "http://192.168.0.112:8080" - SIGNOZ_URL: "http://192.168.0.188:3301" + SIGNOZ_INTERNAL_URL: "http://192.168.0.110:8080" + SIGNOZ_PUBLIC_URL: "https://signoz.wooo.work" LANGFUSE_URL: "http://192.168.0.110:3100" # 2026-04-01 Claude Code: 開啟 Langfuse LLMOps 追蹤 (Phase 15.1 key 已注入 K8s Secret) LANGFUSE_ENABLED: "true" diff --git a/ops/signoz/organization-canonical-route-migration.yaml b/ops/signoz/organization-canonical-route-migration.yaml index 7aeebbc3c..51e6aee59 100644 --- a/ops/signoz/organization-canonical-route-migration.yaml +++ b/ops/signoz/organization-canonical-route-migration.yaml @@ -170,4 +170,5 @@ terminal: - api_runtime_cooldown_pending - cd_probe_safety_fix_not_deployed - backup_collector_restore_guard_not_deployed + - signoz_control_plane_health_route_not_deployed - wave_b_not_authorized diff --git a/ops/signoz/p0-obs-002-post-closure-regression.yaml b/ops/signoz/p0-obs-002-post-closure-regression.yaml index c550b9774..7934161f4 100644 --- a/ops/signoz/p0-obs-002-post-closure-regression.yaml +++ b/ops/signoz/p0-obs-002-post-closure-regression.yaml @@ -73,6 +73,13 @@ source_candidates: restore_armed_before_stop: true bounded_exit_retry: true deployment_status: pending + canonical_health_route: + internal_url: http://192.168.0.110:8080 + public_url: https://signoz.wooo.work + production_networkpolicy_tcp_port: 8080 + legacy_query_route_removed: http://192.168.0.188:3301 + otlp_producer_route_changed: false + deployment_status: pending scope_boundaries: grpc_bridge_rollback_indicated: false diff --git a/scripts/reboot-recovery/signoz-canonical-route-preflight.py b/scripts/reboot-recovery/signoz-canonical-route-preflight.py index 5661913e7..b4b0fae6a 100644 --- a/scripts/reboot-recovery/signoz-canonical-route-preflight.py +++ b/scripts/reboot-recovery/signoz-canonical-route-preflight.py @@ -25,6 +25,7 @@ EXPECTED_BLOCKERS = [ "api_runtime_cooldown_pending", "cd_probe_safety_fix_not_deployed", "backup_collector_restore_guard_not_deployed", + "signoz_control_plane_health_route_not_deployed", "wave_b_not_authorized", ] diff --git a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py index edb71cfbe..bacfc6ffc 100644 --- a/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py +++ b/scripts/reboot-recovery/tests/test_signoz_canonical_route_migration.py @@ -69,6 +69,7 @@ def test_preflight_is_no_write_and_reports_explicit_partial_runtime() -> None: assert payload["active_route"] == "incumbent" assert payload["checks"] and all(payload["checks"].values()) assert "organization_not_initialized" in payload["blockers"] + assert "signoz_control_plane_health_route_not_deployed" in payload["blockers"] assert "wave_b_not_authorized" in payload["blockers"]