From 3023918d61adbbe8cf8310c717933ead3734d0d7 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 20:23:38 +0800 Subject: [PATCH] fix(ops): verify EwoooC metrics payload --- docs/AI_INTELLIGENCE_MODULE_SOT.md | 2 +- .../edge/ewoooc_metrics_nginx_bridge.conf | 2 +- monitoring/edge/ewoooc_prometheus_scrape.yml | 2 +- .../metrics_edge_controlled_apply_service.py | 28 ++++++++++++++++--- ...t_metrics_edge_controlled_apply_service.py | 19 +++++++++++-- tests/test_metrics_edge_security.py | 4 +-- 6 files changed, 45 insertions(+), 12 deletions(-) diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index 38c2e18..d843f5f 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -16,7 +16,7 @@ - production 安全與治理完成度必須分開顯示 program、asset coverage、runtime closure；目前結論是 partial，不是 complete。 - 目前 ordered P0 以 `docs/guides/ai_automation_mainline_work_items.md` 為準：先關閉 access exposure，再做 identity/RBAC、webhook trust、supply chain、asset reconciliation、controlled-apply envelope、internal RAG canary 與 MCP/RAG runtime closure。 - V10.786 起 `/metrics` 不再是 public exception；應用中央政策與 Nginx exact-match edge policy 僅允許 loopback / Docker transport，Prometheus 必須以內部 canary 證明 target `up`，且 public/LAN readback 必須拒絕。監控 Compose 不得保存明文 Grafana 管理密碼，Prometheus 對外埠預設只綁 loopback。 -- V10.787 依 live ingress 修正 metrics transport：production ingress 實際位於 188，110 global Prometheus 只能經 `172.20.0.1:9191` Docker-only Nginx bridge 抓取；bridge 以系統 CA、`mo.wooo.work` SNI/Host 轉送至 188，應用僅額外允許 `192.168.0.110/32` 單一監控主機，不得放行整段 LAN，也不得把 110 的其他 vhost 誤當 EwoooC production ingress。controlled apply 必須支援新 include 建立、Prometheus canary 與失敗自動移除/回滾。 +- V10.787 依 live ingress 修正 metrics transport：production ingress 實際位於 188，110 global Prometheus 只能經專用 `172.20.0.1:19191` Docker-only Nginx bridge 抓取；bridge 以系統 CA、`mo.wooo.work` SNI/Host 轉送至 188，應用僅額外允許 `192.168.0.110/32` 單一監控主機，不得放行整段 LAN，也不得把 110 的其他 vhost 誤當 EwoooC production ingress。controlled apply 必須支援新 include 建立、Prometheus bind-mount reconciliation、`momo_app_info`/health/database product marker canary 與失敗自動移除/回滾；單純 HTTP 200 或 target up 不得算 closure。 - V10.777 起 web process 只能註冊 OpenClaw Blueprint，不得在 import 或 `record_once` 啟動 APScheduler、timed jobs 或 embedding thread；10 個 OpenClaw timed jobs 與唯一 embedding worker runtime owner 必須由 `momo-scheduler` 明確啟動。每日 04:15 的 codebase inventory 只掃描程式與前端資產並寫 artifact receipt，不讀 secret、不寫業務 DB。 --- diff --git a/monitoring/edge/ewoooc_metrics_nginx_bridge.conf b/monitoring/edge/ewoooc_metrics_nginx_bridge.conf index f5e2871..3628b72 100644 --- a/monitoring/edge/ewoooc_metrics_nginx_bridge.conf +++ b/monitoring/edge/ewoooc_metrics_nginx_bridge.conf @@ -1,6 +1,6 @@ # Internal bridge from the 110 Prometheus network to the 188 EwoooC ingress. server { - listen 172.20.0.1:9191; + listen 172.20.0.1:19191; server_name ewooooc-metrics.internal; access_log off; diff --git a/monitoring/edge/ewoooc_prometheus_scrape.yml b/monitoring/edge/ewoooc_prometheus_scrape.yml index 2e6ed77..a6d89c0 100644 --- a/monitoring/edge/ewoooc_prometheus_scrape.yml +++ b/monitoring/edge/ewoooc_prometheus_scrape.yml @@ -4,7 +4,7 @@ scrape_interval: 30s scrape_timeout: 15s static_configs: - - targets: ['172.20.0.1:9191'] + - targets: ['172.20.0.1:19191'] labels: environment: 'production' product: 'ewoooc' diff --git a/services/metrics_edge_controlled_apply_service.py b/services/metrics_edge_controlled_apply_service.py index fe3cb1b..eb989f8 100644 --- a/services/metrics_edge_controlled_apply_service.py +++ b/services/metrics_edge_controlled_apply_service.py @@ -310,6 +310,19 @@ def _wait_prometheus_ready(timeout_seconds: int = 45) -> None: raise RuntimeError("Prometheus did not become ready after controlled rebind") +def _parse_metrics_canary(output: str) -> tuple[str, bool]: + marker = "\n__EWOOOC_HTTP_STATUS__:" + if marker not in output: + raise RuntimeError("metrics canary response has no HTTP status marker") + body, status = output.rsplit(marker, 1) + product_marker_present = ( + "momo_app_info" in body + and "momo_app_health" in body + and "momo_database_up" in body + ) + return status.strip(), product_marker_present + + def apply_metrics_edge_plan( plan: dict[str, Any], *, @@ -400,24 +413,31 @@ def apply_metrics_edge_plan( if metrics_canary_resolve: canary_command.extend(["--resolve", metrics_canary_resolve]) canary_command.extend([ - "-o", "/dev/null", "-w", "%{http_code}", metrics_canary_url, + "-w", "\n__EWOOOC_HTTP_STATUS__:%{http_code}", metrics_canary_url, ]) internal = _run(canary_command) - internal_code = internal.stdout.strip() + internal_code, product_marker_present = _parse_metrics_canary(internal.stdout) target_health = _prometheus_target_health(max(10, canary_timeout)) receipt["post_verifier"] = { "nginx_config_valid": apply_scope == "full", "prometheus_config_valid": True, "internal_metrics_http_code": internal_code, "metrics_internal_canary": internal_code == "200", + "metrics_product_marker_present": product_marker_present, "metrics_target_health": target_health, "prometheus_config_inode_reconciled": True, "prometheus_identity_preserved": identity_preserved, } - if internal_code != "200" or target_health != "up" or not identity_preserved: + if ( + internal_code != "200" + or not product_marker_present + or target_health != "up" + or not identity_preserved + ): raise RuntimeError( "post-verifier failed: " - f"internal={internal_code}, target={target_health}, " + f"internal={internal_code}, product_marker={product_marker_present}, " + f"target={target_health}, " f"identity_preserved={identity_preserved}" ) receipt["status"] = "pass" diff --git a/tests/test_metrics_edge_controlled_apply_service.py b/tests/test_metrics_edge_controlled_apply_service.py index c46f6c4..1af46ac 100644 --- a/tests/test_metrics_edge_controlled_apply_service.py +++ b/tests/test_metrics_edge_controlled_apply_service.py @@ -40,11 +40,11 @@ PROMETHEUS_SNIPPET = """ - job_name: 'ewoooc-app' scheme: http metrics_path: /metrics static_configs: - - targets: ['172.20.0.1:9191'] + - targets: ['172.20.0.1:19191'] """ NGINX_BRIDGE = """server { - listen 172.20.0.1:9191; + listen 172.20.0.1:19191; server_name ewooooc-metrics.internal; location = /metrics { allow 172.16.0.0/12; @@ -84,7 +84,7 @@ def test_render_standalone_nginx_bridge_is_idempotent(): second = render_nginx_config(rendered, NGINX_BRIDGE) assert rendered == second - assert rendered.count("listen 172.20.0.1:9191") == 1 + assert rendered.count("listen 172.20.0.1:19191") == 1 def test_build_plan_is_no_write_and_commit_safe(tmp_path: Path): @@ -144,6 +144,19 @@ def test_prometheus_config_sha_detects_stale_container_mount(tmp_path: Path, mon assert container_sha == stale_sha +def test_metrics_canary_requires_ewoooc_product_markers(): + from services.metrics_edge_controlled_apply_service import _parse_metrics_canary + + valid = ( + "momo_app_info 1\nmomo_app_health 1\nmomo_database_up 1" + "\n__EWOOOC_HTTP_STATUS__:200" + ) + wrong_exporter = "fail2ban_total_failed 0\n__EWOOOC_HTTP_STATUS__:200" + + assert _parse_metrics_canary(valid) == ("200", True) + assert _parse_metrics_canary(wrong_exporter) == ("200", False) + + def test_prometheus_identity_verifier_requires_same_image_and_data_volume(): from services.metrics_edge_controlled_apply_service import ( _prometheus_identity_preserved, diff --git a/tests/test_metrics_edge_security.py b/tests/test_metrics_edge_security.py index 21234cc..62f9d85 100644 --- a/tests/test_metrics_edge_security.py +++ b/tests/test_metrics_edge_security.py @@ -75,7 +75,7 @@ def test_source_owned_edge_and_prometheus_templates_are_closed_by_default(): encoding="utf-8" ) - assert "listen 172.20.0.1:9191;" in nginx + assert "listen 172.20.0.1:19191;" in nginx assert "location = /metrics" in nginx assert "allow 172.16.0.0/12;" in nginx assert "deny all;" in nginx @@ -83,7 +83,7 @@ def test_source_owned_edge_and_prometheus_templates_are_closed_by_default(): assert "proxy_ssl_verify on;" in nginx assert "job_name: 'ewoooc-app'" in prometheus assert "metrics_path: /metrics" in prometheus - assert "targets: ['172.20.0.1:9191']" in prometheus + assert "targets: ['172.20.0.1:19191']" in prometheus def test_monitoring_compose_has_no_plaintext_admin_password_or_public_prometheus_port():