fix(ops): verify EwoooC metrics payload

docs/AI_INTELLIGENCE_MODULE_SOT.md

@@ -16,7 +16,7 @@
 - production 安全與治理完成度必須分開顯示 program、asset coverage、runtime closure；目前結論是 partial，不是 complete。
 - 目前 ordered P0 以 `docs/guides/ai_automation_mainline_work_items.md` 為準：先關閉 access exposure，再做 identity/RBAC、webhook trust、supply chain、asset reconciliation、controlled-apply envelope、internal RAG canary 與 MCP/RAG runtime closure。
 - V10.786 起 `/metrics` 不再是 public exception；應用中央政策與 Nginx exact-match edge policy 僅允許 loopback / Docker transport，Prometheus 必須以內部 canary 證明 target `up`，且 public/LAN readback 必須拒絕。監控 Compose 不得保存明文 Grafana 管理密碼，Prometheus 對外埠預設只綁 loopback。
-- V10.787 依 live ingress 修正 metrics transport：production ingress 實際位於 188，110 global Prometheus 只能經 `172.20.0.1:9191` Docker-only Nginx bridge 抓取；bridge 以系統 CA、`mo.wooo.work` SNI/Host 轉送至 188，應用僅額外允許 `192.168.0.110/32` 單一監控主機，不得放行整段 LAN，也不得把 110 的其他 vhost 誤當 EwoooC production ingress。controlled apply 必須支援新 include 建立、Prometheus canary 與失敗自動移除/回滾。
+- V10.787 依 live ingress 修正 metrics transport：production ingress 實際位於 188，110 global Prometheus 只能經專用 `172.20.0.1:19191` Docker-only Nginx bridge 抓取；bridge 以系統 CA、`mo.wooo.work` SNI/Host 轉送至 188，應用僅額外允許 `192.168.0.110/32` 單一監控主機，不得放行整段 LAN，也不得把 110 的其他 vhost 誤當 EwoooC production ingress。controlled apply 必須支援新 include 建立、Prometheus bind-mount reconciliation、`momo_app_info`/health/database product marker canary 與失敗自動移除/回滾；單純 HTTP 200 或 target up 不得算 closure。
 - V10.777 起 web process 只能註冊 OpenClaw Blueprint，不得在 import 或 `record_once` 啟動 APScheduler、timed jobs 或 embedding thread；10 個 OpenClaw timed jobs 與唯一 embedding worker runtime owner 必須由 `momo-scheduler` 明確啟動。每日 04:15 的 codebase inventory 只掃描程式與前端資產並寫 artifact receipt，不讀 secret、不寫業務 DB。
 
 ---

monitoring/edge/ewoooc_metrics_nginx_bridge.conf

@@ -1,6 +1,6 @@
 # Internal bridge from the 110 Prometheus network to the 188 EwoooC ingress.
 server {
-    listen 172.20.0.1:9191;
+    listen 172.20.0.1:19191;
     server_name ewooooc-metrics.internal;
     access_log off;
 

monitoring/edge/ewoooc_prometheus_scrape.yml

@@ -4,7 +4,7 @@
     scrape_interval: 30s
     scrape_timeout: 15s
     static_configs:
-      - targets: ['172.20.0.1:9191']
+      - targets: ['172.20.0.1:19191']
         labels:
           environment: 'production'
           product: 'ewoooc'

services/metrics_edge_controlled_apply_service.py

@@ -310,6 +310,19 @@ def _wait_prometheus_ready(timeout_seconds: int = 45) -> None:
     raise RuntimeError("Prometheus did not become ready after controlled rebind")
 
 
+def _parse_metrics_canary(output: str) -> tuple[str, bool]:
+    marker = "\n__EWOOOC_HTTP_STATUS__:"
+    if marker not in output:
+        raise RuntimeError("metrics canary response has no HTTP status marker")
+    body, status = output.rsplit(marker, 1)
+    product_marker_present = (
+        "momo_app_info" in body
+        and "momo_app_health" in body
+        and "momo_database_up" in body
+    )
+    return status.strip(), product_marker_present
+
+
 def apply_metrics_edge_plan(
     plan: dict[str, Any],
     *,
@@ -400,24 +413,31 @@ def apply_metrics_edge_plan(
         if metrics_canary_resolve:
             canary_command.extend(["--resolve", metrics_canary_resolve])
         canary_command.extend([
-            "-o", "/dev/null", "-w", "%{http_code}", metrics_canary_url,
+            "-w", "\n__EWOOOC_HTTP_STATUS__:%{http_code}", metrics_canary_url,
         ])
         internal = _run(canary_command)
-        internal_code = internal.stdout.strip()
+        internal_code, product_marker_present = _parse_metrics_canary(internal.stdout)
         target_health = _prometheus_target_health(max(10, canary_timeout))
         receipt["post_verifier"] = {
             "nginx_config_valid": apply_scope == "full",
             "prometheus_config_valid": True,
             "internal_metrics_http_code": internal_code,
             "metrics_internal_canary": internal_code == "200",
+            "metrics_product_marker_present": product_marker_present,
             "metrics_target_health": target_health,
             "prometheus_config_inode_reconciled": True,
             "prometheus_identity_preserved": identity_preserved,
         }
-        if internal_code != "200" or target_health != "up" or not identity_preserved:
+        if (
+            internal_code != "200"
+            or not product_marker_present
+            or target_health != "up"
+            or not identity_preserved
+        ):
             raise RuntimeError(
                 "post-verifier failed: "
-                f"internal={internal_code}, target={target_health}, "
+                f"internal={internal_code}, product_marker={product_marker_present}, "
+                f"target={target_health}, "
                 f"identity_preserved={identity_preserved}"
             )
         receipt["status"] = "pass"

tests/test_metrics_edge_controlled_apply_service.py

@@ -40,11 +40,11 @@ PROMETHEUS_SNIPPET = """  - job_name: 'ewoooc-app'
     scheme: http
     metrics_path: /metrics
     static_configs:
-      - targets: ['172.20.0.1:9191']
+      - targets: ['172.20.0.1:19191']
 """
 
 NGINX_BRIDGE = """server {
-    listen 172.20.0.1:9191;
+    listen 172.20.0.1:19191;
     server_name ewooooc-metrics.internal;
     location = /metrics {
         allow 172.16.0.0/12;
@@ -84,7 +84,7 @@ def test_render_standalone_nginx_bridge_is_idempotent():
     second = render_nginx_config(rendered, NGINX_BRIDGE)
 
     assert rendered == second
-    assert rendered.count("listen 172.20.0.1:9191") == 1
+    assert rendered.count("listen 172.20.0.1:19191") == 1
 
 
 def test_build_plan_is_no_write_and_commit_safe(tmp_path: Path):
@@ -144,6 +144,19 @@ def test_prometheus_config_sha_detects_stale_container_mount(tmp_path: Path, mon
     assert container_sha == stale_sha
 
 
+def test_metrics_canary_requires_ewoooc_product_markers():
+    from services.metrics_edge_controlled_apply_service import _parse_metrics_canary
+
+    valid = (
+        "momo_app_info 1\nmomo_app_health 1\nmomo_database_up 1"
+        "\n__EWOOOC_HTTP_STATUS__:200"
+    )
+    wrong_exporter = "fail2ban_total_failed 0\n__EWOOOC_HTTP_STATUS__:200"
+
+    assert _parse_metrics_canary(valid) == ("200", True)
+    assert _parse_metrics_canary(wrong_exporter) == ("200", False)
+
+
 def test_prometheus_identity_verifier_requires_same_image_and_data_volume():
     from services.metrics_edge_controlled_apply_service import (
         _prometheus_identity_preserved,

tests/test_metrics_edge_security.py

@@ -75,7 +75,7 @@ def test_source_owned_edge_and_prometheus_templates_are_closed_by_default():
         encoding="utf-8"
     )
 
-    assert "listen 172.20.0.1:9191;" in nginx
+    assert "listen 172.20.0.1:19191;" in nginx
     assert "location = /metrics" in nginx
     assert "allow 172.16.0.0/12;" in nginx
     assert "deny all;" in nginx
@@ -83,7 +83,7 @@ def test_source_owned_edge_and_prometheus_templates_are_closed_by_default():
     assert "proxy_ssl_verify on;" in nginx
     assert "job_name: 'ewoooc-app'" in prometheus
     assert "metrics_path: /metrics" in prometheus
-    assert "targets: ['172.20.0.1:9191']" in prometheus
+    assert "targets: ['172.20.0.1:19191']" in prometheus
 
 
 def test_monitoring_compose_has_no_plaintext_admin_password_or_public_prometheus_port():