From bd5350349d025f1e28f5c4cfa31bbd4503b91d15 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 20:08:21 +0800 Subject: [PATCH] fix(ops): bridge production metrics securely --- config.py | 2 +- docs/AI_INTELLIGENCE_MODULE_SOT.md | 3 +- .../edge/ewoooc_metrics_nginx_bridge.conf | 24 +++++ .../edge/ewoooc_metrics_nginx_location.conf | 13 --- monitoring/edge/ewoooc_prometheus_scrape.yml | 4 +- scripts/ops/apply_metrics_edge_policy.py | 16 ++++ services/http_access_policy_service.py | 7 +- .../metrics_edge_controlled_apply_service.py | 89 ++++++++++++++----- ...t_metrics_edge_controlled_apply_service.py | 37 +++++++- tests/test_metrics_edge_security.py | 19 ++-- 10 files changed, 168 insertions(+), 46 deletions(-) create mode 100644 monitoring/edge/ewoooc_metrics_nginx_bridge.conf delete mode 100644 monitoring/edge/ewoooc_metrics_nginx_location.conf diff --git a/config.py b/config.py index 862311c..6f117e4 100644 --- a/config.py +++ b/config.py @@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '') # ========================================== # 系統版本與路徑 # ========================================== -SYSTEM_VERSION = "V10.786" +SYSTEM_VERSION = "V10.787" LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log') public_url = PUBLIC_URL # 用於模板顯示 diff --git a/docs/AI_INTELLIGENCE_MODULE_SOT.md b/docs/AI_INTELLIGENCE_MODULE_SOT.md index c29a28b..38c2e18 100644 --- a/docs/AI_INTELLIGENCE_MODULE_SOT.md +++ b/docs/AI_INTELLIGENCE_MODULE_SOT.md @@ -2,7 +2,7 @@ > **最後更新**: 2026-07-11 (台北時間) > **狀態**: 🟠 Partial。四 Agent、PixelRAG、MCP/RAG registry、PChome controlled-preview families 與多項 smoke/readback 已建立；但 access control、database identity/RBAC、full asset reconciliation、software supply chain、same-run controlled apply、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。 -> **適用版本**: V10.786 +> **適用版本**: V10.787 --- @@ -16,6 +16,7 @@ - production 安全與治理完成度必須分開顯示 program、asset coverage、runtime closure；目前結論是 partial，不是 complete。 - 目前 ordered P0 以 `docs/guides/ai_automation_mainline_work_items.md` 為準：先關閉 access exposure，再做 identity/RBAC、webhook trust、supply chain、asset reconciliation、controlled-apply envelope、internal RAG canary 與 MCP/RAG runtime closure。 - V10.786 起 `/metrics` 不再是 public exception；應用中央政策與 Nginx exact-match edge policy 僅允許 loopback / Docker transport，Prometheus 必須以內部 canary 證明 target `up`，且 public/LAN readback 必須拒絕。監控 Compose 不得保存明文 Grafana 管理密碼，Prometheus 對外埠預設只綁 loopback。 +- V10.787 依 live ingress 修正 metrics transport：production ingress 實際位於 188，110 global Prometheus 只能經 `172.20.0.1:9191` Docker-only Nginx bridge 抓取；bridge 以系統 CA、`mo.wooo.work` SNI/Host 轉送至 188，應用僅額外允許 `192.168.0.110/32` 單一監控主機，不得放行整段 LAN，也不得把 110 的其他 vhost 誤當 EwoooC production ingress。controlled apply 必須支援新 include 建立、Prometheus canary 與失敗自動移除/回滾。 - V10.777 起 web process 只能註冊 OpenClaw Blueprint，不得在 import 或 `record_once` 啟動 APScheduler、timed jobs 或 embedding thread；10 個 OpenClaw timed jobs 與唯一 embedding worker runtime owner 必須由 `momo-scheduler` 明確啟動。每日 04:15 的 codebase inventory 只掃描程式與前端資產並寫 artifact receipt，不讀 secret、不寫業務 DB。 --- diff --git a/monitoring/edge/ewoooc_metrics_nginx_bridge.conf b/monitoring/edge/ewoooc_metrics_nginx_bridge.conf new file mode 100644 index 0000000..f5e2871 --- /dev/null +++ b/monitoring/edge/ewoooc_metrics_nginx_bridge.conf @@ -0,0 +1,24 @@ +# Internal bridge from the 110 Prometheus network to the 188 EwoooC ingress. +server { + listen 172.20.0.1:9191; + server_name ewooooc-metrics.internal; + access_log off; + + location = /metrics { + allow 172.16.0.0/12; + deny all; + + proxy_pass https://192.168.0.188/metrics; + proxy_http_version 1.1; + proxy_ssl_server_name on; + proxy_ssl_name mo.wooo.work; + proxy_ssl_verify on; + proxy_ssl_verify_depth 3; + proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + proxy_set_header Host mo.wooo.work; + } + + location / { + return 404; + } +} diff --git a/monitoring/edge/ewoooc_metrics_nginx_location.conf b/monitoring/edge/ewoooc_metrics_nginx_location.conf deleted file mode 100644 index 7883d56..0000000 --- a/monitoring/edge/ewoooc_metrics_nginx_location.conf +++ /dev/null @@ -1,13 +0,0 @@ -# Source-owned exact-match edge policy for EwoooC Prometheus metrics. -location = /metrics { - allow 127.0.0.1; - allow ::1; - allow 172.16.0.0/12; - deny all; - - proxy_pass http://127.0.0.1:5003/metrics; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; -} diff --git a/monitoring/edge/ewoooc_prometheus_scrape.yml b/monitoring/edge/ewoooc_prometheus_scrape.yml index db1713b..2e6ed77 100644 --- a/monitoring/edge/ewoooc_prometheus_scrape.yml +++ b/monitoring/edge/ewoooc_prometheus_scrape.yml @@ -1,10 +1,10 @@ - job_name: 'ewoooc-app' - scheme: https + scheme: http metrics_path: /metrics scrape_interval: 30s scrape_timeout: 15s static_configs: - - targets: ['mo.wooo.work'] + - targets: ['172.20.0.1:9191'] labels: environment: 'production' product: 'ewoooc' diff --git a/scripts/ops/apply_metrics_edge_policy.py b/scripts/ops/apply_metrics_edge_policy.py index 78d7d0c..f6c71d4 100755 --- a/scripts/ops/apply_metrics_edge_policy.py +++ b/scripts/ops/apply_metrics_edge_policy.py @@ -28,6 +28,19 @@ def main() -> int: parser.add_argument("--run-id", required=True) parser.add_argument("--work-item-id", default="SEC-P0-001") parser.add_argument("--apply", action="store_true") + parser.add_argument( + "--apply-scope", + choices=("full", "prometheus_only"), + default="full", + ) + parser.add_argument( + "--metrics-canary-url", + default="https://mo.wooo.work/metrics", + ) + parser.add_argument( + "--metrics-canary-resolve", + default="mo.wooo.work:443:127.0.0.1", + ) parser.add_argument("--canary-timeout", type=int, default=90) parser.add_argument("--receipt", type=Path) args = parser.parse_args() @@ -43,6 +56,9 @@ def main() -> int: payload = apply_metrics_edge_plan( plan, canary_timeout=args.canary_timeout, + apply_scope=args.apply_scope, + metrics_canary_url=args.metrics_canary_url, + metrics_canary_resolve=args.metrics_canary_resolve, ) if args.apply else {key: value for key, value in plan.items() if not key.startswith("_")} output = json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) if args.receipt: diff --git a/services/http_access_policy_service.py b/services/http_access_policy_service.py index 0b7ecc0..d03027b 100644 --- a/services/http_access_policy_service.py +++ b/services/http_access_policy_service.py @@ -51,7 +51,12 @@ INTERNAL_ONLY_ENDPOINTS = frozenset({ }) INTERNAL_METRICS_NETWORKS = tuple( ipaddress.ip_network(cidr) - for cidr in ("127.0.0.0/8", "172.16.0.0/12", "::1/128") + for cidr in ( + "127.0.0.0/8", + "172.16.0.0/12", + "192.168.0.110/32", + "::1/128", + ) ) diff --git a/services/metrics_edge_controlled_apply_service.py b/services/metrics_edge_controlled_apply_service.py index 4425514..4f0aeaa 100644 --- a/services/metrics_edge_controlled_apply_service.py +++ b/services/metrics_edge_controlled_apply_service.py @@ -60,6 +60,15 @@ def render_nginx_config(config_text: str, fragment_text: str) -> str: fragment = fragment_text.strip() if "location = /metrics" not in fragment or "deny all;" not in fragment: raise ValueError("metrics nginx fragment is incomplete") + if re.search(r"(?m)^server\s*\{", fragment): + managed = f"{NGINX_BEGIN}\n{fragment}\n{NGINX_END}" + replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed) + if replaced is not None: + return f"{replaced.rstrip()}\n" + if config_text.strip(): + raise ValueError("standalone nginx fragment requires a new or managed config") + return f"{managed}\n" + indented_fragment = "\n".join(f" {line}" if line else "" for line in fragment.splitlines()) managed = f"{NGINX_BEGIN}\n{indented_fragment}\n {NGINX_END}" replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed) @@ -121,7 +130,8 @@ def build_metrics_edge_plan( ) -> dict[str, Any]: if not IDENTITY_PATTERN.fullmatch(run_id) or not IDENTITY_PATTERN.fullmatch(work_item_id): raise ValueError("run_id and work_item_id must use bounded safe characters") - nginx_before = nginx_config.read_text(encoding="utf-8") + nginx_existed_before = nginx_config.exists() + nginx_before = nginx_config.read_text(encoding="utf-8") if nginx_existed_before else "" prometheus_before = prometheus_config.read_text(encoding="utf-8") nginx_after = render_nginx_config( nginx_before, @@ -140,6 +150,7 @@ def build_metrics_edge_plan( "generated_at": datetime.now(timezone.utc).isoformat(), "nginx": { "path": str(nginx_config), + "existed_before": nginx_existed_before, "before_sha256": _sha256(nginx_before), "candidate_sha256": _sha256(nginx_after), "changed": nginx_before != nginx_after, @@ -174,7 +185,7 @@ def _run(command: list[str], timeout: int = 30) -> subprocess.CompletedProcess[s def _atomic_write(path: Path, content: str) -> None: - current = path.stat() + current = path.stat() if path.exists() else None with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", @@ -184,11 +195,23 @@ def _atomic_write(path: Path, content: str) -> None: ) as handle: handle.write(content) temporary = Path(handle.name) - os.chmod(temporary, current.st_mode) - os.chown(temporary, current.st_uid, current.st_gid) + os.chmod(temporary, current.st_mode if current else 0o644) + os.chown( + temporary, + current.st_uid if current else os.geteuid(), + current.st_gid if current else os.getegid(), + ) os.replace(temporary, path) +def _inplace_write(path: Path, content: str) -> None: + """Update a file bind mount without replacing its host inode.""" + with path.open("w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + + def _prometheus_target_health(timeout_seconds: int) -> str: deadline = time.monotonic() + timeout_seconds url = "http://127.0.0.1:9090/api/v1/targets?state=active" @@ -212,14 +235,25 @@ def _prometheus_target_health(timeout_seconds: int) -> str: return last_health -def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) -> dict[str, Any]: +def apply_metrics_edge_plan( + plan: dict[str, Any], + *, + canary_timeout: int = 90, + apply_scope: str = "full", + metrics_canary_url: str = "https://mo.wooo.work/metrics", + metrics_canary_resolve: str = "mo.wooo.work:443:127.0.0.1", +) -> dict[str, Any]: + if apply_scope not in {"full", "prometheus_only"}: + raise ValueError("apply_scope must be full or prometheus_only") candidate = plan.pop("_candidate_text") nginx_path = Path(plan["nginx"]["path"]) prometheus_path = Path(plan["prometheus"]["path"]) suffix = re.sub(r"[^A-Za-z0-9_.-]", "_", plan["run_id"]) nginx_backup = nginx_path.with_name(f"{nginx_path.name}.bak.{suffix}") prometheus_backup = prometheus_path.with_name(f"{prometheus_path.name}.bak.{suffix}") - shutil.copy2(nginx_path, nginx_backup) + nginx_existed_before = bool(plan["nginx"].get("existed_before")) + if apply_scope == "full" and nginx_existed_before: + shutil.copy2(nginx_path, nginx_backup) shutil.copy2(prometheus_path, prometheus_backup) receipt = { key: value for key, value in plan.items() @@ -228,33 +262,42 @@ def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) - receipt.update({ "receipt_kind": "metrics_edge_controlled_apply_receipt", "mode": "apply", + "apply_scope": apply_scope, "status": "in_progress", "backup_paths": { - "nginx": str(nginx_backup), "prometheus": str(prometheus_backup), }, "rollback_performed": False, "post_verifier": {}, }) + if apply_scope == "full" and nginx_existed_before: + receipt["backup_paths"]["nginx"] = str(nginx_backup) + elif apply_scope == "full": + receipt["nginx_created_new_file"] = True try: - _atomic_write(nginx_path, candidate["nginx"]) - _atomic_write(prometheus_path, candidate["prometheus"]) - _run(["nginx", "-t"]) + if apply_scope == "full": + _atomic_write(nginx_path, candidate["nginx"]) + _inplace_write(prometheus_path, candidate["prometheus"]) + if apply_scope == "full": + _run(["nginx", "-t"]) _run([ "docker", "exec", "prometheus", "promtool", "check", "config", "/etc/prometheus/prometheus.yml", ]) - _run(["systemctl", "reload", "nginx"]) + if apply_scope == "full": + _run(["systemctl", "reload", "nginx"]) _run(["docker", "kill", "--signal=SIGHUP", "prometheus"]) - internal = _run([ - "curl", "-ksS", "--max-time", "20", "--resolve", - "mo.wooo.work:443:127.0.0.1", "-o", "/dev/null", "-w", "%{http_code}", - "https://mo.wooo.work/metrics", + canary_command = ["curl", "--noproxy", "*", "-sS", "--max-time", "20"] + if metrics_canary_resolve: + canary_command.extend(["--resolve", metrics_canary_resolve]) + canary_command.extend([ + "-o", "/dev/null", "-w", "%{http_code}", metrics_canary_url, ]) + internal = _run(canary_command) internal_code = internal.stdout.strip() target_health = _prometheus_target_health(max(10, canary_timeout)) receipt["post_verifier"] = { - "nginx_config_valid": True, + "nginx_config_valid": apply_scope == "full", "prometheus_config_valid": True, "internal_metrics_http_code": internal_code, "metrics_internal_canary": internal_code == "200", @@ -268,14 +311,16 @@ def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) - receipt["completed_at"] = datetime.now(timezone.utc).isoformat() return receipt except Exception as exc: - shutil.copy2(nginx_backup, nginx_path) + if apply_scope == "full" and nginx_existed_before: + shutil.copy2(nginx_backup, nginx_path) + elif apply_scope == "full": + nginx_path.unlink(missing_ok=True) shutil.copy2(prometheus_backup, prometheus_path) rollback_errors: list[str] = [] - for command in ( - ["nginx", "-t"], - ["systemctl", "reload", "nginx"], - ["docker", "kill", "--signal=SIGHUP", "prometheus"], - ): + rollback_commands = [["docker", "kill", "--signal=SIGHUP", "prometheus"]] + if apply_scope == "full": + rollback_commands[:0] = [["nginx", "-t"], ["systemctl", "reload", "nginx"]] + for command in rollback_commands: try: _run(command) except Exception as rollback_exc: diff --git a/tests/test_metrics_edge_controlled_apply_service.py b/tests/test_metrics_edge_controlled_apply_service.py index ad7e8e5..b6b9c22 100644 --- a/tests/test_metrics_edge_controlled_apply_service.py +++ b/tests/test_metrics_edge_controlled_apply_service.py @@ -37,9 +37,21 @@ scrape_configs: """ PROMETHEUS_SNIPPET = """ - job_name: 'ewoooc-app' + scheme: http metrics_path: /metrics static_configs: - - targets: ['mo.wooo.work'] + - targets: ['172.20.0.1:9191'] +""" + +NGINX_BRIDGE = """server { + listen 172.20.0.1:9191; + server_name ewooooc-metrics.internal; + location = /metrics { + allow 172.16.0.0/12; + deny all; + proxy_pass https://192.168.0.188/metrics; + } +} """ @@ -65,6 +77,16 @@ def test_render_prometheus_config_is_idempotent(): assert rendered.count("job_name: 'ewoooc-app'") == 1 +def test_render_standalone_nginx_bridge_is_idempotent(): + from services.metrics_edge_controlled_apply_service import render_nginx_config + + rendered = render_nginx_config("", NGINX_BRIDGE) + second = render_nginx_config(rendered, NGINX_BRIDGE) + + assert rendered == second + assert rendered.count("listen 172.20.0.1:9191") == 1 + + def test_build_plan_is_no_write_and_commit_safe(tmp_path: Path): from services.metrics_edge_controlled_apply_service import build_metrics_edge_plan @@ -91,6 +113,19 @@ def test_build_plan_is_no_write_and_commit_safe(tmp_path: Path): assert paths["nginx_config"].read_text(encoding="utf-8") == NGINX_CONFIG +def test_prometheus_write_preserves_bind_mount_inode(tmp_path: Path): + from services.metrics_edge_controlled_apply_service import _inplace_write + + config = tmp_path / "prometheus.yml" + config.write_text(PROMETHEUS_CONFIG, encoding="utf-8") + inode_before = config.stat().st_ino + + _inplace_write(config, "global:\n scrape_interval: 30s\n") + + assert config.stat().st_ino == inode_before + assert config.read_text(encoding="utf-8") == "global:\n scrape_interval: 30s\n" + + def test_render_rejects_incomplete_templates(): from services.metrics_edge_controlled_apply_service import ( render_nginx_config, diff --git a/tests/test_metrics_edge_security.py b/tests/test_metrics_edge_security.py index e9aa91c..21234cc 100644 --- a/tests/test_metrics_edge_security.py +++ b/tests/test_metrics_edge_security.py @@ -28,7 +28,8 @@ def test_metrics_network_classifier_is_deny_by_default(): assert is_internal_metrics_request("172.20.0.4") is True assert is_internal_metrics_request("127.0.0.1", "172.20.0.4") is True assert is_internal_metrics_request("127.0.0.1", "203.0.113.10") is False - assert is_internal_metrics_request("127.0.0.1", "192.168.0.110") is False + assert is_internal_metrics_request("127.0.0.1", "192.168.0.110") is True + assert is_internal_metrics_request("127.0.0.1", "192.168.0.111") is False assert is_internal_metrics_request("203.0.113.10", "172.20.0.4") is False assert is_internal_metrics_request("127.0.0.1", "not-an-ip") is False @@ -44,37 +45,45 @@ def test_metrics_route_denies_public_and_lan_but_allows_internal_transport(): lan = client.get( "/metrics", environ_base={"REMOTE_ADDR": "127.0.0.1"}, - headers={"X-Real-IP": "192.168.0.110"}, + headers={"X-Real-IP": "192.168.0.111"}, ) internal = client.get( "/metrics", environ_base={"REMOTE_ADDR": "127.0.0.1"}, headers={"X-Real-IP": "172.20.0.4"}, ) + monitoring = client.get( + "/metrics", + environ_base={"REMOTE_ADDR": "127.0.0.1"}, + headers={"X-Real-IP": "192.168.0.110"}, + ) direct = client.get("/metrics", environ_base={"REMOTE_ADDR": "172.18.0.5"}) assert public.status_code == 404 assert public.get_json() == {"error": "not_found"} assert lan.status_code == 404 assert internal.status_code == 200 + assert monitoring.status_code == 200 assert direct.status_code == 200 def test_source_owned_edge_and_prometheus_templates_are_closed_by_default(): - nginx = (ROOT / "monitoring/edge/ewoooc_metrics_nginx_location.conf").read_text( + nginx = (ROOT / "monitoring/edge/ewoooc_metrics_nginx_bridge.conf").read_text( encoding="utf-8" ) prometheus = (ROOT / "monitoring/edge/ewoooc_prometheus_scrape.yml").read_text( encoding="utf-8" ) + assert "listen 172.20.0.1:9191;" in nginx assert "location = /metrics" in nginx assert "allow 172.16.0.0/12;" in nginx assert "deny all;" in nginx - assert "proxy_pass http://127.0.0.1:5003/metrics;" in nginx + assert "proxy_pass https://192.168.0.188/metrics;" in nginx + assert "proxy_ssl_verify on;" in nginx assert "job_name: 'ewoooc-app'" in prometheus assert "metrics_path: /metrics" in prometheus - assert "targets: ['mo.wooo.work']" in prometheus + assert "targets: ['172.20.0.1:9191']" in prometheus def test_monitoring_compose_has_no_plaintext_admin_password_or_public_prometheus_port():