wooo/ewoooc
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(ops): bridge production metrics securely

Browse Source
This commit is contained in:
ogt
2026-07-11 20:08:21 +08:00
parent 2e46ee4722
commit bd5350349d
10 changed files with 168 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config.py
View File

@@ -402,7 +402,7 @@ YOUTUBE_API_KEY = os.getenv('YOUTUBE_API_KEY', '')
# ==========================================
# 系統版本與路徑
# ==========================================
SYSTEM_VERSION = "V10.786"
SYSTEM_VERSION = "V10.787"
LOG_FILE_PATH = os.path.join(BASE_DIR, 'logs/system.log')
public_url = PUBLIC_URL # 用於模板顯示

3
docs/AI_INTELLIGENCE_MODULE_SOT.md
View File

@@ -2,7 +2,7 @@
> **最後更新**: 2026-07-11 (台北時間)
> **狀態**: 🟠 Partial。四 Agent、PixelRAG、MCP/RAG registry、PChome controlled-preview families 與多項 smoke/readback 已建立；但 access control、database identity/RBAC、full asset reconciliation、software supply chain、same-run controlled apply、restore drill、internal RAG canary 與 MCP/RAG runtime closure 尚未全部完成。任何局部 receipt 不得代表整體閉環完成。
> **適用版本**: V10.786
> **適用版本**: V10.787
---
@@ -16,6 +16,7 @@
- production 安全與治理完成度必須分開顯示 program、asset coverage、runtime closure；目前結論是 partial，不是 complete。
- 目前 ordered P0 以 `docs/guides/ai_automation_mainline_work_items.md` 為準：先關閉 access exposure，再做 identity/RBAC、webhook trust、supply chain、asset reconciliation、controlled-apply envelope、internal RAG canary 與 MCP/RAG runtime closure。
- V10.786 起 `/metrics` 不再是 public exception；應用中央政策與 Nginx exact-match edge policy 僅允許 loopback / Docker transport，Prometheus 必須以內部 canary 證明 target `up`，且 public/LAN readback 必須拒絕。監控 Compose 不得保存明文 Grafana 管理密碼，Prometheus 對外埠預設只綁 loopback。
- V10.787 依 live ingress 修正 metrics transport：production ingress 實際位於 188，110 global Prometheus 只能經 `172.20.0.1:9191` Docker-only Nginx bridge 抓取；bridge 以系統 CA、`mo.wooo.work` SNI/Host 轉送至 188，應用僅額外允許 `192.168.0.110/32` 單一監控主機，不得放行整段 LAN，也不得把 110 的其他 vhost 誤當 EwoooC production ingress。controlled apply 必須支援新 include 建立、Prometheus canary 與失敗自動移除/回滾。
- V10.777 起 web process 只能註冊 OpenClaw Blueprint，不得在 import 或 `record_once` 啟動 APScheduler、timed jobs 或 embedding thread；10 個 OpenClaw timed jobs 與唯一 embedding worker runtime owner 必須由 `momo-scheduler` 明確啟動。每日 04:15 的 codebase inventory 只掃描程式與前端資產並寫 artifact receipt，不讀 secret、不寫業務 DB。
---

24
monitoring/edge/ewoooc_metrics_nginx_bridge.conf Normal file
View File

@@ -0,0 +1,24 @@
# Internal bridge from the 110 Prometheus network to the 188 EwoooC ingress.
server {
listen 172.20.0.1:9191;
server_name ewooooc-metrics.internal;
access_log off;
location = /metrics {
allow 172.16.0.0/12;
deny all;
proxy_pass https://192.168.0.188/metrics;
proxy_http_version 1.1;
proxy_ssl_server_name on;
proxy_ssl_name mo.wooo.work;
proxy_ssl_verify on;
proxy_ssl_verify_depth 3;
proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
proxy_set_header Host mo.wooo.work;
}
location / {
return 404;
}
}

13
monitoring/edge/ewoooc_metrics_nginx_location.conf
View File

@@ -1,13 +0,0 @@
# Source-owned exact-match edge policy for EwoooC Prometheus metrics.
location = /metrics {
allow 127.0.0.1;
allow ::1;
allow 172.16.0.0/12;
deny all;
proxy_pass http://127.0.0.1:5003/metrics;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}

4
monitoring/edge/ewoooc_prometheus_scrape.yml
View File

@@ -1,10 +1,10 @@
- job_name: 'ewoooc-app'
scheme: https
scheme: http
metrics_path: /metrics
scrape_interval: 30s
scrape_timeout: 15s
static_configs:
- targets: ['mo.wooo.work']
- targets: ['172.20.0.1:9191']
labels:
environment: 'production'
product: 'ewoooc'

16
scripts/ops/apply_metrics_edge_policy.py
View File

@@ -28,6 +28,19 @@ def main() -> int:
parser.add_argument("--run-id", required=True)
parser.add_argument("--work-item-id", default="SEC-P0-001")
parser.add_argument("--apply", action="store_true")
parser.add_argument(
"--apply-scope",
choices=("full", "prometheus_only"),
default="full",
)
parser.add_argument(
"--metrics-canary-url",
default="https://mo.wooo.work/metrics",
)
parser.add_argument(
"--metrics-canary-resolve",
default="mo.wooo.work:443:127.0.0.1",
)
parser.add_argument("--canary-timeout", type=int, default=90)
parser.add_argument("--receipt", type=Path)
args = parser.parse_args()
@@ -43,6 +56,9 @@ def main() -> int:
payload = apply_metrics_edge_plan(
plan,
canary_timeout=args.canary_timeout,
apply_scope=args.apply_scope,
metrics_canary_url=args.metrics_canary_url,
metrics_canary_resolve=args.metrics_canary_resolve,
) if args.apply else {key: value for key, value in plan.items() if not key.startswith("_")}
output = json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)
if args.receipt:

7
services/http_access_policy_service.py
View File

@@ -51,7 +51,12 @@ INTERNAL_ONLY_ENDPOINTS = frozenset({
})
INTERNAL_METRICS_NETWORKS = tuple(
ipaddress.ip_network(cidr)
for cidr in ("127.0.0.0/8", "172.16.0.0/12", "::1/128")
for cidr in (
"127.0.0.0/8",
"172.16.0.0/12",
"192.168.0.110/32",
"::1/128",
)
)

89
services/metrics_edge_controlled_apply_service.py
View File

@@ -60,6 +60,15 @@ def render_nginx_config(config_text: str, fragment_text: str) -> str:
fragment = fragment_text.strip()
if "location = /metrics" not in fragment or "deny all;" not in fragment:
raise ValueError("metrics nginx fragment is incomplete")
if re.search(r"(?m)^server\s*\{", fragment):
managed = f"{NGINX_BEGIN}\n{fragment}\n{NGINX_END}"
replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed)
if replaced is not None:
return f"{replaced.rstrip()}\n"
if config_text.strip():
raise ValueError("standalone nginx fragment requires a new or managed config")
return f"{managed}\n"
indented_fragment = "\n".join(f" {line}" if line else "" for line in fragment.splitlines())
managed = f"{NGINX_BEGIN}\n{indented_fragment}\n {NGINX_END}"
replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed)
@@ -121,7 +130,8 @@ def build_metrics_edge_plan(
) -> dict[str, Any]:
if not IDENTITY_PATTERN.fullmatch(run_id) or not IDENTITY_PATTERN.fullmatch(work_item_id):
raise ValueError("run_id and work_item_id must use bounded safe characters")
nginx_before = nginx_config.read_text(encoding="utf-8")
nginx_existed_before = nginx_config.exists()
nginx_before = nginx_config.read_text(encoding="utf-8") if nginx_existed_before else ""
prometheus_before = prometheus_config.read_text(encoding="utf-8")
nginx_after = render_nginx_config(
nginx_before,
@@ -140,6 +150,7 @@ def build_metrics_edge_plan(
"generated_at": datetime.now(timezone.utc).isoformat(),
"nginx": {
"path": str(nginx_config),
"existed_before": nginx_existed_before,
"before_sha256": _sha256(nginx_before),
"candidate_sha256": _sha256(nginx_after),
"changed": nginx_before != nginx_after,
@@ -174,7 +185,7 @@ def _run(command: list[str], timeout: int = 30) -> subprocess.CompletedProcess[s
def _atomic_write(path: Path, content: str) -> None:
current = path.stat()
current = path.stat() if path.exists() else None
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
@@ -184,11 +195,23 @@ def _atomic_write(path: Path, content: str) -> None:
) as handle:
handle.write(content)
temporary = Path(handle.name)
os.chmod(temporary, current.st_mode)
os.chown(temporary, current.st_uid, current.st_gid)
os.chmod(temporary, current.st_mode if current else 0o644)
os.chown(
temporary,
current.st_uid if current else os.geteuid(),
current.st_gid if current else os.getegid(),
)
os.replace(temporary, path)
def _inplace_write(path: Path, content: str) -> None:
"""Update a file bind mount without replacing its host inode."""
with path.open("w", encoding="utf-8") as handle:
handle.write(content)
handle.flush()
os.fsync(handle.fileno())
def _prometheus_target_health(timeout_seconds: int) -> str:
deadline = time.monotonic() + timeout_seconds
url = "http://127.0.0.1:9090/api/v1/targets?state=active"
@@ -212,14 +235,25 @@ def _prometheus_target_health(timeout_seconds: int) -> str:
return last_health
def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) -> dict[str, Any]:
def apply_metrics_edge_plan(
plan: dict[str, Any],
*,
canary_timeout: int = 90,
apply_scope: str = "full",
metrics_canary_url: str = "https://mo.wooo.work/metrics",
metrics_canary_resolve: str = "mo.wooo.work:443:127.0.0.1",
) -> dict[str, Any]:
if apply_scope not in {"full", "prometheus_only"}:
raise ValueError("apply_scope must be full or prometheus_only")
candidate = plan.pop("_candidate_text")
nginx_path = Path(plan["nginx"]["path"])
prometheus_path = Path(plan["prometheus"]["path"])
suffix = re.sub(r"[^A-Za-z0-9_.-]", "_", plan["run_id"])
nginx_backup = nginx_path.with_name(f"{nginx_path.name}.bak.{suffix}")
prometheus_backup = prometheus_path.with_name(f"{prometheus_path.name}.bak.{suffix}")
shutil.copy2(nginx_path, nginx_backup)
nginx_existed_before = bool(plan["nginx"].get("existed_before"))
if apply_scope == "full" and nginx_existed_before:
shutil.copy2(nginx_path, nginx_backup)
shutil.copy2(prometheus_path, prometheus_backup)
receipt = {
key: value for key, value in plan.items()
@@ -228,33 +262,42 @@ def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) -
receipt.update({
"receipt_kind": "metrics_edge_controlled_apply_receipt",
"mode": "apply",
"apply_scope": apply_scope,
"status": "in_progress",
"backup_paths": {
"nginx": str(nginx_backup),
"prometheus": str(prometheus_backup),
},
"rollback_performed": False,
"post_verifier": {},
})
if apply_scope == "full" and nginx_existed_before:
receipt["backup_paths"]["nginx"] = str(nginx_backup)
elif apply_scope == "full":
receipt["nginx_created_new_file"] = True
try:
_atomic_write(nginx_path, candidate["nginx"])
_atomic_write(prometheus_path, candidate["prometheus"])
_run(["nginx", "-t"])
if apply_scope == "full":
_atomic_write(nginx_path, candidate["nginx"])
_inplace_write(prometheus_path, candidate["prometheus"])
if apply_scope == "full":
_run(["nginx", "-t"])
_run([
"docker", "exec", "prometheus", "promtool", "check", "config",
"/etc/prometheus/prometheus.yml",
])
_run(["systemctl", "reload", "nginx"])
if apply_scope == "full":
_run(["systemctl", "reload", "nginx"])
_run(["docker", "kill", "--signal=SIGHUP", "prometheus"])
internal = _run([
"curl", "-ksS", "--max-time", "20", "--resolve",
"mo.wooo.work:443:127.0.0.1", "-o", "/dev/null", "-w", "%{http_code}",
"https://mo.wooo.work/metrics",
canary_command = ["curl", "--noproxy", "*", "-sS", "--max-time", "20"]
if metrics_canary_resolve:
canary_command.extend(["--resolve", metrics_canary_resolve])
canary_command.extend([
"-o", "/dev/null", "-w", "%{http_code}", metrics_canary_url,
])
internal = _run(canary_command)
internal_code = internal.stdout.strip()
target_health = _prometheus_target_health(max(10, canary_timeout))
receipt["post_verifier"] = {
"nginx_config_valid": True,
"nginx_config_valid": apply_scope == "full",
"prometheus_config_valid": True,
"internal_metrics_http_code": internal_code,
"metrics_internal_canary": internal_code == "200",
@@ -268,14 +311,16 @@ def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) -
receipt["completed_at"] = datetime.now(timezone.utc).isoformat()
return receipt
except Exception as exc:
shutil.copy2(nginx_backup, nginx_path)
if apply_scope == "full" and nginx_existed_before:
shutil.copy2(nginx_backup, nginx_path)
elif apply_scope == "full":
nginx_path.unlink(missing_ok=True)
shutil.copy2(prometheus_backup, prometheus_path)
rollback_errors: list[str] = []
for command in (
["nginx", "-t"],
["systemctl", "reload", "nginx"],
["docker", "kill", "--signal=SIGHUP", "prometheus"],
):
rollback_commands = [["docker", "kill", "--signal=SIGHUP", "prometheus"]]
if apply_scope == "full":
rollback_commands[:0] = [["nginx", "-t"], ["systemctl", "reload", "nginx"]]
for command in rollback_commands:
try:
_run(command)
except Exception as rollback_exc:

37
tests/test_metrics_edge_controlled_apply_service.py
View File

@@ -37,9 +37,21 @@ scrape_configs:
"""
PROMETHEUS_SNIPPET = """ - job_name: 'ewoooc-app'
scheme: http
metrics_path: /metrics
static_configs:
- targets: ['mo.wooo.work']
- targets: ['172.20.0.1:9191']
"""
NGINX_BRIDGE = """server {
listen 172.20.0.1:9191;
server_name ewooooc-metrics.internal;
location = /metrics {
allow 172.16.0.0/12;
deny all;
proxy_pass https://192.168.0.188/metrics;
}
}
"""
@@ -65,6 +77,16 @@ def test_render_prometheus_config_is_idempotent():
assert rendered.count("job_name: 'ewoooc-app'") == 1
def test_render_standalone_nginx_bridge_is_idempotent():
from services.metrics_edge_controlled_apply_service import render_nginx_config
rendered = render_nginx_config("", NGINX_BRIDGE)
second = render_nginx_config(rendered, NGINX_BRIDGE)
assert rendered == second
assert rendered.count("listen 172.20.0.1:9191") == 1
def test_build_plan_is_no_write_and_commit_safe(tmp_path: Path):
from services.metrics_edge_controlled_apply_service import build_metrics_edge_plan
@@ -91,6 +113,19 @@ def test_build_plan_is_no_write_and_commit_safe(tmp_path: Path):
assert paths["nginx_config"].read_text(encoding="utf-8") == NGINX_CONFIG
def test_prometheus_write_preserves_bind_mount_inode(tmp_path: Path):
from services.metrics_edge_controlled_apply_service import _inplace_write
config = tmp_path / "prometheus.yml"
config.write_text(PROMETHEUS_CONFIG, encoding="utf-8")
inode_before = config.stat().st_ino
_inplace_write(config, "global:\n scrape_interval: 30s\n")
assert config.stat().st_ino == inode_before
assert config.read_text(encoding="utf-8") == "global:\n scrape_interval: 30s\n"
def test_render_rejects_incomplete_templates():
from services.metrics_edge_controlled_apply_service import (
render_nginx_config,

19
tests/test_metrics_edge_security.py
View File

@@ -28,7 +28,8 @@ def test_metrics_network_classifier_is_deny_by_default():
assert is_internal_metrics_request("172.20.0.4") is True
assert is_internal_metrics_request("127.0.0.1", "172.20.0.4") is True
assert is_internal_metrics_request("127.0.0.1", "203.0.113.10") is False
assert is_internal_metrics_request("127.0.0.1", "192.168.0.110") is False
assert is_internal_metrics_request("127.0.0.1", "192.168.0.110") is True
assert is_internal_metrics_request("127.0.0.1", "192.168.0.111") is False
assert is_internal_metrics_request("203.0.113.10", "172.20.0.4") is False
assert is_internal_metrics_request("127.0.0.1", "not-an-ip") is False
@@ -44,37 +45,45 @@ def test_metrics_route_denies_public_and_lan_but_allows_internal_transport():
lan = client.get(
"/metrics",
environ_base={"REMOTE_ADDR": "127.0.0.1"},
headers={"X-Real-IP": "192.168.0.110"},
headers={"X-Real-IP": "192.168.0.111"},
)
internal = client.get(
"/metrics",
environ_base={"REMOTE_ADDR": "127.0.0.1"},
headers={"X-Real-IP": "172.20.0.4"},
)
monitoring = client.get(
"/metrics",
environ_base={"REMOTE_ADDR": "127.0.0.1"},
headers={"X-Real-IP": "192.168.0.110"},
)
direct = client.get("/metrics", environ_base={"REMOTE_ADDR": "172.18.0.5"})
assert public.status_code == 404
assert public.get_json() == {"error": "not_found"}
assert lan.status_code == 404
assert internal.status_code == 200
assert monitoring.status_code == 200
assert direct.status_code == 200
def test_source_owned_edge_and_prometheus_templates_are_closed_by_default():
nginx = (ROOT / "monitoring/edge/ewoooc_metrics_nginx_location.conf").read_text(
nginx = (ROOT / "monitoring/edge/ewoooc_metrics_nginx_bridge.conf").read_text(
encoding="utf-8"
)
prometheus = (ROOT / "monitoring/edge/ewoooc_prometheus_scrape.yml").read_text(
encoding="utf-8"
)
assert "listen 172.20.0.1:9191;" in nginx
assert "location = /metrics" in nginx
assert "allow 172.16.0.0/12;" in nginx
assert "deny all;" in nginx
assert "proxy_pass http://127.0.0.1:5003/metrics;" in nginx
assert "proxy_pass https://192.168.0.188/metrics;" in nginx
assert "proxy_ssl_verify on;" in nginx
assert "job_name: 'ewoooc-app'" in prometheus
assert "metrics_path: /metrics" in prometheus
assert "targets: ['mo.wooo.work']" in prometheus
assert "targets: ['172.20.0.1:9191']" in prometheus
def test_monitoring_compose_has_no_plaintext_admin_password_or_public_prometheus_port():