From e7fd82111c568cc624c97977fd72071f34b1e5a1 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 20:17:36 +0800 Subject: [PATCH] fix(ops): reconcile Prometheus bind mount --- scripts/ops/apply_metrics_edge_policy.py | 2 + .../metrics_edge_controlled_apply_service.py | 99 ++++++++++++++++++- ...t_metrics_edge_controlled_apply_service.py | 18 ++++ 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/scripts/ops/apply_metrics_edge_policy.py b/scripts/ops/apply_metrics_edge_policy.py index f6c71d4..729dd0d 100755 --- a/scripts/ops/apply_metrics_edge_policy.py +++ b/scripts/ops/apply_metrics_edge_policy.py @@ -42,6 +42,7 @@ def main() -> int: default="mo.wooo.work:443:127.0.0.1", ) parser.add_argument("--canary-timeout", type=int, default=90) + parser.add_argument("--prometheus-compose-file", default="") parser.add_argument("--receipt", type=Path) args = parser.parse_args() @@ -59,6 +60,7 @@ def main() -> int: apply_scope=args.apply_scope, metrics_canary_url=args.metrics_canary_url, metrics_canary_resolve=args.metrics_canary_resolve, + prometheus_compose_file=args.prometheus_compose_file, ) if args.apply else {key: value for key, value in plan.items() if not key.startswith("_")} output = json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) if args.receipt: diff --git a/services/metrics_edge_controlled_apply_service.py b/services/metrics_edge_controlled_apply_service.py index 4f0aeaa..cfc78af 100644 --- a/services/metrics_edge_controlled_apply_service.py +++ b/services/metrics_edge_controlled_apply_service.py @@ -235,6 +235,69 @@ def _prometheus_target_health(timeout_seconds: int) -> str: return last_health +def _prometheus_identity() -> dict[str, str]: + completed = _run([ + "docker", + "inspect", + "prometheus", + "--format", + "{{.Id}}|{{.Image}}|{{range .Mounts}}{{if eq .Destination \"/prometheus\"}}{{.Name}}{{end}}{{end}}", + ]) + container_id, image_id, data_volume = (completed.stdout.strip().split("|", 2) + ["", ""])[:3] + return { + "container_id": container_id, + "image_id": image_id, + "data_volume": data_volume, + } + + +def _prometheus_config_sha(path: Path) -> tuple[str, str]: + host_sha = hashlib.sha256(path.read_bytes()).hexdigest() + completed = _run([ + "docker", "exec", "prometheus", "sha256sum", + "/etc/prometheus/prometheus.yml", + ]) + container_sha = completed.stdout.strip().split(maxsplit=1)[0] + return host_sha, container_sha + + +def _validate_prometheus_candidate(path: Path) -> None: + candidate_path = "/etc/prometheus/ewoooc-candidate.yml" + _run(["docker", "cp", str(path), f"prometheus:{candidate_path}"]) + try: + _run([ + "docker", "exec", "prometheus", "promtool", "check", "config", + candidate_path, + ]) + finally: + subprocess.run( + ["docker", "exec", "prometheus", "rm", "-f", candidate_path], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + + +def _wait_prometheus_ready(timeout_seconds: int = 45) -> None: + deadline = time.monotonic() + timeout_seconds + while time.monotonic() < deadline: + completed = subprocess.run( + [ + "curl", "--noproxy", "*", "-fsS", "--max-time", "5", + "http://127.0.0.1:9090/-/ready", + ], + capture_output=True, + text=True, + timeout=8, + check=False, + ) + if completed.returncode == 0: + return + time.sleep(2) + raise RuntimeError("Prometheus did not become ready after controlled rebind") + + def apply_metrics_edge_plan( plan: dict[str, Any], *, @@ -242,6 +305,7 @@ def apply_metrics_edge_plan( apply_scope: str = "full", metrics_canary_url: str = "https://mo.wooo.work/metrics", metrics_canary_resolve: str = "mo.wooo.work:443:127.0.0.1", + prometheus_compose_file: str = "", ) -> dict[str, Any]: if apply_scope not in {"full", "prometheus_only"}: raise ValueError("apply_scope must be full or prometheus_only") @@ -269,6 +333,7 @@ def apply_metrics_edge_plan( }, "rollback_performed": False, "post_verifier": {}, + "prometheus_rebound": False, }) if apply_scope == "full" and nginx_existed_before: receipt["backup_paths"]["nginx"] = str(nginx_backup) @@ -280,13 +345,40 @@ def apply_metrics_edge_plan( _inplace_write(prometheus_path, candidate["prometheus"]) if apply_scope == "full": _run(["nginx", "-t"]) + _validate_prometheus_candidate(prometheus_path) + if apply_scope == "full": + _run(["systemctl", "reload", "nginx"]) + prometheus_before = _prometheus_identity() + host_sha, container_sha = _prometheus_config_sha(prometheus_path) + if host_sha != container_sha: + if not prometheus_compose_file: + raise RuntimeError("Prometheus bind mount inode drift requires compose rebind") + _run([ + "docker", "compose", "-f", prometheus_compose_file, "up", "-d", + "--no-deps", "--force-recreate", "--pull", "never", "prometheus", + ], timeout=120) + _wait_prometheus_ready() + receipt["prometheus_rebound"] = True + else: + _run(["docker", "kill", "--signal=SIGHUP", "prometheus"]) _run([ "docker", "exec", "prometheus", "promtool", "check", "config", "/etc/prometheus/prometheus.yml", ]) - if apply_scope == "full": - _run(["systemctl", "reload", "nginx"]) - _run(["docker", "kill", "--signal=SIGHUP", "prometheus"]) + host_sha, container_sha = _prometheus_config_sha(prometheus_path) + if host_sha != container_sha: + raise RuntimeError("Prometheus config SHA mismatch after controlled rebind") + prometheus_after = _prometheus_identity() + receipt["prometheus_runtime"] = { + "before": prometheus_before, + "after": prometheus_after, + "config_sha256": host_sha, + "config_inode_reconciled": True, + "data_volume_unchanged": ( + prometheus_before["data_volume"] == prometheus_after["data_volume"] + ), + "image_unchanged": prometheus_before["image_id"] == prometheus_after["image_id"], + } canary_command = ["curl", "--noproxy", "*", "-sS", "--max-time", "20"] if metrics_canary_resolve: canary_command.extend(["--resolve", metrics_canary_resolve]) @@ -302,6 +394,7 @@ def apply_metrics_edge_plan( "internal_metrics_http_code": internal_code, "metrics_internal_canary": internal_code == "200", "metrics_target_health": target_health, + "prometheus_config_inode_reconciled": True, } if internal_code != "200" or target_health != "up": raise RuntimeError( diff --git a/tests/test_metrics_edge_controlled_apply_service.py b/tests/test_metrics_edge_controlled_apply_service.py index b6b9c22..ca00eb1 100644 --- a/tests/test_metrics_edge_controlled_apply_service.py +++ b/tests/test_metrics_edge_controlled_apply_service.py @@ -126,6 +126,24 @@ def test_prometheus_write_preserves_bind_mount_inode(tmp_path: Path): assert config.read_text(encoding="utf-8") == "global:\n scrape_interval: 30s\n" +def test_prometheus_config_sha_detects_stale_container_mount(tmp_path: Path, monkeypatch): + from services import metrics_edge_controlled_apply_service as service + + config = tmp_path / "prometheus.yml" + config.write_text("candidate", encoding="utf-8") + stale_sha = "0" * 64 + monkeypatch.setattr( + service, + "_run", + lambda _command: type("Completed", (), {"stdout": f"{stale_sha} config"})(), + ) + + host_sha, container_sha = service._prometheus_config_sha(config) + + assert host_sha != container_sha + assert container_sha == stale_sha + + def test_render_rejects_incomplete_templates(): from services.metrics_edge_controlled_apply_service import ( render_nginx_config,