fix(ops): reconcile Prometheus bind mount

scripts/ops/apply_metrics_edge_policy.py

@@ -42,6 +42,7 @@ def main() -> int:
         default="mo.wooo.work:443:127.0.0.1",
     )
     parser.add_argument("--canary-timeout", type=int, default=90)
+    parser.add_argument("--prometheus-compose-file", default="")
     parser.add_argument("--receipt", type=Path)
     args = parser.parse_args()
 
@@ -59,6 +60,7 @@ def main() -> int:
         apply_scope=args.apply_scope,
         metrics_canary_url=args.metrics_canary_url,
         metrics_canary_resolve=args.metrics_canary_resolve,
+        prometheus_compose_file=args.prometheus_compose_file,
     ) if args.apply else {key: value for key, value in plan.items() if not key.startswith("_")}
     output = json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)
     if args.receipt:

services/metrics_edge_controlled_apply_service.py

@@ -235,6 +235,69 @@ def _prometheus_target_health(timeout_seconds: int) -> str:
     return last_health
 
 
+def _prometheus_identity() -> dict[str, str]:
+    completed = _run([
+        "docker",
+        "inspect",
+        "prometheus",
+        "--format",
+        "{{.Id}}|{{.Image}}|{{range .Mounts}}{{if eq .Destination \"/prometheus\"}}{{.Name}}{{end}}{{end}}",
+    ])
+    container_id, image_id, data_volume = (completed.stdout.strip().split("|", 2) + ["", ""])[:3]
+    return {
+        "container_id": container_id,
+        "image_id": image_id,
+        "data_volume": data_volume,
+    }
+
+
+def _prometheus_config_sha(path: Path) -> tuple[str, str]:
+    host_sha = hashlib.sha256(path.read_bytes()).hexdigest()
+    completed = _run([
+        "docker", "exec", "prometheus", "sha256sum",
+        "/etc/prometheus/prometheus.yml",
+    ])
+    container_sha = completed.stdout.strip().split(maxsplit=1)[0]
+    return host_sha, container_sha
+
+
+def _validate_prometheus_candidate(path: Path) -> None:
+    candidate_path = "/etc/prometheus/ewoooc-candidate.yml"
+    _run(["docker", "cp", str(path), f"prometheus:{candidate_path}"])
+    try:
+        _run([
+            "docker", "exec", "prometheus", "promtool", "check", "config",
+            candidate_path,
+        ])
+    finally:
+        subprocess.run(
+            ["docker", "exec", "prometheus", "rm", "-f", candidate_path],
+            capture_output=True,
+            text=True,
+            timeout=15,
+            check=False,
+        )
+
+
+def _wait_prometheus_ready(timeout_seconds: int = 45) -> None:
+    deadline = time.monotonic() + timeout_seconds
+    while time.monotonic() < deadline:
+        completed = subprocess.run(
+            [
+                "curl", "--noproxy", "*", "-fsS", "--max-time", "5",
+                "http://127.0.0.1:9090/-/ready",
+            ],
+            capture_output=True,
+            text=True,
+            timeout=8,
+            check=False,
+        )
+        if completed.returncode == 0:
+            return
+        time.sleep(2)
+    raise RuntimeError("Prometheus did not become ready after controlled rebind")
+
+
 def apply_metrics_edge_plan(
     plan: dict[str, Any],
     *,
@@ -242,6 +305,7 @@ def apply_metrics_edge_plan(
     apply_scope: str = "full",
     metrics_canary_url: str = "https://mo.wooo.work/metrics",
     metrics_canary_resolve: str = "mo.wooo.work:443:127.0.0.1",
+    prometheus_compose_file: str = "",
 ) -> dict[str, Any]:
     if apply_scope not in {"full", "prometheus_only"}:
         raise ValueError("apply_scope must be full or prometheus_only")
@@ -269,6 +333,7 @@ def apply_metrics_edge_plan(
         },
         "rollback_performed": False,
         "post_verifier": {},
+        "prometheus_rebound": False,
     })
     if apply_scope == "full" and nginx_existed_before:
         receipt["backup_paths"]["nginx"] = str(nginx_backup)
@@ -280,13 +345,40 @@ def apply_metrics_edge_plan(
         _inplace_write(prometheus_path, candidate["prometheus"])
         if apply_scope == "full":
             _run(["nginx", "-t"])
+        _validate_prometheus_candidate(prometheus_path)
+        if apply_scope == "full":
+            _run(["systemctl", "reload", "nginx"])
+        prometheus_before = _prometheus_identity()
+        host_sha, container_sha = _prometheus_config_sha(prometheus_path)
+        if host_sha != container_sha:
+            if not prometheus_compose_file:
+                raise RuntimeError("Prometheus bind mount inode drift requires compose rebind")
+            _run([
+                "docker", "compose", "-f", prometheus_compose_file, "up", "-d",
+                "--no-deps", "--force-recreate", "--pull", "never", "prometheus",
+            ], timeout=120)
+            _wait_prometheus_ready()
+            receipt["prometheus_rebound"] = True
+        else:
+            _run(["docker", "kill", "--signal=SIGHUP", "prometheus"])
         _run([
             "docker", "exec", "prometheus", "promtool", "check", "config",
             "/etc/prometheus/prometheus.yml",
         ])
-        if apply_scope == "full":
-            _run(["systemctl", "reload", "nginx"])
-        _run(["docker", "kill", "--signal=SIGHUP", "prometheus"])
+        host_sha, container_sha = _prometheus_config_sha(prometheus_path)
+        if host_sha != container_sha:
+            raise RuntimeError("Prometheus config SHA mismatch after controlled rebind")
+        prometheus_after = _prometheus_identity()
+        receipt["prometheus_runtime"] = {
+            "before": prometheus_before,
+            "after": prometheus_after,
+            "config_sha256": host_sha,
+            "config_inode_reconciled": True,
+            "data_volume_unchanged": (
+                prometheus_before["data_volume"] == prometheus_after["data_volume"]
+            ),
+            "image_unchanged": prometheus_before["image_id"] == prometheus_after["image_id"],
+        }
         canary_command = ["curl", "--noproxy", "*", "-sS", "--max-time", "20"]
         if metrics_canary_resolve:
             canary_command.extend(["--resolve", metrics_canary_resolve])
@@ -302,6 +394,7 @@ def apply_metrics_edge_plan(
             "internal_metrics_http_code": internal_code,
             "metrics_internal_canary": internal_code == "200",
             "metrics_target_health": target_health,
+            "prometheus_config_inode_reconciled": True,
         }
         if internal_code != "200" or target_health != "up":
             raise RuntimeError(

tests/test_metrics_edge_controlled_apply_service.py

@@ -126,6 +126,24 @@ def test_prometheus_write_preserves_bind_mount_inode(tmp_path: Path):
     assert config.read_text(encoding="utf-8") == "global:\n  scrape_interval: 30s\n"
 
 
+def test_prometheus_config_sha_detects_stale_container_mount(tmp_path: Path, monkeypatch):
+    from services import metrics_edge_controlled_apply_service as service
+
+    config = tmp_path / "prometheus.yml"
+    config.write_text("candidate", encoding="utf-8")
+    stale_sha = "0" * 64
+    monkeypatch.setattr(
+        service,
+        "_run",
+        lambda _command: type("Completed", (), {"stdout": f"{stale_sha}  config"})(),
+    )
+
+    host_sha, container_sha = service._prometheus_config_sha(config)
+
+    assert host_sha != container_sha
+    assert container_sha == stale_sha
+
+
 def test_render_rejects_incomplete_templates():
     from services.metrics_edge_controlled_apply_service import (
         render_nginx_config,