"""Controlled apply for the EwoooC metrics edge and Prometheus target.""" from __future__ import annotations import hashlib import json import os import re import shutil import subprocess import tempfile import time import urllib.request from datetime import datetime, timezone from pathlib import Path from typing import Any POLICY = "ewoooc_metrics_edge_controlled_apply_v1" NGINX_BEGIN = "# BEGIN MANAGED EWOOOC METRICS" NGINX_END = "# END MANAGED EWOOOC METRICS" PROMETHEUS_BEGIN = "# BEGIN MANAGED EWOOOC METRICS" PROMETHEUS_END = "# END MANAGED EWOOOC METRICS" IDENTITY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{2,127}$") def _managed_replace(text: str, begin: str, end: str, managed: str) -> str | None: begin_count = text.count(begin) end_count = text.count(end) if begin_count == 0 and end_count == 0: return None if begin_count != 1 or end_count != 1: raise ValueError(f"invalid managed marker count for {begin}") start = text.index(begin) finish = text.index(end, start) + len(end) return f"{text[:start]}{managed}{text[finish:]}" def _nginx_server_blocks(lines: list[str]) -> list[tuple[int, int]]: blocks: list[tuple[int, int]] = [] start: int | None = None depth = 0 for index, line in enumerate(lines): code = line.split("#", 1)[0] if start is None and re.match(r"^\s*server\s*\{", code): start = index depth = 0 if start is None: continue depth += code.count("{") - code.count("}") if depth == 0: blocks.append((start, index + 1)) start = None if start is not None: raise ValueError("unbalanced nginx server block") return blocks def render_nginx_config(config_text: str, fragment_text: str) -> str: fragment = fragment_text.strip() if "location = /metrics" not in fragment or "deny all;" not in fragment: raise ValueError("metrics nginx fragment is incomplete") if re.search(r"(?m)^server\s*\{", fragment): managed = f"{NGINX_BEGIN}\n{fragment}\n{NGINX_END}" replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed) if replaced is not None: return f"{replaced.rstrip()}\n" if config_text.strip(): raise ValueError("standalone nginx fragment requires a new or managed config") return f"{managed}\n" indented_fragment = "\n".join(f" {line}" if line else "" for line in fragment.splitlines()) managed = f"{NGINX_BEGIN}\n{indented_fragment}\n {NGINX_END}" replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed) if replaced is not None: return replaced lines = config_text.splitlines(keepends=True) candidates: list[tuple[int, int]] = [] for start, end in _nginx_server_blocks(lines): block = "".join(lines[start:end]) if "server_name mo.wooo.work;" in block and re.search(r"\blisten\s+443\b", block): candidates.append((start, end)) if len(candidates) != 1: raise ValueError(f"expected one mo.wooo.work TLS server, found {len(candidates)}") start, end = candidates[0] location_indexes = [ index for index in range(start, end) if re.match(r"^\s*location\s+/\s*\{", lines[index]) ] if len(location_indexes) != 1: raise ValueError(f"expected one fallback location, found {len(location_indexes)}") insert_at = location_indexes[0] lines.insert(insert_at, f" {managed}\n\n") return "".join(lines) def render_prometheus_config(config_text: str, snippet_text: str) -> str: snippet = snippet_text.rstrip() if "job_name: 'ewoooc-app'" not in snippet or "metrics_path: /metrics" not in snippet: raise ValueError("EwoooC Prometheus snippet is incomplete") if not re.search(r"(?m)^scrape_configs:\s*$", config_text): raise ValueError("Prometheus config has no scrape_configs section") managed = f"{PROMETHEUS_BEGIN}\n{snippet}\n{PROMETHEUS_END}" replaced = _managed_replace( config_text, PROMETHEUS_BEGIN, PROMETHEUS_END, managed, ) if replaced is not None: return replaced return f"{config_text.rstrip()}\n\n{managed}\n" def _sha256(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() def build_metrics_edge_plan( *, nginx_config: Path, prometheus_config: Path, nginx_fragment: Path, prometheus_snippet: Path, run_id: str, work_item_id: str, ) -> dict[str, Any]: if not IDENTITY_PATTERN.fullmatch(run_id) or not IDENTITY_PATTERN.fullmatch(work_item_id): raise ValueError("run_id and work_item_id must use bounded safe characters") nginx_existed_before = nginx_config.exists() nginx_before = nginx_config.read_text(encoding="utf-8") if nginx_existed_before else "" prometheus_before = prometheus_config.read_text(encoding="utf-8") nginx_after = render_nginx_config( nginx_before, nginx_fragment.read_text(encoding="utf-8"), ) prometheus_after = render_prometheus_config( prometheus_before, prometheus_snippet.read_text(encoding="utf-8"), ) return { "policy": POLICY, "trace_id": run_id, "run_id": run_id, "work_item_id": work_item_id, "mode": "check", "generated_at": datetime.now(timezone.utc).isoformat(), "nginx": { "path": str(nginx_config), "existed_before": nginx_existed_before, "before_sha256": _sha256(nginx_before), "candidate_sha256": _sha256(nginx_after), "changed": nginx_before != nginx_after, }, "prometheus": { "path": str(prometheus_config), "before_sha256": _sha256(prometheus_before), "candidate_sha256": _sha256(prometheus_after), "changed": prometheus_before != prometheus_after, }, "_candidate_text": { "nginx": nginx_after, "prometheus": prometheus_after, }, "controlled_apply_allowed": True, "risk": "medium", } def _run(command: list[str], timeout: int = 30) -> subprocess.CompletedProcess[str]: completed = subprocess.run( command, capture_output=True, text=True, timeout=timeout, check=False, ) if completed.returncode != 0: message = (completed.stderr or completed.stdout or "command failed").strip() raise RuntimeError(f"{command[0]} failed: {message[:500]}") return completed def _atomic_write(path: Path, content: str) -> None: current = path.stat() if path.exists() else None with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", dir=path.parent, prefix=f".{path.name}.", delete=False, ) as handle: handle.write(content) temporary = Path(handle.name) os.chmod(temporary, current.st_mode if current else 0o644) os.chown( temporary, current.st_uid if current else os.geteuid(), current.st_gid if current else os.getegid(), ) os.replace(temporary, path) def _inplace_write(path: Path, content: str) -> None: """Update a file bind mount without replacing its host inode.""" with path.open("w", encoding="utf-8") as handle: handle.write(content) handle.flush() os.fsync(handle.fileno()) def _target_health_from_payload(payload: dict[str, Any], expected_scrape_url: str) -> str: targets = ((payload.get("data") or {}).get("activeTargets") or []) matching = [ target for target in targets if (target.get("labels") or {}).get("job") == "ewoooc-app" and ( not expected_scrape_url or target.get("scrapeUrl") == expected_scrape_url ) ] return str(matching[0].get("health") or "unknown") if matching else "missing" def _prometheus_target_health( timeout_seconds: int, expected_scrape_url: str = "", ) -> str: deadline = time.monotonic() + timeout_seconds url = "http://127.0.0.1:9090/api/v1/targets?state=active" last_health = "missing" while time.monotonic() < deadline: try: with urllib.request.urlopen(url, timeout=8) as response: payload = json.load(response) last_health = _target_health_from_payload(payload, expected_scrape_url) if last_health == "up": return last_health except (OSError, ValueError, TypeError): last_health = "query_error" time.sleep(3) return last_health def _prometheus_identity() -> dict[str, str]: completed = _run([ "docker", "inspect", "prometheus", "--format", "{{.Id}}|{{.Image}}|{{range .Mounts}}{{if eq .Destination \"/prometheus\"}}{{.Name}}{{end}}{{end}}", ]) container_id, image_id, data_volume = (completed.stdout.strip().split("|", 2) + ["", ""])[:3] if not container_id or not image_id or not data_volume: raise RuntimeError("Prometheus container identity is incomplete") return { "container_id": container_id, "image_id": image_id, "data_volume": data_volume, } def _prometheus_identity_preserved( before: dict[str, str], after: dict[str, str], ) -> bool: return bool( before.get("image_id") == after.get("image_id") and before.get("data_volume") == after.get("data_volume") ) def _prometheus_config_sha(path: Path) -> tuple[str, str]: host_sha = hashlib.sha256(path.read_bytes()).hexdigest() completed = _run([ "docker", "exec", "prometheus", "sha256sum", "/etc/prometheus/prometheus.yml", ]) container_sha = completed.stdout.strip().split(maxsplit=1)[0] return host_sha, container_sha def _validate_prometheus_candidate(path: Path) -> None: candidate_path = "/etc/prometheus/ewoooc-candidate.yml" _run(["docker", "cp", str(path), f"prometheus:{candidate_path}"]) try: _run([ "docker", "exec", "prometheus", "promtool", "check", "config", candidate_path, ]) finally: subprocess.run( ["docker", "exec", "prometheus", "rm", "-f", candidate_path], capture_output=True, text=True, timeout=15, check=False, ) def _wait_prometheus_ready(timeout_seconds: int = 45) -> None: deadline = time.monotonic() + timeout_seconds while time.monotonic() < deadline: completed = subprocess.run( [ "curl", "--noproxy", "*", "-fsS", "--max-time", "5", "http://127.0.0.1:9090/-/ready", ], capture_output=True, text=True, timeout=8, check=False, ) if completed.returncode == 0: return time.sleep(2) raise RuntimeError("Prometheus did not become ready after controlled rebind") def _parse_metrics_canary(output: str) -> tuple[str, bool]: marker = "\n__EWOOOC_HTTP_STATUS__:" if marker not in output: raise RuntimeError("metrics canary response has no HTTP status marker") body, status = output.rsplit(marker, 1) product_marker_present = ( "momo_app_info" in body and "momo_app_health" in body and "momo_database_up" in body ) return status.strip(), product_marker_present def _wait_metrics_canary( command: list[str], timeout_seconds: int, ) -> tuple[str, bool, int]: deadline = time.monotonic() + timeout_seconds last_status = "000" last_product_marker = False attempts = 0 while time.monotonic() < deadline: attempts += 1 try: completed = _run(command) last_status, last_product_marker = _parse_metrics_canary( completed.stdout ) if last_status == "200" and last_product_marker: return last_status, last_product_marker, attempts except RuntimeError: pass time.sleep(2) return last_status, last_product_marker, attempts def apply_metrics_edge_plan( plan: dict[str, Any], *, canary_timeout: int = 90, apply_scope: str = "full", metrics_canary_url: str = "https://mo.wooo.work/metrics", metrics_canary_resolve: str = "mo.wooo.work:443:127.0.0.1", prometheus_compose_file: str = "", ) -> dict[str, Any]: if apply_scope not in {"full", "prometheus_only"}: raise ValueError("apply_scope must be full or prometheus_only") candidate = plan.pop("_candidate_text") nginx_path = Path(plan["nginx"]["path"]) prometheus_path = Path(plan["prometheus"]["path"]) suffix = re.sub(r"[^A-Za-z0-9_.-]", "_", plan["run_id"]) nginx_backup = nginx_path.with_name(f"{nginx_path.name}.bak.{suffix}") prometheus_backup = prometheus_path.with_name(f"{prometheus_path.name}.bak.{suffix}") nginx_existed_before = bool(plan["nginx"].get("existed_before")) if apply_scope == "full" and nginx_existed_before: shutil.copy2(nginx_path, nginx_backup) shutil.copy2(prometheus_path, prometheus_backup) receipt = { key: value for key, value in plan.items() if not key.startswith("_") } receipt.update({ "receipt_kind": "metrics_edge_controlled_apply_receipt", "mode": "apply", "apply_scope": apply_scope, "status": "in_progress", "backup_paths": { "prometheus": str(prometheus_backup), }, "rollback_performed": False, "post_verifier": {}, "prometheus_rebound": False, }) if apply_scope == "full" and nginx_existed_before: receipt["backup_paths"]["nginx"] = str(nginx_backup) elif apply_scope == "full": receipt["nginx_created_new_file"] = True try: if apply_scope == "full": _atomic_write(nginx_path, candidate["nginx"]) _inplace_write(prometheus_path, candidate["prometheus"]) if apply_scope == "full": _run(["nginx", "-t"]) _validate_prometheus_candidate(prometheus_path) if apply_scope == "full": _run(["systemctl", "reload", "nginx"]) prometheus_before = _prometheus_identity() host_sha, container_sha = _prometheus_config_sha(prometheus_path) if host_sha != container_sha: if not prometheus_compose_file: raise RuntimeError("Prometheus bind mount inode drift requires compose rebind") _run([ "docker", "compose", "-f", prometheus_compose_file, "up", "-d", "--no-deps", "--force-recreate", "--pull", "never", "prometheus", ], timeout=120) _wait_prometheus_ready() receipt["prometheus_rebound"] = True else: _run(["docker", "kill", "--signal=SIGHUP", "prometheus"]) _run([ "docker", "exec", "prometheus", "promtool", "check", "config", "/etc/prometheus/prometheus.yml", ]) host_sha, container_sha = _prometheus_config_sha(prometheus_path) if host_sha != container_sha: raise RuntimeError("Prometheus config SHA mismatch after controlled rebind") prometheus_after = _prometheus_identity() identity_preserved = _prometheus_identity_preserved( prometheus_before, prometheus_after, ) receipt["prometheus_runtime"] = { "before": prometheus_before, "after": prometheus_after, "config_sha256": host_sha, "config_inode_reconciled": True, "data_volume_unchanged": ( prometheus_before["data_volume"] == prometheus_after["data_volume"] ), "image_unchanged": prometheus_before["image_id"] == prometheus_after["image_id"], "identity_preserved": identity_preserved, } canary_command = ["curl", "--noproxy", "*", "-sS", "--max-time", "20"] if metrics_canary_resolve: canary_command.extend(["--resolve", metrics_canary_resolve]) canary_command.extend([ "-w", "\n__EWOOOC_HTTP_STATUS__:%{http_code}", metrics_canary_url, ]) internal_code, product_marker_present, canary_attempts = _wait_metrics_canary( canary_command, min(max(10, canary_timeout), 45), ) target_health = _prometheus_target_health( max(10, canary_timeout), metrics_canary_url, ) receipt["post_verifier"] = { "nginx_config_valid": apply_scope == "full", "prometheus_config_valid": True, "internal_metrics_http_code": internal_code, "metrics_internal_canary": internal_code == "200", "metrics_product_marker_present": product_marker_present, "metrics_canary_attempts": canary_attempts, "metrics_target_health": target_health, "prometheus_config_inode_reconciled": True, "prometheus_identity_preserved": identity_preserved, } if ( internal_code != "200" or not product_marker_present or target_health != "up" or not identity_preserved ): raise RuntimeError( "post-verifier failed: " f"internal={internal_code}, product_marker={product_marker_present}, " f"target={target_health}, " f"identity_preserved={identity_preserved}" ) receipt["status"] = "pass" receipt["completed_at"] = datetime.now(timezone.utc).isoformat() return receipt except Exception as exc: if apply_scope == "full" and nginx_existed_before: shutil.copy2(nginx_backup, nginx_path) elif apply_scope == "full": nginx_path.unlink(missing_ok=True) shutil.copy2(prometheus_backup, prometheus_path) rollback_errors: list[str] = [] rollback_commands = [["docker", "kill", "--signal=SIGHUP", "prometheus"]] if apply_scope == "full": rollback_commands[:0] = [["nginx", "-t"], ["systemctl", "reload", "nginx"]] for command in rollback_commands: try: _run(command) except Exception as rollback_exc: rollback_errors.append(str(rollback_exc)) receipt["status"] = "rolled_back" if not rollback_errors else "rollback_degraded" receipt["rollback_performed"] = True receipt["error"] = str(exc) receipt["rollback_errors"] = rollback_errors receipt["completed_at"] = datetime.now(timezone.utc).isoformat() return receipt __all__ = [ "POLICY", "apply_metrics_edge_plan", "build_metrics_edge_plan", "render_nginx_config", "render_prometheus_config", ]