515 lines
18 KiB
Python
515 lines
18 KiB
Python
"""Controlled apply for the EwoooC metrics edge and Prometheus target."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
POLICY = "ewoooc_metrics_edge_controlled_apply_v1"
|
|
NGINX_BEGIN = "# BEGIN MANAGED EWOOOC METRICS"
|
|
NGINX_END = "# END MANAGED EWOOOC METRICS"
|
|
PROMETHEUS_BEGIN = "# BEGIN MANAGED EWOOOC METRICS"
|
|
PROMETHEUS_END = "# END MANAGED EWOOOC METRICS"
|
|
IDENTITY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{2,127}$")
|
|
|
|
|
|
def _managed_replace(text: str, begin: str, end: str, managed: str) -> str | None:
|
|
begin_count = text.count(begin)
|
|
end_count = text.count(end)
|
|
if begin_count == 0 and end_count == 0:
|
|
return None
|
|
if begin_count != 1 or end_count != 1:
|
|
raise ValueError(f"invalid managed marker count for {begin}")
|
|
start = text.index(begin)
|
|
finish = text.index(end, start) + len(end)
|
|
return f"{text[:start]}{managed}{text[finish:]}"
|
|
|
|
|
|
def _nginx_server_blocks(lines: list[str]) -> list[tuple[int, int]]:
|
|
blocks: list[tuple[int, int]] = []
|
|
start: int | None = None
|
|
depth = 0
|
|
for index, line in enumerate(lines):
|
|
code = line.split("#", 1)[0]
|
|
if start is None and re.match(r"^\s*server\s*\{", code):
|
|
start = index
|
|
depth = 0
|
|
if start is None:
|
|
continue
|
|
depth += code.count("{") - code.count("}")
|
|
if depth == 0:
|
|
blocks.append((start, index + 1))
|
|
start = None
|
|
if start is not None:
|
|
raise ValueError("unbalanced nginx server block")
|
|
return blocks
|
|
|
|
|
|
def render_nginx_config(config_text: str, fragment_text: str) -> str:
|
|
fragment = fragment_text.strip()
|
|
if "location = /metrics" not in fragment or "deny all;" not in fragment:
|
|
raise ValueError("metrics nginx fragment is incomplete")
|
|
if re.search(r"(?m)^server\s*\{", fragment):
|
|
managed = f"{NGINX_BEGIN}\n{fragment}\n{NGINX_END}"
|
|
replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed)
|
|
if replaced is not None:
|
|
return f"{replaced.rstrip()}\n"
|
|
if config_text.strip():
|
|
raise ValueError("standalone nginx fragment requires a new or managed config")
|
|
return f"{managed}\n"
|
|
|
|
indented_fragment = "\n".join(f" {line}" if line else "" for line in fragment.splitlines())
|
|
managed = f"{NGINX_BEGIN}\n{indented_fragment}\n {NGINX_END}"
|
|
replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed)
|
|
if replaced is not None:
|
|
return replaced
|
|
|
|
lines = config_text.splitlines(keepends=True)
|
|
candidates: list[tuple[int, int]] = []
|
|
for start, end in _nginx_server_blocks(lines):
|
|
block = "".join(lines[start:end])
|
|
if "server_name mo.wooo.work;" in block and re.search(r"\blisten\s+443\b", block):
|
|
candidates.append((start, end))
|
|
if len(candidates) != 1:
|
|
raise ValueError(f"expected one mo.wooo.work TLS server, found {len(candidates)}")
|
|
|
|
start, end = candidates[0]
|
|
location_indexes = [
|
|
index
|
|
for index in range(start, end)
|
|
if re.match(r"^\s*location\s+/\s*\{", lines[index])
|
|
]
|
|
if len(location_indexes) != 1:
|
|
raise ValueError(f"expected one fallback location, found {len(location_indexes)}")
|
|
insert_at = location_indexes[0]
|
|
lines.insert(insert_at, f" {managed}\n\n")
|
|
return "".join(lines)
|
|
|
|
|
|
def render_prometheus_config(config_text: str, snippet_text: str) -> str:
|
|
snippet = snippet_text.rstrip()
|
|
if "job_name: 'ewoooc-app'" not in snippet or "metrics_path: /metrics" not in snippet:
|
|
raise ValueError("EwoooC Prometheus snippet is incomplete")
|
|
if not re.search(r"(?m)^scrape_configs:\s*$", config_text):
|
|
raise ValueError("Prometheus config has no scrape_configs section")
|
|
managed = f"{PROMETHEUS_BEGIN}\n{snippet}\n{PROMETHEUS_END}"
|
|
replaced = _managed_replace(
|
|
config_text,
|
|
PROMETHEUS_BEGIN,
|
|
PROMETHEUS_END,
|
|
managed,
|
|
)
|
|
if replaced is not None:
|
|
return replaced
|
|
return f"{config_text.rstrip()}\n\n{managed}\n"
|
|
|
|
|
|
def _sha256(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def build_metrics_edge_plan(
|
|
*,
|
|
nginx_config: Path,
|
|
prometheus_config: Path,
|
|
nginx_fragment: Path,
|
|
prometheus_snippet: Path,
|
|
run_id: str,
|
|
work_item_id: str,
|
|
) -> dict[str, Any]:
|
|
if not IDENTITY_PATTERN.fullmatch(run_id) or not IDENTITY_PATTERN.fullmatch(work_item_id):
|
|
raise ValueError("run_id and work_item_id must use bounded safe characters")
|
|
nginx_existed_before = nginx_config.exists()
|
|
nginx_before = nginx_config.read_text(encoding="utf-8") if nginx_existed_before else ""
|
|
prometheus_before = prometheus_config.read_text(encoding="utf-8")
|
|
nginx_after = render_nginx_config(
|
|
nginx_before,
|
|
nginx_fragment.read_text(encoding="utf-8"),
|
|
)
|
|
prometheus_after = render_prometheus_config(
|
|
prometheus_before,
|
|
prometheus_snippet.read_text(encoding="utf-8"),
|
|
)
|
|
return {
|
|
"policy": POLICY,
|
|
"trace_id": run_id,
|
|
"run_id": run_id,
|
|
"work_item_id": work_item_id,
|
|
"mode": "check",
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"nginx": {
|
|
"path": str(nginx_config),
|
|
"existed_before": nginx_existed_before,
|
|
"before_sha256": _sha256(nginx_before),
|
|
"candidate_sha256": _sha256(nginx_after),
|
|
"changed": nginx_before != nginx_after,
|
|
},
|
|
"prometheus": {
|
|
"path": str(prometheus_config),
|
|
"before_sha256": _sha256(prometheus_before),
|
|
"candidate_sha256": _sha256(prometheus_after),
|
|
"changed": prometheus_before != prometheus_after,
|
|
},
|
|
"_candidate_text": {
|
|
"nginx": nginx_after,
|
|
"prometheus": prometheus_after,
|
|
},
|
|
"controlled_apply_allowed": True,
|
|
"risk": "medium",
|
|
}
|
|
|
|
|
|
def _run(command: list[str], timeout: int = 30) -> subprocess.CompletedProcess[str]:
|
|
completed = subprocess.run(
|
|
command,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
)
|
|
if completed.returncode != 0:
|
|
message = (completed.stderr or completed.stdout or "command failed").strip()
|
|
raise RuntimeError(f"{command[0]} failed: {message[:500]}")
|
|
return completed
|
|
|
|
|
|
def _atomic_write(path: Path, content: str) -> None:
|
|
current = path.stat() if path.exists() else None
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="w",
|
|
encoding="utf-8",
|
|
dir=path.parent,
|
|
prefix=f".{path.name}.",
|
|
delete=False,
|
|
) as handle:
|
|
handle.write(content)
|
|
temporary = Path(handle.name)
|
|
os.chmod(temporary, current.st_mode if current else 0o644)
|
|
os.chown(
|
|
temporary,
|
|
current.st_uid if current else os.geteuid(),
|
|
current.st_gid if current else os.getegid(),
|
|
)
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def _inplace_write(path: Path, content: str) -> None:
|
|
"""Update a file bind mount without replacing its host inode."""
|
|
with path.open("w", encoding="utf-8") as handle:
|
|
handle.write(content)
|
|
handle.flush()
|
|
os.fsync(handle.fileno())
|
|
|
|
|
|
def _target_health_from_payload(payload: dict[str, Any], expected_scrape_url: str) -> str:
|
|
targets = ((payload.get("data") or {}).get("activeTargets") or [])
|
|
matching = [
|
|
target for target in targets
|
|
if (target.get("labels") or {}).get("job") == "ewoooc-app"
|
|
and (
|
|
not expected_scrape_url
|
|
or target.get("scrapeUrl") == expected_scrape_url
|
|
)
|
|
]
|
|
return str(matching[0].get("health") or "unknown") if matching else "missing"
|
|
|
|
|
|
def _prometheus_target_health(
|
|
timeout_seconds: int,
|
|
expected_scrape_url: str = "",
|
|
) -> str:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
url = "http://127.0.0.1:9090/api/v1/targets?state=active"
|
|
last_health = "missing"
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=8) as response:
|
|
payload = json.load(response)
|
|
last_health = _target_health_from_payload(payload, expected_scrape_url)
|
|
if last_health == "up":
|
|
return last_health
|
|
except (OSError, ValueError, TypeError):
|
|
last_health = "query_error"
|
|
time.sleep(3)
|
|
return last_health
|
|
|
|
|
|
def _prometheus_identity() -> dict[str, str]:
|
|
completed = _run([
|
|
"docker",
|
|
"inspect",
|
|
"prometheus",
|
|
"--format",
|
|
"{{.Id}}|{{.Image}}|{{range .Mounts}}{{if eq .Destination \"/prometheus\"}}{{.Name}}{{end}}{{end}}",
|
|
])
|
|
container_id, image_id, data_volume = (completed.stdout.strip().split("|", 2) + ["", ""])[:3]
|
|
if not container_id or not image_id or not data_volume:
|
|
raise RuntimeError("Prometheus container identity is incomplete")
|
|
return {
|
|
"container_id": container_id,
|
|
"image_id": image_id,
|
|
"data_volume": data_volume,
|
|
}
|
|
|
|
|
|
def _prometheus_identity_preserved(
|
|
before: dict[str, str],
|
|
after: dict[str, str],
|
|
) -> bool:
|
|
return bool(
|
|
before.get("image_id") == after.get("image_id")
|
|
and before.get("data_volume") == after.get("data_volume")
|
|
)
|
|
|
|
|
|
def _prometheus_config_sha(path: Path) -> tuple[str, str]:
|
|
host_sha = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
completed = _run([
|
|
"docker", "exec", "prometheus", "sha256sum",
|
|
"/etc/prometheus/prometheus.yml",
|
|
])
|
|
container_sha = completed.stdout.strip().split(maxsplit=1)[0]
|
|
return host_sha, container_sha
|
|
|
|
|
|
def _validate_prometheus_candidate(path: Path) -> None:
|
|
candidate_path = "/etc/prometheus/ewoooc-candidate.yml"
|
|
_run(["docker", "cp", str(path), f"prometheus:{candidate_path}"])
|
|
try:
|
|
_run([
|
|
"docker", "exec", "prometheus", "promtool", "check", "config",
|
|
candidate_path,
|
|
])
|
|
finally:
|
|
subprocess.run(
|
|
["docker", "exec", "prometheus", "rm", "-f", candidate_path],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def _wait_prometheus_ready(timeout_seconds: int = 45) -> None:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
while time.monotonic() < deadline:
|
|
completed = subprocess.run(
|
|
[
|
|
"curl", "--noproxy", "*", "-fsS", "--max-time", "5",
|
|
"http://127.0.0.1:9090/-/ready",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=8,
|
|
check=False,
|
|
)
|
|
if completed.returncode == 0:
|
|
return
|
|
time.sleep(2)
|
|
raise RuntimeError("Prometheus did not become ready after controlled rebind")
|
|
|
|
|
|
def _parse_metrics_canary(output: str) -> tuple[str, bool]:
|
|
marker = "\n__EWOOOC_HTTP_STATUS__:"
|
|
if marker not in output:
|
|
raise RuntimeError("metrics canary response has no HTTP status marker")
|
|
body, status = output.rsplit(marker, 1)
|
|
product_marker_present = (
|
|
"momo_app_info" in body
|
|
and "momo_app_health" in body
|
|
and "momo_database_up" in body
|
|
)
|
|
return status.strip(), product_marker_present
|
|
|
|
|
|
def _wait_metrics_canary(
|
|
command: list[str],
|
|
timeout_seconds: int,
|
|
) -> tuple[str, bool, int]:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
last_status = "000"
|
|
last_product_marker = False
|
|
attempts = 0
|
|
while time.monotonic() < deadline:
|
|
attempts += 1
|
|
try:
|
|
completed = _run(command)
|
|
last_status, last_product_marker = _parse_metrics_canary(
|
|
completed.stdout
|
|
)
|
|
if last_status == "200" and last_product_marker:
|
|
return last_status, last_product_marker, attempts
|
|
except RuntimeError:
|
|
pass
|
|
time.sleep(2)
|
|
return last_status, last_product_marker, attempts
|
|
|
|
|
|
def apply_metrics_edge_plan(
|
|
plan: dict[str, Any],
|
|
*,
|
|
canary_timeout: int = 90,
|
|
apply_scope: str = "full",
|
|
metrics_canary_url: str = "https://mo.wooo.work/metrics",
|
|
metrics_canary_resolve: str = "mo.wooo.work:443:127.0.0.1",
|
|
prometheus_compose_file: str = "",
|
|
) -> dict[str, Any]:
|
|
if apply_scope not in {"full", "prometheus_only"}:
|
|
raise ValueError("apply_scope must be full or prometheus_only")
|
|
candidate = plan.pop("_candidate_text")
|
|
nginx_path = Path(plan["nginx"]["path"])
|
|
prometheus_path = Path(plan["prometheus"]["path"])
|
|
suffix = re.sub(r"[^A-Za-z0-9_.-]", "_", plan["run_id"])
|
|
nginx_backup = nginx_path.with_name(f"{nginx_path.name}.bak.{suffix}")
|
|
prometheus_backup = prometheus_path.with_name(f"{prometheus_path.name}.bak.{suffix}")
|
|
nginx_existed_before = bool(plan["nginx"].get("existed_before"))
|
|
if apply_scope == "full" and nginx_existed_before:
|
|
shutil.copy2(nginx_path, nginx_backup)
|
|
shutil.copy2(prometheus_path, prometheus_backup)
|
|
receipt = {
|
|
key: value for key, value in plan.items()
|
|
if not key.startswith("_")
|
|
}
|
|
receipt.update({
|
|
"receipt_kind": "metrics_edge_controlled_apply_receipt",
|
|
"mode": "apply",
|
|
"apply_scope": apply_scope,
|
|
"status": "in_progress",
|
|
"backup_paths": {
|
|
"prometheus": str(prometheus_backup),
|
|
},
|
|
"rollback_performed": False,
|
|
"post_verifier": {},
|
|
"prometheus_rebound": False,
|
|
})
|
|
if apply_scope == "full" and nginx_existed_before:
|
|
receipt["backup_paths"]["nginx"] = str(nginx_backup)
|
|
elif apply_scope == "full":
|
|
receipt["nginx_created_new_file"] = True
|
|
try:
|
|
if apply_scope == "full":
|
|
_atomic_write(nginx_path, candidate["nginx"])
|
|
_inplace_write(prometheus_path, candidate["prometheus"])
|
|
if apply_scope == "full":
|
|
_run(["nginx", "-t"])
|
|
_validate_prometheus_candidate(prometheus_path)
|
|
if apply_scope == "full":
|
|
_run(["systemctl", "reload", "nginx"])
|
|
prometheus_before = _prometheus_identity()
|
|
host_sha, container_sha = _prometheus_config_sha(prometheus_path)
|
|
if host_sha != container_sha:
|
|
if not prometheus_compose_file:
|
|
raise RuntimeError("Prometheus bind mount inode drift requires compose rebind")
|
|
_run([
|
|
"docker", "compose", "-f", prometheus_compose_file, "up", "-d",
|
|
"--no-deps", "--force-recreate", "--pull", "never", "prometheus",
|
|
], timeout=120)
|
|
_wait_prometheus_ready()
|
|
receipt["prometheus_rebound"] = True
|
|
else:
|
|
_run(["docker", "kill", "--signal=SIGHUP", "prometheus"])
|
|
_run([
|
|
"docker", "exec", "prometheus", "promtool", "check", "config",
|
|
"/etc/prometheus/prometheus.yml",
|
|
])
|
|
host_sha, container_sha = _prometheus_config_sha(prometheus_path)
|
|
if host_sha != container_sha:
|
|
raise RuntimeError("Prometheus config SHA mismatch after controlled rebind")
|
|
prometheus_after = _prometheus_identity()
|
|
identity_preserved = _prometheus_identity_preserved(
|
|
prometheus_before,
|
|
prometheus_after,
|
|
)
|
|
receipt["prometheus_runtime"] = {
|
|
"before": prometheus_before,
|
|
"after": prometheus_after,
|
|
"config_sha256": host_sha,
|
|
"config_inode_reconciled": True,
|
|
"data_volume_unchanged": (
|
|
prometheus_before["data_volume"] == prometheus_after["data_volume"]
|
|
),
|
|
"image_unchanged": prometheus_before["image_id"] == prometheus_after["image_id"],
|
|
"identity_preserved": identity_preserved,
|
|
}
|
|
canary_command = ["curl", "--noproxy", "*", "-sS", "--max-time", "20"]
|
|
if metrics_canary_resolve:
|
|
canary_command.extend(["--resolve", metrics_canary_resolve])
|
|
canary_command.extend([
|
|
"-w", "\n__EWOOOC_HTTP_STATUS__:%{http_code}", metrics_canary_url,
|
|
])
|
|
internal_code, product_marker_present, canary_attempts = _wait_metrics_canary(
|
|
canary_command,
|
|
min(max(10, canary_timeout), 45),
|
|
)
|
|
target_health = _prometheus_target_health(
|
|
max(10, canary_timeout),
|
|
metrics_canary_url,
|
|
)
|
|
receipt["post_verifier"] = {
|
|
"nginx_config_valid": apply_scope == "full",
|
|
"prometheus_config_valid": True,
|
|
"internal_metrics_http_code": internal_code,
|
|
"metrics_internal_canary": internal_code == "200",
|
|
"metrics_product_marker_present": product_marker_present,
|
|
"metrics_canary_attempts": canary_attempts,
|
|
"metrics_target_health": target_health,
|
|
"prometheus_config_inode_reconciled": True,
|
|
"prometheus_identity_preserved": identity_preserved,
|
|
}
|
|
if (
|
|
internal_code != "200"
|
|
or not product_marker_present
|
|
or target_health != "up"
|
|
or not identity_preserved
|
|
):
|
|
raise RuntimeError(
|
|
"post-verifier failed: "
|
|
f"internal={internal_code}, product_marker={product_marker_present}, "
|
|
f"target={target_health}, "
|
|
f"identity_preserved={identity_preserved}"
|
|
)
|
|
receipt["status"] = "pass"
|
|
receipt["completed_at"] = datetime.now(timezone.utc).isoformat()
|
|
return receipt
|
|
except Exception as exc:
|
|
if apply_scope == "full" and nginx_existed_before:
|
|
shutil.copy2(nginx_backup, nginx_path)
|
|
elif apply_scope == "full":
|
|
nginx_path.unlink(missing_ok=True)
|
|
shutil.copy2(prometheus_backup, prometheus_path)
|
|
rollback_errors: list[str] = []
|
|
rollback_commands = [["docker", "kill", "--signal=SIGHUP", "prometheus"]]
|
|
if apply_scope == "full":
|
|
rollback_commands[:0] = [["nginx", "-t"], ["systemctl", "reload", "nginx"]]
|
|
for command in rollback_commands:
|
|
try:
|
|
_run(command)
|
|
except Exception as rollback_exc:
|
|
rollback_errors.append(str(rollback_exc))
|
|
receipt["status"] = "rolled_back" if not rollback_errors else "rollback_degraded"
|
|
receipt["rollback_performed"] = True
|
|
receipt["error"] = str(exc)
|
|
receipt["rollback_errors"] = rollback_errors
|
|
receipt["completed_at"] = datetime.now(timezone.utc).isoformat()
|
|
return receipt
|
|
|
|
|
|
__all__ = [
|
|
"POLICY",
|
|
"apply_metrics_edge_plan",
|
|
"build_metrics_edge_plan",
|
|
"render_nginx_config",
|
|
"render_prometheus_config",
|
|
]
|