298 lines
11 KiB
Python
298 lines
11 KiB
Python
"""Controlled apply for the EwoooC metrics edge and Prometheus target."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
POLICY = "ewoooc_metrics_edge_controlled_apply_v1"
|
|
NGINX_BEGIN = "# BEGIN MANAGED EWOOOC METRICS"
|
|
NGINX_END = "# END MANAGED EWOOOC METRICS"
|
|
PROMETHEUS_BEGIN = "# BEGIN MANAGED EWOOOC METRICS"
|
|
PROMETHEUS_END = "# END MANAGED EWOOOC METRICS"
|
|
IDENTITY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{2,127}$")
|
|
|
|
|
|
def _managed_replace(text: str, begin: str, end: str, managed: str) -> str | None:
|
|
begin_count = text.count(begin)
|
|
end_count = text.count(end)
|
|
if begin_count == 0 and end_count == 0:
|
|
return None
|
|
if begin_count != 1 or end_count != 1:
|
|
raise ValueError(f"invalid managed marker count for {begin}")
|
|
start = text.index(begin)
|
|
finish = text.index(end, start) + len(end)
|
|
return f"{text[:start]}{managed}{text[finish:]}"
|
|
|
|
|
|
def _nginx_server_blocks(lines: list[str]) -> list[tuple[int, int]]:
|
|
blocks: list[tuple[int, int]] = []
|
|
start: int | None = None
|
|
depth = 0
|
|
for index, line in enumerate(lines):
|
|
code = line.split("#", 1)[0]
|
|
if start is None and re.match(r"^\s*server\s*\{", code):
|
|
start = index
|
|
depth = 0
|
|
if start is None:
|
|
continue
|
|
depth += code.count("{") - code.count("}")
|
|
if depth == 0:
|
|
blocks.append((start, index + 1))
|
|
start = None
|
|
if start is not None:
|
|
raise ValueError("unbalanced nginx server block")
|
|
return blocks
|
|
|
|
|
|
def render_nginx_config(config_text: str, fragment_text: str) -> str:
|
|
fragment = fragment_text.strip()
|
|
if "location = /metrics" not in fragment or "deny all;" not in fragment:
|
|
raise ValueError("metrics nginx fragment is incomplete")
|
|
indented_fragment = "\n".join(f" {line}" if line else "" for line in fragment.splitlines())
|
|
managed = f"{NGINX_BEGIN}\n{indented_fragment}\n {NGINX_END}"
|
|
replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed)
|
|
if replaced is not None:
|
|
return replaced
|
|
|
|
lines = config_text.splitlines(keepends=True)
|
|
candidates: list[tuple[int, int]] = []
|
|
for start, end in _nginx_server_blocks(lines):
|
|
block = "".join(lines[start:end])
|
|
if "server_name mo.wooo.work;" in block and re.search(r"\blisten\s+443\b", block):
|
|
candidates.append((start, end))
|
|
if len(candidates) != 1:
|
|
raise ValueError(f"expected one mo.wooo.work TLS server, found {len(candidates)}")
|
|
|
|
start, end = candidates[0]
|
|
location_indexes = [
|
|
index
|
|
for index in range(start, end)
|
|
if re.match(r"^\s*location\s+/\s*\{", lines[index])
|
|
]
|
|
if len(location_indexes) != 1:
|
|
raise ValueError(f"expected one fallback location, found {len(location_indexes)}")
|
|
insert_at = location_indexes[0]
|
|
lines.insert(insert_at, f" {managed}\n\n")
|
|
return "".join(lines)
|
|
|
|
|
|
def render_prometheus_config(config_text: str, snippet_text: str) -> str:
|
|
snippet = snippet_text.rstrip()
|
|
if "job_name: 'ewoooc-app'" not in snippet or "metrics_path: /metrics" not in snippet:
|
|
raise ValueError("EwoooC Prometheus snippet is incomplete")
|
|
if not re.search(r"(?m)^scrape_configs:\s*$", config_text):
|
|
raise ValueError("Prometheus config has no scrape_configs section")
|
|
managed = f"{PROMETHEUS_BEGIN}\n{snippet}\n{PROMETHEUS_END}"
|
|
replaced = _managed_replace(
|
|
config_text,
|
|
PROMETHEUS_BEGIN,
|
|
PROMETHEUS_END,
|
|
managed,
|
|
)
|
|
if replaced is not None:
|
|
return replaced
|
|
return f"{config_text.rstrip()}\n\n{managed}\n"
|
|
|
|
|
|
def _sha256(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def build_metrics_edge_plan(
|
|
*,
|
|
nginx_config: Path,
|
|
prometheus_config: Path,
|
|
nginx_fragment: Path,
|
|
prometheus_snippet: Path,
|
|
run_id: str,
|
|
work_item_id: str,
|
|
) -> dict[str, Any]:
|
|
if not IDENTITY_PATTERN.fullmatch(run_id) or not IDENTITY_PATTERN.fullmatch(work_item_id):
|
|
raise ValueError("run_id and work_item_id must use bounded safe characters")
|
|
nginx_before = nginx_config.read_text(encoding="utf-8")
|
|
prometheus_before = prometheus_config.read_text(encoding="utf-8")
|
|
nginx_after = render_nginx_config(
|
|
nginx_before,
|
|
nginx_fragment.read_text(encoding="utf-8"),
|
|
)
|
|
prometheus_after = render_prometheus_config(
|
|
prometheus_before,
|
|
prometheus_snippet.read_text(encoding="utf-8"),
|
|
)
|
|
return {
|
|
"policy": POLICY,
|
|
"trace_id": run_id,
|
|
"run_id": run_id,
|
|
"work_item_id": work_item_id,
|
|
"mode": "check",
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"nginx": {
|
|
"path": str(nginx_config),
|
|
"before_sha256": _sha256(nginx_before),
|
|
"candidate_sha256": _sha256(nginx_after),
|
|
"changed": nginx_before != nginx_after,
|
|
},
|
|
"prometheus": {
|
|
"path": str(prometheus_config),
|
|
"before_sha256": _sha256(prometheus_before),
|
|
"candidate_sha256": _sha256(prometheus_after),
|
|
"changed": prometheus_before != prometheus_after,
|
|
},
|
|
"_candidate_text": {
|
|
"nginx": nginx_after,
|
|
"prometheus": prometheus_after,
|
|
},
|
|
"controlled_apply_allowed": True,
|
|
"risk": "medium",
|
|
}
|
|
|
|
|
|
def _run(command: list[str], timeout: int = 30) -> subprocess.CompletedProcess[str]:
|
|
completed = subprocess.run(
|
|
command,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
check=False,
|
|
)
|
|
if completed.returncode != 0:
|
|
message = (completed.stderr or completed.stdout or "command failed").strip()
|
|
raise RuntimeError(f"{command[0]} failed: {message[:500]}")
|
|
return completed
|
|
|
|
|
|
def _atomic_write(path: Path, content: str) -> None:
|
|
current = path.stat()
|
|
with tempfile.NamedTemporaryFile(
|
|
mode="w",
|
|
encoding="utf-8",
|
|
dir=path.parent,
|
|
prefix=f".{path.name}.",
|
|
delete=False,
|
|
) as handle:
|
|
handle.write(content)
|
|
temporary = Path(handle.name)
|
|
os.chmod(temporary, current.st_mode)
|
|
os.chown(temporary, current.st_uid, current.st_gid)
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def _prometheus_target_health(timeout_seconds: int) -> str:
|
|
deadline = time.monotonic() + timeout_seconds
|
|
url = "http://127.0.0.1:9090/api/v1/targets?state=active"
|
|
last_health = "missing"
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=8) as response:
|
|
payload = json.load(response)
|
|
targets = ((payload.get("data") or {}).get("activeTargets") or [])
|
|
matching = [
|
|
target for target in targets
|
|
if (target.get("labels") or {}).get("job") == "ewoooc-app"
|
|
]
|
|
if matching:
|
|
last_health = str(matching[0].get("health") or "unknown")
|
|
if last_health == "up":
|
|
return last_health
|
|
except (OSError, ValueError, TypeError):
|
|
last_health = "query_error"
|
|
time.sleep(3)
|
|
return last_health
|
|
|
|
|
|
def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) -> dict[str, Any]:
|
|
candidate = plan.pop("_candidate_text")
|
|
nginx_path = Path(plan["nginx"]["path"])
|
|
prometheus_path = Path(plan["prometheus"]["path"])
|
|
suffix = re.sub(r"[^A-Za-z0-9_.-]", "_", plan["run_id"])
|
|
nginx_backup = nginx_path.with_name(f"{nginx_path.name}.bak.{suffix}")
|
|
prometheus_backup = prometheus_path.with_name(f"{prometheus_path.name}.bak.{suffix}")
|
|
shutil.copy2(nginx_path, nginx_backup)
|
|
shutil.copy2(prometheus_path, prometheus_backup)
|
|
receipt = {
|
|
key: value for key, value in plan.items()
|
|
if not key.startswith("_")
|
|
}
|
|
receipt.update({
|
|
"receipt_kind": "metrics_edge_controlled_apply_receipt",
|
|
"mode": "apply",
|
|
"status": "in_progress",
|
|
"backup_paths": {
|
|
"nginx": str(nginx_backup),
|
|
"prometheus": str(prometheus_backup),
|
|
},
|
|
"rollback_performed": False,
|
|
"post_verifier": {},
|
|
})
|
|
try:
|
|
_atomic_write(nginx_path, candidate["nginx"])
|
|
_atomic_write(prometheus_path, candidate["prometheus"])
|
|
_run(["nginx", "-t"])
|
|
_run([
|
|
"docker", "exec", "prometheus", "promtool", "check", "config",
|
|
"/etc/prometheus/prometheus.yml",
|
|
])
|
|
_run(["systemctl", "reload", "nginx"])
|
|
_run(["docker", "kill", "--signal=SIGHUP", "prometheus"])
|
|
internal = _run([
|
|
"curl", "-ksS", "--max-time", "20", "--resolve",
|
|
"mo.wooo.work:443:127.0.0.1", "-o", "/dev/null", "-w", "%{http_code}",
|
|
"https://mo.wooo.work/metrics",
|
|
])
|
|
internal_code = internal.stdout.strip()
|
|
target_health = _prometheus_target_health(max(10, canary_timeout))
|
|
receipt["post_verifier"] = {
|
|
"nginx_config_valid": True,
|
|
"prometheus_config_valid": True,
|
|
"internal_metrics_http_code": internal_code,
|
|
"metrics_internal_canary": internal_code == "200",
|
|
"metrics_target_health": target_health,
|
|
}
|
|
if internal_code != "200" or target_health != "up":
|
|
raise RuntimeError(
|
|
f"post-verifier failed: internal={internal_code}, target={target_health}"
|
|
)
|
|
receipt["status"] = "pass"
|
|
receipt["completed_at"] = datetime.now(timezone.utc).isoformat()
|
|
return receipt
|
|
except Exception as exc:
|
|
shutil.copy2(nginx_backup, nginx_path)
|
|
shutil.copy2(prometheus_backup, prometheus_path)
|
|
rollback_errors: list[str] = []
|
|
for command in (
|
|
["nginx", "-t"],
|
|
["systemctl", "reload", "nginx"],
|
|
["docker", "kill", "--signal=SIGHUP", "prometheus"],
|
|
):
|
|
try:
|
|
_run(command)
|
|
except Exception as rollback_exc:
|
|
rollback_errors.append(str(rollback_exc))
|
|
receipt["status"] = "rolled_back" if not rollback_errors else "rollback_degraded"
|
|
receipt["rollback_performed"] = True
|
|
receipt["error"] = str(exc)
|
|
receipt["rollback_errors"] = rollback_errors
|
|
receipt["completed_at"] = datetime.now(timezone.utc).isoformat()
|
|
return receipt
|
|
|
|
|
|
__all__ = [
|
|
"POLICY",
|
|
"apply_metrics_edge_plan",
|
|
"build_metrics_edge_plan",
|
|
"render_nginx_config",
|
|
"render_prometheus_config",
|
|
]
|