wooo/ewoooc
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix(ops): bridge production metrics securely

Browse Source
This commit is contained in:
ogt
2026-07-11 20:08:21 +08:00
parent 2e46ee4722
commit bd5350349d
10 changed files with 168 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
services/http_access_policy_service.py
View File

@@ -51,7 +51,12 @@ INTERNAL_ONLY_ENDPOINTS = frozenset({
})
INTERNAL_METRICS_NETWORKS = tuple(
ipaddress.ip_network(cidr)
for cidr in ("127.0.0.0/8", "172.16.0.0/12", "::1/128")
for cidr in (
"127.0.0.0/8",
"172.16.0.0/12",
"192.168.0.110/32",
"::1/128",
)
)

89
services/metrics_edge_controlled_apply_service.py
View File

@@ -60,6 +60,15 @@ def render_nginx_config(config_text: str, fragment_text: str) -> str:
fragment = fragment_text.strip()
if "location = /metrics" not in fragment or "deny all;" not in fragment:
raise ValueError("metrics nginx fragment is incomplete")
if re.search(r"(?m)^server\s*\{", fragment):
managed = f"{NGINX_BEGIN}\n{fragment}\n{NGINX_END}"
replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed)
if replaced is not None:
return f"{replaced.rstrip()}\n"
if config_text.strip():
raise ValueError("standalone nginx fragment requires a new or managed config")
return f"{managed}\n"
indented_fragment = "\n".join(f" {line}" if line else "" for line in fragment.splitlines())
managed = f"{NGINX_BEGIN}\n{indented_fragment}\n {NGINX_END}"
replaced = _managed_replace(config_text, NGINX_BEGIN, NGINX_END, managed)
@@ -121,7 +130,8 @@ def build_metrics_edge_plan(
) -> dict[str, Any]:
if not IDENTITY_PATTERN.fullmatch(run_id) or not IDENTITY_PATTERN.fullmatch(work_item_id):
raise ValueError("run_id and work_item_id must use bounded safe characters")
nginx_before = nginx_config.read_text(encoding="utf-8")
nginx_existed_before = nginx_config.exists()
nginx_before = nginx_config.read_text(encoding="utf-8") if nginx_existed_before else ""
prometheus_before = prometheus_config.read_text(encoding="utf-8")
nginx_after = render_nginx_config(
nginx_before,
@@ -140,6 +150,7 @@ def build_metrics_edge_plan(
"generated_at": datetime.now(timezone.utc).isoformat(),
"nginx": {
"path": str(nginx_config),
"existed_before": nginx_existed_before,
"before_sha256": _sha256(nginx_before),
"candidate_sha256": _sha256(nginx_after),
"changed": nginx_before != nginx_after,
@@ -174,7 +185,7 @@ def _run(command: list[str], timeout: int = 30) -> subprocess.CompletedProcess[s
def _atomic_write(path: Path, content: str) -> None:
current = path.stat()
current = path.stat() if path.exists() else None
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
@@ -184,11 +195,23 @@ def _atomic_write(path: Path, content: str) -> None:
) as handle:
handle.write(content)
temporary = Path(handle.name)
os.chmod(temporary, current.st_mode)
os.chown(temporary, current.st_uid, current.st_gid)
os.chmod(temporary, current.st_mode if current else 0o644)
os.chown(
temporary,
current.st_uid if current else os.geteuid(),
current.st_gid if current else os.getegid(),
)
os.replace(temporary, path)
def _inplace_write(path: Path, content: str) -> None:
"""Update a file bind mount without replacing its host inode."""
with path.open("w", encoding="utf-8") as handle:
handle.write(content)
handle.flush()
os.fsync(handle.fileno())
def _prometheus_target_health(timeout_seconds: int) -> str:
deadline = time.monotonic() + timeout_seconds
url = "http://127.0.0.1:9090/api/v1/targets?state=active"
@@ -212,14 +235,25 @@ def _prometheus_target_health(timeout_seconds: int) -> str:
return last_health
def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) -> dict[str, Any]:
def apply_metrics_edge_plan(
plan: dict[str, Any],
*,
canary_timeout: int = 90,
apply_scope: str = "full",
metrics_canary_url: str = "https://mo.wooo.work/metrics",
metrics_canary_resolve: str = "mo.wooo.work:443:127.0.0.1",
) -> dict[str, Any]:
if apply_scope not in {"full", "prometheus_only"}:
raise ValueError("apply_scope must be full or prometheus_only")
candidate = plan.pop("_candidate_text")
nginx_path = Path(plan["nginx"]["path"])
prometheus_path = Path(plan["prometheus"]["path"])
suffix = re.sub(r"[^A-Za-z0-9_.-]", "_", plan["run_id"])
nginx_backup = nginx_path.with_name(f"{nginx_path.name}.bak.{suffix}")
prometheus_backup = prometheus_path.with_name(f"{prometheus_path.name}.bak.{suffix}")
shutil.copy2(nginx_path, nginx_backup)
nginx_existed_before = bool(plan["nginx"].get("existed_before"))
if apply_scope == "full" and nginx_existed_before:
shutil.copy2(nginx_path, nginx_backup)
shutil.copy2(prometheus_path, prometheus_backup)
receipt = {
key: value for key, value in plan.items()
@@ -228,33 +262,42 @@ def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) -
receipt.update({
"receipt_kind": "metrics_edge_controlled_apply_receipt",
"mode": "apply",
"apply_scope": apply_scope,
"status": "in_progress",
"backup_paths": {
"nginx": str(nginx_backup),
"prometheus": str(prometheus_backup),
},
"rollback_performed": False,
"post_verifier": {},
})
if apply_scope == "full" and nginx_existed_before:
receipt["backup_paths"]["nginx"] = str(nginx_backup)
elif apply_scope == "full":
receipt["nginx_created_new_file"] = True
try:
_atomic_write(nginx_path, candidate["nginx"])
_atomic_write(prometheus_path, candidate["prometheus"])
_run(["nginx", "-t"])
if apply_scope == "full":
_atomic_write(nginx_path, candidate["nginx"])
_inplace_write(prometheus_path, candidate["prometheus"])
if apply_scope == "full":
_run(["nginx", "-t"])
_run([
"docker", "exec", "prometheus", "promtool", "check", "config",
"/etc/prometheus/prometheus.yml",
])
_run(["systemctl", "reload", "nginx"])
if apply_scope == "full":
_run(["systemctl", "reload", "nginx"])
_run(["docker", "kill", "--signal=SIGHUP", "prometheus"])
internal = _run([
"curl", "-ksS", "--max-time", "20", "--resolve",
"mo.wooo.work:443:127.0.0.1", "-o", "/dev/null", "-w", "%{http_code}",
"https://mo.wooo.work/metrics",
canary_command = ["curl", "--noproxy", "*", "-sS", "--max-time", "20"]
if metrics_canary_resolve:
canary_command.extend(["--resolve", metrics_canary_resolve])
canary_command.extend([
"-o", "/dev/null", "-w", "%{http_code}", metrics_canary_url,
])
internal = _run(canary_command)
internal_code = internal.stdout.strip()
target_health = _prometheus_target_health(max(10, canary_timeout))
receipt["post_verifier"] = {
"nginx_config_valid": True,
"nginx_config_valid": apply_scope == "full",
"prometheus_config_valid": True,
"internal_metrics_http_code": internal_code,
"metrics_internal_canary": internal_code == "200",
@@ -268,14 +311,16 @@ def apply_metrics_edge_plan(plan: dict[str, Any], *, canary_timeout: int = 90) -
receipt["completed_at"] = datetime.now(timezone.utc).isoformat()
return receipt
except Exception as exc:
shutil.copy2(nginx_backup, nginx_path)
if apply_scope == "full" and nginx_existed_before:
shutil.copy2(nginx_backup, nginx_path)
elif apply_scope == "full":
nginx_path.unlink(missing_ok=True)
shutil.copy2(prometheus_backup, prometheus_path)
rollback_errors: list[str] = []
for command in (
["nginx", "-t"],
["systemctl", "reload", "nginx"],
["docker", "kill", "--signal=SIGHUP", "prometheus"],
):
rollback_commands = [["docker", "kill", "--signal=SIGHUP", "prometheus"]]
if apply_scope == "full":
rollback_commands[:0] = [["nginx", "-t"], ["systemctl", "reload", "nginx"]]
for command in rollback_commands:
try:
_run(command)
except Exception as rollback_exc: