fix(ops): bind verifier to exact metrics target

This commit is contained in:
ogt
2026-07-11 20:27:12 +08:00
parent 3023918d61
commit f55a257766
4 changed files with 56 additions and 11 deletions

View File

@@ -5,7 +5,9 @@ server {
access_log off;
location = /metrics {
allow 127.0.0.1;
allow 172.16.0.0/12;
allow 192.168.0.110/32;
deny all;
proxy_pass https://192.168.0.188/metrics;

View File

@@ -212,7 +212,23 @@ def _inplace_write(path: Path, content: str) -> None:
os.fsync(handle.fileno())
def _prometheus_target_health(timeout_seconds: int) -> str:
def _target_health_from_payload(payload: dict[str, Any], expected_scrape_url: str) -> str:
targets = ((payload.get("data") or {}).get("activeTargets") or [])
matching = [
target for target in targets
if (target.get("labels") or {}).get("job") == "ewoooc-app"
and (
not expected_scrape_url
or target.get("scrapeUrl") == expected_scrape_url
)
]
return str(matching[0].get("health") or "unknown") if matching else "missing"
def _prometheus_target_health(
timeout_seconds: int,
expected_scrape_url: str = "",
) -> str:
deadline = time.monotonic() + timeout_seconds
url = "http://127.0.0.1:9090/api/v1/targets?state=active"
last_health = "missing"
@@ -220,15 +236,9 @@ def _prometheus_target_health(timeout_seconds: int) -> str:
try:
with urllib.request.urlopen(url, timeout=8) as response:
payload = json.load(response)
targets = ((payload.get("data") or {}).get("activeTargets") or [])
matching = [
target for target in targets
if (target.get("labels") or {}).get("job") == "ewoooc-app"
]
if matching:
last_health = str(matching[0].get("health") or "unknown")
if last_health == "up":
return last_health
last_health = _target_health_from_payload(payload, expected_scrape_url)
if last_health == "up":
return last_health
except (OSError, ValueError, TypeError):
last_health = "query_error"
time.sleep(3)
@@ -417,7 +427,10 @@ def apply_metrics_edge_plan(
])
internal = _run(canary_command)
internal_code, product_marker_present = _parse_metrics_canary(internal.stdout)
target_health = _prometheus_target_health(max(10, canary_timeout))
target_health = _prometheus_target_health(
max(10, canary_timeout),
metrics_canary_url,
)
receipt["post_verifier"] = {
"nginx_config_valid": apply_scope == "full",
"prometheus_config_valid": True,

View File

@@ -157,6 +157,34 @@ def test_metrics_canary_requires_ewoooc_product_markers():
assert _parse_metrics_canary(wrong_exporter) == ("200", False)
def test_target_health_rejects_stale_scrape_url():
from services.metrics_edge_controlled_apply_service import _target_health_from_payload
payload = {
"data": {
"activeTargets": [
{
"labels": {"job": "ewoooc-app"},
"scrapeUrl": "http://172.20.0.1:9191/metrics",
"health": "up",
},
{
"labels": {"job": "ewoooc-app"},
"scrapeUrl": "http://172.20.0.1:19191/metrics",
"health": "down",
},
]
}
}
health = _target_health_from_payload(
payload,
"http://172.20.0.1:19191/metrics",
)
assert health == "down"
def test_prometheus_identity_verifier_requires_same_image_and_data_volume():
from services.metrics_edge_controlled_apply_service import (
_prometheus_identity_preserved,

View File

@@ -77,7 +77,9 @@ def test_source_owned_edge_and_prometheus_templates_are_closed_by_default():
assert "listen 172.20.0.1:19191;" in nginx
assert "location = /metrics" in nginx
assert "allow 127.0.0.1;" in nginx
assert "allow 172.16.0.0/12;" in nginx
assert "allow 192.168.0.110/32;" in nginx
assert "deny all;" in nginx
assert "proxy_pass https://192.168.0.188/metrics;" in nginx
assert "proxy_ssl_verify on;" in nginx