fix(ops): bind verifier to exact metrics target
This commit is contained in:
@@ -5,7 +5,9 @@ server {
|
||||
access_log off;
|
||||
|
||||
location = /metrics {
|
||||
allow 127.0.0.1;
|
||||
allow 172.16.0.0/12;
|
||||
allow 192.168.0.110/32;
|
||||
deny all;
|
||||
|
||||
proxy_pass https://192.168.0.188/metrics;
|
||||
|
||||
@@ -212,7 +212,23 @@ def _inplace_write(path: Path, content: str) -> None:
|
||||
os.fsync(handle.fileno())
|
||||
|
||||
|
||||
def _prometheus_target_health(timeout_seconds: int) -> str:
|
||||
def _target_health_from_payload(payload: dict[str, Any], expected_scrape_url: str) -> str:
|
||||
targets = ((payload.get("data") or {}).get("activeTargets") or [])
|
||||
matching = [
|
||||
target for target in targets
|
||||
if (target.get("labels") or {}).get("job") == "ewoooc-app"
|
||||
and (
|
||||
not expected_scrape_url
|
||||
or target.get("scrapeUrl") == expected_scrape_url
|
||||
)
|
||||
]
|
||||
return str(matching[0].get("health") or "unknown") if matching else "missing"
|
||||
|
||||
|
||||
def _prometheus_target_health(
|
||||
timeout_seconds: int,
|
||||
expected_scrape_url: str = "",
|
||||
) -> str:
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
url = "http://127.0.0.1:9090/api/v1/targets?state=active"
|
||||
last_health = "missing"
|
||||
@@ -220,15 +236,9 @@ def _prometheus_target_health(timeout_seconds: int) -> str:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=8) as response:
|
||||
payload = json.load(response)
|
||||
targets = ((payload.get("data") or {}).get("activeTargets") or [])
|
||||
matching = [
|
||||
target for target in targets
|
||||
if (target.get("labels") or {}).get("job") == "ewoooc-app"
|
||||
]
|
||||
if matching:
|
||||
last_health = str(matching[0].get("health") or "unknown")
|
||||
if last_health == "up":
|
||||
return last_health
|
||||
last_health = _target_health_from_payload(payload, expected_scrape_url)
|
||||
if last_health == "up":
|
||||
return last_health
|
||||
except (OSError, ValueError, TypeError):
|
||||
last_health = "query_error"
|
||||
time.sleep(3)
|
||||
@@ -417,7 +427,10 @@ def apply_metrics_edge_plan(
|
||||
])
|
||||
internal = _run(canary_command)
|
||||
internal_code, product_marker_present = _parse_metrics_canary(internal.stdout)
|
||||
target_health = _prometheus_target_health(max(10, canary_timeout))
|
||||
target_health = _prometheus_target_health(
|
||||
max(10, canary_timeout),
|
||||
metrics_canary_url,
|
||||
)
|
||||
receipt["post_verifier"] = {
|
||||
"nginx_config_valid": apply_scope == "full",
|
||||
"prometheus_config_valid": True,
|
||||
|
||||
@@ -157,6 +157,34 @@ def test_metrics_canary_requires_ewoooc_product_markers():
|
||||
assert _parse_metrics_canary(wrong_exporter) == ("200", False)
|
||||
|
||||
|
||||
def test_target_health_rejects_stale_scrape_url():
|
||||
from services.metrics_edge_controlled_apply_service import _target_health_from_payload
|
||||
|
||||
payload = {
|
||||
"data": {
|
||||
"activeTargets": [
|
||||
{
|
||||
"labels": {"job": "ewoooc-app"},
|
||||
"scrapeUrl": "http://172.20.0.1:9191/metrics",
|
||||
"health": "up",
|
||||
},
|
||||
{
|
||||
"labels": {"job": "ewoooc-app"},
|
||||
"scrapeUrl": "http://172.20.0.1:19191/metrics",
|
||||
"health": "down",
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
health = _target_health_from_payload(
|
||||
payload,
|
||||
"http://172.20.0.1:19191/metrics",
|
||||
)
|
||||
|
||||
assert health == "down"
|
||||
|
||||
|
||||
def test_prometheus_identity_verifier_requires_same_image_and_data_volume():
|
||||
from services.metrics_edge_controlled_apply_service import (
|
||||
_prometheus_identity_preserved,
|
||||
|
||||
@@ -77,7 +77,9 @@ def test_source_owned_edge_and_prometheus_templates_are_closed_by_default():
|
||||
|
||||
assert "listen 172.20.0.1:19191;" in nginx
|
||||
assert "location = /metrics" in nginx
|
||||
assert "allow 127.0.0.1;" in nginx
|
||||
assert "allow 172.16.0.0/12;" in nginx
|
||||
assert "allow 192.168.0.110/32;" in nginx
|
||||
assert "deny all;" in nginx
|
||||
assert "proxy_pass https://192.168.0.188/metrics;" in nginx
|
||||
assert "proxy_ssl_verify on;" in nginx
|
||||
|
||||
Reference in New Issue
Block a user