fix(ops): wait for metrics bridge reload

services/metrics_edge_controlled_apply_service.py

@@ -333,6 +333,29 @@ def _parse_metrics_canary(output: str) -> tuple[str, bool]:
     return status.strip(), product_marker_present
 
 
+def _wait_metrics_canary(
+    command: list[str],
+    timeout_seconds: int,
+) -> tuple[str, bool, int]:
+    deadline = time.monotonic() + timeout_seconds
+    last_status = "000"
+    last_product_marker = False
+    attempts = 0
+    while time.monotonic() < deadline:
+        attempts += 1
+        try:
+            completed = _run(command)
+            last_status, last_product_marker = _parse_metrics_canary(
+                completed.stdout
+            )
+            if last_status == "200" and last_product_marker:
+                return last_status, last_product_marker, attempts
+        except RuntimeError:
+            pass
+        time.sleep(2)
+    return last_status, last_product_marker, attempts
+
+
 def apply_metrics_edge_plan(
     plan: dict[str, Any],
     *,
@@ -425,8 +448,10 @@ def apply_metrics_edge_plan(
         canary_command.extend([
             "-w", "\n__EWOOOC_HTTP_STATUS__:%{http_code}", metrics_canary_url,
         ])
-        internal = _run(canary_command)
-        internal_code, product_marker_present = _parse_metrics_canary(internal.stdout)
+        internal_code, product_marker_present, canary_attempts = _wait_metrics_canary(
+            canary_command,
+            min(max(10, canary_timeout), 45),
+        )
         target_health = _prometheus_target_health(
             max(10, canary_timeout),
             metrics_canary_url,
@@ -437,6 +462,7 @@ def apply_metrics_edge_plan(
             "internal_metrics_http_code": internal_code,
             "metrics_internal_canary": internal_code == "200",
             "metrics_product_marker_present": product_marker_present,
+            "metrics_canary_attempts": canary_attempts,
             "metrics_target_health": target_health,
             "prometheus_config_inode_reconciled": True,
             "prometheus_identity_preserved": identity_preserved,

tests/test_metrics_edge_controlled_apply_service.py

@@ -157,6 +157,27 @@ def test_metrics_canary_requires_ewoooc_product_markers():
     assert _parse_metrics_canary(wrong_exporter) == ("200", False)
 
 
+def test_metrics_canary_waits_for_new_nginx_worker(monkeypatch):
+    from services import metrics_edge_controlled_apply_service as service
+
+    outputs = iter([
+        "forbidden\n__EWOOOC_HTTP_STATUS__:403",
+        (
+            "momo_app_info 1\nmomo_app_health 1\nmomo_database_up 1"
+            "\n__EWOOOC_HTTP_STATUS__:200"
+        ),
+    ])
+
+    monkeypatch.setattr(
+        service,
+        "_run",
+        lambda _command: type("Completed", (), {"stdout": next(outputs)})(),
+    )
+    monkeypatch.setattr(service.time, "sleep", lambda _seconds: None)
+
+    assert service._wait_metrics_canary(["curl"], 10) == ("200", True, 2)
+
+
 def test_target_health_rejects_stale_scrape_url():
     from services.metrics_edge_controlled_apply_service import _target_health_from_payload