fix(ops): wait for metrics bridge reload
This commit is contained in:
@@ -333,6 +333,29 @@ def _parse_metrics_canary(output: str) -> tuple[str, bool]:
|
||||
return status.strip(), product_marker_present
|
||||
|
||||
|
||||
def _wait_metrics_canary(
|
||||
command: list[str],
|
||||
timeout_seconds: int,
|
||||
) -> tuple[str, bool, int]:
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
last_status = "000"
|
||||
last_product_marker = False
|
||||
attempts = 0
|
||||
while time.monotonic() < deadline:
|
||||
attempts += 1
|
||||
try:
|
||||
completed = _run(command)
|
||||
last_status, last_product_marker = _parse_metrics_canary(
|
||||
completed.stdout
|
||||
)
|
||||
if last_status == "200" and last_product_marker:
|
||||
return last_status, last_product_marker, attempts
|
||||
except RuntimeError:
|
||||
pass
|
||||
time.sleep(2)
|
||||
return last_status, last_product_marker, attempts
|
||||
|
||||
|
||||
def apply_metrics_edge_plan(
|
||||
plan: dict[str, Any],
|
||||
*,
|
||||
@@ -425,8 +448,10 @@ def apply_metrics_edge_plan(
|
||||
canary_command.extend([
|
||||
"-w", "\n__EWOOOC_HTTP_STATUS__:%{http_code}", metrics_canary_url,
|
||||
])
|
||||
internal = _run(canary_command)
|
||||
internal_code, product_marker_present = _parse_metrics_canary(internal.stdout)
|
||||
internal_code, product_marker_present, canary_attempts = _wait_metrics_canary(
|
||||
canary_command,
|
||||
min(max(10, canary_timeout), 45),
|
||||
)
|
||||
target_health = _prometheus_target_health(
|
||||
max(10, canary_timeout),
|
||||
metrics_canary_url,
|
||||
@@ -437,6 +462,7 @@ def apply_metrics_edge_plan(
|
||||
"internal_metrics_http_code": internal_code,
|
||||
"metrics_internal_canary": internal_code == "200",
|
||||
"metrics_product_marker_present": product_marker_present,
|
||||
"metrics_canary_attempts": canary_attempts,
|
||||
"metrics_target_health": target_health,
|
||||
"prometheus_config_inode_reconciled": True,
|
||||
"prometheus_identity_preserved": identity_preserved,
|
||||
|
||||
Reference in New Issue
Block a user