fix(reboot): preserve edge maintenance blocker in api overlay
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / build-and-deploy (push) Successful in 4m34s
CD Pipeline / post-deploy-checks (push) Successful in 1m45s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / build-and-deploy (push) Successful in 4m34s
CD Pipeline / post-deploy-checks (push) Successful in 1m45s
This commit is contained in:
@@ -75,12 +75,17 @@ _PROMETHEUS_RUNTIME_COMBINED_QUERY = (
|
||||
+ "|".join(_PROMETHEUS_RUNTIME_QUERIES.values())
|
||||
+ '"}'
|
||||
)
|
||||
_PUBLIC_MAINTENANCE_BLOCKERS = {
|
||||
"edge_public_maintenance_live_config_drift",
|
||||
_PUBLIC_MAINTENANCE_ROUTE_BLOCKERS = {
|
||||
"public_maintenance_fallback_runtime_readback_missing",
|
||||
"public_route_raw_5xx_without_maintenance_fallback",
|
||||
"public_route_unreachable_without_external_l1_fallback",
|
||||
}
|
||||
_PUBLIC_MAINTENANCE_EDGE_BLOCKERS = {
|
||||
"edge_public_maintenance_live_config_drift",
|
||||
}
|
||||
_PUBLIC_MAINTENANCE_BLOCKERS = (
|
||||
_PUBLIC_MAINTENANCE_ROUTE_BLOCKERS | _PUBLIC_MAINTENANCE_EDGE_BLOCKERS
|
||||
)
|
||||
_RUNTIME_BOOLEAN_READY_BLOCKERS = {
|
||||
"backup_core_green_not_1": "backup_core_green",
|
||||
"host_188_service_green_not_1": "host_188_service_green",
|
||||
@@ -895,17 +900,9 @@ def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks(
|
||||
) -> list[str]:
|
||||
runtime_ready_blockers: set[str] = set()
|
||||
public_maintenance = _dict(payload.get("public_maintenance_fallback"))
|
||||
public_maintenance_ready = (
|
||||
public_maintenance.get("runtime_readback_present") is True
|
||||
and public_maintenance.get("ready") is True
|
||||
and _int(public_maintenance.get("raw_5xx_without_fallback_count")) == 0
|
||||
and _int(
|
||||
public_maintenance.get("route_unreachable_without_external_fallback_count")
|
||||
)
|
||||
== 0
|
||||
runtime_ready_blockers.update(
|
||||
_public_maintenance_reconciled_metric_blockers(public_maintenance)
|
||||
)
|
||||
if public_maintenance_ready:
|
||||
runtime_ready_blockers.update(_PUBLIC_MAINTENANCE_BLOCKERS)
|
||||
|
||||
service_backup = _dict(payload.get("controlled_service_data_backup_readback"))
|
||||
rollups = _dict(payload.get("rollups"))
|
||||
@@ -949,6 +946,37 @@ def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks(
|
||||
return _unique_strings(reconciled)
|
||||
|
||||
|
||||
def _public_maintenance_reconciled_metric_blockers(
|
||||
public_maintenance: dict[str, Any],
|
||||
) -> set[str]:
|
||||
reconciled: set[str] = set()
|
||||
if public_maintenance.get("runtime_readback_present") is not True:
|
||||
return reconciled
|
||||
|
||||
reconciled.add("public_maintenance_fallback_runtime_readback_missing")
|
||||
public_route_ready = (
|
||||
_int(public_maintenance.get("raw_5xx_without_fallback_count")) == 0
|
||||
and _int(
|
||||
public_maintenance.get("route_unreachable_without_external_fallback_count")
|
||||
)
|
||||
== 0
|
||||
)
|
||||
if public_route_ready:
|
||||
reconciled.update(
|
||||
blocker
|
||||
for blocker in _PUBLIC_MAINTENANCE_ROUTE_BLOCKERS
|
||||
if blocker != "public_maintenance_fallback_runtime_readback_missing"
|
||||
)
|
||||
|
||||
edge_fallback_ready = (
|
||||
public_maintenance.get("edge_fallback_runtime_readback_present") is True
|
||||
and public_maintenance.get("edge_fallback_ready") is True
|
||||
)
|
||||
if edge_fallback_ready:
|
||||
reconciled.update(_PUBLIC_MAINTENANCE_EDGE_BLOCKERS)
|
||||
return reconciled
|
||||
|
||||
|
||||
def _add_runtime_readback_active_blockers_missing_from_metric(
|
||||
payload: dict[str, Any],
|
||||
active_blockers: list[str],
|
||||
@@ -1273,10 +1301,13 @@ def apply_public_maintenance_runtime_readback(
|
||||
public_maintenance = _build_public_maintenance_fallback_readback(runtime_readback)
|
||||
payload["public_maintenance_fallback"] = public_maintenance
|
||||
|
||||
reconciled_blockers = _public_maintenance_reconciled_metric_blockers(
|
||||
public_maintenance
|
||||
)
|
||||
active_blockers = [
|
||||
blocker
|
||||
for blocker in _strings(payload.get("active_blockers"))
|
||||
if blocker not in _PUBLIC_MAINTENANCE_BLOCKERS
|
||||
if blocker not in reconciled_blockers
|
||||
]
|
||||
active_blockers.extend(_strings(public_maintenance.get("blockers")))
|
||||
active_blockers = _unique_strings(active_blockers)
|
||||
|
||||
Reference in New Issue
Block a user