fix(reboot): preserve edge maintenance blocker in api overlay
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / build-and-deploy (push) Successful in 4m34s
CD Pipeline / post-deploy-checks (push) Successful in 1m45s

This commit is contained in:
Your Name
2026-07-03 15:39:22 +08:00
parent 8fec4d55f5
commit 80d209b6e5
6 changed files with 115 additions and 17 deletions

View File

@@ -75,12 +75,17 @@ _PROMETHEUS_RUNTIME_COMBINED_QUERY = (
+ "|".join(_PROMETHEUS_RUNTIME_QUERIES.values())
+ '"}'
)
_PUBLIC_MAINTENANCE_BLOCKERS = {
"edge_public_maintenance_live_config_drift",
_PUBLIC_MAINTENANCE_ROUTE_BLOCKERS = {
"public_maintenance_fallback_runtime_readback_missing",
"public_route_raw_5xx_without_maintenance_fallback",
"public_route_unreachable_without_external_l1_fallback",
}
_PUBLIC_MAINTENANCE_EDGE_BLOCKERS = {
"edge_public_maintenance_live_config_drift",
}
_PUBLIC_MAINTENANCE_BLOCKERS = (
_PUBLIC_MAINTENANCE_ROUTE_BLOCKERS | _PUBLIC_MAINTENANCE_EDGE_BLOCKERS
)
_RUNTIME_BOOLEAN_READY_BLOCKERS = {
"backup_core_green_not_1": "backup_core_green",
"host_188_service_green_not_1": "host_188_service_green",
@@ -895,17 +900,9 @@ def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks(
) -> list[str]:
runtime_ready_blockers: set[str] = set()
public_maintenance = _dict(payload.get("public_maintenance_fallback"))
public_maintenance_ready = (
public_maintenance.get("runtime_readback_present") is True
and public_maintenance.get("ready") is True
and _int(public_maintenance.get("raw_5xx_without_fallback_count")) == 0
and _int(
public_maintenance.get("route_unreachable_without_external_fallback_count")
)
== 0
runtime_ready_blockers.update(
_public_maintenance_reconciled_metric_blockers(public_maintenance)
)
if public_maintenance_ready:
runtime_ready_blockers.update(_PUBLIC_MAINTENANCE_BLOCKERS)
service_backup = _dict(payload.get("controlled_service_data_backup_readback"))
rollups = _dict(payload.get("rollups"))
@@ -949,6 +946,37 @@ def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks(
return _unique_strings(reconciled)
def _public_maintenance_reconciled_metric_blockers(
public_maintenance: dict[str, Any],
) -> set[str]:
reconciled: set[str] = set()
if public_maintenance.get("runtime_readback_present") is not True:
return reconciled
reconciled.add("public_maintenance_fallback_runtime_readback_missing")
public_route_ready = (
_int(public_maintenance.get("raw_5xx_without_fallback_count")) == 0
and _int(
public_maintenance.get("route_unreachable_without_external_fallback_count")
)
== 0
)
if public_route_ready:
reconciled.update(
blocker
for blocker in _PUBLIC_MAINTENANCE_ROUTE_BLOCKERS
if blocker != "public_maintenance_fallback_runtime_readback_missing"
)
edge_fallback_ready = (
public_maintenance.get("edge_fallback_runtime_readback_present") is True
and public_maintenance.get("edge_fallback_ready") is True
)
if edge_fallback_ready:
reconciled.update(_PUBLIC_MAINTENANCE_EDGE_BLOCKERS)
return reconciled
def _add_runtime_readback_active_blockers_missing_from_metric(
payload: dict[str, Any],
active_blockers: list[str],
@@ -1273,10 +1301,13 @@ def apply_public_maintenance_runtime_readback(
public_maintenance = _build_public_maintenance_fallback_readback(runtime_readback)
payload["public_maintenance_fallback"] = public_maintenance
reconciled_blockers = _public_maintenance_reconciled_metric_blockers(
public_maintenance
)
active_blockers = [
blocker
for blocker in _strings(payload.get("active_blockers"))
if blocker not in _PUBLIC_MAINTENANCE_BLOCKERS
if blocker not in reconciled_blockers
]
active_blockers.extend(_strings(public_maintenance.get("blockers")))
active_blockers = _unique_strings(active_blockers)