fix(reboot): reconcile runtime-cleared maintenance blockers

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -413,6 +413,8 @@ def _annotate_prometheus_metric_readback(
     )
     payload["runtime_metric_source_control_reconciled_blockers"] = []
     payload["runtime_metric_source_control_reconciled_blocker_count"] = 0
+    payload["runtime_metric_runtime_readback_reconciled_blockers"] = []
+    payload["runtime_metric_runtime_readback_reconciled_blocker_count"] = 0
 
     readback = _dict(payload.setdefault("readback", {}))
     readback["runtime_metric_readback_present"] = present
@@ -425,6 +427,8 @@ def _annotate_prometheus_metric_readback(
     ]
     readback["runtime_metric_source_control_reconciled_blockers"] = []
     readback["runtime_metric_source_control_reconciled_blocker_count"] = 0
+    readback["runtime_metric_runtime_readback_reconciled_blockers"] = []
+    readback["runtime_metric_runtime_readback_reconciled_blocker_count"] = 0
 
     rollups = _dict(payload.setdefault("rollups", {}))
     rollups["runtime_metric_readback_present"] = present
@@ -435,6 +439,7 @@ def _annotate_prometheus_metric_readback(
         "runtime_metric_last_run_timestamp"
     ]
     rollups["runtime_metric_source_control_reconciled_blocker_count"] = 0
+    rollups["runtime_metric_runtime_readback_reconciled_blocker_count"] = 0
     _apply_prometheus_windows99_vmware_readback(payload, metric_readback)
 
 
@@ -563,6 +568,55 @@ def _reconcile_prometheus_metric_active_blockers_with_source_controls(
     return _unique_strings(reconciled)
 
 
+def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks(
+    payload: dict[str, Any],
+    active_blockers: list[str],
+) -> list[str]:
+    public_maintenance = _dict(payload.get("public_maintenance_fallback"))
+    public_maintenance_ready = (
+        public_maintenance.get("runtime_readback_present") is True
+        and public_maintenance.get("ready") is True
+        and _int(public_maintenance.get("raw_5xx_without_fallback_count")) == 0
+        and _int(
+            public_maintenance.get("route_unreachable_without_external_fallback_count")
+        )
+        == 0
+    )
+    runtime_ready_blockers = (
+        _PUBLIC_MAINTENANCE_BLOCKERS if public_maintenance_ready else set()
+    )
+    if not runtime_ready_blockers:
+        return _unique_strings(active_blockers)
+
+    reconciled: list[str] = []
+    removed: list[str] = []
+    for blocker in active_blockers:
+        if blocker in runtime_ready_blockers:
+            removed.append(blocker)
+            continue
+        reconciled.append(blocker)
+
+    removed = _unique_strings(removed)
+    if removed:
+        payload["runtime_metric_runtime_readback_reconciled_blockers"] = removed
+        payload["runtime_metric_runtime_readback_reconciled_blocker_count"] = len(
+            removed
+        )
+
+        readback = _dict(payload.setdefault("readback", {}))
+        readback["runtime_metric_runtime_readback_reconciled_blockers"] = removed
+        readback["runtime_metric_runtime_readback_reconciled_blocker_count"] = len(
+            removed
+        )
+
+        rollups = _dict(payload.setdefault("rollups", {}))
+        rollups["runtime_metric_runtime_readback_reconciled_blocker_count"] = len(
+            removed
+        )
+
+    return _unique_strings(reconciled)
+
+
 def _apply_prometheus_metric_active_blockers(
     payload: dict[str, Any],
     metric_readback: dict[str, Any],
@@ -576,6 +630,10 @@ def _apply_prometheus_metric_active_blockers(
         payload,
         active_blockers,
     )
+    active_blockers = _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks(
+        payload,
+        active_blockers,
+    )
 
     can_claim_slo = metric_readback.get("ready") is True and not active_blockers
     primary_blocker = str(

apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py

@@ -157,6 +157,39 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_public_maintenance_runtime_
     assert payload["reboot_sop_progress"]["active_blocker_count"] == 11
 
 
+def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_ready_public_maintenance_metric_blocker(
+    tmp_path,
+):
+    runtime_path = tmp_path / "public-maintenance-fallback.json"
+    runtime_path.write_text(
+        json.dumps(PUBLIC_MAINTENANCE_RUNTIME_READY),
+        encoding="utf-8",
+    )
+    metric_readback = dict(PROMETHEUS_RUNTIME_READBACK)
+    metric_readback["active_blockers"] = [
+        *PROMETHEUS_RUNTIME_BLOCKERS,
+        "public_route_raw_5xx_without_maintenance_fallback",
+    ]
+    metric_readback["active_blocker_count"] = len(metric_readback["active_blockers"])
+
+    payload = load_latest_reboot_auto_recovery_slo_scorecard(
+        public_maintenance_runtime_path=runtime_path,
+        prometheus_metric_readback=metric_readback,
+    )
+
+    assert "public_route_raw_5xx_without_maintenance_fallback" not in payload[
+        "active_blockers"
+    ]
+    assert payload["public_maintenance_fallback"]["ready"] is True
+    assert payload["readback"]["public_maintenance_fallback_ready"] is True
+    assert payload["readback"][
+        "runtime_metric_runtime_readback_reconciled_blockers"
+    ] == ["public_route_raw_5xx_without_maintenance_fallback"]
+    assert payload["rollups"][
+        "runtime_metric_runtime_readback_reconciled_blocker_count"
+    ] == 1
+
+
 def test_reboot_auto_recovery_slo_scorecard_overlays_runtime_scorecard_artifact(
     tmp_path,
 ):