From 149d52b6e20dd9fd71238ff522775efbce1b369b Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 02:40:16 +0800 Subject: [PATCH] fix(reboot): reconcile runtime-cleared maintenance blockers --- .../reboot_auto_recovery_slo_scorecard.py | 58 +++++++++++++++++++ ..._reboot_auto_recovery_slo_scorecard_api.py | 33 +++++++++++ 2 files changed, 91 insertions(+) diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 2cb852dd5..b9121c142 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -413,6 +413,8 @@ def _annotate_prometheus_metric_readback( ) payload["runtime_metric_source_control_reconciled_blockers"] = [] payload["runtime_metric_source_control_reconciled_blocker_count"] = 0 + payload["runtime_metric_runtime_readback_reconciled_blockers"] = [] + payload["runtime_metric_runtime_readback_reconciled_blocker_count"] = 0 readback = _dict(payload.setdefault("readback", {})) readback["runtime_metric_readback_present"] = present @@ -425,6 +427,8 @@ def _annotate_prometheus_metric_readback( ] readback["runtime_metric_source_control_reconciled_blockers"] = [] readback["runtime_metric_source_control_reconciled_blocker_count"] = 0 + readback["runtime_metric_runtime_readback_reconciled_blockers"] = [] + readback["runtime_metric_runtime_readback_reconciled_blocker_count"] = 0 rollups = _dict(payload.setdefault("rollups", {})) rollups["runtime_metric_readback_present"] = present @@ -435,6 +439,7 @@ def _annotate_prometheus_metric_readback( "runtime_metric_last_run_timestamp" ] rollups["runtime_metric_source_control_reconciled_blocker_count"] = 0 + rollups["runtime_metric_runtime_readback_reconciled_blocker_count"] = 0 _apply_prometheus_windows99_vmware_readback(payload, metric_readback) @@ -563,6 +568,55 @@ def _reconcile_prometheus_metric_active_blockers_with_source_controls( return _unique_strings(reconciled) +def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks( + payload: dict[str, Any], + active_blockers: list[str], +) -> list[str]: + public_maintenance = _dict(payload.get("public_maintenance_fallback")) + public_maintenance_ready = ( + public_maintenance.get("runtime_readback_present") is True + and public_maintenance.get("ready") is True + and _int(public_maintenance.get("raw_5xx_without_fallback_count")) == 0 + and _int( + public_maintenance.get("route_unreachable_without_external_fallback_count") + ) + == 0 + ) + runtime_ready_blockers = ( + _PUBLIC_MAINTENANCE_BLOCKERS if public_maintenance_ready else set() + ) + if not runtime_ready_blockers: + return _unique_strings(active_blockers) + + reconciled: list[str] = [] + removed: list[str] = [] + for blocker in active_blockers: + if blocker in runtime_ready_blockers: + removed.append(blocker) + continue + reconciled.append(blocker) + + removed = _unique_strings(removed) + if removed: + payload["runtime_metric_runtime_readback_reconciled_blockers"] = removed + payload["runtime_metric_runtime_readback_reconciled_blocker_count"] = len( + removed + ) + + readback = _dict(payload.setdefault("readback", {})) + readback["runtime_metric_runtime_readback_reconciled_blockers"] = removed + readback["runtime_metric_runtime_readback_reconciled_blocker_count"] = len( + removed + ) + + rollups = _dict(payload.setdefault("rollups", {})) + rollups["runtime_metric_runtime_readback_reconciled_blocker_count"] = len( + removed + ) + + return _unique_strings(reconciled) + + def _apply_prometheus_metric_active_blockers( payload: dict[str, Any], metric_readback: dict[str, Any], @@ -576,6 +630,10 @@ def _apply_prometheus_metric_active_blockers( payload, active_blockers, ) + active_blockers = _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks( + payload, + active_blockers, + ) can_claim_slo = metric_readback.get("ready") is True and not active_blockers primary_blocker = str( diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index e27a92cd5..816713e2b 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -157,6 +157,39 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_public_maintenance_runtime_ assert payload["reboot_sop_progress"]["active_blocker_count"] == 11 +def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_ready_public_maintenance_metric_blocker( + tmp_path, +): + runtime_path = tmp_path / "public-maintenance-fallback.json" + runtime_path.write_text( + json.dumps(PUBLIC_MAINTENANCE_RUNTIME_READY), + encoding="utf-8", + ) + metric_readback = dict(PROMETHEUS_RUNTIME_READBACK) + metric_readback["active_blockers"] = [ + *PROMETHEUS_RUNTIME_BLOCKERS, + "public_route_raw_5xx_without_maintenance_fallback", + ] + metric_readback["active_blocker_count"] = len(metric_readback["active_blockers"]) + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + public_maintenance_runtime_path=runtime_path, + prometheus_metric_readback=metric_readback, + ) + + assert "public_route_raw_5xx_without_maintenance_fallback" not in payload[ + "active_blockers" + ] + assert payload["public_maintenance_fallback"]["ready"] is True + assert payload["readback"]["public_maintenance_fallback_ready"] is True + assert payload["readback"][ + "runtime_metric_runtime_readback_reconciled_blockers" + ] == ["public_route_raw_5xx_without_maintenance_fallback"] + assert payload["rollups"][ + "runtime_metric_runtime_readback_reconciled_blocker_count" + ] == 1 + + def test_reboot_auto_recovery_slo_scorecard_overlays_runtime_scorecard_artifact( tmp_path, ):