fix(reboot): prefer windows99 runtime metrics

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -46,6 +46,9 @@ _PROMETHEUS_RUNTIME_QUERIES = {
     "last_run_timestamp": "awoooi_reboot_auto_recovery_slo_last_run_timestamp",
     "windows99_vmware_readback_present": "awoooi_windows99_vmware_readback_present",
     "windows99_vmware_verify_ready": "awoooi_windows99_vmware_verify_ready",
+    "windows99_vmware_autostart_config_ready": (
+        "awoooi_windows99_vmware_autostart_config_ready"
+    ),
     "windows99_windows_update_ready": (
         "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready"
     ),
@@ -401,12 +404,19 @@ def _load_reboot_slo_prometheus_metric_readback(
         )
         == 1
     )
-    windows99_update_ready = (
-        _prometheus_first_int(
-            scalar_results.get("windows99_windows_update_ready") or [],
-            default=0,
-        )
-        == 1
+    windows99_config_metric = _prometheus_first_int(
+        scalar_results.get("windows99_vmware_autostart_config_ready") or [],
+        default=-1,
+    )
+    windows99_config_ready: bool | None = (
+        None if windows99_config_metric < 0 else windows99_config_metric == 1
+    )
+    windows99_update_metric = _prometheus_first_int(
+        scalar_results.get("windows99_windows_update_ready") or [],
+        default=-1,
+    )
+    windows99_update_ready: bool | None = (
+        None if windows99_update_metric < 0 else windows99_update_metric == 1
     )
     windows99_missing_vmx_aliases = _prometheus_label_values(
         missing_vmx_results,
@@ -440,6 +450,7 @@ def _load_reboot_slo_prometheus_metric_readback(
         "last_run_timestamp": last_run_timestamp,
         "windows99_vmware_readback_present": windows99_readback_present,
         "windows99_vmware_verify_ready": windows99_verify_ready,
+        "windows99_vmware_autostart_config_ready": windows99_config_ready,
         "windows99_windows_update_no_auto_reboot_ready": windows99_update_ready,
         "windows99_missing_vmx_aliases": windows99_missing_vmx_aliases,
         "windows99_powered_off_aliases": windows99_powered_off_aliases,
@@ -714,9 +725,15 @@ def _apply_prometheus_windows99_vmware_readback(
         blocker for blocker in active_blockers if blocker.startswith("windows99_")
     ]
     verify_ready = metric_readback.get("windows99_vmware_verify_ready") is True
-    update_ready = (
-        metric_readback.get("windows99_windows_update_no_auto_reboot_ready") is True
+    metric_config_ready = metric_readback.get(
+        "windows99_vmware_autostart_config_ready"
     )
+    metric_has_config_ready = isinstance(metric_config_ready, bool)
+    metric_update_ready = metric_readback.get(
+        "windows99_windows_update_no_auto_reboot_ready"
+    )
+    metric_has_update_ready = isinstance(metric_update_ready, bool)
+    update_ready = metric_update_ready is True
     windows99 = _dict(payload.setdefault("windows99_vmware_autostart", {}))
     source_service_blockers = _strings(windows99.get("service_blockers"))
     source_policy_blockers = _strings(windows99.get("policy_blockers"))
@@ -724,13 +741,26 @@ def _apply_prometheus_windows99_vmware_readback(
     source_config_blocked = bool(source_service_blockers) or (
         bool(source_task) and source_task.get("ok") is not True
     )
-    if source_config_blocked:
+    if metric_has_config_ready:
+        if metric_config_ready is not True:
+            windows99_blockers.append("windows99_vmware_autostart_config_not_ready")
+    elif source_config_blocked:
         windows99_blockers.append("windows99_vmware_autostart_config_not_ready")
-    if source_policy_blockers:
+    if metric_has_update_ready:
+        if metric_update_ready is not True:
+            windows99_blockers.append(
+                "windows99_update_no_auto_reboot_policy_not_ready"
+            )
+    elif source_policy_blockers:
         windows99_blockers.append("windows99_update_no_auto_reboot_policy_not_ready")
         update_ready = False
     windows99_blockers = _unique_strings(windows99_blockers)
-    config_ready = not missing_vmx_aliases and not source_config_blocked
+    metric_or_source_config_ready = (
+        metric_config_ready is True
+        if metric_has_config_ready
+        else not source_config_blocked
+    )
+    config_ready = not missing_vmx_aliases and metric_or_source_config_ready
     windows99.update(
         {
             "readback_present": True,

apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py

@@ -103,7 +103,6 @@ PROMETHEUS_SOURCE_RECONCILED_BLOCKER = (
 )
 WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS = [
     "windows99_vmware_autostart_config_not_ready",
-    "windows99_update_no_auto_reboot_policy_not_ready",
 ]
 PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS = [
     blocker
@@ -426,7 +425,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     )
     assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1
     assert payload["active_blockers"] == expected_active_blockers
-    assert payload["active_blocker_count"] == 9
+    assert payload["active_blocker_count"] == 8
     assert payload["readiness_percent"] == 47
     assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
     assert payload["next_safe_action"] == (
@@ -439,7 +438,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
         payload["runtime_metric_runtime_readback_added_blockers"]
         == WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS
     )
-    assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 2
+    assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 1
     package = payload["windows99_vmx_source_repair_package"]
     assert package["schema_version"] == (
         "windows99_vmx_source_repair_check_mode_package_v1"
@@ -481,7 +480,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
         is True
     )
     assert payload["controlled_service_data_backup_readback"]["blocking_fields"] == []
-    assert payload["active_blocker_action_matrix"]["item_count"] == 9
+    assert payload["active_blocker_action_matrix"]["item_count"] == 8
     assert payload["windows99_vmware_autostart"]["readback_present"] is True
     assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"]
     assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [
@@ -494,13 +493,13 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     assert payload["windows99_vmware_autostart"]["blockers"] == [
         "windows99_vmware_guest_power_not_ready",
         "windows99_vmware_vmx_missing",
-        *WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS,
+        "windows99_vmware_autostart_config_not_ready",
     ]
     assert payload["readback"]["windows99_vmware_readback_present"] is True
     assert payload["readback"]["windows99_vmrun_present"] is True
     assert payload["readback"]["windows99_vmware_config_ready"] is False
     assert payload["readback"]["windows99_vmware_power_ready"] is False
-    assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is False
+    assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is True
     assert payload["readback"]["windows99_runtime_metric_overlay_applied"] is True
     assert (
         payload["windows99_verify_collection"]["status"]
@@ -515,7 +514,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     assert payload["rollups"]["windows99_vmrun_present"] is True
     assert payload["rollups"]["windows99_vmware_config_ready"] is False
     assert payload["rollups"]["windows99_vmware_power_ready"] is False
-    assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is False
+    assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True
     assert payload["rollups"]["windows99_vmware_missing_vmx_count"] == 1
     assert payload["rollups"]["windows99_vmware_powered_off_count"] == 5
     assert payload["rollups"]["windows99_missing_vmx_alias_count"] == 1
@@ -560,7 +559,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     ] == 5
     assert payload["active_blocker_action_matrix"]["category_counts"][
         "windows99_vmware_autostart"
-    ] == 4
+    ] == 3
     assert payload["active_blocker_action_matrix"]["category_counts"].get(
         "post_reboot_service_readiness",
         0,
@@ -573,7 +572,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
         "security_observability",
         0,
     ) == 0
-    assert payload["readback"]["active_blocker_count"] == 9
+    assert payload["readback"]["active_blocker_count"] == 8
     assert payload["readback"]["runtime_metric_readback_present"] is True
     assert payload["readback"][
         "runtime_metric_source_control_reconciled_blocker_count"
@@ -583,20 +582,63 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     ] == WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS
     assert payload["readback"][
         "runtime_metric_runtime_readback_added_blocker_count"
-    ] == 2
-    assert payload["rollups"]["active_blocker_count"] == 9
+    ] == 1
+    assert payload["rollups"]["active_blocker_count"] == 8
     assert payload["rollups"]["runtime_metric_readback_present"] is True
     assert payload["rollups"][
         "runtime_metric_source_control_reconciled_blocker_count"
     ] == 1
     assert payload["rollups"][
         "runtime_metric_runtime_readback_added_blocker_count"
-    ] == 2
+    ] == 1
     assert payload["rollups"]["primary_blocker_owner_lane"] == (
         "reboot_event_detector_and_host_probe"
     )
 
 
+def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_windows99_config_when_runtime_metric_is_ready():
+    metric_readback = {
+        **PROMETHEUS_RUNTIME_READBACK,
+        "active_blockers": [
+            "all_required_hosts_not_in_10_minute_reboot_window",
+            "fresh_all_host_reboot_event_missing",
+            "host_boot_observation_older_than_target_window",
+            "host_unreachable_after_reboot",
+            "host_uptime_unknown",
+            "reboot_event_required_host_unreachable",
+            "windows99_vmware_guest_power_not_ready",
+        ],
+        "active_blocker_count": 7,
+        "windows99_vmware_autostart_config_ready": True,
+        "windows99_windows_update_no_auto_reboot_ready": True,
+        "windows99_missing_vmx_aliases": [],
+        "windows99_powered_off_aliases": ["110", "112", "120", "121", "188"],
+    }
+
+    payload = load_latest_reboot_auto_recovery_slo_scorecard(
+        prometheus_metric_readback=metric_readback,
+    )
+
+    assert payload["active_blockers"] == metric_readback["active_blockers"]
+    assert payload["active_blocker_count"] == 7
+    assert payload["runtime_metric_runtime_readback_added_blockers"] == []
+    assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 0
+    assert "windows99_vmware_autostart_config_not_ready" not in payload[
+        "active_blockers"
+    ]
+    assert "windows99_update_no_auto_reboot_policy_not_ready" not in payload[
+        "active_blockers"
+    ]
+    assert payload["windows99_vmware_autostart"]["config_ready"] is True
+    assert payload["windows99_vmware_autostart"][
+        "windows_update_no_auto_reboot_ready"
+    ] is True
+    assert payload["readback"]["windows99_vmware_config_ready"] is True
+    assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is True
+    assert payload["rollups"]["windows99_vmware_config_ready"] is True
+    assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True
+
+
 def test_reboot_auto_recovery_slo_scorecard_surfaces_windows99_console_artifact_blocker(
     tmp_path,
 ):
@@ -710,7 +752,7 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when
         *PROMETHEUS_RUNTIME_BLOCKERS,
         *WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS,
     ]
-    assert payload["active_blocker_count"] == 10
+    assert payload["active_blocker_count"] == 9
     assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0
     action_by_blocker = {
         item["blocker"]: item