fix(reboot): surface windows99 vmware alias metrics

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -44,6 +44,13 @@ _PROMETHEUS_RUNTIME_QUERIES = {
     "blocker_count": "awoooi_reboot_auto_recovery_slo_blocker_count",
     "ready": "awoooi_reboot_auto_recovery_slo_ready",
     "last_run_timestamp": "awoooi_reboot_auto_recovery_slo_last_run_timestamp",
+    "windows99_vmware_readback_present": "awoooi_windows99_vmware_readback_present",
+    "windows99_vmware_verify_ready": "awoooi_windows99_vmware_verify_ready",
+    "windows99_windows_update_ready": (
+        "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready"
+    ),
+    "windows99_missing_vmx_alias": "awoooi_windows99_vmware_missing_vmx_alias",
+    "windows99_powered_off_alias": "awoooi_windows99_vmware_powered_off_alias",
 }
 _PROMETHEUS_RUNTIME_COMBINED_QUERY = (
     '{__name__=~"'
@@ -261,8 +268,22 @@ def _load_reboot_slo_prometheus_metric_readback(
     scalar_results = {
         key: results_by_name.get(query, [])
         for key, query in _PROMETHEUS_RUNTIME_QUERIES.items()
-        if key not in {"active_blocker", "primary_blocker"}
+        if key
+        not in {
+            "active_blocker",
+            "primary_blocker",
+            "windows99_missing_vmx_alias",
+            "windows99_powered_off_alias",
+        }
     }
+    missing_vmx_results = results_by_name.get(
+        _PROMETHEUS_RUNTIME_QUERIES["windows99_missing_vmx_alias"],
+        [],
+    )
+    powered_off_results = results_by_name.get(
+        _PROMETHEUS_RUNTIME_QUERIES["windows99_powered_off_alias"],
+        [],
+    )
     active_blockers = _unique_strings(
         [
             str(_dict(result.get("metric")).get("blocker") or "")
@@ -289,6 +310,35 @@ def _load_reboot_slo_prometheus_metric_readback(
         scalar_results.get("last_run_timestamp") or [],
         default=0,
     )
+    windows99_readback_present = (
+        _prometheus_first_int(
+            scalar_results.get("windows99_vmware_readback_present") or [],
+            default=0,
+        )
+        == 1
+    )
+    windows99_verify_ready = (
+        _prometheus_first_int(
+            scalar_results.get("windows99_vmware_verify_ready") or [],
+            default=0,
+        )
+        == 1
+    )
+    windows99_update_ready = (
+        _prometheus_first_int(
+            scalar_results.get("windows99_windows_update_ready") or [],
+            default=0,
+        )
+        == 1
+    )
+    windows99_missing_vmx_aliases = _prometheus_label_values(
+        missing_vmx_results,
+        "alias",
+    )
+    windows99_powered_off_aliases = _prometheus_label_values(
+        powered_off_results,
+        "alias",
+    )
     return {
         "schema_version": "reboot_slo_prometheus_metric_readback_v1",
         "readback_present": bool(combined_results),
@@ -299,6 +349,11 @@ def _load_reboot_slo_prometheus_metric_readback(
         "primary_blocker": primary_blocker,
         "ready": ready,
         "last_run_timestamp": last_run_timestamp,
+        "windows99_vmware_readback_present": windows99_readback_present,
+        "windows99_vmware_verify_ready": windows99_verify_ready,
+        "windows99_windows_update_no_auto_reboot_ready": windows99_update_ready,
+        "windows99_missing_vmx_aliases": windows99_missing_vmx_aliases,
+        "windows99_powered_off_aliases": windows99_powered_off_aliases,
     }
 
 
@@ -329,6 +384,19 @@ def _prometheus_first_int(results: list[dict[str, Any]], *, default: int) -> int
         return default
 
 
+def _prometheus_label_values(
+    results: list[dict[str, Any]],
+    label_name: str,
+) -> list[str]:
+    return _unique_strings(
+        [
+            str(_dict(result.get("metric")).get(label_name) or "")
+            for result in results
+            if str(_dict(result.get("metric")).get(label_name) or "")
+        ]
+    )
+
+
 def _annotate_prometheus_metric_readback(
     payload: dict[str, Any],
     metric_readback: dict[str, Any],
@@ -367,6 +435,59 @@ def _annotate_prometheus_metric_readback(
         "runtime_metric_last_run_timestamp"
     ]
     rollups["runtime_metric_source_control_reconciled_blocker_count"] = 0
+    _apply_prometheus_windows99_vmware_readback(payload, metric_readback)
+
+
+def _apply_prometheus_windows99_vmware_readback(
+    payload: dict[str, Any],
+    metric_readback: dict[str, Any],
+) -> None:
+    if metric_readback.get("readback_present") is not True:
+        return
+    if metric_readback.get("windows99_vmware_readback_present") is not True:
+        return
+
+    active_blockers = _strings(metric_readback.get("active_blockers"))
+    missing_vmx_aliases = _strings(metric_readback.get("windows99_missing_vmx_aliases"))
+    powered_off_aliases = _strings(metric_readback.get("windows99_powered_off_aliases"))
+    windows99_blockers = [
+        blocker for blocker in active_blockers if blocker.startswith("windows99_")
+    ]
+    verify_ready = metric_readback.get("windows99_vmware_verify_ready") is True
+    update_ready = (
+        metric_readback.get("windows99_windows_update_no_auto_reboot_ready") is True
+    )
+    windows99 = _dict(payload.setdefault("windows99_vmware_autostart", {}))
+    windows99.update(
+        {
+            "readback_present": True,
+            "verify_ready": verify_ready,
+            "vmrun_present": True,
+            "config_ready": not missing_vmx_aliases,
+            "power_ready": not powered_off_aliases,
+            "windows_update_no_auto_reboot_ready": update_ready,
+            "missing_vmx_aliases": missing_vmx_aliases,
+            "powered_off_aliases": powered_off_aliases,
+            "blockers": windows99_blockers,
+            "runtime_metric_overlay_applied": True,
+        }
+    )
+    payload["windows99_vmware_verify_ready"] = verify_ready
+    payload["windows99_update_no_auto_reboot_ready"] = update_ready
+
+    readback = _dict(payload.setdefault("readback", {}))
+    readback["windows99_vmware_verify_ready"] = verify_ready
+    readback["windows99_update_no_auto_reboot_ready"] = update_ready
+    readback["windows99_runtime_metric_overlay_applied"] = True
+    readback["windows99_missing_vmx_aliases"] = missing_vmx_aliases
+    readback["windows99_powered_off_aliases"] = powered_off_aliases
+
+    rollups = _dict(payload.setdefault("rollups", {}))
+    rollups["windows99_vmware_verify_ready"] = verify_ready
+    rollups["windows99_update_no_auto_reboot_ready"] = update_ready
+    rollups["windows99_runtime_metric_overlay_applied"] = True
+    rollups["windows99_missing_vmx_alias_count"] = len(missing_vmx_aliases)
+    rollups["windows99_powered_off_alias_count"] = len(powered_off_aliases)
 
 
 def _reconcile_prometheus_metric_active_blockers_with_source_controls(

apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py

@@ -87,7 +87,8 @@ PROMETHEUS_RUNTIME_BLOCKERS = [
     "host_unreachable_after_reboot",
     "host_uptime_unknown",
     "reboot_event_required_host_unreachable",
-    "windows99_vmware_autostart_readback_missing",
+    "windows99_vmware_guest_power_not_ready",
+    "windows99_vmware_vmx_missing",
 ]
 PROMETHEUS_SOURCE_RECONCILED_BLOCKER = (
     "conversation_event_hot_path_index_migration_source_missing"
@@ -104,10 +105,15 @@ PROMETHEUS_RUNTIME_READBACK = {
     "source": "prometheus_query_api",
     "query_url": "http://192.168.0.110:9090/api/v1/query",
     "active_blockers": PROMETHEUS_RUNTIME_BLOCKERS,
-    "active_blocker_count": 7,
+    "active_blocker_count": 8,
     "primary_blocker": "reboot_event_required_host_unreachable",
     "ready": False,
     "last_run_timestamp": 1783010479,
+    "windows99_vmware_readback_present": True,
+    "windows99_vmware_verify_ready": False,
+    "windows99_windows_update_no_auto_reboot_ready": True,
+    "windows99_missing_vmx_aliases": ["111"],
+    "windows99_powered_off_aliases": ["111", "112", "120", "121", "188"],
 }
 
 
@@ -264,7 +270,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
 
     assert payload["runtime_scorecard_readback_present"] is False
     assert payload["runtime_metric_readback_present"] is True
-    assert payload["runtime_metric_active_blocker_count"] == 7
+    assert payload["runtime_metric_active_blocker_count"] == 8
     assert payload["runtime_metric_last_run_timestamp"] == 1783010479
     assert (
         payload["runtime_metric_source_control_reconciled_blockers"]
@@ -272,7 +278,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     )
     assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1
     assert payload["active_blockers"] == PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS
-    assert payload["active_blocker_count"] == 6
+    assert payload["active_blocker_count"] == 7
     assert payload["readiness_percent"] == 47
     assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
     assert payload["next_safe_action"] == (
@@ -281,7 +287,22 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
     assert "backup_core_green_not_1" not in payload["active_blockers"]
     assert "service_green_not_1" not in payload["active_blockers"]
     assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"]
-    assert payload["active_blocker_action_matrix"]["item_count"] == 6
+    assert payload["active_blocker_action_matrix"]["item_count"] == 7
+    assert payload["windows99_vmware_autostart"]["readback_present"] is True
+    assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"]
+    assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [
+        "111",
+        "112",
+        "120",
+        "121",
+        "188",
+    ]
+    assert payload["windows99_vmware_autostart"]["blockers"] == [
+        "windows99_vmware_guest_power_not_ready",
+        "windows99_vmware_vmx_missing",
+    ]
+    assert payload["readback"]["windows99_runtime_metric_overlay_applied"] is True
+    assert payload["rollups"]["windows99_missing_vmx_alias_count"] == 1
     action_by_blocker = {
         item["blocker"]: item
         for item in payload["active_blocker_action_matrix"]["items"]
@@ -291,12 +312,12 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
         "host_cpu_pressure",
         0,
     ) == 0
-    assert payload["readback"]["active_blocker_count"] == 6
+    assert payload["readback"]["active_blocker_count"] == 7
     assert payload["readback"]["runtime_metric_readback_present"] is True
     assert payload["readback"][
         "runtime_metric_source_control_reconciled_blocker_count"
     ] == 1
-    assert payload["rollups"]["active_blocker_count"] == 6
+    assert payload["rollups"]["active_blocker_count"] == 7
     assert payload["rollups"]["runtime_metric_readback_present"] is True
     assert payload["rollups"][
         "runtime_metric_source_control_reconciled_blocker_count"
@@ -324,7 +345,7 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when
     )
 
     assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
-    assert payload["active_blocker_count"] == 7
+    assert payload["active_blocker_count"] == 8
     assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0
     action_by_blocker = {
         item["blocker"]: item
@@ -372,7 +393,7 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
                 "metric": {
                     "__name__": "awoooi_reboot_auto_recovery_slo_blocker_count",
                 },
-                "value": [1783010479, "7"],
+                "value": [1783010479, str(len(PROMETHEUS_RUNTIME_BLOCKERS))],
             },
             {
                 "metric": {"__name__": "awoooi_reboot_auto_recovery_slo_ready"},
@@ -384,6 +405,43 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
                 },
                 "value": [1783010479, "1783010479"],
             },
+            {
+                "metric": {
+                    "__name__": "awoooi_windows99_vmware_readback_present",
+                },
+                "value": [1783010479, "1"],
+            },
+            {
+                "metric": {
+                    "__name__": "awoooi_windows99_vmware_verify_ready",
+                },
+                "value": [1783010479, "0"],
+            },
+            {
+                "metric": {
+                    "__name__": (
+                        "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready"
+                    ),
+                },
+                "value": [1783010479, "1"],
+            },
+            {
+                "metric": {
+                    "__name__": "awoooi_windows99_vmware_missing_vmx_alias",
+                    "alias": "111",
+                },
+                "value": [1783010479, "1"],
+            },
+            *[
+                {
+                    "metric": {
+                        "__name__": "awoooi_windows99_vmware_powered_off_alias",
+                        "alias": alias,
+                    },
+                    "value": [1783010479, "1"],
+                }
+                for alias in ["111", "112", "120", "121", "188"]
+            ],
         ]
 
     monkeypatch.setattr(
@@ -403,10 +461,18 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
     assert "awoooi_reboot_auto_recovery_slo_last_run_timestamp" in calls[0][1]
     assert payload["readback_present"] is True
     assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
-    assert payload["active_blocker_count"] == 7
+    assert payload["active_blocker_count"] == len(PROMETHEUS_RUNTIME_BLOCKERS)
     assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
     assert payload["ready"] is False
     assert payload["last_run_timestamp"] == 1783010479
+    assert payload["windows99_missing_vmx_aliases"] == ["111"]
+    assert payload["windows99_powered_off_aliases"] == [
+        "111",
+        "112",
+        "120",
+        "121",
+        "188",
+    ]
 
 
 def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch):