fix(reboot): bound slo prometheus readback latency

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -45,6 +45,11 @@ _PROMETHEUS_RUNTIME_QUERIES = {
     "ready": "awoooi_reboot_auto_recovery_slo_ready",
     "last_run_timestamp": "awoooi_reboot_auto_recovery_slo_last_run_timestamp",
 }
+_PROMETHEUS_RUNTIME_COMBINED_QUERY = (
+    '{__name__=~"'
+    + "|".join(_PROMETHEUS_RUNTIME_QUERIES.values())
+    + '"}'
+)
 _PUBLIC_MAINTENANCE_BLOCKERS = {
     "public_maintenance_fallback_runtime_readback_missing",
     "public_route_raw_5xx_without_maintenance_fallback",
@@ -213,27 +218,13 @@ def _load_reboot_slo_prometheus_metric_readback(
         or os.environ.get(_PROMETHEUS_QUERY_URL_ENV, "").strip()
         or _DEFAULT_PROMETHEUS_QUERY_URL
     )
-    timeout_seconds = _env_float(_PROMETHEUS_TIMEOUT_SECONDS_ENV, default=3.0)
+    timeout_seconds = _env_float(_PROMETHEUS_TIMEOUT_SECONDS_ENV, default=1.0)
     try:
-        active_results = _query_prometheus_vector(
+        combined_results = _query_prometheus_vector(
             query_url,
-            _PROMETHEUS_RUNTIME_QUERIES["active_blocker"],
+            _PROMETHEUS_RUNTIME_COMBINED_QUERY,
             timeout_seconds=timeout_seconds,
         )
-        primary_results = _query_prometheus_vector(
-            query_url,
-            _PROMETHEUS_RUNTIME_QUERIES["primary_blocker"],
-            timeout_seconds=timeout_seconds,
-        )
-        scalar_results = {
-            key: _query_prometheus_vector(
-                query_url,
-                query,
-                timeout_seconds=timeout_seconds,
-            )
-            for key, query in _PROMETHEUS_RUNTIME_QUERIES.items()
-            if key not in {"active_blocker", "primary_blocker"}
-        }
     except (
         OSError,
         TimeoutError,
@@ -249,6 +240,24 @@ def _load_reboot_slo_prometheus_metric_readback(
             "error_class": exc.__class__.__name__,
         }
 
+    results_by_name: dict[str, list[dict[str, Any]]] = {}
+    for result in combined_results:
+        metric_name = str(_dict(result.get("metric")).get("__name__") or "")
+        if metric_name:
+            results_by_name.setdefault(metric_name, []).append(result)
+    active_results = results_by_name.get(
+        _PROMETHEUS_RUNTIME_QUERIES["active_blocker"],
+        [],
+    )
+    primary_results = results_by_name.get(
+        _PROMETHEUS_RUNTIME_QUERIES["primary_blocker"],
+        [],
+    )
+    scalar_results = {
+        key: results_by_name.get(query, [])
+        for key, query in _PROMETHEUS_RUNTIME_QUERIES.items()
+        if key not in {"active_blocker", "primary_blocker"}
+    }
     active_blockers = _unique_strings(
         [
             str(_dict(result.get("metric")).get("blocker") or "")
@@ -277,12 +286,7 @@ def _load_reboot_slo_prometheus_metric_readback(
     )
     return {
         "schema_version": "reboot_slo_prometheus_metric_readback_v1",
-        "readback_present": bool(
-            active_results
-            or primary_results
-            or scalar_results.get("blocker_count")
-            or scalar_results.get("ready")
-        ),
+        "readback_present": bool(combined_results),
         "source": "prometheus_query_api",
         "query_url": query_url,
         "active_blockers": active_blockers,