fix(reboot): surface windows99 collector metrics

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -51,6 +51,21 @@ _PROMETHEUS_RUNTIME_QUERIES = {
     ),
     "windows99_missing_vmx_alias": "awoooi_windows99_vmware_missing_vmx_alias",
     "windows99_powered_off_alias": "awoooi_windows99_vmware_powered_off_alias",
+    "windows99_no_secret_collector_readback_present": (
+        "awoooi_windows99_no_secret_collector_readback_present"
+    ),
+    "windows99_no_secret_collector_status": (
+        "awoooi_windows99_no_secret_collector_status"
+    ),
+    "windows99_no_secret_collector_ssh_batchmode_auth_ready": (
+        "awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready"
+    ),
+    "windows99_no_secret_collector_port_22_open": (
+        "awoooi_windows99_no_secret_collector_port_22_open"
+    ),
+    "windows99_no_secret_collector_remote_verify_attempted": (
+        "awoooi_windows99_no_secret_collector_remote_verify_attempted"
+    ),
 }
 _PROMETHEUS_RUNTIME_COMBINED_QUERY = (
     '{__name__=~"'
@@ -81,6 +96,13 @@ _PROMETHEUS_SOURCE_CONTROLLED_BLOCKERS = {
         "conversation_event_hot_path_index_migration_source_present"
     ),
 }
+_WINDOWS99_COLLECTOR_BLOCKERS_BY_STATUS = {
+    "blocked_ssh_publickey_auth_missing": "windows99_ssh_publickey_auth_missing",
+    "blocked_ssh_port_closed": "windows99_ssh_port_closed",
+    "blocked_local_verify_script_missing": "windows99_local_verify_script_missing",
+    "blocked_remote_verify_output_invalid": "windows99_remote_verify_output_invalid",
+    "blocked_remote_verify_command_failed": "windows99_remote_verify_command_failed",
+}
 
 
 def load_latest_reboot_auto_recovery_slo_scorecard(
@@ -288,6 +310,7 @@ def _load_reboot_slo_prometheus_metric_readback(
             "primary_blocker",
             "windows99_missing_vmx_alias",
             "windows99_powered_off_alias",
+            "windows99_no_secret_collector_status",
         }
     }
     missing_vmx_results = results_by_name.get(
@@ -353,6 +376,18 @@ def _load_reboot_slo_prometheus_metric_readback(
         powered_off_results,
         "alias",
     )
+    collector_status_results = results_by_name.get(
+        _PROMETHEUS_RUNTIME_QUERIES["windows99_no_secret_collector_status"],
+        [],
+    )
+    windows99_collector_status = _prometheus_first_label_value(
+        collector_status_results,
+        "status",
+    )
+    windows99_collector_safe_next_step = _prometheus_first_label_value(
+        collector_status_results,
+        "safe_next_step",
+    )
     return {
         "schema_version": "reboot_slo_prometheus_metric_readback_v1",
         "readback_present": bool(combined_results),
@@ -368,6 +403,48 @@ def _load_reboot_slo_prometheus_metric_readback(
         "windows99_windows_update_no_auto_reboot_ready": windows99_update_ready,
         "windows99_missing_vmx_aliases": windows99_missing_vmx_aliases,
         "windows99_powered_off_aliases": windows99_powered_off_aliases,
+        "windows99_no_secret_collector_readback_present": (
+            _prometheus_first_int(
+                scalar_results.get(
+                    "windows99_no_secret_collector_readback_present"
+                )
+                or [],
+                default=0,
+            )
+            == 1
+        ),
+        "windows99_no_secret_collector_status": windows99_collector_status,
+        "windows99_no_secret_collector_safe_next_step": (
+            windows99_collector_safe_next_step
+        ),
+        "windows99_no_secret_collector_ssh_batchmode_auth_ready": (
+            _prometheus_first_int(
+                scalar_results.get(
+                    "windows99_no_secret_collector_ssh_batchmode_auth_ready"
+                )
+                or [],
+                default=0,
+            )
+            == 1
+        ),
+        "windows99_no_secret_collector_port_22_open": (
+            _prometheus_first_int(
+                scalar_results.get("windows99_no_secret_collector_port_22_open")
+                or [],
+                default=0,
+            )
+            == 1
+        ),
+        "windows99_no_secret_collector_remote_verify_attempted": (
+            _prometheus_first_int(
+                scalar_results.get(
+                    "windows99_no_secret_collector_remote_verify_attempted"
+                )
+                or [],
+                default=0,
+            )
+            == 1
+        ),
     }
 
 
@@ -411,6 +488,20 @@ def _prometheus_label_values(
     )
 
 
+def _prometheus_first_label_value(
+    results: list[dict[str, Any]],
+    label_name: str,
+) -> str:
+    for result in results:
+        value = _prometheus_first_int([result], default=0)
+        if value != 1:
+            continue
+        label = str(_dict(result.get("metric")).get(label_name) or "")
+        if label:
+            return label
+    return ""
+
+
 def _annotate_prometheus_metric_readback(
     payload: dict[str, Any],
     metric_readback: dict[str, Any],
@@ -459,9 +550,114 @@ def _annotate_prometheus_metric_readback(
     rollups["runtime_metric_source_control_reconciled_blocker_count"] = 0
     rollups["runtime_metric_runtime_readback_reconciled_blocker_count"] = 0
     rollups["runtime_metric_runtime_readback_added_blocker_count"] = 0
+    _apply_prometheus_windows99_no_secret_collector_readback(
+        payload,
+        metric_readback,
+    )
     _apply_prometheus_windows99_vmware_readback(payload, metric_readback)
 
 
+def _apply_prometheus_windows99_no_secret_collector_readback(
+    payload: dict[str, Any],
+    metric_readback: dict[str, Any],
+) -> None:
+    if metric_readback.get("readback_present") is not True:
+        return
+    if (
+        metric_readback.get("windows99_no_secret_collector_readback_present")
+        is not True
+    ):
+        return
+
+    status = str(
+        metric_readback.get("windows99_no_secret_collector_status") or "unknown"
+    )
+    safe_next_step = str(
+        metric_readback.get("windows99_no_secret_collector_safe_next_step") or ""
+    )
+    ssh_ready = (
+        metric_readback.get(
+            "windows99_no_secret_collector_ssh_batchmode_auth_ready"
+        )
+        is True
+    )
+    port_22_open = (
+        metric_readback.get("windows99_no_secret_collector_port_22_open") is True
+    )
+    remote_attempted = (
+        metric_readback.get(
+            "windows99_no_secret_collector_remote_verify_attempted"
+        )
+        is True
+    )
+    blocker = _WINDOWS99_COLLECTOR_BLOCKERS_BY_STATUS.get(status)
+    if not blocker and status.startswith("blocked_"):
+        blocker = "windows99_no_secret_collector_not_ready"
+
+    collection = _dict(payload.setdefault("windows99_verify_collection", {}))
+    available_channels = _strings(collection.get("available_collection_channels"))
+    if ssh_ready:
+        available_channels.append("no_secret_ssh_batchmode_verify_collector")
+    collector_blockers = _strings(collection.get("collection_blockers"))
+    if blocker:
+        collector_blockers.append(blocker)
+    collector = _dict(collection.get("no_secret_collector"))
+    collector.update(
+        {
+            "schema_version": "windows99_vmware_verify_collector_v1",
+            "readback_present": True,
+            "status": status,
+            "safe_next_step": safe_next_step,
+            "ssh_batchmode_auth_ready": ssh_ready,
+            "port_22_open": port_22_open,
+            "remote_verify_attempted": remote_attempted,
+            "source": "prometheus_textfile_metric",
+        }
+    )
+    collection.update(
+        {
+            "no_secret_collector_readback_present": True,
+            "no_secret_collector_status": status,
+            "no_secret_collector_safe_next_step": safe_next_step,
+            "no_secret_collector_ssh_batchmode_auth_ready": ssh_ready,
+            "no_secret_collector_port_22_open": port_22_open,
+            "no_secret_collector_remote_verify_attempted": remote_attempted,
+            "no_secret_collector": collector,
+            "available_collection_channels": _unique_strings(available_channels),
+            "collection_blockers": _unique_strings(collector_blockers),
+        }
+    )
+
+    readback = _dict(payload.setdefault("readback", {}))
+    readback["windows99_no_secret_collector_readback_present"] = True
+    readback["windows99_no_secret_collector_status"] = status
+    readback["windows99_no_secret_collector_ssh_batchmode_auth_ready"] = ssh_ready
+    readback["windows99_no_secret_collector_port_22_open"] = port_22_open
+    readback["windows99_no_secret_collector_remote_verify_attempted"] = (
+        remote_attempted
+    )
+    readback["windows99_no_secret_collector_safe_next_step"] = safe_next_step
+    readback["windows99_available_collection_channels"] = collection[
+        "available_collection_channels"
+    ]
+
+    rollups = _dict(payload.setdefault("rollups", {}))
+    rollups["windows99_no_secret_collector_readback_present"] = True
+    rollups["windows99_no_secret_collector_status"] = status
+    rollups["windows99_no_secret_collector_ssh_batchmode_auth_ready"] = ssh_ready
+    rollups["windows99_no_secret_collector_port_22_open"] = port_22_open
+    rollups["windows99_no_secret_collector_remote_verify_attempted"] = (
+        remote_attempted
+    )
+    rollups["windows99_no_secret_collector_safe_next_step"] = safe_next_step
+    rollups["windows99_available_collection_channels"] = collection[
+        "available_collection_channels"
+    ]
+    rollups["windows99_verify_collection_blocker_count"] = len(
+        _strings(collection.get("collection_blockers"))
+    )
+
+
 def _apply_prometheus_windows99_vmware_readback(
     payload: dict[str, Any],
     metric_readback: dict[str, Any],