diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 68936efc8..40c6fb39a 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -122,6 +122,9 @@ _WINDOWS99_COLLECTOR_BLOCKERS_BY_STATUS = { "blocked_remote_verify_output_invalid": "windows99_remote_verify_output_invalid", "blocked_remote_verify_command_failed": "windows99_remote_verify_command_failed", } +_WINDOWS99_COLLECTOR_BLOCKER_VALUES = set( + _WINDOWS99_COLLECTOR_BLOCKERS_BY_STATUS.values() +) | {"windows99_no_secret_collector_not_ready"} _WINDOWS99_COLLECTED_VERIFY_STATUS = "collected_windows99_vmware_verify_stdout" _WINDOWS99_MISSING_VMX_NEXT_SAFE_ACTION = ( "restore_windows99_missing_vmx_source_for_aliases_then_rerun_no_secret_" @@ -739,10 +742,21 @@ def _apply_prometheus_windows99_no_secret_collector_readback( available_channels = _strings(collection.get("available_collection_channels")) if ssh_ready: available_channels.append("no_secret_ssh_batchmode_verify_collector") - collector_blockers = _strings(collection.get("collection_blockers")) + collector_blockers = [ + item + for item in _strings(collection.get("collection_blockers")) + if item not in _WINDOWS99_COLLECTOR_BLOCKER_VALUES + ] if blocker: collector_blockers.append(blocker) collector = _dict(collection.get("no_secret_collector")) + nested_collector_blockers = [ + item + for item in _strings(collector.get("blockers")) + if item not in _WINDOWS99_COLLECTOR_BLOCKER_VALUES + ] + if blocker: + nested_collector_blockers.append(blocker) collector.update( { "schema_version": "windows99_vmware_verify_collector_v1", @@ -753,6 +767,7 @@ def _apply_prometheus_windows99_no_secret_collector_readback( "port_22_open": port_22_open, "remote_verify_attempted": remote_attempted, "source": "prometheus_textfile_metric", + "blockers": _unique_strings(nested_collector_blockers), } ) collection.update( @@ -794,9 +809,7 @@ def _apply_prometheus_windows99_no_secret_collector_readback( rollups["windows99_available_collection_channels"] = collection[ "available_collection_channels" ] - rollups["windows99_verify_collection_blocker_count"] = len( - _strings(collection.get("collection_blockers")) - ) + _sync_windows99_verify_collection_readback(payload) def _apply_prometheus_windows99_vmware_readback( @@ -814,6 +827,19 @@ def _apply_prometheus_windows99_vmware_readback( windows99_blockers = [ blocker for blocker in active_blockers if blocker.startswith("windows99_") ] + collector_status = str( + _dict(payload.get("windows99_vmware_verify_collector")).get("status") + or _dict(payload.get("windows99_verify_collection")).get( + "no_secret_collector_status" + ) + or "" + ) + if collector_status == _WINDOWS99_COLLECTED_VERIFY_STATUS: + windows99_blockers = [ + blocker + for blocker in windows99_blockers + if blocker not in _WINDOWS99_COLLECTOR_BLOCKER_VALUES + ] verify_ready = metric_readback.get("windows99_vmware_verify_ready") is True metric_config_ready = metric_readback.get( "windows99_vmware_autostart_config_ready" @@ -967,9 +993,7 @@ def _apply_prometheus_windows99_vmware_readback( ) rollups["windows99_autostart_task_never_ran"] = scheduled_task.get("never_ran") rollups["windows99_verify_collection_status"] = collection["status"] - rollups["windows99_verify_collection_blocker_count"] = len( - _strings(collection.get("collection_blockers")) - ) + _sync_windows99_verify_collection_readback(payload) _sync_windows99_vmx_source_repair_package( payload, generated_at=str( @@ -1035,6 +1059,15 @@ def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks( or rollups.get(ready_key) is True ): runtime_ready_blockers.add(blocker) + collector_status = str( + _dict(payload.get("windows99_vmware_verify_collector")).get("status") + or _dict(payload.get("windows99_verify_collection")).get( + "no_secret_collector_status" + ) + or "" + ) + if collector_status == _WINDOWS99_COLLECTED_VERIFY_STATUS: + runtime_ready_blockers.update(_WINDOWS99_COLLECTOR_BLOCKER_VALUES) if not runtime_ready_blockers: return _unique_strings(active_blockers) @@ -1068,6 +1101,27 @@ def _reconcile_prometheus_metric_active_blockers_with_runtime_readbacks( return _unique_strings(reconciled) +def _sync_windows99_verify_collection_readback(payload: dict[str, Any]) -> None: + collection = _dict(payload.get("windows99_verify_collection")) + if not collection: + return + collection_blockers = _strings(collection.get("collection_blockers")) + status = str(collection.get("status") or "unknown") + can_collect_no_secret = collection.get("can_collect_no_secret_verify") is True + + readback = _dict(payload.setdefault("readback", {})) + readback["windows99_verify_collection_status"] = status + readback["windows99_verify_collection_can_collect_no_secret"] = can_collect_no_secret + readback["windows99_verify_collection_blockers"] = collection_blockers + readback["windows99_verify_collection_blocker_count"] = len(collection_blockers) + + rollups = _dict(payload.setdefault("rollups", {})) + rollups["windows99_verify_collection_status"] = status + rollups["windows99_verify_collection_can_collect_no_secret"] = can_collect_no_secret + rollups["windows99_verify_collection_blockers"] = collection_blockers + rollups["windows99_verify_collection_blocker_count"] = len(collection_blockers) + + def _public_maintenance_reconciled_metric_blockers( public_maintenance: dict[str, Any], ) -> set[str]: @@ -2652,6 +2706,11 @@ def _build_windows99_verify_collection_packet( if blocker not in collection_blockers: collection_blockers.append(blocker) for blocker in _strings(windows99_collector.get("blockers")): + if ( + collector_status == _WINDOWS99_COLLECTED_VERIFY_STATUS + and blocker in _WINDOWS99_COLLECTOR_BLOCKER_VALUES + ): + continue if blocker not in collection_blockers: collection_blockers.append(blocker) for blocker in console_artifact_blockers: diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 8c6541508..d57792c4e 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -1274,6 +1274,22 @@ def test_reboot_auto_recovery_slo_scorecard_promotes_collected_windows99_runtime assert payload["windows99_verify_collection"][ "no_secret_collector_status" ] == "collected_windows99_vmware_verify_stdout" + assert "windows99_ssh_publickey_auth_missing" not in payload[ + "windows99_verify_collection" + ]["collection_blockers"] + assert "windows99_ssh_publickey_auth_missing" not in payload["active_blockers"] + assert payload["windows99_verify_collection"]["no_secret_collector"][ + "blockers" + ] == [] + assert "windows99_ssh_publickey_auth_missing" not in payload["readback"][ + "windows99_verify_collection_blockers" + ] + assert "windows99_ssh_publickey_auth_missing" not in payload["rollups"][ + "windows99_verify_collection_blockers" + ] + assert payload["rollups"]["windows99_verify_collection_blocker_count"] == len( + payload["windows99_verify_collection"]["collection_blockers"] + ) action_by_blocker = { item["blocker"]: item for item in payload["active_blocker_action_matrix"]["items"] diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index 45321308f..d7486a2b6 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -1225,6 +1225,14 @@ def build_windows99_verify_collection_packet( windows99_collector.get("ssh_batchmode_auth_ready") is True or collector_status == "collected_windows99_vmware_verify_stdout" ) + collector_blocker_values = { + "windows99_ssh_publickey_auth_missing", + "windows99_ssh_port_closed", + "windows99_local_verify_script_missing", + "windows99_remote_verify_output_invalid", + "windows99_remote_verify_command_failed", + "windows99_no_secret_collector_not_ready", + } blockers = strings(windows99.get("blockers")) collection_blockers: list[str] = [] if not host99_reachable: @@ -1238,6 +1246,10 @@ def build_windows99_verify_collection_packet( blocker for blocker in strings(windows99_collector.get("blockers")) if blocker not in collection_blockers + and not ( + collector_status == "collected_windows99_vmware_verify_stdout" + and blocker in collector_blocker_values + ) ) collection_blockers.extend( blocker