diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index cb6484cce..a4eda337e 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -44,6 +44,13 @@ _PROMETHEUS_RUNTIME_QUERIES = { "blocker_count": "awoooi_reboot_auto_recovery_slo_blocker_count", "ready": "awoooi_reboot_auto_recovery_slo_ready", "last_run_timestamp": "awoooi_reboot_auto_recovery_slo_last_run_timestamp", + "windows99_vmware_readback_present": "awoooi_windows99_vmware_readback_present", + "windows99_vmware_verify_ready": "awoooi_windows99_vmware_verify_ready", + "windows99_windows_update_ready": ( + "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready" + ), + "windows99_missing_vmx_alias": "awoooi_windows99_vmware_missing_vmx_alias", + "windows99_powered_off_alias": "awoooi_windows99_vmware_powered_off_alias", } _PROMETHEUS_RUNTIME_COMBINED_QUERY = ( '{__name__=~"' @@ -261,8 +268,22 @@ def _load_reboot_slo_prometheus_metric_readback( scalar_results = { key: results_by_name.get(query, []) for key, query in _PROMETHEUS_RUNTIME_QUERIES.items() - if key not in {"active_blocker", "primary_blocker"} + if key + not in { + "active_blocker", + "primary_blocker", + "windows99_missing_vmx_alias", + "windows99_powered_off_alias", + } } + missing_vmx_results = results_by_name.get( + _PROMETHEUS_RUNTIME_QUERIES["windows99_missing_vmx_alias"], + [], + ) + powered_off_results = results_by_name.get( + _PROMETHEUS_RUNTIME_QUERIES["windows99_powered_off_alias"], + [], + ) active_blockers = _unique_strings( [ str(_dict(result.get("metric")).get("blocker") or "") @@ -289,6 +310,35 @@ def _load_reboot_slo_prometheus_metric_readback( scalar_results.get("last_run_timestamp") or [], default=0, ) + windows99_readback_present = ( + _prometheus_first_int( + scalar_results.get("windows99_vmware_readback_present") or [], + default=0, + ) + == 1 + ) + windows99_verify_ready = ( + _prometheus_first_int( + scalar_results.get("windows99_vmware_verify_ready") or [], + default=0, + ) + == 1 + ) + windows99_update_ready = ( + _prometheus_first_int( + scalar_results.get("windows99_windows_update_ready") or [], + default=0, + ) + == 1 + ) + windows99_missing_vmx_aliases = _prometheus_label_values( + missing_vmx_results, + "alias", + ) + windows99_powered_off_aliases = _prometheus_label_values( + powered_off_results, + "alias", + ) return { "schema_version": "reboot_slo_prometheus_metric_readback_v1", "readback_present": bool(combined_results), @@ -299,6 +349,11 @@ def _load_reboot_slo_prometheus_metric_readback( "primary_blocker": primary_blocker, "ready": ready, "last_run_timestamp": last_run_timestamp, + "windows99_vmware_readback_present": windows99_readback_present, + "windows99_vmware_verify_ready": windows99_verify_ready, + "windows99_windows_update_no_auto_reboot_ready": windows99_update_ready, + "windows99_missing_vmx_aliases": windows99_missing_vmx_aliases, + "windows99_powered_off_aliases": windows99_powered_off_aliases, } @@ -329,6 +384,19 @@ def _prometheus_first_int(results: list[dict[str, Any]], *, default: int) -> int return default +def _prometheus_label_values( + results: list[dict[str, Any]], + label_name: str, +) -> list[str]: + return _unique_strings( + [ + str(_dict(result.get("metric")).get(label_name) or "") + for result in results + if str(_dict(result.get("metric")).get(label_name) or "") + ] + ) + + def _annotate_prometheus_metric_readback( payload: dict[str, Any], metric_readback: dict[str, Any], @@ -367,6 +435,59 @@ def _annotate_prometheus_metric_readback( "runtime_metric_last_run_timestamp" ] rollups["runtime_metric_source_control_reconciled_blocker_count"] = 0 + _apply_prometheus_windows99_vmware_readback(payload, metric_readback) + + +def _apply_prometheus_windows99_vmware_readback( + payload: dict[str, Any], + metric_readback: dict[str, Any], +) -> None: + if metric_readback.get("readback_present") is not True: + return + if metric_readback.get("windows99_vmware_readback_present") is not True: + return + + active_blockers = _strings(metric_readback.get("active_blockers")) + missing_vmx_aliases = _strings(metric_readback.get("windows99_missing_vmx_aliases")) + powered_off_aliases = _strings(metric_readback.get("windows99_powered_off_aliases")) + windows99_blockers = [ + blocker for blocker in active_blockers if blocker.startswith("windows99_") + ] + verify_ready = metric_readback.get("windows99_vmware_verify_ready") is True + update_ready = ( + metric_readback.get("windows99_windows_update_no_auto_reboot_ready") is True + ) + windows99 = _dict(payload.setdefault("windows99_vmware_autostart", {})) + windows99.update( + { + "readback_present": True, + "verify_ready": verify_ready, + "vmrun_present": True, + "config_ready": not missing_vmx_aliases, + "power_ready": not powered_off_aliases, + "windows_update_no_auto_reboot_ready": update_ready, + "missing_vmx_aliases": missing_vmx_aliases, + "powered_off_aliases": powered_off_aliases, + "blockers": windows99_blockers, + "runtime_metric_overlay_applied": True, + } + ) + payload["windows99_vmware_verify_ready"] = verify_ready + payload["windows99_update_no_auto_reboot_ready"] = update_ready + + readback = _dict(payload.setdefault("readback", {})) + readback["windows99_vmware_verify_ready"] = verify_ready + readback["windows99_update_no_auto_reboot_ready"] = update_ready + readback["windows99_runtime_metric_overlay_applied"] = True + readback["windows99_missing_vmx_aliases"] = missing_vmx_aliases + readback["windows99_powered_off_aliases"] = powered_off_aliases + + rollups = _dict(payload.setdefault("rollups", {})) + rollups["windows99_vmware_verify_ready"] = verify_ready + rollups["windows99_update_no_auto_reboot_ready"] = update_ready + rollups["windows99_runtime_metric_overlay_applied"] = True + rollups["windows99_missing_vmx_alias_count"] = len(missing_vmx_aliases) + rollups["windows99_powered_off_alias_count"] = len(powered_off_aliases) def _reconcile_prometheus_metric_active_blockers_with_source_controls( diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index a5297694c..6937f289d 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -87,7 +87,8 @@ PROMETHEUS_RUNTIME_BLOCKERS = [ "host_unreachable_after_reboot", "host_uptime_unknown", "reboot_event_required_host_unreachable", - "windows99_vmware_autostart_readback_missing", + "windows99_vmware_guest_power_not_ready", + "windows99_vmware_vmx_missing", ] PROMETHEUS_SOURCE_RECONCILED_BLOCKER = ( "conversation_event_hot_path_index_migration_source_missing" @@ -104,10 +105,15 @@ PROMETHEUS_RUNTIME_READBACK = { "source": "prometheus_query_api", "query_url": "http://192.168.0.110:9090/api/v1/query", "active_blockers": PROMETHEUS_RUNTIME_BLOCKERS, - "active_blocker_count": 7, + "active_blocker_count": 8, "primary_blocker": "reboot_event_required_host_unreachable", "ready": False, "last_run_timestamp": 1783010479, + "windows99_vmware_readback_present": True, + "windows99_vmware_verify_ready": False, + "windows99_windows_update_no_auto_reboot_ready": True, + "windows99_missing_vmx_aliases": ["111"], + "windows99_powered_off_aliases": ["111", "112", "120", "121", "188"], } @@ -264,7 +270,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert payload["runtime_scorecard_readback_present"] is False assert payload["runtime_metric_readback_present"] is True - assert payload["runtime_metric_active_blocker_count"] == 7 + assert payload["runtime_metric_active_blocker_count"] == 8 assert payload["runtime_metric_last_run_timestamp"] == 1783010479 assert ( payload["runtime_metric_source_control_reconciled_blockers"] @@ -272,7 +278,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( ) assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1 assert payload["active_blockers"] == PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS - assert payload["active_blocker_count"] == 6 + assert payload["active_blocker_count"] == 7 assert payload["readiness_percent"] == 47 assert payload["primary_blocker"] == "reboot_event_required_host_unreachable" assert payload["next_safe_action"] == ( @@ -281,7 +287,22 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert "backup_core_green_not_1" not in payload["active_blockers"] assert "service_green_not_1" not in payload["active_blockers"] assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"] - assert payload["active_blocker_action_matrix"]["item_count"] == 6 + assert payload["active_blocker_action_matrix"]["item_count"] == 7 + assert payload["windows99_vmware_autostart"]["readback_present"] is True + assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"] + assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [ + "111", + "112", + "120", + "121", + "188", + ] + assert payload["windows99_vmware_autostart"]["blockers"] == [ + "windows99_vmware_guest_power_not_ready", + "windows99_vmware_vmx_missing", + ] + assert payload["readback"]["windows99_runtime_metric_overlay_applied"] is True + assert payload["rollups"]["windows99_missing_vmx_alias_count"] == 1 action_by_blocker = { item["blocker"]: item for item in payload["active_blocker_action_matrix"]["items"] @@ -291,12 +312,12 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( "host_cpu_pressure", 0, ) == 0 - assert payload["readback"]["active_blocker_count"] == 6 + assert payload["readback"]["active_blocker_count"] == 7 assert payload["readback"]["runtime_metric_readback_present"] is True assert payload["readback"][ "runtime_metric_source_control_reconciled_blocker_count" ] == 1 - assert payload["rollups"]["active_blocker_count"] == 6 + assert payload["rollups"]["active_blocker_count"] == 7 assert payload["rollups"]["runtime_metric_readback_present"] is True assert payload["rollups"][ "runtime_metric_source_control_reconciled_blocker_count" @@ -324,7 +345,7 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when ) assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS - assert payload["active_blocker_count"] == 7 + assert payload["active_blocker_count"] == 8 assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0 action_by_blocker = { item["blocker"]: item @@ -372,7 +393,7 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch): "metric": { "__name__": "awoooi_reboot_auto_recovery_slo_blocker_count", }, - "value": [1783010479, "7"], + "value": [1783010479, str(len(PROMETHEUS_RUNTIME_BLOCKERS))], }, { "metric": {"__name__": "awoooi_reboot_auto_recovery_slo_ready"}, @@ -384,6 +405,43 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch): }, "value": [1783010479, "1783010479"], }, + { + "metric": { + "__name__": "awoooi_windows99_vmware_readback_present", + }, + "value": [1783010479, "1"], + }, + { + "metric": { + "__name__": "awoooi_windows99_vmware_verify_ready", + }, + "value": [1783010479, "0"], + }, + { + "metric": { + "__name__": ( + "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready" + ), + }, + "value": [1783010479, "1"], + }, + { + "metric": { + "__name__": "awoooi_windows99_vmware_missing_vmx_alias", + "alias": "111", + }, + "value": [1783010479, "1"], + }, + *[ + { + "metric": { + "__name__": "awoooi_windows99_vmware_powered_off_alias", + "alias": alias, + }, + "value": [1783010479, "1"], + } + for alias in ["111", "112", "120", "121", "188"] + ], ] monkeypatch.setattr( @@ -403,10 +461,18 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch): assert "awoooi_reboot_auto_recovery_slo_last_run_timestamp" in calls[0][1] assert payload["readback_present"] is True assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS - assert payload["active_blocker_count"] == 7 + assert payload["active_blocker_count"] == len(PROMETHEUS_RUNTIME_BLOCKERS) assert payload["primary_blocker"] == "reboot_event_required_host_unreachable" assert payload["ready"] is False assert payload["last_run_timestamp"] == 1783010479 + assert payload["windows99_missing_vmx_aliases"] == ["111"] + assert payload["windows99_powered_off_aliases"] == [ + "111", + "112", + "120", + "121", + "188", + ] def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch): diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 4fa3a9ff1..51be28970 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -175,6 +175,52 @@ payload=json.load(open(sys.argv[1], encoding="utf-8")) print(payload.get("public_maintenance_fallback", {}).get("route_unreachable_without_external_fallback_count", 0) or 0) PY )" +windows99_vmware_metrics="$(python3 - "$scorecard_file" <<'PY' +import json, sys + +payload = json.load(open(sys.argv[1], encoding="utf-8")) +windows99 = payload.get("windows99_vmware_autostart") or {} + + +def label(value: object) -> str: + text = str(value or "") + return ( + text.replace("\\", "\\\\") + .replace("\n", "\\n") + .replace('"', '\\"') + ) + + +def gauge(name: str, value: bool) -> None: + print(f'{name}{{scope="windows99"}} {1 if value else 0}') + + +gauge( + "awoooi_windows99_vmware_readback_present", + windows99.get("readback_present") is True, +) +gauge( + "awoooi_windows99_vmware_verify_ready", + windows99.get("verify_ready") is True, +) +gauge( + "awoooi_windows99_vmware_windows_update_no_auto_reboot_ready", + windows99.get("windows_update_no_auto_reboot_ready") is True, +) +for alias in windows99.get("missing_vmx_aliases") or []: + if alias: + print( + "awoooi_windows99_vmware_missing_vmx_alias" + f'{{scope="windows99",alias="{label(alias)}"}} 1' + ) +for alias in windows99.get("powered_off_aliases") or []: + if alias: + print( + "awoooi_windows99_vmware_powered_off_alias" + f'{{scope="windows99",alias="{label(alias)}"}} 1' + ) +PY +)" active_blocker_metrics="$(python3 - "$scorecard_file" "$TARGET_MINUTES" <<'PY' import json, sys @@ -258,7 +304,20 @@ awoooi_public_route_raw_5xx_without_maintenance_fallback{scope="public_routes"} # HELP awoooi_public_route_unreachable_without_external_l1_fallback Public routes unreachable without external fallback proof. # TYPE awoooi_public_route_unreachable_without_external_l1_fallback gauge awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_routes"} $public_unreachable_count +# HELP awoooi_windows99_vmware_readback_present Whether Windows99 VMware verifier stdout was collected. +# TYPE awoooi_windows99_vmware_readback_present gauge +# HELP awoooi_windows99_vmware_verify_ready Whether Windows99 VMware autostart verifier is fully green. +# TYPE awoooi_windows99_vmware_verify_ready gauge +# HELP awoooi_windows99_vmware_windows_update_no_auto_reboot_ready Whether Windows Update no-auto-reboot policy is verified. +# TYPE awoooi_windows99_vmware_windows_update_no_auto_reboot_ready gauge +# HELP awoooi_windows99_vmware_missing_vmx_alias Missing required Windows99 VMX aliases. +# TYPE awoooi_windows99_vmware_missing_vmx_alias gauge +# HELP awoooi_windows99_vmware_powered_off_alias Required Windows99 VMware guests observed powered off. +# TYPE awoooi_windows99_vmware_powered_off_alias gauge METRICS +if [ -n "$windows99_vmware_metrics" ]; then + printf '%s\n' "$windows99_vmware_metrics" >>"$tmp_metric" +fi if [ -n "$active_blocker_metrics" ]; then printf '%s\n' "$active_blocker_metrics" >>"$tmp_metric" fi diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py index 2abf8a9fa..9b9b7a7d5 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py @@ -109,6 +109,8 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None: assert "scorecard_args+=(--windows99-vmware-file" in text assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text + assert "awoooi_windows99_vmware_missing_vmx_alias" in text + assert "awoooi_windows99_vmware_powered_off_alias" in text assert "active_blocker_metrics" in text assert 'for blocker in payload.get("active_blockers") or []' in text assert "awoooi_reboot_auto_recovery_slo_active_blocker" in text