fix(reboot): promote windows99 nested runtime blockers
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m3s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
Your Name
2026-07-03 10:18:49 +08:00
parent 2fd90ad98a
commit 81cb38cc27
6 changed files with 74 additions and 24 deletions

View File

@@ -93,6 +93,10 @@ PROMETHEUS_RUNTIME_BLOCKERS = [
PROMETHEUS_SOURCE_RECONCILED_BLOCKER = (
"conversation_event_hot_path_index_migration_source_missing"
)
WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS = [
"windows99_vmware_autostart_config_not_ready",
"windows99_update_no_auto_reboot_policy_not_ready",
]
PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS = [
blocker
for blocker in PROMETHEUS_RUNTIME_BLOCKERS
@@ -399,7 +403,10 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
payload = load_latest_reboot_auto_recovery_slo_scorecard(
prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK
)
expected_active_blockers = PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS
expected_active_blockers = [
*PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS,
*WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS,
]
assert payload["runtime_scorecard_readback_present"] is False
assert payload["runtime_metric_readback_present"] is True
@@ -411,7 +418,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
)
assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1
assert payload["active_blockers"] == expected_active_blockers
assert payload["active_blocker_count"] == 7
assert payload["active_blocker_count"] == 9
assert payload["readiness_percent"] == 47
assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
assert payload["next_safe_action"] == (
@@ -420,8 +427,11 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
assert "backup_core_green_not_1" not in payload["active_blockers"]
assert "service_green_not_1" not in payload["active_blockers"]
assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"]
assert payload["runtime_metric_runtime_readback_added_blockers"] == []
assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 0
assert (
payload["runtime_metric_runtime_readback_added_blockers"]
== WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS
)
assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 2
assert payload["service_green"] is True
assert payload["backup_core_green"] is True
assert payload["product_data_green"] is True
@@ -439,7 +449,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
is True
)
assert payload["controlled_service_data_backup_readback"]["blocking_fields"] == []
assert payload["active_blocker_action_matrix"]["item_count"] == 7
assert payload["active_blocker_action_matrix"]["item_count"] == 9
assert payload["windows99_vmware_autostart"]["readback_present"] is True
assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"]
assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [
@@ -452,12 +462,13 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
assert payload["windows99_vmware_autostart"]["blockers"] == [
"windows99_vmware_guest_power_not_ready",
"windows99_vmware_vmx_missing",
*WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS,
]
assert payload["readback"]["windows99_vmware_readback_present"] is True
assert payload["readback"]["windows99_vmrun_present"] is True
assert payload["readback"]["windows99_vmware_config_ready"] is False
assert payload["readback"]["windows99_vmware_power_ready"] is False
assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is True
assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is False
assert payload["readback"]["windows99_runtime_metric_overlay_applied"] is True
assert (
payload["windows99_verify_collection"]["status"]
@@ -472,7 +483,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
assert payload["rollups"]["windows99_vmrun_present"] is True
assert payload["rollups"]["windows99_vmware_config_ready"] is False
assert payload["rollups"]["windows99_vmware_power_ready"] is False
assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is True
assert payload["rollups"]["windows99_update_no_auto_reboot_ready"] is False
assert payload["rollups"]["windows99_vmware_missing_vmx_count"] == 1
assert payload["rollups"]["windows99_vmware_powered_off_count"] == 5
assert payload["rollups"]["windows99_missing_vmx_alias_count"] == 1
@@ -517,7 +528,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
] == 5
assert payload["active_blocker_action_matrix"]["category_counts"][
"windows99_vmware_autostart"
] == 2
] == 4
assert payload["active_blocker_action_matrix"]["category_counts"].get(
"post_reboot_service_readiness",
0,
@@ -530,25 +541,25 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
"security_observability",
0,
) == 0
assert payload["readback"]["active_blocker_count"] == 7
assert payload["readback"]["active_blocker_count"] == 9
assert payload["readback"]["runtime_metric_readback_present"] is True
assert payload["readback"][
"runtime_metric_source_control_reconciled_blocker_count"
] == 1
assert payload["readback"][
"runtime_metric_runtime_readback_added_blockers"
] == []
] == WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS
assert payload["readback"][
"runtime_metric_runtime_readback_added_blocker_count"
] == 0
assert payload["rollups"]["active_blocker_count"] == 7
] == 2
assert payload["rollups"]["active_blocker_count"] == 9
assert payload["rollups"]["runtime_metric_readback_present"] is True
assert payload["rollups"][
"runtime_metric_source_control_reconciled_blocker_count"
] == 1
assert payload["rollups"][
"runtime_metric_runtime_readback_added_blocker_count"
] == 0
] == 2
assert payload["rollups"]["primary_blocker_owner_lane"] == (
"reboot_event_detector_and_host_probe"
)
@@ -663,8 +674,11 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when
prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK,
)
assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
assert payload["active_blocker_count"] == 8
assert payload["active_blockers"] == [
*PROMETHEUS_RUNTIME_BLOCKERS,
*WINDOWS99_RUNTIME_READBACK_ADDED_BLOCKERS,
]
assert payload["active_blocker_count"] == 10
assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0
action_by_blocker = {
item["blocker"]: item