fix(reboot): surface windows99 vmware alias metrics
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 58s
CD Pipeline / build-and-deploy (push) Successful in 5m15s
CD Pipeline / post-deploy-checks (push) Successful in 1m51s

This commit is contained in:
Your Name
2026-07-03 02:26:16 +08:00
parent 86a01df058
commit 8a0ddc8d71
4 changed files with 259 additions and 11 deletions

View File

@@ -87,7 +87,8 @@ PROMETHEUS_RUNTIME_BLOCKERS = [
"host_unreachable_after_reboot",
"host_uptime_unknown",
"reboot_event_required_host_unreachable",
"windows99_vmware_autostart_readback_missing",
"windows99_vmware_guest_power_not_ready",
"windows99_vmware_vmx_missing",
]
PROMETHEUS_SOURCE_RECONCILED_BLOCKER = (
"conversation_event_hot_path_index_migration_source_missing"
@@ -104,10 +105,15 @@ PROMETHEUS_RUNTIME_READBACK = {
"source": "prometheus_query_api",
"query_url": "http://192.168.0.110:9090/api/v1/query",
"active_blockers": PROMETHEUS_RUNTIME_BLOCKERS,
"active_blocker_count": 7,
"active_blocker_count": 8,
"primary_blocker": "reboot_event_required_host_unreachable",
"ready": False,
"last_run_timestamp": 1783010479,
"windows99_vmware_readback_present": True,
"windows99_vmware_verify_ready": False,
"windows99_windows_update_no_auto_reboot_ready": True,
"windows99_missing_vmx_aliases": ["111"],
"windows99_powered_off_aliases": ["111", "112", "120", "121", "188"],
}
@@ -264,7 +270,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
assert payload["runtime_scorecard_readback_present"] is False
assert payload["runtime_metric_readback_present"] is True
assert payload["runtime_metric_active_blocker_count"] == 7
assert payload["runtime_metric_active_blocker_count"] == 8
assert payload["runtime_metric_last_run_timestamp"] == 1783010479
assert (
payload["runtime_metric_source_control_reconciled_blockers"]
@@ -272,7 +278,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
)
assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1
assert payload["active_blockers"] == PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS
assert payload["active_blocker_count"] == 6
assert payload["active_blocker_count"] == 7
assert payload["readiness_percent"] == 47
assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
assert payload["next_safe_action"] == (
@@ -281,7 +287,22 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
assert "backup_core_green_not_1" not in payload["active_blockers"]
assert "service_green_not_1" not in payload["active_blockers"]
assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"]
assert payload["active_blocker_action_matrix"]["item_count"] == 6
assert payload["active_blocker_action_matrix"]["item_count"] == 7
assert payload["windows99_vmware_autostart"]["readback_present"] is True
assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"]
assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [
"111",
"112",
"120",
"121",
"188",
]
assert payload["windows99_vmware_autostart"]["blockers"] == [
"windows99_vmware_guest_power_not_ready",
"windows99_vmware_vmx_missing",
]
assert payload["readback"]["windows99_runtime_metric_overlay_applied"] is True
assert payload["rollups"]["windows99_missing_vmx_alias_count"] == 1
action_by_blocker = {
item["blocker"]: item
for item in payload["active_blocker_action_matrix"]["items"]
@@ -291,12 +312,12 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
"host_cpu_pressure",
0,
) == 0
assert payload["readback"]["active_blocker_count"] == 6
assert payload["readback"]["active_blocker_count"] == 7
assert payload["readback"]["runtime_metric_readback_present"] is True
assert payload["readback"][
"runtime_metric_source_control_reconciled_blocker_count"
] == 1
assert payload["rollups"]["active_blocker_count"] == 6
assert payload["rollups"]["active_blocker_count"] == 7
assert payload["rollups"]["runtime_metric_readback_present"] is True
assert payload["rollups"][
"runtime_metric_source_control_reconciled_blocker_count"
@@ -324,7 +345,7 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when
)
assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
assert payload["active_blocker_count"] == 7
assert payload["active_blocker_count"] == 8
assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0
action_by_blocker = {
item["blocker"]: item
@@ -372,7 +393,7 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
"metric": {
"__name__": "awoooi_reboot_auto_recovery_slo_blocker_count",
},
"value": [1783010479, "7"],
"value": [1783010479, str(len(PROMETHEUS_RUNTIME_BLOCKERS))],
},
{
"metric": {"__name__": "awoooi_reboot_auto_recovery_slo_ready"},
@@ -384,6 +405,43 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
},
"value": [1783010479, "1783010479"],
},
{
"metric": {
"__name__": "awoooi_windows99_vmware_readback_present",
},
"value": [1783010479, "1"],
},
{
"metric": {
"__name__": "awoooi_windows99_vmware_verify_ready",
},
"value": [1783010479, "0"],
},
{
"metric": {
"__name__": (
"awoooi_windows99_vmware_windows_update_no_auto_reboot_ready"
),
},
"value": [1783010479, "1"],
},
{
"metric": {
"__name__": "awoooi_windows99_vmware_missing_vmx_alias",
"alias": "111",
},
"value": [1783010479, "1"],
},
*[
{
"metric": {
"__name__": "awoooi_windows99_vmware_powered_off_alias",
"alias": alias,
},
"value": [1783010479, "1"],
}
for alias in ["111", "112", "120", "121", "188"]
],
]
monkeypatch.setattr(
@@ -403,10 +461,18 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
assert "awoooi_reboot_auto_recovery_slo_last_run_timestamp" in calls[0][1]
assert payload["readback_present"] is True
assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
assert payload["active_blocker_count"] == 7
assert payload["active_blocker_count"] == len(PROMETHEUS_RUNTIME_BLOCKERS)
assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
assert payload["ready"] is False
assert payload["last_run_timestamp"] == 1783010479
assert payload["windows99_missing_vmx_aliases"] == ["111"]
assert payload["windows99_powered_off_aliases"] == [
"111",
"112",
"120",
"121",
"188",
]
def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch):