wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(reboot): expose windows99 autostart task run state
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 1m0s
Details
CD Pipeline / build-and-deploy (push) Failing after 1m1s
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details

Browse Source
This commit is contained in:
Your Name
2026-07-03 16:11:33 +08:00
parent 344862b71a
commit 37dda358a3
11 changed files with 447 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
apps/api/src/services/reboot_auto_recovery_slo_scorecard.py
View File

@@ -52,6 +52,15 @@ _PROMETHEUS_RUNTIME_QUERIES = {
"windows99_windows_update_ready": (
"awoooi_windows99_vmware_windows_update_no_auto_reboot_ready"
),
"windows99_autostart_task_start_script_present": (
"awoooi_windows99_vmware_autostart_task_start_script_present"
),
"windows99_autostart_task_last_result": (
"awoooi_windows99_vmware_autostart_task_last_result"
),
"windows99_autostart_task_never_ran": (
"awoooi_windows99_vmware_autostart_task_never_ran"
),
"windows99_missing_vmx_alias": "awoooi_windows99_vmware_missing_vmx_alias",
"windows99_powered_off_alias": "awoooi_windows99_vmware_powered_off_alias",
"windows99_no_secret_collector_readback_present": (
@@ -434,6 +443,54 @@ def _load_reboot_slo_prometheus_metric_readback(
windows99_update_ready: bool | None = (
None if windows99_update_metric < 0 else windows99_update_metric == 1
)
windows99_task_start_script_metric = _prometheus_first_int(
scalar_results.get("windows99_autostart_task_start_script_present") or [],
default=-1,
)
windows99_task_start_script_present: bool | None = (
None
if windows99_task_start_script_metric < 0
else windows99_task_start_script_metric == 1
)
windows99_task_last_result_results = results_by_name.get(
_PROMETHEUS_RUNTIME_QUERIES["windows99_autostart_task_last_result"],
[],
)
windows99_task_last_result = _prometheus_first_int(
windows99_task_last_result_results,
default=-1,
)
windows99_task_last_result_hex = _prometheus_first_label_value(
windows99_task_last_result_results,
"result_hex",
)
windows99_task_last_run_time = _prometheus_first_label_value(
windows99_task_last_result_results,
"last_run_time",
)
windows99_task_never_ran_results = results_by_name.get(
_PROMETHEUS_RUNTIME_QUERIES["windows99_autostart_task_never_ran"],
[],
)
windows99_task_never_ran_metric = _prometheus_first_int(
windows99_task_never_ran_results,
default=-1,
)
windows99_task_never_ran: bool | None = (
None
if windows99_task_never_ran_metric < 0
else windows99_task_never_ran_metric == 1
)
if not windows99_task_last_result_hex:
windows99_task_last_result_hex = _prometheus_first_label_value(
windows99_task_never_ran_results,
"result_hex",
)
if not windows99_task_last_run_time:
windows99_task_last_run_time = _prometheus_first_label_value(
windows99_task_never_ran_results,
"last_run_time",
)
windows99_missing_vmx_aliases = _prometheus_label_values(
missing_vmx_results,
"alias",
@@ -468,6 +525,13 @@ def _load_reboot_slo_prometheus_metric_readback(
"windows99_vmware_verify_ready": windows99_verify_ready,
"windows99_vmware_autostart_config_ready": windows99_config_ready,
"windows99_windows_update_no_auto_reboot_ready": windows99_update_ready,
"windows99_autostart_task_start_script_present": (
windows99_task_start_script_present
),
"windows99_autostart_task_last_result": windows99_task_last_result,
"windows99_autostart_task_last_result_hex": windows99_task_last_result_hex,
"windows99_autostart_task_last_run_time": windows99_task_last_run_time,
"windows99_autostart_task_never_ran": windows99_task_never_ran,
"windows99_missing_vmx_aliases": windows99_missing_vmx_aliases,
"windows99_powered_off_aliases": windows99_powered_off_aliases,
"windows99_no_secret_collector_readback_present": (
@@ -791,6 +855,31 @@ def _apply_prometheus_windows99_vmware_readback(
"runtime_metric_overlay_applied": True,
}
)
scheduled_task = _dict(windows99.setdefault("scheduled_task", {}))
task_start_script_present = metric_readback.get(
"windows99_autostart_task_start_script_present"
)
if isinstance(task_start_script_present, bool):
scheduled_task["start_script_present"] = task_start_script_present
raw_task_last_result = metric_readback.get("windows99_autostart_task_last_result")
task_last_result = (
int(raw_task_last_result)
if isinstance(raw_task_last_result, int | float)
else -1
)
if task_last_result >= 0:
scheduled_task["last_task_result"] = str(task_last_result)
scheduled_task["last_task_result_hex"] = str(
metric_readback.get("windows99_autostart_task_last_result_hex") or ""
)
task_last_run_time = str(
metric_readback.get("windows99_autostart_task_last_run_time") or ""
)
if task_last_run_time:
scheduled_task["last_run_time"] = task_last_run_time
task_never_ran = metric_readback.get("windows99_autostart_task_never_ran")
if isinstance(task_never_ran, bool):
scheduled_task["never_ran"] = task_never_ran
payload["windows99_vmware_verify_ready"] = verify_ready
payload["windows99_update_no_auto_reboot_ready"] = update_ready
@@ -810,6 +899,19 @@ def _apply_prometheus_windows99_vmware_readback(
readback["windows99_runtime_metric_overlay_applied"] = True
readback["windows99_missing_vmx_aliases"] = missing_vmx_aliases
readback["windows99_powered_off_aliases"] = powered_off_aliases
readback["windows99_autostart_task_start_script_present"] = scheduled_task.get(
"start_script_present"
)
readback["windows99_autostart_task_last_result"] = scheduled_task.get(
"last_task_result"
)
readback["windows99_autostart_task_last_result_hex"] = scheduled_task.get(
"last_task_result_hex"
)
readback["windows99_autostart_task_last_run_time"] = scheduled_task.get(
"last_run_time"
)
readback["windows99_autostart_task_never_ran"] = scheduled_task.get("never_ran")
collection = _dict(payload.setdefault("windows99_verify_collection", {}))
preserved_collection_blockers = [
@@ -844,6 +946,16 @@ def _apply_prometheus_windows99_vmware_readback(
rollups["windows99_vmware_powered_off_count"] = len(powered_off_aliases)
rollups["windows99_missing_vmx_alias_count"] = len(missing_vmx_aliases)
rollups["windows99_powered_off_alias_count"] = len(powered_off_aliases)
rollups["windows99_autostart_task_start_script_present"] = scheduled_task.get(
"start_script_present"
)
rollups["windows99_autostart_task_last_result"] = scheduled_task.get(
"last_task_result"
)
rollups["windows99_autostart_task_last_result_hex"] = scheduled_task.get(
"last_task_result_hex"
)
rollups["windows99_autostart_task_never_ran"] = scheduled_task.get("never_ran")
rollups["windows99_verify_collection_status"] = collection["status"]
rollups["windows99_verify_collection_blocker_count"] = len(
_strings(collection.get("collection_blockers"))
@@ -2567,6 +2679,8 @@ def _build_windows99_verify_collection_packet(
"VMX alias=<alias> present=<0|1>",
"VMWARE_SERVICE name=<name> ok=<0|1>",
"VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs ok=<0|1>",
"VMWARE_AUTOSTART_TASK_DETAIL last_task_result=<code>",
"VMWARE_AUTOSTART_TASK_ACTION execute=<path>",
"WINDOWS_UPDATE_POLICY name=<name> ok=<0|1>",
"VM_POWER alias=<alias> running=<0|1>",
"VMWARE_AUTOSTART_CONFIG_READY",
@@ -3322,6 +3436,7 @@ def _reboot_sop_current_phase(active_blockers: list[str], can_claim_slo: bool) -
"windows99_vmrun_missing",
"windows99_vmware_vmx_missing",
"windows99_vmware_autostart_config_not_ready",
"windows99_vmware_autostart_task_never_ran",
"windows99_vmware_guest_power_not_ready",
"windows99_update_no_auto_reboot_policy_not_ready",
"windows99_console_clipboard_unreliable",
@@ -3376,6 +3491,7 @@ def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str:
"windows99_vmrun_missing",
"windows99_vmware_vmx_missing",
"windows99_vmware_autostart_config_not_ready",
"windows99_vmware_autostart_task_never_ran",
"windows99_vmware_guest_power_not_ready",
"windows99_update_no_auto_reboot_policy_not_ready",
"windows99_console_clipboard_unreliable",
@@ -3505,7 +3621,10 @@ def _active_blocker_action_row(
)
elif blocker == "windows99_vmware_vmx_missing":
next_safe_action = _WINDOWS99_MISSING_VMX_NEXT_SAFE_ACTION
elif blocker == "windows99_vmware_guest_power_not_ready":
elif blocker in {
"windows99_vmware_guest_power_not_ready",
"windows99_vmware_autostart_task_never_ran",
}:
next_safe_action = _WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION
elif blocker in {
"windows99_console_clipboard_unreliable",

107
apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py
View File

@@ -699,6 +699,77 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
)
def test_reboot_auto_recovery_slo_scorecard_overlays_windows99_task_run_state():
metric_readback = {
**PROMETHEUS_RUNTIME_READBACK,
"active_blockers": [
"all_required_hosts_not_in_10_minute_reboot_window",
"fresh_all_host_reboot_event_missing",
"host_boot_observation_older_than_target_window",
"host_unreachable_after_reboot",
"host_uptime_unknown",
"reboot_event_required_host_unreachable",
"windows99_vmware_guest_power_not_ready",
"windows99_vmware_autostart_task_never_ran",
],
"active_blocker_count": 8,
"windows99_vmware_autostart_config_ready": True,
"windows99_windows_update_no_auto_reboot_ready": True,
"windows99_missing_vmx_aliases": [],
"windows99_powered_off_aliases": ["110", "112", "120", "121", "188"],
"windows99_autostart_task_start_script_present": True,
"windows99_autostart_task_last_result": 267011,
"windows99_autostart_task_last_result_hex": "0x41303",
"windows99_autostart_task_last_run_time": "1999-11-30T00:00:00",
"windows99_autostart_task_never_ran": True,
}
payload = load_latest_reboot_auto_recovery_slo_scorecard(
prometheus_metric_readback=metric_readback,
)
task = payload["windows99_vmware_autostart"]["scheduled_task"]
assert task["start_script_present"] is True
assert task["last_task_result"] == "267011"
assert task["last_task_result_hex"] == "0x41303"
assert task["last_run_time"] == "1999-11-30T00:00:00"
assert task["never_ran"] is True
assert payload["windows99_vmware_autostart"]["config_ready"] is True
assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [
"110",
"112",
"120",
"121",
"188",
]
assert "windows99_vmware_autostart_task_never_ran" in payload[
"active_blockers"
]
assert payload["readback"]["windows99_autostart_task_last_result"] == "267011"
assert payload["readback"]["windows99_autostart_task_last_result_hex"] == (
"0x41303"
)
assert payload["readback"]["windows99_autostart_task_last_run_time"] == (
"1999-11-30T00:00:00"
)
assert payload["readback"]["windows99_autostart_task_never_ran"] is True
assert payload["rollups"]["windows99_autostart_task_last_result"] == "267011"
assert payload["rollups"]["windows99_autostart_task_last_result_hex"] == (
"0x41303"
)
assert payload["rollups"]["windows99_autostart_task_never_ran"] is True
action_by_blocker = {
item["blocker"]: item
for item in payload["active_blocker_action_matrix"]["items"]
}
assert action_by_blocker["windows99_vmware_autostart_task_never_ran"][
"category"
] == "windows99_vmware_autostart"
assert action_by_blocker["windows99_vmware_autostart_task_never_ran"][
"next_safe_action"
] == EXPECTED_WINDOWS99_POWERED_OFF_NEXT_STEP
def test_reboot_auto_recovery_slo_scorecard_does_not_reopen_windows99_config_when_runtime_metric_is_ready():
metric_readback = {
**PROMETHEUS_RUNTIME_READBACK,
@@ -959,6 +1030,35 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
},
"value": [1783010479, "1"],
},
{
"metric": {
"__name__": (
"awoooi_windows99_vmware_autostart_task_"
"start_script_present"
),
},
"value": [1783010479, "1"],
},
{
"metric": {
"__name__": (
"awoooi_windows99_vmware_autostart_task_last_result"
),
"result_hex": "0x41303",
"last_run_time": "1999-11-30T00:00:00",
},
"value": [1783010479, "267011"],
},
{
"metric": {
"__name__": (
"awoooi_windows99_vmware_autostart_task_never_ran"
),
"result_hex": "0x41303",
"last_run_time": "1999-11-30T00:00:00",
},
"value": [1783010479, "1"],
},
{
"metric": {
"__name__": "awoooi_windows99_vmware_missing_vmx_alias",
@@ -1042,6 +1142,13 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch):
assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
assert payload["ready"] is False
assert payload["last_run_timestamp"] == 1783010479
assert payload["windows99_autostart_task_start_script_present"] is True
assert payload["windows99_autostart_task_last_result"] == 267011
assert payload["windows99_autostart_task_last_result_hex"] == "0x41303"
assert payload["windows99_autostart_task_last_run_time"] == (
"1999-11-30T00:00:00"
)
assert payload["windows99_autostart_task_never_ran"] is True
assert payload["windows99_missing_vmx_aliases"] == ["111"]
assert payload["windows99_powered_off_aliases"] == [
"111",