diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index dd7b7e4fa..0a9192b4d 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -103,6 +103,15 @@ _WINDOWS99_COLLECTOR_BLOCKERS_BY_STATUS = { "blocked_remote_verify_output_invalid": "windows99_remote_verify_output_invalid", "blocked_remote_verify_command_failed": "windows99_remote_verify_command_failed", } +_WINDOWS99_COLLECTED_VERIFY_STATUS = "collected_windows99_vmware_verify_stdout" +_WINDOWS99_MISSING_VMX_NEXT_SAFE_ACTION = ( + "restore_windows99_missing_vmx_source_for_aliases_then_rerun_no_secret_" + "collector_and_scorecard_no_vm_power_change" +) +_WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION = ( + "build_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_" + "then_rerun_no_secret_collector_no_vm_power_change" +) def load_latest_reboot_auto_recovery_slo_scorecard( @@ -1054,8 +1063,12 @@ def _apply_prometheus_metric_active_blockers( primary_blocker=primary_blocker, ) primary_action = _dict(action_matrix.get("primary_blocker_action")) + runtime_next_safe_action = _windows99_collected_runtime_next_safe_action( + metric_readback + ) next_safe_action = str( - primary_action.get("next_safe_action") + runtime_next_safe_action + or primary_action.get("next_safe_action") or _dict(payload.get("reboot_sop_progress")).get("next_safe_action") or payload.get("next_safe_action") or "" @@ -1066,6 +1079,8 @@ def _apply_prometheus_metric_active_blockers( payload["primary_blocker"] = primary_blocker payload["current_phase"] = current_phase payload["eta_or_wait_reason"] = eta_or_wait_reason + if runtime_next_safe_action: + payload["safe_next_step"] = runtime_next_safe_action payload["next_safe_action"] = next_safe_action payload["can_claim_all_services_recovered_within_target"] = can_claim_slo payload["blocked_by_fresh_reboot_window_only"] = active_blockers == [ @@ -1098,6 +1113,7 @@ def _apply_prometheus_metric_active_blockers( "telegram_active_blocker_alert_required_count": action_matrix[ "telegram_alert_required_count" ], + "safe_next_step": payload.get("safe_next_step", ""), "next_safe_action": next_safe_action, "blocked_by_fresh_reboot_window_only": payload[ "blocked_by_fresh_reboot_window_only" @@ -1129,6 +1145,28 @@ def _apply_prometheus_metric_active_blockers( ) +def _windows99_collected_runtime_next_safe_action( + metric_readback: dict[str, Any], +) -> str: + if metric_readback.get("readback_present") is not True: + return "" + if ( + metric_readback.get("windows99_no_secret_collector_readback_present") + is not True + ): + return "" + if ( + str(metric_readback.get("windows99_no_secret_collector_status") or "") + != _WINDOWS99_COLLECTED_VERIFY_STATUS + ): + return "" + if _strings(metric_readback.get("windows99_missing_vmx_aliases")): + return _WINDOWS99_MISSING_VMX_NEXT_SAFE_ACTION + if _strings(metric_readback.get("windows99_powered_off_aliases")): + return _WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION + return "" + + def _path_from_env(name: str) -> Path | None: value = os.environ.get(name, "").strip() return Path(value) if value else None @@ -2565,6 +2603,10 @@ def _active_blocker_action_row( "restore_windows99_no_secret_management_channel_or_collect_local_" "console_verify_readback_then_rerun_reboot_scorecard_no_reboot" ) + elif blocker == "windows99_vmware_vmx_missing": + next_safe_action = _WINDOWS99_MISSING_VMX_NEXT_SAFE_ACTION + elif blocker == "windows99_vmware_guest_power_not_ready": + next_safe_action = _WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION elif blocker in { "windows99_console_clipboard_unreliable", "windows99_console_focus_unreliable", diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index f8e3e1fd8..869ad10f1 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -42,6 +42,14 @@ EXPECTED_WINDOWS99_NEXT_STEP = ( "restore_windows99_no_secret_management_channel_or_collect_local_console_" "verify_readback_then_rerun_reboot_scorecard_no_reboot" ) +EXPECTED_WINDOWS99_MISSING_VMX_NEXT_STEP = ( + "restore_windows99_missing_vmx_source_for_aliases_then_rerun_no_secret_" + "collector_and_scorecard_no_vm_power_change" +) +EXPECTED_WINDOWS99_POWERED_OFF_NEXT_STEP = ( + "build_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_" + "then_rerun_no_secret_collector_no_vm_power_change" +) PUBLIC_MAINTENANCE_RUNTIME_READY = { "schema_version": "awoooi_public_maintenance_fallback_probe_v1", @@ -912,6 +920,55 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_collector_blocke assert payload["rollups"]["windows99_no_secret_collector_readback_present"] is True +def test_reboot_auto_recovery_slo_scorecard_promotes_collected_windows99_runtime_next_step(): + metric_readback = dict(PROMETHEUS_RUNTIME_READBACK) + metric_readback.update( + { + "windows99_no_secret_collector_readback_present": True, + "windows99_no_secret_collector_status": ( + "collected_windows99_vmware_verify_stdout" + ), + "windows99_no_secret_collector_safe_next_step": ( + "commit_no_secret_verify_artifact_then_rerun_reboot_auto_" + "recovery_slo_scorecard" + ), + "windows99_no_secret_collector_ssh_batchmode_auth_ready": True, + "windows99_no_secret_collector_port_22_open": True, + "windows99_no_secret_collector_remote_verify_attempted": True, + } + ) + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + prometheus_metric_readback=metric_readback + ) + + assert payload["primary_blocker"] == "reboot_event_required_host_unreachable" + assert payload["safe_next_step"] == EXPECTED_WINDOWS99_MISSING_VMX_NEXT_STEP + assert payload["next_safe_action"] == EXPECTED_WINDOWS99_MISSING_VMX_NEXT_STEP + assert payload["readback"]["safe_next_step"] == ( + EXPECTED_WINDOWS99_MISSING_VMX_NEXT_STEP + ) + assert payload["readback"]["next_safe_action"] == ( + EXPECTED_WINDOWS99_MISSING_VMX_NEXT_STEP + ) + assert payload["reboot_sop_progress"]["next_safe_action"] == ( + EXPECTED_WINDOWS99_MISSING_VMX_NEXT_STEP + ) + assert payload["windows99_verify_collection"][ + "no_secret_collector_status" + ] == "collected_windows99_vmware_verify_stdout" + action_by_blocker = { + item["blocker"]: item + for item in payload["active_blocker_action_matrix"]["items"] + } + assert action_by_blocker["windows99_vmware_vmx_missing"][ + "next_safe_action" + ] == EXPECTED_WINDOWS99_MISSING_VMX_NEXT_STEP + assert action_by_blocker["windows99_vmware_guest_power_not_ready"][ + "next_safe_action" + ] == EXPECTED_WINDOWS99_POWERED_OFF_NEXT_STEP + + def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch): monkeypatch.setattr( agents, diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 1c50b5f85..80ba36e05 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,19 @@ +## 2026-07-03 — 10:58 Windows99 collected verifier 下一步上卷到 production API + +**完成內容**: +- 修正 `reboot-auto-recovery-slo-scorecard` API overlay:Prometheus / no-secret collector 已讀回 `collected_windows99_vmware_verify_stdout` 時,`safe_next_step` 與 `next_safe_action` 會優先指向 runtime 已驗出的 Windows99 blocker。 +- 若 `windows99_missing_vmx_aliases` 非空,下一步固定為 `restore_windows99_missing_vmx_source_for_aliases_then_rerun_no_secret_collector_and_scorecard_no_vm_power_change`;若只剩 powered-off alias,下一步固定為 check-mode package,不授權 VM power change。 +- `active_blocker_action_matrix` 同步把 `windows99_vmware_vmx_missing` 與 `windows99_vmware_guest_power_not_ready` 指到各自具體下一步,避免 production API 退回泛用 management-channel blocker 或泛用 host-probe action。 + +**已跑驗證**: +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py -q -p no:cacheprovider`:`38 passed`。 +- `python3.11 -m py_compile apps/api/src/services/reboot_auto_recovery_slo_scorecard.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py`:通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`:`GITEA_RUNNER_PRESSURE_GUARD_OK`。 +- `git diff --check`:通過。 + +**仍維持**: +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth;未使用 GitHub / gh;未 workflow_dispatch;未啟動或關閉 VM;未重啟 host / service;未 Docker / Nginx / K3s / DB / firewall restart;未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 10:15 Windows99 nested blocker 上卷到 active blockers **完成內容**: