From f973cd12b2005f30df0e3413ee40130d1cae9d2c Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 2 Jul 2026 21:41:56 +0800 Subject: [PATCH] fix(reboot): expose public maintenance fallback in api scorecard --- .../reboot_auto_recovery_slo_scorecard.py | 104 +++++++++++++++++- ...awoooi_priority_work_order_readback_api.py | 2 +- ..._reboot_auto_recovery_slo_scorecard_api.py | 41 ++++--- docs/LOGBOOK.md | 12 ++ 4 files changed, 144 insertions(+), 15 deletions(-) diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 6cf7733ec..6ad6af568 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -173,11 +173,17 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: host_boot_detection = _dict(scorecard.get("host_boot_detection")) post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness")) stockplatform = _dict(scorecard.get("stockplatform_data_freshness")) + public_maintenance = _build_public_maintenance_fallback_readback( + scorecard.get("public_maintenance_fallback") + ) windows99 = _dict(scorecard.get("windows99_vmware_autostart")) windows99_management = _dict(scorecard.get("windows99_management_channel")) windows99_ssh_batch = _dict(windows99_management.get("ssh_batch")) source_controls = _dict(scorecard.get("source_controls")) - active_blockers = _strings(scorecard.get("active_blockers")) + active_blockers = _unique_strings( + _strings(scorecard.get("active_blockers")) + + _strings(public_maintenance.get("blockers")) + ) required_checks = { "source_controls_present": all(source_controls.values()), "required_hosts_observed": not _strings(host_boot_detection.get("missing_hosts")), @@ -196,6 +202,9 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "host_188_service_green": post_reboot_readiness.get("host_188_service_green") is True, "stockplatform_freshness_ok": stockplatform.get("freshness_status") == "ok", "stockplatform_ingestion_ok": stockplatform.get("ingestion_status") in {"ok", "unknown"}, + "public_maintenance_fallback_runtime_ready": ( + public_maintenance.get("ready") is True + ), "fresh_reboot_window_observed": not _strings(host_boot_detection.get("stale_hosts")), "can_claim_slo": scorecard.get("can_claim_all_services_recovered_within_target") is True, @@ -291,6 +300,23 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "required" ) is True, + "public_maintenance_runtime_readback_present": ( + public_maintenance.get("runtime_readback_present") is True + ), + "public_maintenance_fallback_ready": ( + public_maintenance.get("ready") is True + ), + "public_route_raw_5xx_without_fallback_count": _int( + public_maintenance.get("raw_5xx_without_fallback_count") + ), + "public_route_unreachable_without_external_fallback_count": _int( + public_maintenance.get( + "route_unreachable_without_external_fallback_count" + ) + ), + "public_maintenance_fallback_route_count": _int( + public_maintenance.get("maintenance_fallback_route_count") + ), "controlled_service_data_backup_readback_present": ( controlled_service_data_backup_readback["readback_present"] is True ), @@ -438,6 +464,15 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "readiness_percent": readiness_percent, "blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only, "latest_verify_only_metric_present": latest_verify_only_metric_present, + "public_maintenance_runtime_readback_present": rollups[ + "public_maintenance_runtime_readback_present" + ], + "public_maintenance_fallback_ready": rollups[ + "public_maintenance_fallback_ready" + ], + "public_route_raw_5xx_without_fallback_count": rollups[ + "public_route_raw_5xx_without_fallback_count" + ], "controlled_service_data_backup_readback_status": rollups[ "controlled_service_data_backup_readback_status" ], @@ -477,6 +512,7 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "host_boot_detection": host_boot_detection, "post_reboot_readiness": post_reboot_readiness, "stockplatform_data_freshness": stockplatform, + "public_maintenance_fallback": public_maintenance, "windows99_vmware_autostart": windows99, "windows99_verify_collection": windows99_verify_collection, "windows99_management_channel": windows99_management, @@ -499,6 +535,66 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: } +def _build_public_maintenance_fallback_readback(value: Any) -> dict[str, Any]: + public_maintenance = _dict(value) + if not public_maintenance: + return { + "runtime_readback_present": False, + "ready": False, + "target_url_count": 0, + "route_count": 0, + "raw_5xx_without_fallback_count": 0, + "route_unreachable_without_external_fallback_count": 0, + "maintenance_fallback_route_count": 0, + "raw_5xx_without_fallback_urls": [], + "route_unreachable_without_external_fallback_urls": [], + "maintenance_fallback_urls": [], + "routes": [], + "blockers": ["public_maintenance_fallback_runtime_readback_missing"], + } + + raw_5xx_count = _int(public_maintenance.get("raw_5xx_without_fallback_count")) + unreachable_count = _int( + public_maintenance.get("route_unreachable_without_external_fallback_count") + ) + blockers = _strings(public_maintenance.get("blockers")) + if ( + raw_5xx_count + and "public_route_raw_5xx_without_maintenance_fallback" not in blockers + ): + blockers.append("public_route_raw_5xx_without_maintenance_fallback") + if ( + unreachable_count + and "public_route_unreachable_without_external_l1_fallback" not in blockers + ): + blockers.append("public_route_unreachable_without_external_l1_fallback") + + return { + "runtime_readback_present": True, + "schema_version": str(public_maintenance.get("schema_version") or "unknown"), + "generated_at": str(public_maintenance.get("generated_at") or ""), + "ready": public_maintenance.get("ready") is True and not blockers, + "target_url_count": _int(public_maintenance.get("target_url_count")), + "route_count": _int(public_maintenance.get("route_count")), + "raw_5xx_without_fallback_count": raw_5xx_count, + "route_unreachable_without_external_fallback_count": unreachable_count, + "maintenance_fallback_route_count": _int( + public_maintenance.get("maintenance_fallback_route_count") + ), + "raw_5xx_without_fallback_urls": _strings( + public_maintenance.get("raw_5xx_without_fallback_urls") + ), + "route_unreachable_without_external_fallback_urls": _strings( + public_maintenance.get("route_unreachable_without_external_fallback_urls") + ), + "maintenance_fallback_urls": _strings( + public_maintenance.get("maintenance_fallback_urls") + ), + "routes": _list_of_dicts(public_maintenance.get("routes")), + "blockers": _unique_strings(blockers), + } + + def _build_controlled_service_data_backup_readback( *, post_reboot_readiness: dict[str, Any], @@ -893,6 +989,12 @@ def _strings(value: Any) -> list[str]: return [str(item) for item in value if item is not None] +def _list_of_dicts(value: Any) -> list[dict[str, Any]]: + if not isinstance(value, list): + return [] + return [item for item in value if isinstance(item, dict)] + + def _unique_strings(values: list[str]) -> list[str]: seen: set[str] = set() unique: list[str] = [] diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index ba80a4f07..623ad6c6a 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -69,7 +69,7 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order(): "blockers_open" ) assert payload["mainline_execution_state"]["active_p0_immediate_apply_gap_count"] == 0 - assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 43 + assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 40 assert ( payload["mainline_execution_state"][ "controlled_service_data_backup_readback_present" diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 680c6809f..f8cb45c74 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -24,6 +24,7 @@ EXPECTED_REBOOT_SLO_BLOCKERS = [ "wazuh_dashboard_degraded", "windows99_remote_execution_channel_unavailable", "windows99_vmware_autostart_readback_missing", + "public_maintenance_fallback_runtime_readback_missing", ] EXPECTED_DRILL_PREFLIGHT_BLOCKERS = [ @@ -95,8 +96,8 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_overlays_live_stockplatform assert payload["required_checks"]["stockplatform_freshness_ok"] is False assert payload["required_checks"]["stockplatform_ingestion_ok"] is False assert payload["required_checks"]["product_data_green"] is False - assert payload["readiness_percent"] == 21 - assert payload["active_blocker_count"] == 15 + assert payload["readiness_percent"] == 20 + assert payload["active_blocker_count"] == 16 assert "product_data_green_not_1" in payload["active_blockers"] assert "stockplatform_freshness_core_margin_short_daily_missing" in payload[ "active_blockers" @@ -149,8 +150,8 @@ def _assert_reboot_slo_payload(payload: dict): assert payload["safe_next_step"] == EXPECTED_WINDOWS99_NEXT_STEP assert payload["next_safe_action"] == payload["safe_next_step"] assert payload["can_claim_all_services_recovered_within_target"] is False - assert payload["active_blocker_count"] == 11 - assert payload["readiness_percent"] == 43 + assert payload["active_blocker_count"] == 12 + assert payload["readiness_percent"] == 40 assert payload["service_green"] is False assert payload["product_data_green"] is True assert payload["backup_core_green"] is False @@ -167,6 +168,14 @@ def _assert_reboot_slo_payload(payload: dict): assert payload["latest_verify_only_metric_last_run_timestamp"] == 0 assert payload["stockplatform_freshness_status"] == "ok" assert payload["stockplatform_ingestion_status"] == "ok" + assert payload["required_checks"][ + "public_maintenance_fallback_runtime_ready" + ] is False + assert payload["public_maintenance_fallback"]["runtime_readback_present"] is False + assert payload["public_maintenance_fallback"]["ready"] is False + assert payload["public_maintenance_fallback"]["blockers"] == [ + "public_maintenance_fallback_runtime_readback_missing" + ] assert payload["windows99_vmware_verify_ready"] is False assert payload["windows99_update_no_auto_reboot_ready"] is False assert payload["windows99_remote_execution_channel_ready"] is False @@ -196,10 +205,13 @@ def _assert_reboot_slo_payload(payload: dict): ) assert payload["readback"]["safe_next_step"] == EXPECTED_WINDOWS99_NEXT_STEP assert payload["readback"]["next_safe_action"] == payload["safe_next_step"] - assert payload["readback"]["active_blocker_count"] == 11 - assert payload["readback"]["readiness_percent"] == 43 + assert payload["readback"]["active_blocker_count"] == 12 + assert payload["readback"]["readiness_percent"] == 40 assert payload["readback"]["blocked_by_fresh_reboot_window_only"] is False assert payload["readback"]["latest_verify_only_metric_present"] is False + assert payload["readback"]["public_maintenance_runtime_readback_present"] is False + assert payload["readback"]["public_maintenance_fallback_ready"] is False + assert payload["readback"]["public_route_raw_5xx_without_fallback_count"] == 0 assert payload["readback"]["windows99_vmware_verify_ready"] is False assert payload["readback"]["windows99_update_no_auto_reboot_ready"] is False assert payload["readback"]["windows99_verify_collection_status"] == ( @@ -211,8 +223,8 @@ def _assert_reboot_slo_payload(payload: dict): ) assert payload["readback"]["windows99_remote_execution_channel_ready"] is False assert payload["readback"]["windows99_ssh_batch_status"] == "permission_denied" - assert payload["rollups"]["active_blocker_count"] == 11 - assert payload["rollups"]["readiness_percent"] == 43 + assert payload["rollups"]["active_blocker_count"] == 12 + assert payload["rollups"]["readiness_percent"] == 40 assert payload["rollups"]["observed_host_count"] == 7 assert payload["rollups"]["missing_host_count"] == 0 assert payload["rollups"]["unreachable_host_count"] == 1 @@ -229,6 +241,9 @@ def _assert_reboot_slo_payload(payload: dict): assert payload["rollups"]["stockplatform_ingestion_status"] == "ok" assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 0 assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 0 + assert payload["rollups"]["public_maintenance_runtime_readback_present"] is False + assert payload["rollups"]["public_maintenance_fallback_ready"] is False + assert payload["rollups"]["public_route_raw_5xx_without_fallback_count"] == 0 assert ( payload["rollups"]["controlled_service_data_backup_readback_present"] is True @@ -282,8 +297,8 @@ def _assert_reboot_slo_payload(payload: dict): "event_and_probe" ) assert progress["primary_blocker"] == "reboot_event_required_host_unreachable" - assert progress["active_blocker_count"] == 11 - assert progress["readiness_percent"] == 43 + assert progress["active_blocker_count"] == 12 + assert progress["readiness_percent"] == 40 assert progress["next_safe_action"] == payload["safe_next_step"] assert progress["fixed_triage_order"][0] == "99_vmware_autostart_and_vm_power" service_data_backup = payload["controlled_service_data_backup_readback"] @@ -451,8 +466,8 @@ def _assert_drill_preflight_payload(payload: dict): } current = payload["current_readback"] assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready" - assert current["readiness_percent"] == 43 - assert current["active_blocker_count"] == 11 + assert current["readiness_percent"] == 40 + assert current["active_blocker_count"] == 12 assert current["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS assert current["latest_verify_only_metric_ready"] == 0 assert current["latest_verify_only_metric_blocker_count"] == 0 @@ -477,7 +492,7 @@ def _assert_drill_preflight_payload(payload: dict): assert readback["workplan_id"] == "P0-006" assert readback["preflight_ready"] is False assert readback["target_selector"]["required_host_count"] == 7 - assert readback["current_readback"]["active_blocker_count"] == 11 + assert readback["current_readback"]["active_blocker_count"] == 12 assert readback["safe_next_step"] == payload["safe_next_step"] rollups = payload["rollups"] assert rollups["preflight_ready"] is False diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 0ec624215..f645686bf 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -10,6 +10,18 @@ **仍維持**： - 未使用 GitHub / `gh` / GitHub API；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未 workflow_dispatch；未重啟主機 / VM / Docker / Nginx / K3s / DB / firewall；未切 DNS / CDN / provider。 +## 2026-07-02 — 21:41 P0-006 public maintenance fallback API readback parity + +**完成內容**： +- `apps/api/src/services/reboot_auto_recovery_slo_scorecard.py` 已同步 script scorecard 的 public maintenance fallback gate：缺 runtime readback 時 fail-closed 加入 `public_maintenance_fallback_runtime_readback_missing`，並上卷到 `required_checks` / `rollups` / `readback` / `public_maintenance_fallback`。 +- `awoooi-priority-work-order-readback` 上卷 readiness 已同步新 required check：目前 P0-006 readiness 由 `43` 收斂為 `40`，但 primary blocker 仍是 host/windows99/reboot SLO，不會把主線帶回 StockPlatform。 + +**驗證**： +- `DATABASE_URL=sqlite+aiosqlite:////tmp/awoooi-codex-api-test.db PYTHONPATH=apps/api python3.11 -m pytest -q apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py apps/api/tests/test_stockplatform_public_api_runtime_readback.py apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py apps/api/tests/test_platform_read_model_fallbacks.py -p no:cacheprovider`：`58 passed`。 + +**仍維持**： +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 SSH、未 Docker / K8s / firewall / Wazuh 寫入、未重啟 host / VM / service、未切 DNS / CDN / provider。 + ## 2026-07-02 — 21:38 P0-006 public maintenance fallback test fixture closure **完成內容**：