diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index aa991fbea..43c65f397 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -24,10 +24,13 @@ _PUBLIC_MAINTENANCE_RUNTIME_FILE_ENV = ( "AWOOOI_PUBLIC_MAINTENANCE_FALLBACK_READBACK_FILE" ) _PUBLIC_MAINTENANCE_RUNTIME_DIR_ENV = "AWOOOI_REBOOT_RECOVERY_LOG_DIR" +_RUNTIME_SCORECARD_FILE_ENV = "AWOOOI_REBOOT_RECOVERY_SLO_SCORECARD_FILE" +_RUNTIME_SCORECARD_DIR_ENV = "AWOOOI_REBOOT_RECOVERY_LOG_DIR" _DEFAULT_REBOOT_RECOVERY_LOG_DIR = Path("/home/wooo/reboot-recovery") _PUBLIC_MAINTENANCE_RUNTIME_PATTERN = ( "reboot-auto-recovery-slo-*/public-maintenance-fallback.json" ) +_RUNTIME_SCORECARD_PATTERN = "reboot-auto-recovery-slo-*/scorecard.json" _PUBLIC_MAINTENANCE_BLOCKERS = { "public_maintenance_fallback_runtime_readback_missing", "public_route_raw_5xx_without_maintenance_fallback", @@ -39,6 +42,8 @@ def load_latest_reboot_auto_recovery_slo_scorecard( operations_dir: Path | None = None, public_maintenance_runtime_path: Path | None = None, public_maintenance_runtime_dir: Path | None = None, + runtime_scorecard_path: Path | None = None, + runtime_scorecard_dir: Path | None = None, ) -> dict[str, Any]: """Load and validate the committed P0-006 reboot recovery scorecard.""" directory = operations_dir or _DEFAULT_OPERATIONS_DIR @@ -51,7 +56,21 @@ def load_latest_reboot_auto_recovery_slo_scorecard( if scorecard.get("schema_version") != _SOURCE_SCHEMA_VERSION: raise ValueError(f"{path}: unsupported schema_version={scorecard.get('schema_version')!r}") - payload = _build_payload(scorecard, path) + source_scorecard = scorecard + runtime_scorecard, runtime_scorecard_source = _load_latest_runtime_scorecard( + runtime_path=runtime_scorecard_path, + runtime_dir=runtime_scorecard_dir, + ) + if runtime_scorecard: + source_scorecard = runtime_scorecard + + payload = _build_payload(source_scorecard, path) + _annotate_runtime_scorecard_readback( + payload=payload, + committed_scorecard=scorecard, + runtime_scorecard=runtime_scorecard, + runtime_scorecard_source=runtime_scorecard_source, + ) public_maintenance_runtime = _load_latest_public_maintenance_runtime_readback( runtime_path=public_maintenance_runtime_path, runtime_dir=public_maintenance_runtime_dir, @@ -62,6 +81,70 @@ def load_latest_reboot_auto_recovery_slo_scorecard( return payload +def _load_latest_runtime_scorecard( + *, + runtime_path: Path | None = None, + runtime_dir: Path | None = None, +) -> tuple[dict[str, Any], Path | None]: + explicit_path = runtime_path or _path_from_env(_RUNTIME_SCORECARD_FILE_ENV) + if explicit_path: + payload = _read_json_file(explicit_path) + return _runtime_scorecard_if_supported(payload), explicit_path + + directory = ( + runtime_dir + or _path_from_env(_RUNTIME_SCORECARD_DIR_ENV) + or _DEFAULT_REBOOT_RECOVERY_LOG_DIR + ) + try: + candidates = [ + path + for path in directory.glob(_RUNTIME_SCORECARD_PATTERN) + if path.is_file() + ] + except OSError: + return {}, None + if not candidates: + return {}, None + latest = max(candidates, key=lambda path: path.stat().st_mtime) + payload = _read_json_file(latest) + return _runtime_scorecard_if_supported(payload), latest + + +def _runtime_scorecard_if_supported(payload: dict[str, Any]) -> dict[str, Any]: + if payload.get("schema_version") != _SOURCE_SCHEMA_VERSION: + return {} + return payload + + +def _annotate_runtime_scorecard_readback( + *, + payload: dict[str, Any], + committed_scorecard: dict[str, Any], + runtime_scorecard: dict[str, Any], + runtime_scorecard_source: Path | None, +) -> None: + runtime_present = bool(runtime_scorecard) + committed_generated_at = str(committed_scorecard.get("generated_at") or "") + runtime_generated_at = str(runtime_scorecard.get("generated_at") or "") + runtime_source_ref = str(runtime_scorecard_source or "") + + payload["snapshot_generated_at"] = committed_generated_at + payload["runtime_scorecard_readback_present"] = runtime_present + payload["runtime_scorecard_generated_at"] = runtime_generated_at + payload["runtime_scorecard_source_ref"] = runtime_source_ref + + readback = _dict(payload.setdefault("readback", {})) + readback["snapshot_generated_at"] = committed_generated_at + readback["runtime_scorecard_readback_present"] = runtime_present + readback["runtime_scorecard_generated_at"] = runtime_generated_at + readback["runtime_scorecard_source_ref"] = runtime_source_ref + + rollups = _dict(payload.setdefault("rollups", {})) + rollups["runtime_scorecard_readback_present"] = runtime_present + rollups["runtime_scorecard_generated_at"] = runtime_generated_at + + def _load_latest_public_maintenance_runtime_readback( *, runtime_path: Path | None = None, diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 432caca49..522c4849f 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from pathlib import Path from fastapi import FastAPI from fastapi.testclient import TestClient @@ -111,6 +112,49 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_public_maintenance_runtime_ assert payload["reboot_sop_progress"]["active_blocker_count"] == 11 +def test_reboot_auto_recovery_slo_scorecard_overlays_runtime_scorecard_artifact( + tmp_path, +): + runtime_scorecard = _runtime_scorecard_with_service_data_backup_green() + runtime_path = tmp_path / "scorecard.json" + runtime_path.write_text(json.dumps(runtime_scorecard), encoding="utf-8") + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + runtime_scorecard_path=runtime_path + ) + + service_blockers = { + "backup_core_green_not_1", + "post_start_blocked_not_zero", + "service_green_not_1", + "wazuh_dashboard_degraded", + } + assert payload["runtime_scorecard_readback_present"] is True + assert payload["runtime_scorecard_generated_at"] == ( + "2026-07-02T23:59:00+08:00" + ) + assert payload["snapshot_generated_at"] == "2026-07-02T15:08:44+08:00" + assert not service_blockers.intersection(payload["active_blockers"]) + assert payload["active_blocker_count"] == 7 + assert payload["readiness_percent"] == 60 + assert payload["service_green"] is True + assert payload["backup_core_green"] is True + assert payload["controlled_service_data_backup_readback"]["status"] == ( + "ready_service_data_backup_green" + ) + assert payload["controlled_service_data_backup_readback"]["blocking_fields"] == [] + assert ( + payload["controlled_service_data_backup_readback"][ + "can_clear_service_data_backup_blockers" + ] + is True + ) + assert payload["windows99_vmware_verify_ready"] is False + assert payload["primary_blocker"] == "reboot_event_required_host_unreachable" + assert payload["readback"]["runtime_scorecard_readback_present"] is True + assert payload["rollups"]["runtime_scorecard_readback_present"] is True + + def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch): monkeypatch.setattr( agents, @@ -629,3 +673,33 @@ def _stockplatform_runtime_blocked() -> dict: "freshness_non_ok_source_count": 3, }, } + + +def _runtime_scorecard_with_service_data_backup_green() -> dict: + source_path = ( + Path(__file__).resolve().parents[3] + / "docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json" + ) + payload = json.loads(source_path.read_text(encoding="utf-8")) + payload["generated_at"] = "2026-07-02T23:59:00+08:00" + payload["active_blockers"] = [ + blocker + for blocker in payload["active_blockers"] + if blocker + not in { + "backup_core_green_not_1", + "post_start_blocked_not_zero", + "service_green_not_1", + "wazuh_dashboard_degraded", + } + ] + payload["post_reboot_readiness"].update( + { + "backup_core_green": True, + "post_start_blocked": 0, + "post_start_result": "FULL_STACK_GREEN_REBOOT_SLO_HOST_BLOCKED", + "service_green": True, + "wazuh_dashboard_degraded": False, + } + ) + return payload diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 3b9401d27..854278c2f 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -14,6 +14,23 @@ **仍維持**： - 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未改 production DB schema、未寫 KM 資料、未 DROP / TRUNCATE / restore / prune；下一步需推 Gitea CD 後重驗 `/api/v1/knowledge?project_id=awoooi` 與 `/zh-TW/knowledge-base` desktop / mobile。 +## 2026-07-02 — 23:59 P0-006 runtime scorecard artifact overlay + +**完成內容**： +- `apps/api/src/services/reboot_auto_recovery_slo_scorecard.py` 新增 `AWOOOI_REBOOT_RECOVERY_SLO_SCORECARD_FILE` 與 `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-*/scorecard.json` runtime artifact 讀取，讓 production API 可吃進同一個 reboot exporter 產出的 no-secret `scorecard.json`。 +- 合法 artifact 必須維持 `awoooi_reboot_auto_recovery_slo_scorecard_v1`；缺 artifact 或 schema 不符時維持 committed snapshot fail-closed，不清 blocker。 +- API readback / rollups 新增 `runtime_scorecard_readback_present`、`runtime_scorecard_generated_at`、`runtime_scorecard_source_ref`，用來分離 committed snapshot truth 與 exporter runtime truth。 +- 測試 fixture 證明 service / backup / post-start / Wazuh 類 readback 若由 runtime scorecard 轉綠，可清相關 blocker 並把 `controlled_service_data_backup_readback.status` 轉為 `ready_service_data_backup_green`；Windows99 / host blocker 仍保留，不假清。 + +**驗證**： +- `DATABASE_URL=sqlite+aiosqlite:////tmp/awoooi-codex-api-test.db PYTHONPATH=apps/api python3.11 -m pytest -q apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py -p no:cacheprovider`：`7 passed`。 +- `DATABASE_URL=sqlite+aiosqlite:////tmp/awoooi-codex-api-test.db PYTHONPATH=apps/api python3.11 -m pytest -q apps/api/tests/test_delivery_closure_workbench_api.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py -p no:cacheprovider`：`30 passed`。 +- `python3.11 -m py_compile apps/api/src/services/reboot_auto_recovery_slo_scorecard.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/src/services/delivery_closure_workbench.py apps/api/src/services/awoooi_priority_work_order_readback.py`：通過。 +- `git diff --check`：通過。 + +**仍維持**： +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未重啟 host / VM / service；未 Docker / K8s / firewall / Wazuh 寫入；未 DROP / TRUNCATE / restore / prune。 + ## 2026-07-02 — 22:22 P0-006 reboot SLO per-blocker 告警投影 **完成內容**：