From f06840ba12e0b38091899b7637523c1455378ac2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 29 Jun 2026 23:45:35 +0800 Subject: [PATCH] feat(api): expose reboot drill preflight --- .gitea/workflows/cd.yaml | 3 + apps/api/src/api/v1/agents.py | 35 ++++ .../reboot_auto_recovery_drill_preflight.py | 183 ++++++++++++++++++ ..._reboot_auto_recovery_slo_scorecard_api.py | 94 +++++++++ docs/LOGBOOK.md | 13 ++ .../test_cd_controlled_runtime_profile.py | 2 + 6 files changed, 330 insertions(+) create mode 100644 apps/api/src/services/reboot_auto_recovery_drill_preflight.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index b04137c1b..01d8fbd32 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -282,6 +282,8 @@ jobs: ;; apps/api/src/services/reboot_auto_recovery_slo_scorecard.py) ;; + apps/api/src/services/reboot_auto_recovery_drill_preflight.py) + ;; apps/api/src/services/iwooos_security_operating_system.py) ;; apps/api/Dockerfile) @@ -529,6 +531,7 @@ jobs: src/services/gitea_private_inventory_closeout_validation.py \ src/services/gitea_private_inventory_p0_scorecard.py \ src/services/reboot_auto_recovery_slo_scorecard.py \ + src/services/reboot_auto_recovery_drill_preflight.py \ src/services/iwooos_security_operating_system.py \ src/services/awoooi_gitea_onboarding_warning_step_dashboard.py \ src/services/awoooi_gitea_onboarding_warning_step_owner_package.py \ diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index b761ae621..6f298c356 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -403,6 +403,9 @@ from src.services.product_code_review_gate import ( load_latest_product_code_review_gate, ) from src.services.public_redaction import redact_public_lan_topology +from src.services.reboot_auto_recovery_drill_preflight import ( + load_latest_reboot_auto_recovery_drill_preflight, +) from src.services.reboot_auto_recovery_slo_scorecard import ( load_latest_reboot_auto_recovery_slo_scorecard, ) @@ -1125,6 +1128,38 @@ async def get_reboot_auto_recovery_slo_scorecard() -> dict[str, Any]: ) from exc +@router.get( + "/reboot-auto-recovery-drill-preflight", + response_model=dict[str, Any], + summary="取得 P0-006 reboot drill preflight", + description=( + "讀取 P0-006 reboot auto-recovery SLO scorecard，產生 approved reboot " + "drill 的只讀 preflight contract；此端點只回傳 target selector、" + "preconditions、verify-only、rollback/post-verifier 與 break-glass 邊界。" + "它不重啟主機、不 restart service、不寫資料庫、不觸發 workflow、" + "不讀 secret、不呼叫 GitHub，也不授權 runtime execution。" + ), +) +async def get_reboot_auto_recovery_drill_preflight() -> dict[str, Any]: + """回傳 P0-006 reboot drill 只讀 preflight contract。""" + try: + payload = await asyncio.to_thread( + load_latest_reboot_auto_recovery_drill_preflight + ) + return redact_public_lan_topology(payload) + except FileNotFoundError as exc: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=str(exc), + ) from exc + except (json.JSONDecodeError, ValueError) as exc: + logger.error("reboot_auto_recovery_drill_preflight_invalid", error=str(exc)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="P0-006 reboot drill preflight 快照無效", + ) from exc + + @router.get( "/gitea-private-inventory-p0-scorecard", response_model=dict[str, Any], diff --git a/apps/api/src/services/reboot_auto_recovery_drill_preflight.py b/apps/api/src/services/reboot_auto_recovery_drill_preflight.py new file mode 100644 index 000000000..a6cc06f14 --- /dev/null +++ b/apps/api/src/services/reboot_auto_recovery_drill_preflight.py @@ -0,0 +1,183 @@ +"""P0-006 reboot drill preflight readback. + +This service turns the remaining fresh-boot-window blocker into a machine +readable preflight package. It does not reboot hosts, restart services, trigger +workflows, or write runtime state. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from src.services.reboot_auto_recovery_slo_scorecard import ( + load_latest_reboot_auto_recovery_slo_scorecard, +) + +_API_SCHEMA_VERSION = "reboot_auto_recovery_drill_preflight_readback_v1" +_FRESH_BOOT_BLOCKER = "host_boot_observation_older_than_target_window" + + +def load_latest_reboot_auto_recovery_drill_preflight( + operations_dir: Path | None = None, +) -> dict[str, Any]: + """Build a read-only drill preflight contract from the latest scorecard.""" + scorecard = load_latest_reboot_auto_recovery_slo_scorecard(operations_dir) + return _build_payload(scorecard) + + +def _build_payload(scorecard: dict[str, Any]) -> dict[str, Any]: + host_boot = _dict(scorecard.get("host_boot_detection")) + rollups = _dict(scorecard.get("rollups")) + operation_boundaries = _dict(scorecard.get("operation_boundaries")) + active_blockers = _strings(scorecard.get("active_blockers")) + required_hosts = _strings(host_boot.get("required_hosts")) + target_minutes = _int(_dict(scorecard.get("readback")).get("target_minutes")) + target_seconds = target_minutes * 60 + + preconditions = { + "source_controls_present": _dict(scorecard.get("required_checks")).get( + "source_controls_present" + ) + is True, + "service_green": scorecard.get("service_green") is True, + "product_data_green": scorecard.get("product_data_green") is True, + "backup_core_green": scorecard.get("backup_core_green") is True, + "host_188_service_green": scorecard.get("host_188_service_green") is True, + "all_required_hosts_observed": _int(scorecard.get("observed_host_count")) + == len(required_hosts) + and _int(scorecard.get("missing_host_count")) == 0, + "all_required_hosts_reachable": _int(scorecard.get("unreachable_host_count")) + == 0, + "latest_verify_only_metric_present": scorecard.get( + "latest_verify_only_metric_present" + ) + is True, + "stockplatform_freshness_ok": scorecard.get("stockplatform_freshness_status") + == "ok", + "stockplatform_ingestion_ok": scorecard.get("stockplatform_ingestion_status") + == "ok", + "blocked_only_by_fresh_reboot_window": active_blockers == [_FRESH_BOOT_BLOCKER], + } + preflight_ready = all(preconditions.values()) + blocker_count = sum(1 for value in preconditions.values() if not value) + status = ( + "ready_for_break_glass_reboot_drill_authorization" + if preflight_ready + else "blocked_reboot_drill_preflight_not_ready" + ) + return { + "schema_version": _API_SCHEMA_VERSION, + "generated_at": str(scorecard.get("generated_at") or ""), + "priority": "P0-006", + "scope": "reboot_auto_recovery_drill_preflight", + "status": status, + "preflight_ready": preflight_ready, + "preflight_blocker_count": blocker_count, + "break_glass_authorization_required": True, + "execution_authorized_by_this_endpoint": False, + "safe_next_step": ( + "collect_separate_reboot_drill_authorization_or_wait_for_next_" + "real_all_host_reboot_event_then_rerun_verify_only" + ), + "target_selector": { + "scope": "awoooi_p0_reboot_slo_hosts", + "required_host_aliases": required_hosts, + "required_host_count": len(required_hosts), + "observed_host_count": _int(scorecard.get("observed_host_count")), + "missing_host_count": _int(scorecard.get("missing_host_count")), + "unreachable_host_count": _int(scorecard.get("unreachable_host_count")), + "stale_host_count": _int(scorecard.get("stale_host_count")), + "selector_source": "P0-006 committed reboot auto-recovery scorecard", + }, + "preconditions": preconditions, + "current_readback": { + "scorecard_status": str(scorecard.get("status") or ""), + "readiness_percent": _int(scorecard.get("readiness_percent")), + "active_blocker_count": _int(scorecard.get("active_blocker_count")), + "active_blockers": active_blockers, + "service_green": scorecard.get("service_green") is True, + "product_data_green": scorecard.get("product_data_green") is True, + "backup_core_green": scorecard.get("backup_core_green") is True, + "post_start_blocked": _int(rollups.get("post_start_blocked")), + "latest_verify_only_metric_ready": _int( + scorecard.get("latest_verify_only_metric_ready") + ), + "latest_verify_only_metric_blocker_count": _int( + scorecard.get("latest_verify_only_metric_blocker_count") + ), + "latest_verify_only_metric_max_host_uptime_seconds": _int( + scorecard.get("latest_verify_only_metric_max_host_uptime_seconds") + ), + "stockplatform_freshness_status": str( + scorecard.get("stockplatform_freshness_status") or "" + ), + "stockplatform_ingestion_status": str( + scorecard.get("stockplatform_ingestion_status") or "" + ), + }, + "check_mode": { + "verify_only_available": True, + "verify_only_source": ( + "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py" + ), + "post_apply_verifier_endpoint": ( + "/api/v1/agents/reboot-auto-recovery-slo-scorecard" + ), + "expected_after_real_fresh_boot_or_approved_drill": { + "status": "slo_ready", + "active_blocker_count": 0, + "latest_verify_only_metric_ready": 1, + "latest_verify_only_metric_blocker_count": 0, + "max_host_uptime_seconds_lte": target_seconds, + }, + }, + "rollback_plan": { + "preflight_is_read_only": True, + "rollback_required_for_this_endpoint": False, + "if_separately_approved_drill_fails": [ + "stop further reboot waves", + "run post-reboot readiness summary", + "keep startup recovery units as the recovery source of truth", + "rerun SLO verify-only and expose blockers without manual DB writes", + ], + }, + "operation_boundaries": { + **operation_boundaries, + "host_reboot_authorized_by_this_endpoint": False, + "host_reboot_performed": False, + "service_restart_performed": False, + "database_write_or_restore_performed": False, + "workflow_trigger_performed": False, + "secret_value_collection_allowed": False, + "github_api_used": False, + "runtime_write_allowed": False, + }, + "forbidden_without_separate_break_glass": [ + "host_reboot", + "node_drain", + "service_restart", + "database_write_or_restore", + "firewall_cutover", + "workflow_dispatch", + "secret_or_token_read", + ], + } + + +def _dict(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _int(value: Any) -> int: + if isinstance(value, bool): + return int(value) + if isinstance(value, int | float): + return int(value) + return 0 + + +def _strings(value: Any) -> list[str]: + if not isinstance(value, list): + return [] + return [str(item) for item in value if item is not None] diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index f110d0257..90eff29b1 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -4,6 +4,9 @@ from fastapi import FastAPI from fastapi.testclient import TestClient from src.api.v1.agents import router +from src.services.reboot_auto_recovery_drill_preflight import ( + load_latest_reboot_auto_recovery_drill_preflight, +) from src.services.reboot_auto_recovery_slo_scorecard import ( load_latest_reboot_auto_recovery_slo_scorecard, ) @@ -26,6 +29,23 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(): _assert_reboot_slo_payload(response.json()) +def test_reboot_auto_recovery_drill_preflight_loader_returns_break_glass_package(): + payload = load_latest_reboot_auto_recovery_drill_preflight() + + _assert_drill_preflight_payload(payload) + + +def test_reboot_auto_recovery_drill_preflight_endpoint_returns_readback(): + app = FastAPI() + app.include_router(router, prefix="/api/v1") + client = TestClient(app) + + response = client.get("/api/v1/agents/reboot-auto-recovery-drill-preflight") + + assert response.status_code == 200 + _assert_drill_preflight_payload(response.json()) + + def _assert_reboot_slo_payload(payload: dict): assert payload["schema_version"] == "reboot_auto_recovery_slo_scorecard_readback_v1" assert payload["priority"] == "P0-006" @@ -110,3 +130,77 @@ def _assert_reboot_slo_payload(payload: dict): assert boundaries["stockplatform_manual_data_write_performed"] is False assert boundaries["secret_value_collection_allowed"] is False assert boundaries["github_api_used"] is False + + +def _assert_drill_preflight_payload(payload: dict): + assert ( + payload["schema_version"] + == "reboot_auto_recovery_drill_preflight_readback_v1" + ) + assert payload["priority"] == "P0-006" + assert payload["status"] == "ready_for_break_glass_reboot_drill_authorization" + assert payload["preflight_ready"] is True + assert payload["preflight_blocker_count"] == 0 + assert payload["break_glass_authorization_required"] is True + assert payload["execution_authorized_by_this_endpoint"] is False + assert payload["safe_next_step"] == ( + "collect_separate_reboot_drill_authorization_or_wait_for_next_" + "real_all_host_reboot_event_then_rerun_verify_only" + ) + target_selector = payload["target_selector"] + assert target_selector["required_host_aliases"] == ["110", "120", "121", "188"] + assert target_selector["required_host_count"] == 4 + assert target_selector["observed_host_count"] == 4 + assert target_selector["missing_host_count"] == 0 + assert target_selector["unreachable_host_count"] == 0 + assert target_selector["stale_host_count"] == 4 + assert set(payload["preconditions"]) == { + "source_controls_present", + "service_green", + "product_data_green", + "backup_core_green", + "host_188_service_green", + "all_required_hosts_observed", + "all_required_hosts_reachable", + "latest_verify_only_metric_present", + "stockplatform_freshness_ok", + "stockplatform_ingestion_ok", + "blocked_only_by_fresh_reboot_window", + } + assert all(payload["preconditions"].values()) + current = payload["current_readback"] + assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready" + assert current["readiness_percent"] == 82 + assert current["active_blocker_count"] == 1 + assert current["active_blockers"] == [ + "host_boot_observation_older_than_target_window" + ] + assert current["latest_verify_only_metric_ready"] == 0 + assert current["latest_verify_only_metric_blocker_count"] == 1 + assert current["stockplatform_freshness_status"] == "ok" + assert current["stockplatform_ingestion_status"] == "ok" + check_mode = payload["check_mode"] + assert check_mode["verify_only_available"] is True + assert ( + check_mode["post_apply_verifier_endpoint"] + == "/api/v1/agents/reboot-auto-recovery-slo-scorecard" + ) + expected = check_mode["expected_after_real_fresh_boot_or_approved_drill"] + assert expected["status"] == "slo_ready" + assert expected["active_blocker_count"] == 0 + assert expected["latest_verify_only_metric_ready"] == 1 + assert expected["latest_verify_only_metric_blocker_count"] == 0 + assert expected["max_host_uptime_seconds_lte"] == 600 + rollback = payload["rollback_plan"] + assert rollback["preflight_is_read_only"] is True + assert rollback["rollback_required_for_this_endpoint"] is False + boundaries = payload["operation_boundaries"] + assert boundaries["host_reboot_authorized_by_this_endpoint"] is False + assert boundaries["host_reboot_performed"] is False + assert boundaries["service_restart_performed"] is False + assert boundaries["database_write_or_restore_performed"] is False + assert boundaries["workflow_trigger_performed"] is False + assert boundaries["secret_value_collection_allowed"] is False + assert boundaries["github_api_used"] is False + assert boundaries["runtime_write_allowed"] is False + assert "host_reboot" in payload["forbidden_without_separate_break_glass"] diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 338c7f35f..994f9bc77 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,16 @@ +## 2026-06-29 — 22:05 P0-006 reboot drill preflight API + +**照優先順序完成的實作**： +- P0-006 仍是 active P0，service/data/backup 維持 green，唯一 blocker 是 `host_boot_observation_older_than_target_window`。 +- 新增 `/api/v1/agents/reboot-auto-recovery-drill-preflight`，把 approved reboot drill 的 target selector、preconditions、verify-only、post-verifier、rollback 與 break-glass 邊界變成機器可讀 API。 +- 此 endpoint 明確 `preflight_ready=true`、`break_glass_authorization_required=true`、`execution_authorized_by_this_endpoint=false`，不把「全面授權」誤解成可直接重啟主機。 + +**驗證**： +- Focused pytest：P0-006 scorecard / drill preflight / CD profile `22 passed`。 +- `ruff check`、`py_compile`、`git diff --check`、Gitea runner pressure guard、Gitea secret env guard：通過。 + +**邊界**：未操作 host / Docker / K8s / DB / firewall / Wazuh runtime；未觸發 workflow_dispatch；未使用 GitHub / `gh` / GitHub API；未讀 secret / token / raw sessions / SQLite / `.env`。 + ## 2026-06-29 — 21:43 P0-006 reboot SLO API readback promoted **照優先順序完成的實作**： diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 624858533..9240f2c4c 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -206,8 +206,10 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() - expected_sources = [ "docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json)", "apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)", + "apps/api/src/services/reboot_auto_recovery_drill_preflight.py)", "apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py)", "src/services/reboot_auto_recovery_slo_scorecard.py", + "src/services/reboot_auto_recovery_drill_preflight.py", "tests/test_reboot_auto_recovery_slo_scorecard_api.py", "scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service)", "scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer)",