feat(api): expose reboot drill preflight
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 20s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
Your Name
2026-06-29 23:45:35 +08:00
parent 6efb5fc97c
commit f06840ba12
6 changed files with 330 additions and 0 deletions

View File

@@ -282,6 +282,8 @@ jobs:
;;
apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)
;;
apps/api/src/services/reboot_auto_recovery_drill_preflight.py)
;;
apps/api/src/services/iwooos_security_operating_system.py)
;;
apps/api/Dockerfile)
@@ -529,6 +531,7 @@ jobs:
src/services/gitea_private_inventory_closeout_validation.py \
src/services/gitea_private_inventory_p0_scorecard.py \
src/services/reboot_auto_recovery_slo_scorecard.py \
src/services/reboot_auto_recovery_drill_preflight.py \
src/services/iwooos_security_operating_system.py \
src/services/awoooi_gitea_onboarding_warning_step_dashboard.py \
src/services/awoooi_gitea_onboarding_warning_step_owner_package.py \

View File

@@ -403,6 +403,9 @@ from src.services.product_code_review_gate import (
load_latest_product_code_review_gate,
)
from src.services.public_redaction import redact_public_lan_topology
from src.services.reboot_auto_recovery_drill_preflight import (
load_latest_reboot_auto_recovery_drill_preflight,
)
from src.services.reboot_auto_recovery_slo_scorecard import (
load_latest_reboot_auto_recovery_slo_scorecard,
)
@@ -1125,6 +1128,38 @@ async def get_reboot_auto_recovery_slo_scorecard() -> dict[str, Any]:
) from exc
@router.get(
"/reboot-auto-recovery-drill-preflight",
response_model=dict[str, Any],
summary="取得 P0-006 reboot drill preflight",
description=(
"讀取 P0-006 reboot auto-recovery SLO scorecard產生 approved reboot "
"drill 的只讀 preflight contract此端點只回傳 target selector、"
"preconditions、verify-only、rollback/post-verifier 與 break-glass 邊界。"
"它不重啟主機、不 restart service、不寫資料庫、不觸發 workflow、"
"不讀 secret、不呼叫 GitHub也不授權 runtime execution。"
),
)
async def get_reboot_auto_recovery_drill_preflight() -> dict[str, Any]:
"""回傳 P0-006 reboot drill 只讀 preflight contract。"""
try:
payload = await asyncio.to_thread(
load_latest_reboot_auto_recovery_drill_preflight
)
return redact_public_lan_topology(payload)
except FileNotFoundError as exc:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=str(exc),
) from exc
except (json.JSONDecodeError, ValueError) as exc:
logger.error("reboot_auto_recovery_drill_preflight_invalid", error=str(exc))
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="P0-006 reboot drill preflight 快照無效",
) from exc
@router.get(
"/gitea-private-inventory-p0-scorecard",
response_model=dict[str, Any],

View File

@@ -0,0 +1,183 @@
"""P0-006 reboot drill preflight readback.
This service turns the remaining fresh-boot-window blocker into a machine
readable preflight package. It does not reboot hosts, restart services, trigger
workflows, or write runtime state.
"""
from __future__ import annotations
from pathlib import Path
from typing import Any
from src.services.reboot_auto_recovery_slo_scorecard import (
load_latest_reboot_auto_recovery_slo_scorecard,
)
_API_SCHEMA_VERSION = "reboot_auto_recovery_drill_preflight_readback_v1"
_FRESH_BOOT_BLOCKER = "host_boot_observation_older_than_target_window"
def load_latest_reboot_auto_recovery_drill_preflight(
operations_dir: Path | None = None,
) -> dict[str, Any]:
"""Build a read-only drill preflight contract from the latest scorecard."""
scorecard = load_latest_reboot_auto_recovery_slo_scorecard(operations_dir)
return _build_payload(scorecard)
def _build_payload(scorecard: dict[str, Any]) -> dict[str, Any]:
host_boot = _dict(scorecard.get("host_boot_detection"))
rollups = _dict(scorecard.get("rollups"))
operation_boundaries = _dict(scorecard.get("operation_boundaries"))
active_blockers = _strings(scorecard.get("active_blockers"))
required_hosts = _strings(host_boot.get("required_hosts"))
target_minutes = _int(_dict(scorecard.get("readback")).get("target_minutes"))
target_seconds = target_minutes * 60
preconditions = {
"source_controls_present": _dict(scorecard.get("required_checks")).get(
"source_controls_present"
)
is True,
"service_green": scorecard.get("service_green") is True,
"product_data_green": scorecard.get("product_data_green") is True,
"backup_core_green": scorecard.get("backup_core_green") is True,
"host_188_service_green": scorecard.get("host_188_service_green") is True,
"all_required_hosts_observed": _int(scorecard.get("observed_host_count"))
== len(required_hosts)
and _int(scorecard.get("missing_host_count")) == 0,
"all_required_hosts_reachable": _int(scorecard.get("unreachable_host_count"))
== 0,
"latest_verify_only_metric_present": scorecard.get(
"latest_verify_only_metric_present"
)
is True,
"stockplatform_freshness_ok": scorecard.get("stockplatform_freshness_status")
== "ok",
"stockplatform_ingestion_ok": scorecard.get("stockplatform_ingestion_status")
== "ok",
"blocked_only_by_fresh_reboot_window": active_blockers == [_FRESH_BOOT_BLOCKER],
}
preflight_ready = all(preconditions.values())
blocker_count = sum(1 for value in preconditions.values() if not value)
status = (
"ready_for_break_glass_reboot_drill_authorization"
if preflight_ready
else "blocked_reboot_drill_preflight_not_ready"
)
return {
"schema_version": _API_SCHEMA_VERSION,
"generated_at": str(scorecard.get("generated_at") or ""),
"priority": "P0-006",
"scope": "reboot_auto_recovery_drill_preflight",
"status": status,
"preflight_ready": preflight_ready,
"preflight_blocker_count": blocker_count,
"break_glass_authorization_required": True,
"execution_authorized_by_this_endpoint": False,
"safe_next_step": (
"collect_separate_reboot_drill_authorization_or_wait_for_next_"
"real_all_host_reboot_event_then_rerun_verify_only"
),
"target_selector": {
"scope": "awoooi_p0_reboot_slo_hosts",
"required_host_aliases": required_hosts,
"required_host_count": len(required_hosts),
"observed_host_count": _int(scorecard.get("observed_host_count")),
"missing_host_count": _int(scorecard.get("missing_host_count")),
"unreachable_host_count": _int(scorecard.get("unreachable_host_count")),
"stale_host_count": _int(scorecard.get("stale_host_count")),
"selector_source": "P0-006 committed reboot auto-recovery scorecard",
},
"preconditions": preconditions,
"current_readback": {
"scorecard_status": str(scorecard.get("status") or ""),
"readiness_percent": _int(scorecard.get("readiness_percent")),
"active_blocker_count": _int(scorecard.get("active_blocker_count")),
"active_blockers": active_blockers,
"service_green": scorecard.get("service_green") is True,
"product_data_green": scorecard.get("product_data_green") is True,
"backup_core_green": scorecard.get("backup_core_green") is True,
"post_start_blocked": _int(rollups.get("post_start_blocked")),
"latest_verify_only_metric_ready": _int(
scorecard.get("latest_verify_only_metric_ready")
),
"latest_verify_only_metric_blocker_count": _int(
scorecard.get("latest_verify_only_metric_blocker_count")
),
"latest_verify_only_metric_max_host_uptime_seconds": _int(
scorecard.get("latest_verify_only_metric_max_host_uptime_seconds")
),
"stockplatform_freshness_status": str(
scorecard.get("stockplatform_freshness_status") or ""
),
"stockplatform_ingestion_status": str(
scorecard.get("stockplatform_ingestion_status") or ""
),
},
"check_mode": {
"verify_only_available": True,
"verify_only_source": (
"scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"
),
"post_apply_verifier_endpoint": (
"/api/v1/agents/reboot-auto-recovery-slo-scorecard"
),
"expected_after_real_fresh_boot_or_approved_drill": {
"status": "slo_ready",
"active_blocker_count": 0,
"latest_verify_only_metric_ready": 1,
"latest_verify_only_metric_blocker_count": 0,
"max_host_uptime_seconds_lte": target_seconds,
},
},
"rollback_plan": {
"preflight_is_read_only": True,
"rollback_required_for_this_endpoint": False,
"if_separately_approved_drill_fails": [
"stop further reboot waves",
"run post-reboot readiness summary",
"keep startup recovery units as the recovery source of truth",
"rerun SLO verify-only and expose blockers without manual DB writes",
],
},
"operation_boundaries": {
**operation_boundaries,
"host_reboot_authorized_by_this_endpoint": False,
"host_reboot_performed": False,
"service_restart_performed": False,
"database_write_or_restore_performed": False,
"workflow_trigger_performed": False,
"secret_value_collection_allowed": False,
"github_api_used": False,
"runtime_write_allowed": False,
},
"forbidden_without_separate_break_glass": [
"host_reboot",
"node_drain",
"service_restart",
"database_write_or_restore",
"firewall_cutover",
"workflow_dispatch",
"secret_or_token_read",
],
}
def _dict(value: Any) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
def _int(value: Any) -> int:
if isinstance(value, bool):
return int(value)
if isinstance(value, int | float):
return int(value)
return 0
def _strings(value: Any) -> list[str]:
if not isinstance(value, list):
return []
return [str(item) for item in value if item is not None]

View File

@@ -4,6 +4,9 @@ from fastapi import FastAPI
from fastapi.testclient import TestClient
from src.api.v1.agents import router
from src.services.reboot_auto_recovery_drill_preflight import (
load_latest_reboot_auto_recovery_drill_preflight,
)
from src.services.reboot_auto_recovery_slo_scorecard import (
load_latest_reboot_auto_recovery_slo_scorecard,
)
@@ -26,6 +29,23 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback():
_assert_reboot_slo_payload(response.json())
def test_reboot_auto_recovery_drill_preflight_loader_returns_break_glass_package():
payload = load_latest_reboot_auto_recovery_drill_preflight()
_assert_drill_preflight_payload(payload)
def test_reboot_auto_recovery_drill_preflight_endpoint_returns_readback():
app = FastAPI()
app.include_router(router, prefix="/api/v1")
client = TestClient(app)
response = client.get("/api/v1/agents/reboot-auto-recovery-drill-preflight")
assert response.status_code == 200
_assert_drill_preflight_payload(response.json())
def _assert_reboot_slo_payload(payload: dict):
assert payload["schema_version"] == "reboot_auto_recovery_slo_scorecard_readback_v1"
assert payload["priority"] == "P0-006"
@@ -110,3 +130,77 @@ def _assert_reboot_slo_payload(payload: dict):
assert boundaries["stockplatform_manual_data_write_performed"] is False
assert boundaries["secret_value_collection_allowed"] is False
assert boundaries["github_api_used"] is False
def _assert_drill_preflight_payload(payload: dict):
assert (
payload["schema_version"]
== "reboot_auto_recovery_drill_preflight_readback_v1"
)
assert payload["priority"] == "P0-006"
assert payload["status"] == "ready_for_break_glass_reboot_drill_authorization"
assert payload["preflight_ready"] is True
assert payload["preflight_blocker_count"] == 0
assert payload["break_glass_authorization_required"] is True
assert payload["execution_authorized_by_this_endpoint"] is False
assert payload["safe_next_step"] == (
"collect_separate_reboot_drill_authorization_or_wait_for_next_"
"real_all_host_reboot_event_then_rerun_verify_only"
)
target_selector = payload["target_selector"]
assert target_selector["required_host_aliases"] == ["110", "120", "121", "188"]
assert target_selector["required_host_count"] == 4
assert target_selector["observed_host_count"] == 4
assert target_selector["missing_host_count"] == 0
assert target_selector["unreachable_host_count"] == 0
assert target_selector["stale_host_count"] == 4
assert set(payload["preconditions"]) == {
"source_controls_present",
"service_green",
"product_data_green",
"backup_core_green",
"host_188_service_green",
"all_required_hosts_observed",
"all_required_hosts_reachable",
"latest_verify_only_metric_present",
"stockplatform_freshness_ok",
"stockplatform_ingestion_ok",
"blocked_only_by_fresh_reboot_window",
}
assert all(payload["preconditions"].values())
current = payload["current_readback"]
assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready"
assert current["readiness_percent"] == 82
assert current["active_blocker_count"] == 1
assert current["active_blockers"] == [
"host_boot_observation_older_than_target_window"
]
assert current["latest_verify_only_metric_ready"] == 0
assert current["latest_verify_only_metric_blocker_count"] == 1
assert current["stockplatform_freshness_status"] == "ok"
assert current["stockplatform_ingestion_status"] == "ok"
check_mode = payload["check_mode"]
assert check_mode["verify_only_available"] is True
assert (
check_mode["post_apply_verifier_endpoint"]
== "/api/v1/agents/reboot-auto-recovery-slo-scorecard"
)
expected = check_mode["expected_after_real_fresh_boot_or_approved_drill"]
assert expected["status"] == "slo_ready"
assert expected["active_blocker_count"] == 0
assert expected["latest_verify_only_metric_ready"] == 1
assert expected["latest_verify_only_metric_blocker_count"] == 0
assert expected["max_host_uptime_seconds_lte"] == 600
rollback = payload["rollback_plan"]
assert rollback["preflight_is_read_only"] is True
assert rollback["rollback_required_for_this_endpoint"] is False
boundaries = payload["operation_boundaries"]
assert boundaries["host_reboot_authorized_by_this_endpoint"] is False
assert boundaries["host_reboot_performed"] is False
assert boundaries["service_restart_performed"] is False
assert boundaries["database_write_or_restore_performed"] is False
assert boundaries["workflow_trigger_performed"] is False
assert boundaries["secret_value_collection_allowed"] is False
assert boundaries["github_api_used"] is False
assert boundaries["runtime_write_allowed"] is False
assert "host_reboot" in payload["forbidden_without_separate_break_glass"]

View File

@@ -1,3 +1,16 @@
## 2026-06-29 — 22:05 P0-006 reboot drill preflight API
**照優先順序完成的實作**
- P0-006 仍是 active P0service/data/backup 維持 green唯一 blocker 是 `host_boot_observation_older_than_target_window`
- 新增 `/api/v1/agents/reboot-auto-recovery-drill-preflight`,把 approved reboot drill 的 target selector、preconditions、verify-only、post-verifier、rollback 與 break-glass 邊界變成機器可讀 API。
- 此 endpoint 明確 `preflight_ready=true``break_glass_authorization_required=true``execution_authorized_by_this_endpoint=false`,不把「全面授權」誤解成可直接重啟主機。
**驗證**
- Focused pytestP0-006 scorecard / drill preflight / CD profile `22 passed`
- `ruff check``py_compile``git diff --check`、Gitea runner pressure guard、Gitea secret env guard通過。
**邊界**:未操作 host / Docker / K8s / DB / firewall / Wazuh runtime未觸發 workflow_dispatch未使用 GitHub / `gh` / GitHub API未讀 secret / token / raw sessions / SQLite / `.env`
## 2026-06-29 — 21:43 P0-006 reboot SLO API readback promoted
**照優先順序完成的實作**

View File

@@ -206,8 +206,10 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() -
expected_sources = [
"docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json)",
"apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)",
"apps/api/src/services/reboot_auto_recovery_drill_preflight.py)",
"apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py)",
"src/services/reboot_auto_recovery_slo_scorecard.py",
"src/services/reboot_auto_recovery_drill_preflight.py",
"tests/test_reboot_auto_recovery_slo_scorecard_api.py",
"scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service)",
"scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer)",