feat(api): expose reboot drill preflight
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 20s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 20s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -282,6 +282,8 @@ jobs:
|
||||
;;
|
||||
apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)
|
||||
;;
|
||||
apps/api/src/services/reboot_auto_recovery_drill_preflight.py)
|
||||
;;
|
||||
apps/api/src/services/iwooos_security_operating_system.py)
|
||||
;;
|
||||
apps/api/Dockerfile)
|
||||
@@ -529,6 +531,7 @@ jobs:
|
||||
src/services/gitea_private_inventory_closeout_validation.py \
|
||||
src/services/gitea_private_inventory_p0_scorecard.py \
|
||||
src/services/reboot_auto_recovery_slo_scorecard.py \
|
||||
src/services/reboot_auto_recovery_drill_preflight.py \
|
||||
src/services/iwooos_security_operating_system.py \
|
||||
src/services/awoooi_gitea_onboarding_warning_step_dashboard.py \
|
||||
src/services/awoooi_gitea_onboarding_warning_step_owner_package.py \
|
||||
|
||||
@@ -403,6 +403,9 @@ from src.services.product_code_review_gate import (
|
||||
load_latest_product_code_review_gate,
|
||||
)
|
||||
from src.services.public_redaction import redact_public_lan_topology
|
||||
from src.services.reboot_auto_recovery_drill_preflight import (
|
||||
load_latest_reboot_auto_recovery_drill_preflight,
|
||||
)
|
||||
from src.services.reboot_auto_recovery_slo_scorecard import (
|
||||
load_latest_reboot_auto_recovery_slo_scorecard,
|
||||
)
|
||||
@@ -1125,6 +1128,38 @@ async def get_reboot_auto_recovery_slo_scorecard() -> dict[str, Any]:
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/reboot-auto-recovery-drill-preflight",
|
||||
response_model=dict[str, Any],
|
||||
summary="取得 P0-006 reboot drill preflight",
|
||||
description=(
|
||||
"讀取 P0-006 reboot auto-recovery SLO scorecard,產生 approved reboot "
|
||||
"drill 的只讀 preflight contract;此端點只回傳 target selector、"
|
||||
"preconditions、verify-only、rollback/post-verifier 與 break-glass 邊界。"
|
||||
"它不重啟主機、不 restart service、不寫資料庫、不觸發 workflow、"
|
||||
"不讀 secret、不呼叫 GitHub,也不授權 runtime execution。"
|
||||
),
|
||||
)
|
||||
async def get_reboot_auto_recovery_drill_preflight() -> dict[str, Any]:
|
||||
"""回傳 P0-006 reboot drill 只讀 preflight contract。"""
|
||||
try:
|
||||
payload = await asyncio.to_thread(
|
||||
load_latest_reboot_auto_recovery_drill_preflight
|
||||
)
|
||||
return redact_public_lan_topology(payload)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
logger.error("reboot_auto_recovery_drill_preflight_invalid", error=str(exc))
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="P0-006 reboot drill preflight 快照無效",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/gitea-private-inventory-p0-scorecard",
|
||||
response_model=dict[str, Any],
|
||||
|
||||
183
apps/api/src/services/reboot_auto_recovery_drill_preflight.py
Normal file
183
apps/api/src/services/reboot_auto_recovery_drill_preflight.py
Normal file
@@ -0,0 +1,183 @@
|
||||
"""P0-006 reboot drill preflight readback.
|
||||
|
||||
This service turns the remaining fresh-boot-window blocker into a machine
|
||||
readable preflight package. It does not reboot hosts, restart services, trigger
|
||||
workflows, or write runtime state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.services.reboot_auto_recovery_slo_scorecard import (
|
||||
load_latest_reboot_auto_recovery_slo_scorecard,
|
||||
)
|
||||
|
||||
_API_SCHEMA_VERSION = "reboot_auto_recovery_drill_preflight_readback_v1"
|
||||
_FRESH_BOOT_BLOCKER = "host_boot_observation_older_than_target_window"
|
||||
|
||||
|
||||
def load_latest_reboot_auto_recovery_drill_preflight(
|
||||
operations_dir: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a read-only drill preflight contract from the latest scorecard."""
|
||||
scorecard = load_latest_reboot_auto_recovery_slo_scorecard(operations_dir)
|
||||
return _build_payload(scorecard)
|
||||
|
||||
|
||||
def _build_payload(scorecard: dict[str, Any]) -> dict[str, Any]:
|
||||
host_boot = _dict(scorecard.get("host_boot_detection"))
|
||||
rollups = _dict(scorecard.get("rollups"))
|
||||
operation_boundaries = _dict(scorecard.get("operation_boundaries"))
|
||||
active_blockers = _strings(scorecard.get("active_blockers"))
|
||||
required_hosts = _strings(host_boot.get("required_hosts"))
|
||||
target_minutes = _int(_dict(scorecard.get("readback")).get("target_minutes"))
|
||||
target_seconds = target_minutes * 60
|
||||
|
||||
preconditions = {
|
||||
"source_controls_present": _dict(scorecard.get("required_checks")).get(
|
||||
"source_controls_present"
|
||||
)
|
||||
is True,
|
||||
"service_green": scorecard.get("service_green") is True,
|
||||
"product_data_green": scorecard.get("product_data_green") is True,
|
||||
"backup_core_green": scorecard.get("backup_core_green") is True,
|
||||
"host_188_service_green": scorecard.get("host_188_service_green") is True,
|
||||
"all_required_hosts_observed": _int(scorecard.get("observed_host_count"))
|
||||
== len(required_hosts)
|
||||
and _int(scorecard.get("missing_host_count")) == 0,
|
||||
"all_required_hosts_reachable": _int(scorecard.get("unreachable_host_count"))
|
||||
== 0,
|
||||
"latest_verify_only_metric_present": scorecard.get(
|
||||
"latest_verify_only_metric_present"
|
||||
)
|
||||
is True,
|
||||
"stockplatform_freshness_ok": scorecard.get("stockplatform_freshness_status")
|
||||
== "ok",
|
||||
"stockplatform_ingestion_ok": scorecard.get("stockplatform_ingestion_status")
|
||||
== "ok",
|
||||
"blocked_only_by_fresh_reboot_window": active_blockers == [_FRESH_BOOT_BLOCKER],
|
||||
}
|
||||
preflight_ready = all(preconditions.values())
|
||||
blocker_count = sum(1 for value in preconditions.values() if not value)
|
||||
status = (
|
||||
"ready_for_break_glass_reboot_drill_authorization"
|
||||
if preflight_ready
|
||||
else "blocked_reboot_drill_preflight_not_ready"
|
||||
)
|
||||
return {
|
||||
"schema_version": _API_SCHEMA_VERSION,
|
||||
"generated_at": str(scorecard.get("generated_at") or ""),
|
||||
"priority": "P0-006",
|
||||
"scope": "reboot_auto_recovery_drill_preflight",
|
||||
"status": status,
|
||||
"preflight_ready": preflight_ready,
|
||||
"preflight_blocker_count": blocker_count,
|
||||
"break_glass_authorization_required": True,
|
||||
"execution_authorized_by_this_endpoint": False,
|
||||
"safe_next_step": (
|
||||
"collect_separate_reboot_drill_authorization_or_wait_for_next_"
|
||||
"real_all_host_reboot_event_then_rerun_verify_only"
|
||||
),
|
||||
"target_selector": {
|
||||
"scope": "awoooi_p0_reboot_slo_hosts",
|
||||
"required_host_aliases": required_hosts,
|
||||
"required_host_count": len(required_hosts),
|
||||
"observed_host_count": _int(scorecard.get("observed_host_count")),
|
||||
"missing_host_count": _int(scorecard.get("missing_host_count")),
|
||||
"unreachable_host_count": _int(scorecard.get("unreachable_host_count")),
|
||||
"stale_host_count": _int(scorecard.get("stale_host_count")),
|
||||
"selector_source": "P0-006 committed reboot auto-recovery scorecard",
|
||||
},
|
||||
"preconditions": preconditions,
|
||||
"current_readback": {
|
||||
"scorecard_status": str(scorecard.get("status") or ""),
|
||||
"readiness_percent": _int(scorecard.get("readiness_percent")),
|
||||
"active_blocker_count": _int(scorecard.get("active_blocker_count")),
|
||||
"active_blockers": active_blockers,
|
||||
"service_green": scorecard.get("service_green") is True,
|
||||
"product_data_green": scorecard.get("product_data_green") is True,
|
||||
"backup_core_green": scorecard.get("backup_core_green") is True,
|
||||
"post_start_blocked": _int(rollups.get("post_start_blocked")),
|
||||
"latest_verify_only_metric_ready": _int(
|
||||
scorecard.get("latest_verify_only_metric_ready")
|
||||
),
|
||||
"latest_verify_only_metric_blocker_count": _int(
|
||||
scorecard.get("latest_verify_only_metric_blocker_count")
|
||||
),
|
||||
"latest_verify_only_metric_max_host_uptime_seconds": _int(
|
||||
scorecard.get("latest_verify_only_metric_max_host_uptime_seconds")
|
||||
),
|
||||
"stockplatform_freshness_status": str(
|
||||
scorecard.get("stockplatform_freshness_status") or ""
|
||||
),
|
||||
"stockplatform_ingestion_status": str(
|
||||
scorecard.get("stockplatform_ingestion_status") or ""
|
||||
),
|
||||
},
|
||||
"check_mode": {
|
||||
"verify_only_available": True,
|
||||
"verify_only_source": (
|
||||
"scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"
|
||||
),
|
||||
"post_apply_verifier_endpoint": (
|
||||
"/api/v1/agents/reboot-auto-recovery-slo-scorecard"
|
||||
),
|
||||
"expected_after_real_fresh_boot_or_approved_drill": {
|
||||
"status": "slo_ready",
|
||||
"active_blocker_count": 0,
|
||||
"latest_verify_only_metric_ready": 1,
|
||||
"latest_verify_only_metric_blocker_count": 0,
|
||||
"max_host_uptime_seconds_lte": target_seconds,
|
||||
},
|
||||
},
|
||||
"rollback_plan": {
|
||||
"preflight_is_read_only": True,
|
||||
"rollback_required_for_this_endpoint": False,
|
||||
"if_separately_approved_drill_fails": [
|
||||
"stop further reboot waves",
|
||||
"run post-reboot readiness summary",
|
||||
"keep startup recovery units as the recovery source of truth",
|
||||
"rerun SLO verify-only and expose blockers without manual DB writes",
|
||||
],
|
||||
},
|
||||
"operation_boundaries": {
|
||||
**operation_boundaries,
|
||||
"host_reboot_authorized_by_this_endpoint": False,
|
||||
"host_reboot_performed": False,
|
||||
"service_restart_performed": False,
|
||||
"database_write_or_restore_performed": False,
|
||||
"workflow_trigger_performed": False,
|
||||
"secret_value_collection_allowed": False,
|
||||
"github_api_used": False,
|
||||
"runtime_write_allowed": False,
|
||||
},
|
||||
"forbidden_without_separate_break_glass": [
|
||||
"host_reboot",
|
||||
"node_drain",
|
||||
"service_restart",
|
||||
"database_write_or_restore",
|
||||
"firewall_cutover",
|
||||
"workflow_dispatch",
|
||||
"secret_or_token_read",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _dict(value: Any) -> dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _int(value: Any) -> int:
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, int | float):
|
||||
return int(value)
|
||||
return 0
|
||||
|
||||
|
||||
def _strings(value: Any) -> list[str]:
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [str(item) for item in value if item is not None]
|
||||
@@ -4,6 +4,9 @@ from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.api.v1.agents import router
|
||||
from src.services.reboot_auto_recovery_drill_preflight import (
|
||||
load_latest_reboot_auto_recovery_drill_preflight,
|
||||
)
|
||||
from src.services.reboot_auto_recovery_slo_scorecard import (
|
||||
load_latest_reboot_auto_recovery_slo_scorecard,
|
||||
)
|
||||
@@ -26,6 +29,23 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback():
|
||||
_assert_reboot_slo_payload(response.json())
|
||||
|
||||
|
||||
def test_reboot_auto_recovery_drill_preflight_loader_returns_break_glass_package():
|
||||
payload = load_latest_reboot_auto_recovery_drill_preflight()
|
||||
|
||||
_assert_drill_preflight_payload(payload)
|
||||
|
||||
|
||||
def test_reboot_auto_recovery_drill_preflight_endpoint_returns_readback():
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api/v1")
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/api/v1/agents/reboot-auto-recovery-drill-preflight")
|
||||
|
||||
assert response.status_code == 200
|
||||
_assert_drill_preflight_payload(response.json())
|
||||
|
||||
|
||||
def _assert_reboot_slo_payload(payload: dict):
|
||||
assert payload["schema_version"] == "reboot_auto_recovery_slo_scorecard_readback_v1"
|
||||
assert payload["priority"] == "P0-006"
|
||||
@@ -110,3 +130,77 @@ def _assert_reboot_slo_payload(payload: dict):
|
||||
assert boundaries["stockplatform_manual_data_write_performed"] is False
|
||||
assert boundaries["secret_value_collection_allowed"] is False
|
||||
assert boundaries["github_api_used"] is False
|
||||
|
||||
|
||||
def _assert_drill_preflight_payload(payload: dict):
|
||||
assert (
|
||||
payload["schema_version"]
|
||||
== "reboot_auto_recovery_drill_preflight_readback_v1"
|
||||
)
|
||||
assert payload["priority"] == "P0-006"
|
||||
assert payload["status"] == "ready_for_break_glass_reboot_drill_authorization"
|
||||
assert payload["preflight_ready"] is True
|
||||
assert payload["preflight_blocker_count"] == 0
|
||||
assert payload["break_glass_authorization_required"] is True
|
||||
assert payload["execution_authorized_by_this_endpoint"] is False
|
||||
assert payload["safe_next_step"] == (
|
||||
"collect_separate_reboot_drill_authorization_or_wait_for_next_"
|
||||
"real_all_host_reboot_event_then_rerun_verify_only"
|
||||
)
|
||||
target_selector = payload["target_selector"]
|
||||
assert target_selector["required_host_aliases"] == ["110", "120", "121", "188"]
|
||||
assert target_selector["required_host_count"] == 4
|
||||
assert target_selector["observed_host_count"] == 4
|
||||
assert target_selector["missing_host_count"] == 0
|
||||
assert target_selector["unreachable_host_count"] == 0
|
||||
assert target_selector["stale_host_count"] == 4
|
||||
assert set(payload["preconditions"]) == {
|
||||
"source_controls_present",
|
||||
"service_green",
|
||||
"product_data_green",
|
||||
"backup_core_green",
|
||||
"host_188_service_green",
|
||||
"all_required_hosts_observed",
|
||||
"all_required_hosts_reachable",
|
||||
"latest_verify_only_metric_present",
|
||||
"stockplatform_freshness_ok",
|
||||
"stockplatform_ingestion_ok",
|
||||
"blocked_only_by_fresh_reboot_window",
|
||||
}
|
||||
assert all(payload["preconditions"].values())
|
||||
current = payload["current_readback"]
|
||||
assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready"
|
||||
assert current["readiness_percent"] == 82
|
||||
assert current["active_blocker_count"] == 1
|
||||
assert current["active_blockers"] == [
|
||||
"host_boot_observation_older_than_target_window"
|
||||
]
|
||||
assert current["latest_verify_only_metric_ready"] == 0
|
||||
assert current["latest_verify_only_metric_blocker_count"] == 1
|
||||
assert current["stockplatform_freshness_status"] == "ok"
|
||||
assert current["stockplatform_ingestion_status"] == "ok"
|
||||
check_mode = payload["check_mode"]
|
||||
assert check_mode["verify_only_available"] is True
|
||||
assert (
|
||||
check_mode["post_apply_verifier_endpoint"]
|
||||
== "/api/v1/agents/reboot-auto-recovery-slo-scorecard"
|
||||
)
|
||||
expected = check_mode["expected_after_real_fresh_boot_or_approved_drill"]
|
||||
assert expected["status"] == "slo_ready"
|
||||
assert expected["active_blocker_count"] == 0
|
||||
assert expected["latest_verify_only_metric_ready"] == 1
|
||||
assert expected["latest_verify_only_metric_blocker_count"] == 0
|
||||
assert expected["max_host_uptime_seconds_lte"] == 600
|
||||
rollback = payload["rollback_plan"]
|
||||
assert rollback["preflight_is_read_only"] is True
|
||||
assert rollback["rollback_required_for_this_endpoint"] is False
|
||||
boundaries = payload["operation_boundaries"]
|
||||
assert boundaries["host_reboot_authorized_by_this_endpoint"] is False
|
||||
assert boundaries["host_reboot_performed"] is False
|
||||
assert boundaries["service_restart_performed"] is False
|
||||
assert boundaries["database_write_or_restore_performed"] is False
|
||||
assert boundaries["workflow_trigger_performed"] is False
|
||||
assert boundaries["secret_value_collection_allowed"] is False
|
||||
assert boundaries["github_api_used"] is False
|
||||
assert boundaries["runtime_write_allowed"] is False
|
||||
assert "host_reboot" in payload["forbidden_without_separate_break_glass"]
|
||||
|
||||
@@ -1,3 +1,16 @@
|
||||
## 2026-06-29 — 22:05 P0-006 reboot drill preflight API
|
||||
|
||||
**照優先順序完成的實作**:
|
||||
- P0-006 仍是 active P0,service/data/backup 維持 green,唯一 blocker 是 `host_boot_observation_older_than_target_window`。
|
||||
- 新增 `/api/v1/agents/reboot-auto-recovery-drill-preflight`,把 approved reboot drill 的 target selector、preconditions、verify-only、post-verifier、rollback 與 break-glass 邊界變成機器可讀 API。
|
||||
- 此 endpoint 明確 `preflight_ready=true`、`break_glass_authorization_required=true`、`execution_authorized_by_this_endpoint=false`,不把「全面授權」誤解成可直接重啟主機。
|
||||
|
||||
**驗證**:
|
||||
- Focused pytest:P0-006 scorecard / drill preflight / CD profile `22 passed`。
|
||||
- `ruff check`、`py_compile`、`git diff --check`、Gitea runner pressure guard、Gitea secret env guard:通過。
|
||||
|
||||
**邊界**:未操作 host / Docker / K8s / DB / firewall / Wazuh runtime;未觸發 workflow_dispatch;未使用 GitHub / `gh` / GitHub API;未讀 secret / token / raw sessions / SQLite / `.env`。
|
||||
|
||||
## 2026-06-29 — 21:43 P0-006 reboot SLO API readback promoted
|
||||
|
||||
**照優先順序完成的實作**:
|
||||
|
||||
@@ -206,8 +206,10 @@ def test_reboot_auto_recovery_slo_sources_stay_on_controlled_runtime_profile() -
|
||||
expected_sources = [
|
||||
"docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json)",
|
||||
"apps/api/src/services/reboot_auto_recovery_slo_scorecard.py)",
|
||||
"apps/api/src/services/reboot_auto_recovery_drill_preflight.py)",
|
||||
"apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py)",
|
||||
"src/services/reboot_auto_recovery_slo_scorecard.py",
|
||||
"src/services/reboot_auto_recovery_drill_preflight.py",
|
||||
"tests/test_reboot_auto_recovery_slo_scorecard_api.py",
|
||||
"scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service)",
|
||||
"scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer)",
|
||||
|
||||
Reference in New Issue
Block a user