wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(api): expose reboot recovery freshness readback

Browse Source
This commit is contained in:
Your Name
2026-06-29 19:57:23 +08:00
parent 8e3449f9e2
commit 813cab2014
12 changed files with 1497 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
apps/api/src/api/v1/agents.py
View File

@@ -338,12 +338,12 @@ from src.services.dependency_upgrade_approval_package_template import (
from src.services.docker_build_surface_inventory import (
load_latest_docker_build_surface_inventory,
)
from src.services.gitea_private_inventory_p0_scorecard import (
load_latest_gitea_private_inventory_p0_scorecard,
)
from src.services.gitea_authenticated_inventory_payload_validation import (
validate_gitea_authenticated_inventory_payload,
)
from src.services.gitea_private_inventory_p0_scorecard import (
load_latest_gitea_private_inventory_p0_scorecard,
)
from src.services.gitea_workflow_runner_health import (
load_latest_gitea_workflow_runner_health,
)
@@ -378,6 +378,9 @@ from src.services.product_code_review_gate import (
load_latest_product_code_review_gate,
)
from src.services.public_redaction import redact_public_lan_topology
from src.services.reboot_auto_recovery_slo_scorecard import (
load_latest_reboot_auto_recovery_slo_scorecard,
)
from src.services.runtime_surface_inventory import (
load_latest_runtime_surface_inventory,
)
@@ -1042,6 +1045,38 @@ async def get_p0_cicd_baseline_source_readiness() -> dict[str, Any]:
) from exc
@router.get(
"/reboot-auto-recovery-slo-scorecard",
response_model=dict[str, Any],
summary="取得 P0-006 reboot auto-recovery SLO scorecard",
description=(
"讀取已提交的 P0-006 reboot auto-recovery 10-minute SLO scorecard；"
"此端點只回傳 host boot probe、post-reboot readiness、StockPlatform freshness / "
"ingestion readback、retry window 與 controlled recovery gate 狀態。"
"它不重啟主機、不 restart service、不寫資料庫、不偽造 freshness marker、"
"不讀 secret、不觸發 workflow、不呼叫 GitHub。"
),
)
async def get_reboot_auto_recovery_slo_scorecard() -> dict[str, Any]:
"""回傳 P0-006 reboot auto-recovery SLO scorecard 只讀快照。"""
try:
payload = await asyncio.to_thread(
load_latest_reboot_auto_recovery_slo_scorecard
)
return redact_public_lan_topology(payload)
except FileNotFoundError as exc:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail=str(exc),
) from exc
except (json.JSONDecodeError, ValueError) as exc:
logger.error("reboot_auto_recovery_slo_scorecard_invalid", error=str(exc))
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="P0-006 reboot auto-recovery SLO scorecard 快照無效",
) from exc
@router.get(
"/gitea-private-inventory-p0-scorecard",
response_model=dict[str, Any],

161
apps/api/src/services/delivery_closure_workbench.py
View File

@@ -30,6 +30,9 @@ from src.services.gitea_workflow_runner_health import (
from src.services.p0_cicd_baseline_source_readiness import (
load_latest_p0_cicd_baseline_source_readiness,
)
from src.services.reboot_auto_recovery_slo_scorecard import (
load_latest_reboot_auto_recovery_slo_scorecard,
)
from src.services.runtime_surface_inventory import (
load_latest_runtime_surface_inventory,
)
@@ -47,6 +50,7 @@ def load_delivery_closure_workbench() -> dict[str, Any]:
runtime = load_latest_runtime_surface_inventory()
backup = load_latest_backup_dr_readiness_matrix()
credential_escrow_intake = load_latest_credential_escrow_evidence_intake_readiness()
reboot_slo = load_latest_reboot_auto_recovery_slo_scorecard()
return build_delivery_closure_workbench(
status_cleanup=status_cleanup,
production_deploy=production_deploy,
@@ -56,6 +60,7 @@ def load_delivery_closure_workbench() -> dict[str, Any]:
runtime=runtime,
backup=backup,
credential_escrow_intake=credential_escrow_intake,
reboot_slo=reboot_slo,
)
@@ -69,6 +74,7 @@ def build_delivery_closure_workbench(
runtime: dict[str, Any],
backup: dict[str, Any],
credential_escrow_intake: dict[str, Any],
reboot_slo: dict[str, Any],
) -> dict[str, Any]:
"""Build the delivery workbench response from already validated snapshots."""
status_summary = _dict(status_cleanup.get("summary"))
@@ -94,6 +100,13 @@ def build_delivery_closure_workbench(
runtime_rollups = _dict(runtime.get("rollups"))
backup_status = _dict(backup.get("program_status"))
backup_rollups = _dict(backup.get("rollups"))
reboot_readback = _dict(reboot_slo.get("readback"))
reboot_rollups = _dict(reboot_slo.get("rollups"))
reboot_stockplatform = _dict(reboot_slo.get("stockplatform_data_freshness"))
reboot_stockplatform_eod = _dict(reboot_stockplatform.get("eod_window"))
reboot_stockplatform_recovery_gate = _dict(
reboot_stockplatform.get("controlled_recovery_gate")
)
credential_intake_rollups = _dict(credential_escrow_intake.get("rollups"))
credential_intake_readback = _dict(credential_escrow_intake.get("readback"))
single_preflight_intake = _dict(
@@ -106,6 +119,7 @@ def build_delivery_closure_workbench(
private_inventory_blockers = _int(
private_inventory_rollups.get("active_blocker_count")
)
reboot_blockers = _int(reboot_rollups.get("active_blocker_count"))
credential_escrow_required_items = _int(
backup_rollups.get("credential_escrow_required_item_count")
)
@@ -397,6 +411,96 @@ def build_delivery_closure_workbench(
"href": "/deployments",
"next_action": _first_string(production_deploy.get("next_actions")),
},
{
"id": "reboot_auto_recovery",
"source_id": "reboot_auto_recovery_slo_scorecard",
"completion_percent": _percent(
reboot_rollups.get("readiness_percent")
),
"status": str(reboot_slo.get("status") or "unknown"),
"blocker_count": reboot_blockers,
"metric": {
"kind": "reboot_auto_recovery_slo",
"workplan_id": str(reboot_readback.get("workplan_id") or "P0-006"),
"target_minutes": _int(reboot_readback.get("target_minutes")),
"can_claim_all_services_recovered_within_target": reboot_rollups.get(
"can_claim_all_services_recovered_within_target"
)
is True,
"active_blockers": _strings(reboot_slo.get("active_blockers")),
"post_start_blocked": _int(
reboot_rollups.get("post_start_blocked")
),
"service_green": reboot_rollups.get("service_green") is True,
"product_data_green": reboot_rollups.get("product_data_green")
is True,
"backup_core_green": reboot_rollups.get("backup_core_green") is True,
"observed_host_count": _int(
reboot_rollups.get("observed_host_count")
),
"missing_host_count": _int(reboot_rollups.get("missing_host_count")),
"unreachable_host_count": _int(
reboot_rollups.get("unreachable_host_count")
),
"stale_host_count": _int(reboot_rollups.get("stale_host_count")),
"stockplatform_freshness_status": str(
reboot_rollups.get("stockplatform_freshness_status") or ""
),
"stockplatform_ingestion_status": str(
reboot_rollups.get("stockplatform_ingestion_status") or ""
),
"stockplatform_freshness_blocker_count": _int(
reboot_rollups.get("stockplatform_freshness_blocker_count")
),
"stockplatform_ingestion_blocker_count": _int(
reboot_rollups.get("stockplatform_ingestion_blocker_count")
),
"stockplatform_freshness_blockers": _strings(
reboot_stockplatform.get("freshness_blockers")
),
"stockplatform_ingestion_blockers": _strings(
reboot_stockplatform.get("ingestion_blockers")
),
"stockplatform_eod_classification": str(
reboot_stockplatform_eod.get("classification") or ""
),
"stockplatform_eod_next_action": str(
reboot_stockplatform_eod.get("next_action") or ""
),
"stockplatform_final_retry_window_end_local": str(
reboot_stockplatform_eod.get("final_retry_window_end_local") or ""
),
"stockplatform_final_retry_window_passed": reboot_rollups.get(
"stockplatform_final_retry_window_passed"
)
is True,
"stockplatform_controlled_recovery_gate_required": reboot_rollups.get(
"stockplatform_controlled_recovery_gate_required"
)
is True,
"stockplatform_controlled_recovery_gate_status": str(
reboot_stockplatform_recovery_gate.get("status") or ""
),
"host_reboot_performed": _dict(
reboot_slo.get("operation_boundaries")
).get("host_reboot_performed")
is True,
"service_restart_performed": _dict(
reboot_slo.get("operation_boundaries")
).get("service_restart_performed")
is True,
"database_write_or_restore_performed": _dict(
reboot_slo.get("operation_boundaries")
).get("database_write_or_restore_performed")
is True,
"secret_value_collection_allowed": _dict(
reboot_slo.get("operation_boundaries")
).get("secret_value_collection_allowed")
is True,
},
"href": "/operations",
"next_action": str(reboot_readback.get("safe_next_step") or ""),
},
{
"id": "credential_escrow",
"source_id": "backup_dr_credential_escrow",
@@ -773,6 +877,7 @@ def build_delivery_closure_workbench(
source_statuses = [
_source_status("status_cleanup", status_cleanup),
_source_status("production_deploy_readback", production_deploy),
_source_status("reboot_auto_recovery_slo_scorecard", reboot_slo),
_source_status("gitea_private_inventory_p0_scorecard", private_inventory),
_source_status("p0_cicd_baseline_source_readiness", cicd_baseline),
_source_status("gitea_ci_cd", gitea),
@@ -815,6 +920,46 @@ def build_delivery_closure_workbench(
"visibility_change_authorized": False,
"refs_sync_authorized": False,
"workflow_trigger_authorized": False,
"reboot_auto_recovery_status": str(reboot_slo.get("status") or ""),
"reboot_auto_recovery_workplan_id": str(
reboot_readback.get("workplan_id") or "P0-006"
),
"reboot_auto_recovery_readiness_percent": _int(
reboot_rollups.get("readiness_percent")
),
"reboot_auto_recovery_active_blocker_count": reboot_blockers,
"reboot_auto_recovery_can_claim_slo": reboot_rollups.get(
"can_claim_all_services_recovered_within_target"
)
is True,
"reboot_auto_recovery_service_green": reboot_rollups.get("service_green")
is True,
"reboot_auto_recovery_product_data_green": reboot_rollups.get(
"product_data_green"
)
is True,
"reboot_auto_recovery_observed_host_count": _int(
reboot_rollups.get("observed_host_count")
),
"reboot_auto_recovery_stale_host_count": _int(
reboot_rollups.get("stale_host_count")
),
"reboot_auto_recovery_stockplatform_freshness_status": str(
reboot_rollups.get("stockplatform_freshness_status") or ""
),
"reboot_auto_recovery_stockplatform_ingestion_status": str(
reboot_rollups.get("stockplatform_ingestion_status") or ""
),
"reboot_auto_recovery_stockplatform_final_retry_window_passed": (
reboot_rollups.get("stockplatform_final_retry_window_passed") is True
),
"reboot_auto_recovery_stockplatform_controlled_recovery_gate_required": (
reboot_rollups.get("stockplatform_controlled_recovery_gate_required")
is True
),
"reboot_auto_recovery_safe_next_step": str(
reboot_readback.get("safe_next_step") or ""
),
"gitea_private_inventory_status": str(private_inventory.get("status") or ""),
"gitea_private_inventory_workplan_id": str(
private_inventory_readback.get("workplan_id") or ""
@@ -1282,6 +1427,22 @@ def build_delivery_closure_workbench(
"secret_value_collection_allowed": False,
"backup_restore_execution_allowed": False,
"active_scan_allowed": False,
"host_reboot_performed": _dict(
reboot_slo.get("operation_boundaries")
).get("host_reboot_performed")
is True,
"service_restart_performed": _dict(
reboot_slo.get("operation_boundaries")
).get("service_restart_performed")
is True,
"database_write_or_restore_performed": _dict(
reboot_slo.get("operation_boundaries")
).get("database_write_or_restore_performed")
is True,
"stockplatform_manual_data_write_performed": _dict(
reboot_slo.get("operation_boundaries")
).get("stockplatform_manual_data_write_performed")
is True,
},
}

178
apps/api/src/services/reboot_auto_recovery_slo_scorecard.py Normal file
View File

@@ -0,0 +1,178 @@
"""P0-006 reboot auto-recovery SLO scorecard readback.
This loader promotes the committed reboot recovery scorecard into a stable API
contract. It does not reboot hosts, restart services, query runtime secrets, or
write StockPlatform data; it only reads the committed JSON scorecard.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from src.services.snapshot_paths import default_operations_dir
_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__))
_SCORECARD_FILE = "awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json"
_SOURCE_SCHEMA_VERSION = "awoooi_reboot_auto_recovery_slo_scorecard_v1"
_API_SCHEMA_VERSION = "reboot_auto_recovery_slo_scorecard_readback_v1"
def load_latest_reboot_auto_recovery_slo_scorecard(
operations_dir: Path | None = None,
) -> dict[str, Any]:
"""Load and validate the committed P0-006 reboot recovery scorecard."""
directory = operations_dir or _DEFAULT_OPERATIONS_DIR
path = directory / _SCORECARD_FILE
with path.open(encoding="utf-8") as handle:
scorecard = json.load(handle)
if not isinstance(scorecard, dict):
raise ValueError(f"{path}: expected JSON object")
if scorecard.get("schema_version") != _SOURCE_SCHEMA_VERSION:
raise ValueError(f"{path}: unsupported schema_version={scorecard.get('schema_version')!r}")
payload = _build_payload(scorecard, path)
_require_operation_boundaries(payload, str(path))
return payload
def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
host_boot_detection = _dict(scorecard.get("host_boot_detection"))
post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness"))
stockplatform = _dict(scorecard.get("stockplatform_data_freshness"))
active_blockers = _strings(scorecard.get("active_blockers"))
required_checks = {
"source_controls_present": all(_dict(scorecard.get("source_controls")).values()),
"required_hosts_observed": not _strings(host_boot_detection.get("missing_hosts")),
"required_hosts_reachable": not _strings(host_boot_detection.get("unreachable_hosts")),
"service_green": post_reboot_readiness.get("service_green") is True,
"product_data_green": post_reboot_readiness.get("product_data_green") is True,
"backup_core_green": post_reboot_readiness.get("backup_core_green") is True,
"host_188_service_green": post_reboot_readiness.get("host_188_service_green") is True,
"stockplatform_freshness_ok": stockplatform.get("freshness_status") == "ok",
"stockplatform_ingestion_ok": stockplatform.get("ingestion_status") in {"ok", "unknown"},
"fresh_reboot_window_observed": not _strings(host_boot_detection.get("stale_hosts")),
"can_claim_slo": scorecard.get("can_claim_all_services_recovered_within_target")
is True,
}
completed_check_count = sum(1 for value in required_checks.values() if value)
readiness_percent = _percent(
completed_check_count / max(len(required_checks), 1) * 100
)
recovery_gate = _dict(stockplatform.get("controlled_recovery_gate"))
return {
"schema_version": _API_SCHEMA_VERSION,
"generated_at": str(scorecard.get("generated_at") or ""),
"priority": "P0-006",
"scope": "reboot_auto_recovery_slo_scorecard",
"status": str(scorecard.get("status") or "unknown"),
"readback": {
"workplan_id": "P0-006",
"workplan_title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO",
"source_scorecard_ref": f"docs/operations/{path.name}",
"target_minutes": _int(scorecard.get("target_minutes")),
"safe_next_step": str(scorecard.get("safe_next_step") or ""),
},
"host_boot_detection": host_boot_detection,
"post_reboot_readiness": post_reboot_readiness,
"stockplatform_data_freshness": stockplatform,
"active_blockers": active_blockers,
"required_checks": required_checks,
"rollups": {
"active_blocker_count": len(active_blockers),
"readiness_percent": readiness_percent,
"completed_check_count": completed_check_count,
"required_check_count": len(required_checks),
"can_claim_all_services_recovered_within_target": scorecard.get(
"can_claim_all_services_recovered_within_target"
)
is True,
"observed_host_count": len(_strings(host_boot_detection.get("observed_hosts"))),
"missing_host_count": len(_strings(host_boot_detection.get("missing_hosts"))),
"unreachable_host_count": len(
_strings(host_boot_detection.get("unreachable_hosts"))
),
"stale_host_count": len(_strings(host_boot_detection.get("stale_hosts"))),
"post_start_blocked": _int(post_reboot_readiness.get("post_start_blocked")),
"service_green": post_reboot_readiness.get("service_green") is True,
"product_data_green": post_reboot_readiness.get("product_data_green") is True,
"backup_core_green": post_reboot_readiness.get("backup_core_green") is True,
"stockplatform_freshness_status": str(
stockplatform.get("freshness_status") or "unknown"
),
"stockplatform_ingestion_status": str(
stockplatform.get("ingestion_status") or "unknown"
),
"stockplatform_freshness_blocker_count": len(
_strings(stockplatform.get("freshness_blockers"))
),
"stockplatform_ingestion_blocker_count": len(
_strings(stockplatform.get("ingestion_blockers"))
),
"stockplatform_final_retry_window_passed": _dict(
stockplatform.get("eod_window")
).get("final_retry_window_passed")
is True,
"stockplatform_controlled_recovery_gate_required": recovery_gate.get(
"required"
)
is True,
},
"operation_boundaries": {
"read_only_api_allowed": True,
"host_reboot_performed": False,
"host_restart_performed": False,
"service_restart_performed": False,
"database_write_or_restore_performed": False,
"stockplatform_manual_data_write_performed": False,
"secret_value_collection_allowed": False,
"github_api_used": False,
"workflow_trigger_performed": False,
"runtime_write_allowed": False,
},
}
def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None:
boundaries = _dict(payload.get("operation_boundaries"))
forbidden_true = [
key
for key in (
"host_reboot_performed",
"host_restart_performed",
"service_restart_performed",
"database_write_or_restore_performed",
"stockplatform_manual_data_write_performed",
"secret_value_collection_allowed",
"github_api_used",
"workflow_trigger_performed",
"runtime_write_allowed",
)
if boundaries.get(key) is True
]
if forbidden_true:
raise ValueError(f"{label}: forbidden operation boundary true: {forbidden_true}")
def _dict(value: Any) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
def _int(value: Any) -> int:
if isinstance(value, bool):
return int(value)
if isinstance(value, int | float):
return int(value)
return 0
def _percent(value: Any) -> int:
return max(0, min(100, round(float(value or 0))))
def _strings(value: Any) -> list[str]:
if not isinstance(value, list):
return []
return [str(item) for item in value if item is not None]