88145ea07c
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 22s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
233 lines
9.7 KiB
Python
233 lines
9.7 KiB
Python
"""P0-006 reboot drill preflight readback.
|
|
|
|
This service turns the remaining fresh-boot-window blocker into a machine
|
|
readable preflight package. It does not reboot hosts, restart services, trigger
|
|
workflows, or write runtime state.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from src.services.reboot_auto_recovery_slo_scorecard import (
|
|
load_latest_reboot_auto_recovery_slo_scorecard,
|
|
)
|
|
|
|
_API_SCHEMA_VERSION = "reboot_auto_recovery_drill_preflight_readback_v1"
|
|
_FRESH_BOOT_BLOCKER = "host_boot_observation_older_than_target_window"
|
|
|
|
|
|
def load_latest_reboot_auto_recovery_drill_preflight(
|
|
operations_dir: Path | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Build a read-only drill preflight contract from the latest scorecard."""
|
|
scorecard = load_latest_reboot_auto_recovery_slo_scorecard(operations_dir)
|
|
return _build_payload(scorecard)
|
|
|
|
|
|
def _build_payload(scorecard: dict[str, Any]) -> dict[str, Any]:
|
|
host_boot = _dict(scorecard.get("host_boot_detection"))
|
|
rollups = _dict(scorecard.get("rollups"))
|
|
operation_boundaries = _dict(scorecard.get("operation_boundaries"))
|
|
active_blockers = _strings(scorecard.get("active_blockers"))
|
|
required_hosts = _strings(host_boot.get("required_hosts"))
|
|
target_minutes = _int(_dict(scorecard.get("readback")).get("target_minutes"))
|
|
target_seconds = target_minutes * 60
|
|
|
|
preconditions = {
|
|
"source_controls_present": _dict(scorecard.get("required_checks")).get(
|
|
"source_controls_present"
|
|
)
|
|
is True,
|
|
"service_green": scorecard.get("service_green") is True,
|
|
"product_data_green": scorecard.get("product_data_green") is True,
|
|
"backup_core_green": scorecard.get("backup_core_green") is True,
|
|
"host_188_service_green": scorecard.get("host_188_service_green") is True,
|
|
"all_required_hosts_observed": _int(scorecard.get("observed_host_count"))
|
|
== len(required_hosts)
|
|
and _int(scorecard.get("missing_host_count")) == 0,
|
|
"all_required_hosts_reachable": _int(scorecard.get("unreachable_host_count"))
|
|
== 0,
|
|
"latest_verify_only_metric_present": scorecard.get(
|
|
"latest_verify_only_metric_present"
|
|
)
|
|
is True,
|
|
"stockplatform_freshness_ok": scorecard.get("stockplatform_freshness_status")
|
|
== "ok",
|
|
"stockplatform_ingestion_ok": scorecard.get("stockplatform_ingestion_status")
|
|
== "ok",
|
|
"blocked_only_by_fresh_reboot_window": active_blockers == [_FRESH_BOOT_BLOCKER],
|
|
}
|
|
preflight_ready = all(preconditions.values())
|
|
preflight_blockers = [
|
|
f"{name}_not_ready" for name, value in preconditions.items() if not value
|
|
]
|
|
blocker_count = len(preflight_blockers)
|
|
status = (
|
|
"ready_for_break_glass_reboot_drill_authorization"
|
|
if preflight_ready
|
|
else "blocked_reboot_drill_preflight_not_ready"
|
|
)
|
|
safe_next_step = (
|
|
"collect_separate_reboot_drill_authorization_or_wait_for_next_"
|
|
"real_all_host_reboot_event_then_rerun_verify_only"
|
|
)
|
|
target_selector = {
|
|
"scope": "awoooi_p0_reboot_slo_hosts",
|
|
"required_host_aliases": required_hosts,
|
|
"required_host_count": len(required_hosts),
|
|
"observed_host_count": _int(scorecard.get("observed_host_count")),
|
|
"missing_host_count": _int(scorecard.get("missing_host_count")),
|
|
"unreachable_host_count": _int(scorecard.get("unreachable_host_count")),
|
|
"stale_host_count": _int(scorecard.get("stale_host_count")),
|
|
"selector_source": "P0-006 committed reboot auto-recovery scorecard",
|
|
}
|
|
current_readback = {
|
|
"scorecard_status": str(scorecard.get("status") or ""),
|
|
"readiness_percent": _int(scorecard.get("readiness_percent")),
|
|
"active_blocker_count": _int(scorecard.get("active_blocker_count")),
|
|
"active_blockers": active_blockers,
|
|
"service_green": scorecard.get("service_green") is True,
|
|
"product_data_green": scorecard.get("product_data_green") is True,
|
|
"backup_core_green": scorecard.get("backup_core_green") is True,
|
|
"post_start_blocked": _int(rollups.get("post_start_blocked")),
|
|
"latest_verify_only_metric_ready": _int(
|
|
scorecard.get("latest_verify_only_metric_ready")
|
|
),
|
|
"latest_verify_only_metric_blocker_count": _int(
|
|
scorecard.get("latest_verify_only_metric_blocker_count")
|
|
),
|
|
"latest_verify_only_metric_max_host_uptime_seconds": _int(
|
|
scorecard.get("latest_verify_only_metric_max_host_uptime_seconds")
|
|
),
|
|
"stockplatform_freshness_status": str(
|
|
scorecard.get("stockplatform_freshness_status") or ""
|
|
),
|
|
"stockplatform_ingestion_status": str(
|
|
scorecard.get("stockplatform_ingestion_status") or ""
|
|
),
|
|
}
|
|
check_mode = {
|
|
"verify_only_available": True,
|
|
"verify_only_source": (
|
|
"scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"
|
|
),
|
|
"post_apply_verifier_endpoint": (
|
|
"/api/v1/agents/reboot-auto-recovery-slo-scorecard"
|
|
),
|
|
"expected_after_real_fresh_boot_or_approved_drill": {
|
|
"status": "slo_ready",
|
|
"active_blocker_count": 0,
|
|
"latest_verify_only_metric_ready": 1,
|
|
"latest_verify_only_metric_blocker_count": 0,
|
|
"max_host_uptime_seconds_lte": target_seconds,
|
|
},
|
|
}
|
|
rollback_plan = {
|
|
"preflight_is_read_only": True,
|
|
"rollback_required_for_this_endpoint": False,
|
|
"if_separately_approved_drill_fails": [
|
|
"stop further reboot waves",
|
|
"run post-reboot readiness summary",
|
|
"keep startup recovery units as the recovery source of truth",
|
|
"rerun SLO verify-only and expose blockers without manual DB writes",
|
|
],
|
|
}
|
|
boundaries = {
|
|
**operation_boundaries,
|
|
"host_reboot_authorized_by_this_endpoint": False,
|
|
"host_reboot_performed": False,
|
|
"service_restart_performed": False,
|
|
"database_write_or_restore_performed": False,
|
|
"workflow_trigger_performed": False,
|
|
"secret_value_collection_allowed": False,
|
|
"github_api_used": False,
|
|
"runtime_write_allowed": False,
|
|
}
|
|
return {
|
|
"schema_version": _API_SCHEMA_VERSION,
|
|
"generated_at": str(scorecard.get("generated_at") or ""),
|
|
"priority": "P0-006",
|
|
"scope": "reboot_auto_recovery_drill_preflight",
|
|
"status": status,
|
|
"preflight_ready": preflight_ready,
|
|
"preflight_blocker_count": blocker_count,
|
|
"break_glass_authorization_required": True,
|
|
"execution_authorized_by_this_endpoint": False,
|
|
"safe_next_step": safe_next_step,
|
|
"target_selector": target_selector,
|
|
"preconditions": preconditions,
|
|
"current_readback": current_readback,
|
|
"check_mode": check_mode,
|
|
"rollback_plan": rollback_plan,
|
|
"readback": {
|
|
"workplan_id": "P0-006",
|
|
"workplan_title": "reboot auto-recovery 10-minute SLO",
|
|
"source_scorecard_status": str(scorecard.get("status") or ""),
|
|
"preflight_ready": preflight_ready,
|
|
"break_glass_authorization_required": True,
|
|
"execution_authorized_by_this_endpoint": False,
|
|
"target_selector": target_selector,
|
|
"current_readback": current_readback,
|
|
"check_mode": check_mode,
|
|
"rollback_plan": rollback_plan,
|
|
"safe_next_step": safe_next_step,
|
|
},
|
|
"rollups": {
|
|
"preflight_ready": preflight_ready,
|
|
"preflight_ready_count": int(preflight_ready),
|
|
"preflight_blocker_count": blocker_count,
|
|
"target_required_host_count": len(required_hosts),
|
|
"target_observed_host_count": target_selector["observed_host_count"],
|
|
"target_missing_host_count": target_selector["missing_host_count"],
|
|
"target_unreachable_host_count": target_selector[
|
|
"unreachable_host_count"
|
|
],
|
|
"target_stale_host_count": target_selector["stale_host_count"],
|
|
"service_green": preconditions["service_green"],
|
|
"product_data_green": preconditions["product_data_green"],
|
|
"backup_core_green": preconditions["backup_core_green"],
|
|
"stockplatform_freshness_ok": preconditions["stockplatform_freshness_ok"],
|
|
"stockplatform_ingestion_ok": preconditions["stockplatform_ingestion_ok"],
|
|
"blocked_only_by_fresh_reboot_window": preconditions[
|
|
"blocked_only_by_fresh_reboot_window"
|
|
],
|
|
"break_glass_authorization_required": True,
|
|
"execution_authorized_by_this_endpoint": False,
|
|
"host_reboot_authorized_by_this_endpoint": False,
|
|
"workflow_trigger_performed": False,
|
|
"secret_value_collection_allowed": False,
|
|
"runtime_write_allowed": False,
|
|
},
|
|
"active_blockers": preflight_blockers,
|
|
"operation_boundaries": boundaries,
|
|
"forbidden_without_separate_break_glass": [
|
|
"host_reboot",
|
|
"node_drain",
|
|
"service_restart",
|
|
"database_write_or_restore",
|
|
"firewall_cutover",
|
|
"workflow_dispatch",
|
|
"secret_or_token_read",
|
|
],
|
|
}
|
|
|
|
|
|
def _dict(value: Any) -> dict[str, Any]:
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
|
|
def _int(value: Any) -> int:
|
|
if isinstance(value, bool):
|
|
return int(value)
|
|
if isinstance(value, int | float):
|
|
return int(value)
|
|
return 0
|
|
|
|
|
|
def _strings(value: Any) -> list[str]:
|
|
if not isinstance(value, list):
|
|
return []
|
|
return [str(item) for item in value if item is not None]
|