feat(reboot): expose service data backup readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 51s
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 51s
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
This commit is contained in:
@@ -2399,9 +2399,18 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
reboot_slo = load_latest_reboot_auto_recovery_slo_scorecard()
|
||||
reboot_slo_rollups = _dict(reboot_slo.get("rollups"))
|
||||
reboot_stockplatform = _dict(reboot_slo.get("stockplatform_data_freshness"))
|
||||
reboot_service_data_backup = _dict(
|
||||
reboot_slo.get("controlled_service_data_backup_readback")
|
||||
)
|
||||
reboot_preflight_rollups = _dict(reboot_preflight.get("rollups"))
|
||||
reboot_preflight_target_selector = _dict(reboot_preflight.get("target_selector"))
|
||||
reboot_active_blockers = _strings(reboot_slo.get("active_blockers"))
|
||||
service_data_backup_readback_present = (
|
||||
reboot_service_data_backup.get("readback_present") is True
|
||||
)
|
||||
service_data_backup_blocking_fields = _strings(
|
||||
reboot_service_data_backup.get("blocking_fields")
|
||||
)
|
||||
p0_006_event_gated = bool(
|
||||
reboot_slo_rollups.get("blocked_by_fresh_reboot_window_only") is True
|
||||
)
|
||||
@@ -2443,6 +2452,25 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
state["active_p0_can_claim_10_minute_recovery"] = bool(
|
||||
reboot_slo.get("can_claim_all_services_recovered_within_target") is True
|
||||
)
|
||||
state["controlled_service_data_backup_readback_present"] = (
|
||||
service_data_backup_readback_present
|
||||
)
|
||||
state["controlled_service_data_backup_readback_status"] = str(
|
||||
reboot_service_data_backup.get("status") or "unknown"
|
||||
)
|
||||
state["controlled_service_data_backup_blocker_count"] = len(
|
||||
service_data_backup_blocking_fields
|
||||
)
|
||||
state["controlled_service_data_backup_blocking_fields"] = (
|
||||
service_data_backup_blocking_fields
|
||||
)
|
||||
state["controlled_service_data_backup_can_clear_blockers"] = bool(
|
||||
reboot_service_data_backup.get("can_clear_service_data_backup_blockers")
|
||||
is True
|
||||
)
|
||||
state["controlled_service_data_backup_next_safe_action"] = str(
|
||||
reboot_service_data_backup.get("next_safe_action") or ""
|
||||
)
|
||||
state["stale_snapshot_or_old_cd_runs_must_not_reopen_closed_work"] = True
|
||||
state["p0_004_template_copy_apply_gate_production_http_status"] = 200
|
||||
state["p0_004_template_copy_apply_gate_runtime_readback_state"] = (
|
||||
@@ -2481,7 +2509,12 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
"or_separate_break_glass_reboot_drill_authorization"
|
||||
)
|
||||
if p0_006_event_gated
|
||||
else "blocked_reboot_slo_scorecard_requires_controlled_service_data_backup_readback"
|
||||
else (
|
||||
"blocked_reboot_slo_scorecard_service_data_backup_readback_present_"
|
||||
"blockers_open"
|
||||
if service_data_backup_readback_present
|
||||
else "blocked_reboot_slo_scorecard_requires_controlled_service_data_backup_readback"
|
||||
)
|
||||
)
|
||||
|
||||
runtime_sha = str(
|
||||
@@ -2574,6 +2607,27 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
reboot_slo_rollups.get("stockplatform_controlled_recovery_gate_required")
|
||||
is True
|
||||
)
|
||||
evidence["controlled_service_data_backup_readback_present"] = (
|
||||
service_data_backup_readback_present
|
||||
)
|
||||
evidence["controlled_service_data_backup_readback_status"] = str(
|
||||
reboot_service_data_backup.get("status") or "unknown"
|
||||
)
|
||||
evidence["controlled_service_data_backup_blocker_count"] = len(
|
||||
service_data_backup_blocking_fields
|
||||
)
|
||||
evidence["controlled_service_data_backup_blocking_fields"] = (
|
||||
service_data_backup_blocking_fields
|
||||
)
|
||||
evidence["controlled_service_data_backup_can_clear_blockers"] = bool(
|
||||
reboot_service_data_backup.get(
|
||||
"can_clear_service_data_backup_blockers"
|
||||
)
|
||||
is True
|
||||
)
|
||||
evidence["controlled_service_data_backup_next_safe_action"] = str(
|
||||
reboot_service_data_backup.get("next_safe_action") or ""
|
||||
)
|
||||
evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "")
|
||||
evidence["drill_preflight_ready"] = (
|
||||
reboot_preflight_rollups.get("preflight_ready") is True
|
||||
@@ -2633,18 +2687,21 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
|
||||
professional_fix["action"] = (
|
||||
"Continue P0-006 from the reboot SLO scorecard truth: close the "
|
||||
"service, product-data, backup, 188 service, host reachability, "
|
||||
"disk, and Wazuh blockers in controlled lanes; for StockPlatform "
|
||||
"freshness/ingestion postgres_not_ready, use the production "
|
||||
"migration/control-channel path with target selector, dry-run, "
|
||||
"rollback, public API verifier, and KM/RAG/MCP/PlayBook writeback. "
|
||||
"disk, and Wazuh blockers in controlled lanes; keep the "
|
||||
"controlled service/data/backup readback visible as the source "
|
||||
"selector for these blockers. For StockPlatform freshness/ingestion "
|
||||
"postgres_not_ready, use the production migration/control-channel "
|
||||
"path with target selector, dry-run, rollback, public API verifier, "
|
||||
"and KM/RAG/MCP/PlayBook writeback. "
|
||||
"Do not reboot, restart services, write DB rows, fake freshness, "
|
||||
"trigger workflows, or read secrets from this lane."
|
||||
)
|
||||
workplan["reason"] = (
|
||||
"The current reboot SLO scorecard is blocked, not merely waiting "
|
||||
"for a fresh reboot window. Service/product-data/backup and "
|
||||
"StockPlatform freshness/ingestion evidence must stay visible in "
|
||||
"the priority readback until controlled recovery evidence closes."
|
||||
"for a fresh reboot window. Controlled service/product-data/backup "
|
||||
"readback is present, but its blocker fields remain open and must "
|
||||
"stay visible in the priority readback until controlled recovery "
|
||||
"evidence closes."
|
||||
)
|
||||
workplan["safe_next_step"] = str(reboot_slo.get("safe_next_step") or "")
|
||||
workplan["status"] = "blocked_reboot_auto_recovery_slo_not_ready"
|
||||
@@ -2808,6 +2865,15 @@ def _set_rollups_and_summary(
|
||||
"active_p0_immediate_apply_gap_count": active_gap_count,
|
||||
"active_p0_event_gated_by_fresh_reboot_window_only": p0_006_event_gated,
|
||||
"active_p0_live_active_blocker_count": len(active_blockers),
|
||||
"controlled_service_data_backup_readback_present": (
|
||||
state.get("controlled_service_data_backup_readback_present") is True
|
||||
),
|
||||
"controlled_service_data_backup_blocker_count": _int(
|
||||
state.get("controlled_service_data_backup_blocker_count")
|
||||
),
|
||||
"controlled_service_data_backup_can_clear_blockers": (
|
||||
state.get("controlled_service_data_backup_can_clear_blockers") is True
|
||||
),
|
||||
"p0_004_runtime_readback_ready": p0_004_ready,
|
||||
"reboot_drill_preflight_runtime_readback_ready": (
|
||||
state.get("reboot_drill_preflight_runtime_readback_state") == "ready"
|
||||
@@ -2825,6 +2891,24 @@ def _set_rollups_and_summary(
|
||||
state.get("active_p0_readiness_percent")
|
||||
),
|
||||
"active_p0_live_active_blockers": active_blockers,
|
||||
"controlled_service_data_backup_readback_present": (
|
||||
state.get("controlled_service_data_backup_readback_present") is True
|
||||
),
|
||||
"controlled_service_data_backup_readback_status": str(
|
||||
state.get("controlled_service_data_backup_readback_status") or ""
|
||||
),
|
||||
"controlled_service_data_backup_blocker_count": _int(
|
||||
state.get("controlled_service_data_backup_blocker_count")
|
||||
),
|
||||
"controlled_service_data_backup_blocking_fields": _strings(
|
||||
state.get("controlled_service_data_backup_blocking_fields")
|
||||
),
|
||||
"controlled_service_data_backup_can_clear_blockers": (
|
||||
state.get("controlled_service_data_backup_can_clear_blockers") is True
|
||||
),
|
||||
"controlled_service_data_backup_next_safe_action": str(
|
||||
state.get("controlled_service_data_backup_next_safe_action") or ""
|
||||
),
|
||||
"latest_successful_deployed_source_sha": latest_source_sha,
|
||||
"latest_successful_deployed_source_short_sha": latest_source_sha[:10],
|
||||
"latest_successful_deploy_marker": str(
|
||||
|
||||
@@ -79,6 +79,13 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
product_data_green = post_reboot_readiness.get("product_data_green") is True
|
||||
backup_core_green = post_reboot_readiness.get("backup_core_green") is True
|
||||
host_188_service_green = post_reboot_readiness.get("host_188_service_green") is True
|
||||
controlled_service_data_backup_readback = (
|
||||
_build_controlled_service_data_backup_readback(
|
||||
post_reboot_readiness=post_reboot_readiness,
|
||||
stockplatform=stockplatform,
|
||||
active_blockers=active_blockers,
|
||||
)
|
||||
)
|
||||
blocked_by_fresh_reboot_window_only = active_blockers == [
|
||||
"host_boot_observation_older_than_target_window"
|
||||
]
|
||||
@@ -136,6 +143,24 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"required"
|
||||
)
|
||||
is True,
|
||||
"controlled_service_data_backup_readback_present": (
|
||||
controlled_service_data_backup_readback["readback_present"] is True
|
||||
),
|
||||
"controlled_service_data_backup_readback_status": str(
|
||||
controlled_service_data_backup_readback["status"]
|
||||
),
|
||||
"controlled_service_data_backup_blocker_count": len(
|
||||
_strings(controlled_service_data_backup_readback.get("blocking_fields"))
|
||||
),
|
||||
"controlled_service_data_backup_can_clear_blockers": (
|
||||
controlled_service_data_backup_readback[
|
||||
"can_clear_service_data_backup_blockers"
|
||||
]
|
||||
is True
|
||||
),
|
||||
"controlled_service_data_backup_next_safe_action": str(
|
||||
controlled_service_data_backup_readback["next_safe_action"]
|
||||
),
|
||||
}
|
||||
return {
|
||||
"schema_version": _API_SCHEMA_VERSION,
|
||||
@@ -187,8 +212,20 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
"readiness_percent": readiness_percent,
|
||||
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
|
||||
"latest_verify_only_metric_present": latest_verify_only_metric_present,
|
||||
"controlled_service_data_backup_readback_status": rollups[
|
||||
"controlled_service_data_backup_readback_status"
|
||||
],
|
||||
"controlled_service_data_backup_blocker_count": rollups[
|
||||
"controlled_service_data_backup_blocker_count"
|
||||
],
|
||||
"controlled_service_data_backup_next_safe_action": rollups[
|
||||
"controlled_service_data_backup_next_safe_action"
|
||||
],
|
||||
},
|
||||
"reboot_sop_progress": sop_progress,
|
||||
"controlled_service_data_backup_readback": (
|
||||
controlled_service_data_backup_readback
|
||||
),
|
||||
"host_boot_detection": host_boot_detection,
|
||||
"post_reboot_readiness": post_reboot_readiness,
|
||||
"stockplatform_data_freshness": stockplatform,
|
||||
@@ -211,6 +248,110 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _build_controlled_service_data_backup_readback(
|
||||
*,
|
||||
post_reboot_readiness: dict[str, Any],
|
||||
stockplatform: dict[str, Any],
|
||||
active_blockers: list[str],
|
||||
) -> dict[str, Any]:
|
||||
service_green = post_reboot_readiness.get("service_green") is True
|
||||
product_data_green = post_reboot_readiness.get("product_data_green") is True
|
||||
backup_core_green = post_reboot_readiness.get("backup_core_green") is True
|
||||
host_188_service_green = post_reboot_readiness.get("host_188_service_green") is True
|
||||
post_start_blocked = _int(post_reboot_readiness.get("post_start_blocked"))
|
||||
wazuh_dashboard_degraded = (
|
||||
post_reboot_readiness.get("wazuh_dashboard_degraded") is True
|
||||
)
|
||||
stockplatform_freshness_status = str(
|
||||
stockplatform.get("freshness_status") or "unknown"
|
||||
)
|
||||
stockplatform_ingestion_status = str(
|
||||
stockplatform.get("ingestion_status") or "unknown"
|
||||
)
|
||||
blocking_fields: list[str] = []
|
||||
if not service_green:
|
||||
blocking_fields.append("service_green")
|
||||
if post_start_blocked != 0:
|
||||
blocking_fields.append("post_start_blocked")
|
||||
if not product_data_green:
|
||||
blocking_fields.append("product_data_green")
|
||||
if not backup_core_green:
|
||||
blocking_fields.append("backup_core_green")
|
||||
if not host_188_service_green:
|
||||
blocking_fields.append("host_188_service_green")
|
||||
if wazuh_dashboard_degraded:
|
||||
blocking_fields.append("wazuh_dashboard_degraded")
|
||||
if stockplatform_freshness_status != "ok":
|
||||
blocking_fields.append("stockplatform_freshness_status")
|
||||
if stockplatform_ingestion_status not in {"ok", "unknown"}:
|
||||
blocking_fields.append("stockplatform_ingestion_status")
|
||||
|
||||
related_blockers = [
|
||||
blocker
|
||||
for blocker in active_blockers
|
||||
if blocker
|
||||
in {
|
||||
"backup_core_green_not_1",
|
||||
"host_188_service_green_not_1",
|
||||
"post_start_blocked_not_zero",
|
||||
"product_data_green_not_1",
|
||||
"service_green_not_1",
|
||||
"stockplatform_freshness_blocked",
|
||||
"stockplatform_ingestion_blocked",
|
||||
"wazuh_dashboard_degraded",
|
||||
}
|
||||
]
|
||||
readback_present = (
|
||||
post_reboot_readiness.get("summary_present") is True
|
||||
or stockplatform.get("freshness_endpoint_readback_present") is True
|
||||
or stockplatform.get("ingestion_endpoint_readback_present") is True
|
||||
)
|
||||
can_clear_blockers = readback_present and not blocking_fields
|
||||
return {
|
||||
"schema_version": "controlled_service_data_backup_readback_v1",
|
||||
"status": (
|
||||
"ready_service_data_backup_green"
|
||||
if can_clear_blockers
|
||||
else "blocked_service_data_backup_readback_not_green"
|
||||
),
|
||||
"readback_present": readback_present,
|
||||
"service_green": service_green,
|
||||
"product_data_green": product_data_green,
|
||||
"backup_core_green": backup_core_green,
|
||||
"host_188_service_green": host_188_service_green,
|
||||
"post_start_blocked": post_start_blocked,
|
||||
"wazuh_dashboard_degraded": wazuh_dashboard_degraded,
|
||||
"stockplatform_freshness_status": stockplatform_freshness_status,
|
||||
"stockplatform_ingestion_status": stockplatform_ingestion_status,
|
||||
"stockplatform_freshness_blockers": _strings(
|
||||
stockplatform.get("freshness_blockers")
|
||||
),
|
||||
"stockplatform_ingestion_blockers": _strings(
|
||||
stockplatform.get("ingestion_blockers")
|
||||
),
|
||||
"blocking_fields": blocking_fields,
|
||||
"related_active_blockers": related_blockers,
|
||||
"can_clear_service_data_backup_blockers": can_clear_blockers,
|
||||
"next_safe_action": (
|
||||
"keep_timer_live_wait_for_fresh_all_host_reboot_event_or_drill_preflight"
|
||||
if can_clear_blockers
|
||||
else "collect_public_service_data_backup_readback_and_refresh_reboot_slo_scorecard_no_reboot_no_db_write"
|
||||
),
|
||||
"controlled_lane": "read_only_verifier_input",
|
||||
"forbidden_actions": [
|
||||
"host_reboot",
|
||||
"service_restart",
|
||||
"manual_db_update",
|
||||
"truncate_or_restore",
|
||||
"fake_freshness_marker",
|
||||
"secret_value_read",
|
||||
"workflow_dispatch",
|
||||
"github_api",
|
||||
],
|
||||
"runtime_write_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
def _build_reboot_sop_progress(
|
||||
*,
|
||||
scorecard: dict[str, Any],
|
||||
|
||||
Reference in New Issue
Block a user