feat(reboot): expose service data backup readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 51s
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled

This commit is contained in:
ogt
2026-07-02 14:31:36 +08:00
parent 3679c1cf41
commit cb442cdbaa
5 changed files with 399 additions and 8 deletions

View File

@@ -2399,9 +2399,18 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
reboot_slo = load_latest_reboot_auto_recovery_slo_scorecard()
reboot_slo_rollups = _dict(reboot_slo.get("rollups"))
reboot_stockplatform = _dict(reboot_slo.get("stockplatform_data_freshness"))
reboot_service_data_backup = _dict(
reboot_slo.get("controlled_service_data_backup_readback")
)
reboot_preflight_rollups = _dict(reboot_preflight.get("rollups"))
reboot_preflight_target_selector = _dict(reboot_preflight.get("target_selector"))
reboot_active_blockers = _strings(reboot_slo.get("active_blockers"))
service_data_backup_readback_present = (
reboot_service_data_backup.get("readback_present") is True
)
service_data_backup_blocking_fields = _strings(
reboot_service_data_backup.get("blocking_fields")
)
p0_006_event_gated = bool(
reboot_slo_rollups.get("blocked_by_fresh_reboot_window_only") is True
)
@@ -2443,6 +2452,25 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
state["active_p0_can_claim_10_minute_recovery"] = bool(
reboot_slo.get("can_claim_all_services_recovered_within_target") is True
)
state["controlled_service_data_backup_readback_present"] = (
service_data_backup_readback_present
)
state["controlled_service_data_backup_readback_status"] = str(
reboot_service_data_backup.get("status") or "unknown"
)
state["controlled_service_data_backup_blocker_count"] = len(
service_data_backup_blocking_fields
)
state["controlled_service_data_backup_blocking_fields"] = (
service_data_backup_blocking_fields
)
state["controlled_service_data_backup_can_clear_blockers"] = bool(
reboot_service_data_backup.get("can_clear_service_data_backup_blockers")
is True
)
state["controlled_service_data_backup_next_safe_action"] = str(
reboot_service_data_backup.get("next_safe_action") or ""
)
state["stale_snapshot_or_old_cd_runs_must_not_reopen_closed_work"] = True
state["p0_004_template_copy_apply_gate_production_http_status"] = 200
state["p0_004_template_copy_apply_gate_runtime_readback_state"] = (
@@ -2481,7 +2509,12 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
"or_separate_break_glass_reboot_drill_authorization"
)
if p0_006_event_gated
else "blocked_reboot_slo_scorecard_requires_controlled_service_data_backup_readback"
else (
"blocked_reboot_slo_scorecard_service_data_backup_readback_present_"
"blockers_open"
if service_data_backup_readback_present
else "blocked_reboot_slo_scorecard_requires_controlled_service_data_backup_readback"
)
)
runtime_sha = str(
@@ -2574,6 +2607,27 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
reboot_slo_rollups.get("stockplatform_controlled_recovery_gate_required")
is True
)
evidence["controlled_service_data_backup_readback_present"] = (
service_data_backup_readback_present
)
evidence["controlled_service_data_backup_readback_status"] = str(
reboot_service_data_backup.get("status") or "unknown"
)
evidence["controlled_service_data_backup_blocker_count"] = len(
service_data_backup_blocking_fields
)
evidence["controlled_service_data_backup_blocking_fields"] = (
service_data_backup_blocking_fields
)
evidence["controlled_service_data_backup_can_clear_blockers"] = bool(
reboot_service_data_backup.get(
"can_clear_service_data_backup_blockers"
)
is True
)
evidence["controlled_service_data_backup_next_safe_action"] = str(
reboot_service_data_backup.get("next_safe_action") or ""
)
evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "")
evidence["drill_preflight_ready"] = (
reboot_preflight_rollups.get("preflight_ready") is True
@@ -2633,18 +2687,21 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
professional_fix["action"] = (
"Continue P0-006 from the reboot SLO scorecard truth: close the "
"service, product-data, backup, 188 service, host reachability, "
"disk, and Wazuh blockers in controlled lanes; for StockPlatform "
"freshness/ingestion postgres_not_ready, use the production "
"migration/control-channel path with target selector, dry-run, "
"rollback, public API verifier, and KM/RAG/MCP/PlayBook writeback. "
"disk, and Wazuh blockers in controlled lanes; keep the "
"controlled service/data/backup readback visible as the source "
"selector for these blockers. For StockPlatform freshness/ingestion "
"postgres_not_ready, use the production migration/control-channel "
"path with target selector, dry-run, rollback, public API verifier, "
"and KM/RAG/MCP/PlayBook writeback. "
"Do not reboot, restart services, write DB rows, fake freshness, "
"trigger workflows, or read secrets from this lane."
)
workplan["reason"] = (
"The current reboot SLO scorecard is blocked, not merely waiting "
"for a fresh reboot window. Service/product-data/backup and "
"StockPlatform freshness/ingestion evidence must stay visible in "
"the priority readback until controlled recovery evidence closes."
"for a fresh reboot window. Controlled service/product-data/backup "
"readback is present, but its blocker fields remain open and must "
"stay visible in the priority readback until controlled recovery "
"evidence closes."
)
workplan["safe_next_step"] = str(reboot_slo.get("safe_next_step") or "")
workplan["status"] = "blocked_reboot_auto_recovery_slo_not_ready"
@@ -2808,6 +2865,15 @@ def _set_rollups_and_summary(
"active_p0_immediate_apply_gap_count": active_gap_count,
"active_p0_event_gated_by_fresh_reboot_window_only": p0_006_event_gated,
"active_p0_live_active_blocker_count": len(active_blockers),
"controlled_service_data_backup_readback_present": (
state.get("controlled_service_data_backup_readback_present") is True
),
"controlled_service_data_backup_blocker_count": _int(
state.get("controlled_service_data_backup_blocker_count")
),
"controlled_service_data_backup_can_clear_blockers": (
state.get("controlled_service_data_backup_can_clear_blockers") is True
),
"p0_004_runtime_readback_ready": p0_004_ready,
"reboot_drill_preflight_runtime_readback_ready": (
state.get("reboot_drill_preflight_runtime_readback_state") == "ready"
@@ -2825,6 +2891,24 @@ def _set_rollups_and_summary(
state.get("active_p0_readiness_percent")
),
"active_p0_live_active_blockers": active_blockers,
"controlled_service_data_backup_readback_present": (
state.get("controlled_service_data_backup_readback_present") is True
),
"controlled_service_data_backup_readback_status": str(
state.get("controlled_service_data_backup_readback_status") or ""
),
"controlled_service_data_backup_blocker_count": _int(
state.get("controlled_service_data_backup_blocker_count")
),
"controlled_service_data_backup_blocking_fields": _strings(
state.get("controlled_service_data_backup_blocking_fields")
),
"controlled_service_data_backup_can_clear_blockers": (
state.get("controlled_service_data_backup_can_clear_blockers") is True
),
"controlled_service_data_backup_next_safe_action": str(
state.get("controlled_service_data_backup_next_safe_action") or ""
),
"latest_successful_deployed_source_sha": latest_source_sha,
"latest_successful_deployed_source_short_sha": latest_source_sha[:10],
"latest_successful_deploy_marker": str(

View File

@@ -79,6 +79,13 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
product_data_green = post_reboot_readiness.get("product_data_green") is True
backup_core_green = post_reboot_readiness.get("backup_core_green") is True
host_188_service_green = post_reboot_readiness.get("host_188_service_green") is True
controlled_service_data_backup_readback = (
_build_controlled_service_data_backup_readback(
post_reboot_readiness=post_reboot_readiness,
stockplatform=stockplatform,
active_blockers=active_blockers,
)
)
blocked_by_fresh_reboot_window_only = active_blockers == [
"host_boot_observation_older_than_target_window"
]
@@ -136,6 +143,24 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"required"
)
is True,
"controlled_service_data_backup_readback_present": (
controlled_service_data_backup_readback["readback_present"] is True
),
"controlled_service_data_backup_readback_status": str(
controlled_service_data_backup_readback["status"]
),
"controlled_service_data_backup_blocker_count": len(
_strings(controlled_service_data_backup_readback.get("blocking_fields"))
),
"controlled_service_data_backup_can_clear_blockers": (
controlled_service_data_backup_readback[
"can_clear_service_data_backup_blockers"
]
is True
),
"controlled_service_data_backup_next_safe_action": str(
controlled_service_data_backup_readback["next_safe_action"]
),
}
return {
"schema_version": _API_SCHEMA_VERSION,
@@ -187,8 +212,20 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"readiness_percent": readiness_percent,
"blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only,
"latest_verify_only_metric_present": latest_verify_only_metric_present,
"controlled_service_data_backup_readback_status": rollups[
"controlled_service_data_backup_readback_status"
],
"controlled_service_data_backup_blocker_count": rollups[
"controlled_service_data_backup_blocker_count"
],
"controlled_service_data_backup_next_safe_action": rollups[
"controlled_service_data_backup_next_safe_action"
],
},
"reboot_sop_progress": sop_progress,
"controlled_service_data_backup_readback": (
controlled_service_data_backup_readback
),
"host_boot_detection": host_boot_detection,
"post_reboot_readiness": post_reboot_readiness,
"stockplatform_data_freshness": stockplatform,
@@ -211,6 +248,110 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
}
def _build_controlled_service_data_backup_readback(
*,
post_reboot_readiness: dict[str, Any],
stockplatform: dict[str, Any],
active_blockers: list[str],
) -> dict[str, Any]:
service_green = post_reboot_readiness.get("service_green") is True
product_data_green = post_reboot_readiness.get("product_data_green") is True
backup_core_green = post_reboot_readiness.get("backup_core_green") is True
host_188_service_green = post_reboot_readiness.get("host_188_service_green") is True
post_start_blocked = _int(post_reboot_readiness.get("post_start_blocked"))
wazuh_dashboard_degraded = (
post_reboot_readiness.get("wazuh_dashboard_degraded") is True
)
stockplatform_freshness_status = str(
stockplatform.get("freshness_status") or "unknown"
)
stockplatform_ingestion_status = str(
stockplatform.get("ingestion_status") or "unknown"
)
blocking_fields: list[str] = []
if not service_green:
blocking_fields.append("service_green")
if post_start_blocked != 0:
blocking_fields.append("post_start_blocked")
if not product_data_green:
blocking_fields.append("product_data_green")
if not backup_core_green:
blocking_fields.append("backup_core_green")
if not host_188_service_green:
blocking_fields.append("host_188_service_green")
if wazuh_dashboard_degraded:
blocking_fields.append("wazuh_dashboard_degraded")
if stockplatform_freshness_status != "ok":
blocking_fields.append("stockplatform_freshness_status")
if stockplatform_ingestion_status not in {"ok", "unknown"}:
blocking_fields.append("stockplatform_ingestion_status")
related_blockers = [
blocker
for blocker in active_blockers
if blocker
in {
"backup_core_green_not_1",
"host_188_service_green_not_1",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"service_green_not_1",
"stockplatform_freshness_blocked",
"stockplatform_ingestion_blocked",
"wazuh_dashboard_degraded",
}
]
readback_present = (
post_reboot_readiness.get("summary_present") is True
or stockplatform.get("freshness_endpoint_readback_present") is True
or stockplatform.get("ingestion_endpoint_readback_present") is True
)
can_clear_blockers = readback_present and not blocking_fields
return {
"schema_version": "controlled_service_data_backup_readback_v1",
"status": (
"ready_service_data_backup_green"
if can_clear_blockers
else "blocked_service_data_backup_readback_not_green"
),
"readback_present": readback_present,
"service_green": service_green,
"product_data_green": product_data_green,
"backup_core_green": backup_core_green,
"host_188_service_green": host_188_service_green,
"post_start_blocked": post_start_blocked,
"wazuh_dashboard_degraded": wazuh_dashboard_degraded,
"stockplatform_freshness_status": stockplatform_freshness_status,
"stockplatform_ingestion_status": stockplatform_ingestion_status,
"stockplatform_freshness_blockers": _strings(
stockplatform.get("freshness_blockers")
),
"stockplatform_ingestion_blockers": _strings(
stockplatform.get("ingestion_blockers")
),
"blocking_fields": blocking_fields,
"related_active_blockers": related_blockers,
"can_clear_service_data_backup_blockers": can_clear_blockers,
"next_safe_action": (
"keep_timer_live_wait_for_fresh_all_host_reboot_event_or_drill_preflight"
if can_clear_blockers
else "collect_public_service_data_backup_readback_and_refresh_reboot_slo_scorecard_no_reboot_no_db_write"
),
"controlled_lane": "read_only_verifier_input",
"forbidden_actions": [
"host_reboot",
"service_restart",
"manual_db_update",
"truncate_or_restore",
"fake_freshness_marker",
"secret_value_read",
"workflow_dispatch",
"github_api",
],
"runtime_write_allowed": False,
}
def _build_reboot_sop_progress(
*,
scorecard: dict[str, Any],