wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(api): align priority work order with reboot slo truth
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 31s
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name
2026-06-30 21:15:10 +08:00
parent 62a4702dd8
commit 454bec0bf2
3 changed files with 141 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
apps/api/src/services/awoooi_priority_work_order_readback.py
View File

@@ -778,35 +778,64 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
from src.services.reboot_auto_recovery_drill_preflight import (
load_latest_reboot_auto_recovery_drill_preflight,
)
from src.services.reboot_auto_recovery_slo_scorecard import (
load_latest_reboot_auto_recovery_slo_scorecard,
)
workbench = load_delivery_closure_workbench()
workbench_summary = _dict(workbench.get("summary"))
workbench_readback = _dict(workbench.get("readback"))
workbench_rollups = _dict(workbench.get("rollups"))
template_apply_gate = load_latest_awoooi_gitea_onboarding_warning_step_template_copy_apply_gate()
template_receipt = load_latest_awoooi_gitea_onboarding_warning_step_template_copy_receipt()
runtime_gate = load_latest_awoooi_gitea_onboarding_warning_step_runtime_enablement_gate()
reboot_preflight = load_latest_reboot_auto_recovery_drill_preflight()
reboot_slo = load_latest_reboot_auto_recovery_slo_scorecard()
reboot_slo_rollups = _dict(reboot_slo.get("rollups"))
reboot_stockplatform = _dict(reboot_slo.get("stockplatform_data_freshness"))
reboot_preflight_rollups = _dict(reboot_preflight.get("rollups"))
reboot_preflight_target_selector = _dict(reboot_preflight.get("target_selector"))
reboot_active_blockers = _strings(reboot_slo.get("active_blockers"))
p0_006_event_gated = bool(
reboot_slo_rollups.get("blocked_by_fresh_reboot_window_only") is True
)
stock_freshness_status = str(
reboot_slo_rollups.get("stockplatform_freshness_status")
or reboot_stockplatform.get("freshness_status")
or "unknown"
)
stock_ingestion_status = str(
reboot_slo_rollups.get("stockplatform_ingestion_status")
or reboot_stockplatform.get("ingestion_status")
or "unknown"
)
stock_blockers = _strings(reboot_stockplatform.get("freshness_blockers"))
stock_ingestion_blockers = _strings(
reboot_stockplatform.get("ingestion_blockers")
)
state = _dict(payload.setdefault("mainline_execution_state", {}))
state["active_p0_workplan_id"] = str(
workbench_readback.get("current_p0_workplan_id") or "P0-006"
)
state["active_p0_state"] = "event_gated_waiting_fresh_all_host_reboot_window"
state["active_p0_state"] = (
"event_gated_waiting_fresh_all_host_reboot_window"
if p0_006_event_gated
else "blocked_reboot_auto_recovery_slo_not_ready"
)
state["active_p0_immediate_apply_gap_count"] = 0
state["active_p0_live_api_status"] = str(
workbench_readback.get("current_p0_status")
reboot_slo.get("status")
or workbench_readback.get("current_p0_status")
or "blocked_reboot_auto_recovery_slo_not_ready"
)
state["active_p0_live_active_blockers"] = _strings(
workbench_readback.get("current_p0_active_blockers")
)
state["active_p0_live_active_blockers"] = reboot_active_blockers
state["active_p0_readiness_percent"] = _int(
workbench_readback.get("current_p0_readiness_percent")
reboot_slo.get("readiness_percent")
or workbench_readback.get("current_p0_readiness_percent")
)
state["active_p0_can_claim_10_minute_recovery"] = bool(
reboot_slo.get("can_claim_all_services_recovered_within_target") is True
)
state["active_p0_can_claim_10_minute_recovery"] = False
state["stale_snapshot_or_old_cd_runs_must_not_reopen_closed_work"] = True
state["p0_004_template_copy_apply_gate_production_http_status"] = 200
state["p0_004_template_copy_apply_gate_runtime_readback_state"] = (
@@ -836,10 +865,16 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
state["reboot_drill_preflight_status"] = str(reboot_preflight.get("status") or "")
state["next_executable_mainline_workplan_id"] = (
"P0-006-REBOOT-DRILL-PREFLIGHT-READBACK"
if p0_006_event_gated
else "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD"
)
state["next_executable_mainline_state"] = (
"production_preflight_ready_wait_for_next_real_all_host_reboot_event_"
"or_separate_break_glass_reboot_drill_authorization"
(
"production_preflight_ready_wait_for_next_real_all_host_reboot_event_"
"or_separate_break_glass_reboot_drill_authorization"
)
if p0_006_event_gated
else "blocked_reboot_slo_scorecard_requires_controlled_service_data_backup_readback"
)
runtime_sha = str(
@@ -907,31 +942,6 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
evidence["production_deploy_governance_fields_present"] = closure_percent == 100
evidence["latest_cd_run_status"] = "Success"
stock_freshness_status = str(
workbench_rollups.get("current_p0_stockplatform_freshness_status") or ""
)
stock_ingestion_status = str(
workbench_rollups.get("current_p0_stockplatform_ingestion_status") or ""
)
stock_blockers = _stockplatform_status_blockers(
freshness_status=stock_freshness_status,
ingestion_status=stock_ingestion_status,
)
stockplatform_green = not stock_blockers
service_green = workbench_rollups.get("current_p0_service_green") is True
product_data_green = (
workbench_rollups.get("current_p0_product_data_green") is True
)
backup_core_green = workbench_rollups.get("current_p0_backup_core_green") is True
drill_preflight_ready = reboot_preflight_rollups.get("preflight_ready") is True
base_readback_green = (
service_green
and product_data_green
and backup_core_green
and stockplatform_green
and drill_preflight_ready
)
for item in _list(payload.get("in_progress_or_blocked_in_priority_order")):
workplan = _dict(item)
if workplan.get("workplan_id") != "P0-006":
@@ -939,24 +949,28 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
evidence = _dict(workplan.setdefault("evidence", {}))
evidence["active_blockers"] = state["active_p0_live_active_blockers"]
evidence["service_green"] = service_green
evidence["product_data_green"] = product_data_green
evidence["backup_core_green"] = backup_core_green
evidence["service_green"] = bool(reboot_slo.get("service_green") is True)
evidence["product_data_green"] = bool(
reboot_slo.get("product_data_green") is True
)
evidence["backup_core_green"] = bool(
reboot_slo.get("backup_core_green") is True
)
evidence["host_188_service_green"] = bool(
reboot_slo.get("host_188_service_green") is True
)
evidence["stock_freshness_status"] = stock_freshness_status
evidence["stock_ingestion_status"] = stock_ingestion_status
evidence["stock_blockers"] = [
blocker
for blocker in stock_blockers
if blocker.startswith("stockplatform_freshness_")
]
evidence["stock_ingestion_blockers"] = [
blocker
for blocker in stock_blockers
if blocker.startswith("stockplatform_ingestion_")
]
evidence["stockplatform_controlled_recovery_gate_required"] = False
evidence["stock_blockers"] = stock_blockers
evidence["stock_ingestion_blockers"] = stock_ingestion_blockers
evidence["stockplatform_controlled_recovery_gate_required"] = bool(
reboot_slo_rollups.get("stockplatform_controlled_recovery_gate_required")
is True
)
evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "")
evidence["drill_preflight_ready"] = drill_preflight_ready
evidence["drill_preflight_ready"] = (
reboot_preflight_rollups.get("preflight_ready") is True
)
evidence["drill_preflight_blocker_count"] = _int(
reboot_preflight_rollups.get("preflight_blocker_count")
)
@@ -985,11 +999,7 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
)
professional_fix = _dict(workplan.setdefault("professional_fix", {}))
professional_fix["owner"] = (
"reboot auto-recovery lane plus drill preflight readback"
)
if base_readback_green:
if p0_006_event_gated:
professional_fix["action"] = (
"Keep the live boot-triggered SLO timer enabled. StockPlatform "
"freshness/ingestion are ok; the only active blocker is the fresh "
@@ -1012,42 +1022,37 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
"approved_reboot_drill_to_prove_10_minute_slo"
)
workplan["status"] = "blocked_waiting_fresh_all_host_reboot_window"
continue
professional_fix["action"] = (
"Keep the live boot-triggered SLO timer enabled, but do not claim "
"reboot-window-only closure while service/data/backup/StockPlatform "
"readback is not green. First run the controlled source freshness and "
"Postgres contract readback, then rerun public health, freshness, "
"ingestion, backup, and drill preflight verifiers. Do not reboot, "
"restart services, write DB rows, fake freshness, trigger workflows, "
"or read secrets from this lane."
else:
professional_fix["action"] = (
"Continue P0-006 from the reboot SLO scorecard truth: close the "
"service, product-data, backup, 188 service, host reachability, "
"disk, and Wazuh blockers in controlled lanes; for StockPlatform "
"freshness/ingestion postgres_not_ready, use the production "
"migration/control-channel path with target selector, dry-run, "
"rollback, public API verifier, and KM/RAG/MCP/PlayBook writeback. "
"Do not reboot, restart services, write DB rows, fake freshness, "
"trigger workflows, or read secrets from this lane."
)
workplan["reason"] = (
"The current reboot SLO scorecard is blocked, not merely waiting "
"for a fresh reboot window. Service/product-data/backup and "
"StockPlatform freshness/ingestion evidence must stay visible in "
"the priority readback until controlled recovery evidence closes."
)
workplan["safe_next_step"] = str(reboot_slo.get("safe_next_step") or "")
workplan["status"] = "blocked_reboot_auto_recovery_slo_not_ready"
professional_fix["owner"] = (
"reboot auto-recovery lane plus drill preflight readback"
)
workplan["reason"] = (
"Current P0-006 readback is not green: service_green="
f"{service_green}, product_data_green={product_data_green}, "
f"backup_core_green={backup_core_green}, "
f"stock_freshness_status={stock_freshness_status}, "
f"stock_ingestion_status={stock_ingestion_status}, "
f"drill_preflight_ready={drill_preflight_ready}."
)
workplan["safe_next_step"] = (
"resolve_stockplatform_source_freshness_and_reboot_slo_readback_"
"blockers_then_rerun_priority_work_order"
)
workplan["status"] = "blocked_p0_006_readback_not_green"
p0_004_ready = (
state["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready"
and state["p0_004_runtime_enablement_runtime_readback_state"] == "ready"
)
p0_006_event_gated = workbench_rollups.get(
"current_p0_blocked_by_fresh_reboot_window_only"
) is True
payload["status"] = (
"p0_006_event_gated_all_immediate_apply_gaps_closed"
if p0_004_ready and p0_006_event_gated
else "mainline_readback_requires_attention"
else "p0_006_blocked_reboot_auto_recovery_slo_not_ready"
)
if p0_006_event_gated:
payload["next_execution_order"] = [
@@ -1075,35 +1080,23 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
),
]
else:
state["active_p0_state"] = "blocked_p0_006_readback_not_green"
state["next_executable_mainline_workplan_id"] = (
"P0-006-STOCKPLATFORM-DATA-DEPENDENCY-READBACK"
if not stockplatform_green
else "P0-006-REBOOT-SLO-READBACK-RECOVERY"
)
state["next_executable_mainline_state"] = (
"controlled_stockplatform_source_freshness_and_postgres_contract_"
"readback_required"
if not stockplatform_green
else "controlled_reboot_slo_readback_recovery_required"
)
payload["next_execution_order"] = [
(
"P0-006: service/data/backup/StockPlatform readback is not green; "
"resolve controlled source freshness, Postgres contract, backup, "
"and reboot SLO readback blockers before claiming reboot-window-only "
"closure. Do not reboot/restart/DB-write/fake freshness from this lane."
"P0-006: reboot SLO scorecard is blocked; close the active "
"service/product-data/backup/host/Wazuh blockers in controlled "
"lanes, and keep StockPlatform postgres_not_ready on the "
"production migration/control-channel path with verifier and "
"metadata writeback."
),
(
"P0-006-STOCKPLATFORM-DATA-DEPENDENCY-READBACK: StockPlatform "
f"freshness={stock_freshness_status} and "
f"ingestion={stock_ingestion_status}; run read-only source "
"freshness/Postgres contract readback and controlled verifier."
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: "
"freshness/ingestion are not_configured with postgres_not_ready; "
"do not manually DB-write or fake freshness."
),
(
"P0-006-REBOOT-DRILL-PREFLIGHT-READBACK: keep preflight as "
"evidence only while base service/data/backup readback is not green; "
"do not reboot without separate break-glass drill authorization."
"P0-006-REBOOT-DRILL-PREFLIGHT-READBACK: drill preflight remains "
"blocked until service/product-data/backup/host readiness turns "
"green; reboot stays break-glass."
),
(
"NEXT: keep this priority-order API as the source of truth before "
@@ -1334,23 +1327,6 @@ def _unique_strings(values: list[str]) -> list[str]:
return unique
def _stockplatform_status_blockers(
*,
freshness_status: str,
ingestion_status: str,
) -> list[str]:
blockers: list[str] = []
if freshness_status != "ok":
blockers.append(
f"stockplatform_freshness_{freshness_status or 'unknown'}"
)
if ingestion_status != "ok":
blockers.append(
f"stockplatform_ingestion_{ingestion_status or 'unknown'}"
)
return blockers
def _is_sha(value: str) -> bool:
return bool(_SHA_RE.fullmatch(value))