diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index f1650ee73..072437f9d 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -778,35 +778,64 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None: from src.services.reboot_auto_recovery_drill_preflight import ( load_latest_reboot_auto_recovery_drill_preflight, ) + from src.services.reboot_auto_recovery_slo_scorecard import ( + load_latest_reboot_auto_recovery_slo_scorecard, + ) workbench = load_delivery_closure_workbench() workbench_summary = _dict(workbench.get("summary")) workbench_readback = _dict(workbench.get("readback")) - workbench_rollups = _dict(workbench.get("rollups")) template_apply_gate = load_latest_awoooi_gitea_onboarding_warning_step_template_copy_apply_gate() template_receipt = load_latest_awoooi_gitea_onboarding_warning_step_template_copy_receipt() runtime_gate = load_latest_awoooi_gitea_onboarding_warning_step_runtime_enablement_gate() reboot_preflight = load_latest_reboot_auto_recovery_drill_preflight() + reboot_slo = load_latest_reboot_auto_recovery_slo_scorecard() + reboot_slo_rollups = _dict(reboot_slo.get("rollups")) + reboot_stockplatform = _dict(reboot_slo.get("stockplatform_data_freshness")) reboot_preflight_rollups = _dict(reboot_preflight.get("rollups")) reboot_preflight_target_selector = _dict(reboot_preflight.get("target_selector")) + reboot_active_blockers = _strings(reboot_slo.get("active_blockers")) + p0_006_event_gated = bool( + reboot_slo_rollups.get("blocked_by_fresh_reboot_window_only") is True + ) + stock_freshness_status = str( + reboot_slo_rollups.get("stockplatform_freshness_status") + or reboot_stockplatform.get("freshness_status") + or "unknown" + ) + stock_ingestion_status = str( + reboot_slo_rollups.get("stockplatform_ingestion_status") + or reboot_stockplatform.get("ingestion_status") + or "unknown" + ) + stock_blockers = _strings(reboot_stockplatform.get("freshness_blockers")) + stock_ingestion_blockers = _strings( + reboot_stockplatform.get("ingestion_blockers") + ) state = _dict(payload.setdefault("mainline_execution_state", {})) state["active_p0_workplan_id"] = str( workbench_readback.get("current_p0_workplan_id") or "P0-006" ) - state["active_p0_state"] = "event_gated_waiting_fresh_all_host_reboot_window" + state["active_p0_state"] = ( + "event_gated_waiting_fresh_all_host_reboot_window" + if p0_006_event_gated + else "blocked_reboot_auto_recovery_slo_not_ready" + ) state["active_p0_immediate_apply_gap_count"] = 0 state["active_p0_live_api_status"] = str( - workbench_readback.get("current_p0_status") + reboot_slo.get("status") + or workbench_readback.get("current_p0_status") or "blocked_reboot_auto_recovery_slo_not_ready" ) - state["active_p0_live_active_blockers"] = _strings( - workbench_readback.get("current_p0_active_blockers") - ) + state["active_p0_live_active_blockers"] = reboot_active_blockers state["active_p0_readiness_percent"] = _int( - workbench_readback.get("current_p0_readiness_percent") + reboot_slo.get("readiness_percent") + or workbench_readback.get("current_p0_readiness_percent") + ) + state["active_p0_can_claim_10_minute_recovery"] = bool( + reboot_slo.get("can_claim_all_services_recovered_within_target") is True ) - state["active_p0_can_claim_10_minute_recovery"] = False state["stale_snapshot_or_old_cd_runs_must_not_reopen_closed_work"] = True state["p0_004_template_copy_apply_gate_production_http_status"] = 200 state["p0_004_template_copy_apply_gate_runtime_readback_state"] = ( @@ -836,10 +865,16 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None: state["reboot_drill_preflight_status"] = str(reboot_preflight.get("status") or "") state["next_executable_mainline_workplan_id"] = ( "P0-006-REBOOT-DRILL-PREFLIGHT-READBACK" + if p0_006_event_gated + else "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD" ) state["next_executable_mainline_state"] = ( - "production_preflight_ready_wait_for_next_real_all_host_reboot_event_" - "or_separate_break_glass_reboot_drill_authorization" + ( + "production_preflight_ready_wait_for_next_real_all_host_reboot_event_" + "or_separate_break_glass_reboot_drill_authorization" + ) + if p0_006_event_gated + else "blocked_reboot_slo_scorecard_requires_controlled_service_data_backup_readback" ) runtime_sha = str( @@ -907,31 +942,6 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None: evidence["production_deploy_governance_fields_present"] = closure_percent == 100 evidence["latest_cd_run_status"] = "Success" - stock_freshness_status = str( - workbench_rollups.get("current_p0_stockplatform_freshness_status") or "" - ) - stock_ingestion_status = str( - workbench_rollups.get("current_p0_stockplatform_ingestion_status") or "" - ) - stock_blockers = _stockplatform_status_blockers( - freshness_status=stock_freshness_status, - ingestion_status=stock_ingestion_status, - ) - stockplatform_green = not stock_blockers - service_green = workbench_rollups.get("current_p0_service_green") is True - product_data_green = ( - workbench_rollups.get("current_p0_product_data_green") is True - ) - backup_core_green = workbench_rollups.get("current_p0_backup_core_green") is True - drill_preflight_ready = reboot_preflight_rollups.get("preflight_ready") is True - base_readback_green = ( - service_green - and product_data_green - and backup_core_green - and stockplatform_green - and drill_preflight_ready - ) - for item in _list(payload.get("in_progress_or_blocked_in_priority_order")): workplan = _dict(item) if workplan.get("workplan_id") != "P0-006": @@ -939,24 +949,28 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None: evidence = _dict(workplan.setdefault("evidence", {})) evidence["active_blockers"] = state["active_p0_live_active_blockers"] - evidence["service_green"] = service_green - evidence["product_data_green"] = product_data_green - evidence["backup_core_green"] = backup_core_green + evidence["service_green"] = bool(reboot_slo.get("service_green") is True) + evidence["product_data_green"] = bool( + reboot_slo.get("product_data_green") is True + ) + evidence["backup_core_green"] = bool( + reboot_slo.get("backup_core_green") is True + ) + evidence["host_188_service_green"] = bool( + reboot_slo.get("host_188_service_green") is True + ) evidence["stock_freshness_status"] = stock_freshness_status evidence["stock_ingestion_status"] = stock_ingestion_status - evidence["stock_blockers"] = [ - blocker - for blocker in stock_blockers - if blocker.startswith("stockplatform_freshness_") - ] - evidence["stock_ingestion_blockers"] = [ - blocker - for blocker in stock_blockers - if blocker.startswith("stockplatform_ingestion_") - ] - evidence["stockplatform_controlled_recovery_gate_required"] = False + evidence["stock_blockers"] = stock_blockers + evidence["stock_ingestion_blockers"] = stock_ingestion_blockers + evidence["stockplatform_controlled_recovery_gate_required"] = bool( + reboot_slo_rollups.get("stockplatform_controlled_recovery_gate_required") + is True + ) evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "") - evidence["drill_preflight_ready"] = drill_preflight_ready + evidence["drill_preflight_ready"] = ( + reboot_preflight_rollups.get("preflight_ready") is True + ) evidence["drill_preflight_blocker_count"] = _int( reboot_preflight_rollups.get("preflight_blocker_count") ) @@ -985,11 +999,7 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None: ) professional_fix = _dict(workplan.setdefault("professional_fix", {})) - professional_fix["owner"] = ( - "reboot auto-recovery lane plus drill preflight readback" - ) - - if base_readback_green: + if p0_006_event_gated: professional_fix["action"] = ( "Keep the live boot-triggered SLO timer enabled. StockPlatform " "freshness/ingestion are ok; the only active blocker is the fresh " @@ -1012,42 +1022,37 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None: "approved_reboot_drill_to_prove_10_minute_slo" ) workplan["status"] = "blocked_waiting_fresh_all_host_reboot_window" - continue - - professional_fix["action"] = ( - "Keep the live boot-triggered SLO timer enabled, but do not claim " - "reboot-window-only closure while service/data/backup/StockPlatform " - "readback is not green. First run the controlled source freshness and " - "Postgres contract readback, then rerun public health, freshness, " - "ingestion, backup, and drill preflight verifiers. Do not reboot, " - "restart services, write DB rows, fake freshness, trigger workflows, " - "or read secrets from this lane." + else: + professional_fix["action"] = ( + "Continue P0-006 from the reboot SLO scorecard truth: close the " + "service, product-data, backup, 188 service, host reachability, " + "disk, and Wazuh blockers in controlled lanes; for StockPlatform " + "freshness/ingestion postgres_not_ready, use the production " + "migration/control-channel path with target selector, dry-run, " + "rollback, public API verifier, and KM/RAG/MCP/PlayBook writeback. " + "Do not reboot, restart services, write DB rows, fake freshness, " + "trigger workflows, or read secrets from this lane." + ) + workplan["reason"] = ( + "The current reboot SLO scorecard is blocked, not merely waiting " + "for a fresh reboot window. Service/product-data/backup and " + "StockPlatform freshness/ingestion evidence must stay visible in " + "the priority readback until controlled recovery evidence closes." + ) + workplan["safe_next_step"] = str(reboot_slo.get("safe_next_step") or "") + workplan["status"] = "blocked_reboot_auto_recovery_slo_not_ready" + professional_fix["owner"] = ( + "reboot auto-recovery lane plus drill preflight readback" ) - workplan["reason"] = ( - "Current P0-006 readback is not green: service_green=" - f"{service_green}, product_data_green={product_data_green}, " - f"backup_core_green={backup_core_green}, " - f"stock_freshness_status={stock_freshness_status}, " - f"stock_ingestion_status={stock_ingestion_status}, " - f"drill_preflight_ready={drill_preflight_ready}." - ) - workplan["safe_next_step"] = ( - "resolve_stockplatform_source_freshness_and_reboot_slo_readback_" - "blockers_then_rerun_priority_work_order" - ) - workplan["status"] = "blocked_p0_006_readback_not_green" p0_004_ready = ( state["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready" and state["p0_004_runtime_enablement_runtime_readback_state"] == "ready" ) - p0_006_event_gated = workbench_rollups.get( - "current_p0_blocked_by_fresh_reboot_window_only" - ) is True payload["status"] = ( "p0_006_event_gated_all_immediate_apply_gaps_closed" if p0_004_ready and p0_006_event_gated - else "mainline_readback_requires_attention" + else "p0_006_blocked_reboot_auto_recovery_slo_not_ready" ) if p0_006_event_gated: payload["next_execution_order"] = [ @@ -1075,35 +1080,23 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None: ), ] else: - state["active_p0_state"] = "blocked_p0_006_readback_not_green" - state["next_executable_mainline_workplan_id"] = ( - "P0-006-STOCKPLATFORM-DATA-DEPENDENCY-READBACK" - if not stockplatform_green - else "P0-006-REBOOT-SLO-READBACK-RECOVERY" - ) - state["next_executable_mainline_state"] = ( - "controlled_stockplatform_source_freshness_and_postgres_contract_" - "readback_required" - if not stockplatform_green - else "controlled_reboot_slo_readback_recovery_required" - ) payload["next_execution_order"] = [ ( - "P0-006: service/data/backup/StockPlatform readback is not green; " - "resolve controlled source freshness, Postgres contract, backup, " - "and reboot SLO readback blockers before claiming reboot-window-only " - "closure. Do not reboot/restart/DB-write/fake freshness from this lane." + "P0-006: reboot SLO scorecard is blocked; close the active " + "service/product-data/backup/host/Wazuh blockers in controlled " + "lanes, and keep StockPlatform postgres_not_ready on the " + "production migration/control-channel path with verifier and " + "metadata writeback." ), ( - "P0-006-STOCKPLATFORM-DATA-DEPENDENCY-READBACK: StockPlatform " - f"freshness={stock_freshness_status} and " - f"ingestion={stock_ingestion_status}; run read-only source " - "freshness/Postgres contract readback and controlled verifier." + "P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: " + "freshness/ingestion are not_configured with postgres_not_ready; " + "do not manually DB-write or fake freshness." ), ( - "P0-006-REBOOT-DRILL-PREFLIGHT-READBACK: keep preflight as " - "evidence only while base service/data/backup readback is not green; " - "do not reboot without separate break-glass drill authorization." + "P0-006-REBOOT-DRILL-PREFLIGHT-READBACK: drill preflight remains " + "blocked until service/product-data/backup/host readiness turns " + "green; reboot stays break-glass." ), ( "NEXT: keep this priority-order API as the source of truth before " @@ -1334,23 +1327,6 @@ def _unique_strings(values: list[str]) -> list[str]: return unique -def _stockplatform_status_blockers( - *, - freshness_status: str, - ingestion_status: str, -) -> list[str]: - blockers: list[str] = [] - if freshness_status != "ok": - blockers.append( - f"stockplatform_freshness_{freshness_status or 'unknown'}" - ) - if ingestion_status != "ok": - blockers.append( - f"stockplatform_ingestion_{ingestion_status or 'unknown'}" - ) - return blockers - - def _is_sha(value: str) -> bool: return bool(_SHA_RE.fullmatch(value)) diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 1868d9c41..7fe039fb7 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -35,28 +35,32 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order(): payload = load_latest_awoooi_priority_work_order_readback() assert payload["schema_version"] == "awoooi_priority_work_order_readback_v1" + assert payload["status"] == "p0_006_blocked_reboot_auto_recovery_slo_not_ready" assert payload["mainline_execution_state"]["active_p0_workplan_id"] == "P0-006" + assert payload["mainline_execution_state"]["active_p0_state"] == ( + "blocked_reboot_auto_recovery_slo_not_ready" + ) + assert payload["mainline_execution_state"][ + "next_executable_mainline_workplan_id" + ] == "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD" assert payload["mainline_execution_state"]["active_p0_immediate_apply_gap_count"] == 0 + assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 18 assert payload["mainline_execution_state"][ "stale_snapshot_or_old_cd_runs_must_not_reopen_closed_work" ] is True assert payload["next_execution_order"][0].startswith("P0-006:") in_progress = payload["in_progress_or_blocked_in_priority_order"][0] assert in_progress["workplan_id"] == "P0-006" - assert payload["mainline_execution_state"]["active_p0_state"] == ( - "blocked_p0_006_readback_not_green" - ) - assert payload["mainline_execution_state"]["next_executable_mainline_workplan_id"] == ( - "P0-006-STOCKPLATFORM-DATA-DEPENDENCY-READBACK" - ) - assert in_progress["status"] == "blocked_p0_006_readback_not_green" + assert in_progress["status"] == "blocked_reboot_auto_recovery_slo_not_ready" + assert in_progress["evidence"]["service_green"] is False + assert in_progress["evidence"]["product_data_green"] is False + assert in_progress["evidence"]["backup_core_green"] is False + assert in_progress["evidence"]["host_188_service_green"] is False assert in_progress["evidence"]["stock_freshness_status"] == "not_configured" assert in_progress["evidence"]["stock_ingestion_status"] == "not_configured" - assert in_progress["evidence"]["stock_blockers"] == [ - "stockplatform_freshness_not_configured" - ] + assert in_progress["evidence"]["stock_blockers"] == ["postgres_not_ready"] assert in_progress["evidence"]["stock_ingestion_blockers"] == [ - "stockplatform_ingestion_not_configured" + "postgres_not_ready" ] assert in_progress["evidence"]["drill_preflight_ready"] is False assert in_progress["evidence"]["drill_preflight_blocker_count"] == 9 @@ -66,12 +70,16 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order(): ] is False ) - assert "service/data/backup/StockPlatform readback is not green" in in_progress[ + assert "reboot SLO scorecard truth" in in_progress[ "professional_fix" ]["action"] - assert "Do not reboot" in in_progress[ + assert "production migration/control-channel" in in_progress[ "professional_fix" ]["action"] + assert "StockPlatform freshness/ingestion are ok" not in in_progress[ + "professional_fix" + ]["action"] + assert "not merely waiting for a fresh reboot window" in in_progress["reason"] assert payload["operation_boundaries"]["github_api_used"] is False assert payload["operation_boundaries"]["github_cli_used"] is False assert payload["operation_boundaries"]["secret_or_runner_token_read"] is False @@ -100,17 +108,19 @@ def test_awoooi_priority_work_order_readback_endpoint_returns_snapshot( assert response.status_code == 200 data = response.json() - assert data["status"] == "mainline_readback_requires_attention" + assert data["status"] == "p0_006_blocked_reboot_auto_recovery_slo_not_ready" assert data["mainline_execution_state"]["active_p0_workplan_id"] == "P0-006" - assert data["mainline_execution_state"]["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready" assert data["mainline_execution_state"]["active_p0_state"] == ( - "blocked_p0_006_readback_not_green" + "blocked_reboot_auto_recovery_slo_not_ready" ) + assert data["mainline_execution_state"][ + "next_executable_mainline_workplan_id" + ] == "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD" + assert data["mainline_execution_state"]["p0_004_template_copy_apply_gate_runtime_readback_state"] == "ready" assert data["mainline_execution_state"]["reboot_drill_preflight_runtime_readback_state"] == "blocked" assert data["rollups"]["stockplatform_public_api_runtime_ready"] is True - assert data["rollups"]["active_p0_event_gated_by_fresh_reboot_window_only"] is False assert data["next_execution_order"][0].startswith("P0-006:") - assert "readback is not green" in data["next_execution_order"][0] + assert "reboot SLO scorecard is blocked" in data["next_execution_order"][0] def test_awoooi_priority_work_order_readback_overlays_harbor_deploy_blocker( diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index ebb6359d0..a07739c7e 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,18 @@ +## 2026-06-30 — 21:13 Priority work-order readback 對齊 reboot SLO fail-closed 真相 + +**照主線修正的問題**： +- 最新 CD `#4070` 失敗點不是 Harbor login，而是 `test_awoooi_priority_work_order_readback_loader_returns_mainline_order` 仍期待 StockPlatform `stock_freshness_status=ok`；但當前 reboot SLO scorecard 已是 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `18%`、active blockers `12`、Stock `not_configured` / `postgres_not_ready`。 +- `apps/api/src/services/awoooi_priority_work_order_readback.py` 改為在 enrichment 時讀 `reboot-auto-recovery-slo-scorecard` 作為 P0-006 權威來源：service/product-data/backup/188 service、Stock freshness/ingestion、drill preflight 與 active blockers 都不再用舊 workbench 綠燈文案覆蓋。 +- Priority work-order 基礎狀態改為 `p0_006_blocked_reboot_auto_recovery_slo_not_ready`，下一步維持 `P0-006`，並明確指向 controlled service/data/backup/host/Wazuh lanes 與 StockPlatform `postgres_not_ready` production migration/control-channel path；不再宣稱只剩 fresh reboot window。 + +**驗證**： +- `pytest apps/api/tests/test_awoooi_priority_work_order_readback_api.py -q` 通過（8 passed）。 +- `pytest apps/api/tests/test_awoooi_priority_work_order_readback_api.py apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py apps/api/tests/test_stockplatform_public_api_runtime_readback.py apps/api/tests/test_stockplatform_public_api_controlled_recovery_preflight.py apps/api/tests/test_delivery_closure_workbench_api.py -q` 通過（25 passed）。 +- `pytest ops/runner/test_cd_controlled_runtime_profile.py -q` 通過（31 passed）。 +- `py_compile`、`ruff check`、`git diff --check` 通過。 + +**邊界**：只改 priority readback loader / API tests / LOGBOOK；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未 SSH / Docker / Nginx / K3s / DB / firewall runtime 寫入。 + ## 2026-06-30 — 21:02 Harbor 110 repair jobs stale / generic label classifier **照主線修正的問題**：