diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index 7260c29e1..ee0f63bba 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -62,6 +62,14 @@ _PARALLEL_MAINLINE_FORBIDDEN_ACTIONS = [ "db_write_or_restore_from_this_wait_lane", "docker_or_host_restart_for_source_pending", ] +_REBOOT_HOST_BOOT_DETECTION_BLOCKERS = { + "all_required_hosts_not_in_10_minute_reboot_window", + "fresh_all_host_reboot_event_missing", + "host_boot_observation_older_than_target_window", + "host_unreachable_after_reboot", + "host_uptime_unknown", + "reboot_event_required_host_unreachable", +} _AI_AUTOMATION_NODE_RECEIPT_SCHEMA_VERSION = "ai_automation_node_receipt_v1" _AI_AUTOMATION_NODE_RECEIPT_REQUIRED_FIELDS = [ "trace_id", @@ -2519,6 +2527,63 @@ def _harbor_registry_next_state(*, status: str, candidate_packaged: bool) -> str return "controlled_recovery_waiting_registry_v2_evidence_and_executor_contract" +def _reboot_slo_next_mainline_state( + *, + event_gated: bool, + service_data_backup_readback_present: bool, + service_data_backup_blocking_fields: list[str], + windows99_verify_collection_blockers: list[str], + active_blockers: list[str], +) -> str: + if event_gated: + return ( + "production_preflight_ready_wait_for_next_real_all_host_reboot_event_" + "or_separate_break_glass_reboot_drill_authorization" + ) + if not service_data_backup_readback_present: + return ( + "blocked_reboot_slo_scorecard_requires_controlled_service_data_" + "backup_readback" + ) + if service_data_backup_blocking_fields: + return ( + "blocked_reboot_slo_scorecard_service_data_backup_readback_present_" + "blockers_open" + ) + + has_windows99_blockers = bool( + windows99_verify_collection_blockers + or [blocker for blocker in active_blockers if blocker.startswith("windows99_")] + ) + has_host_boot_blockers = bool( + [ + blocker + for blocker in active_blockers + if blocker in _REBOOT_HOST_BOOT_DETECTION_BLOCKERS + ] + ) + if has_host_boot_blockers and has_windows99_blockers: + return ( + "blocked_reboot_slo_scorecard_host_boot_and_windows99_verify_" + "collection_not_ready" + ) + if has_windows99_blockers: + return "blocked_reboot_slo_scorecard_windows99_verify_collection_not_ready" + if has_host_boot_blockers: + return "blocked_reboot_slo_scorecard_host_boot_detection_not_ready" + if active_blockers: + return "blocked_reboot_slo_scorecard_active_blockers_open" + return "blocked_reboot_slo_scorecard_waiting_post_verifier_readback" + + +def _service_data_backup_blockers_closed( + *, + service_data_backup_readback_present: bool, + service_data_backup_blocking_fields: list[str], +) -> bool: + return service_data_backup_readback_present and not service_data_backup_blocking_fields + + def _harbor_registry_reason(*, status: str, candidate_packaged: bool) -> str: if candidate_packaged and "upstream_502" in status: return ( @@ -3150,18 +3215,12 @@ def _enrich_from_current_readbacks( if p0_006_event_gated else "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD" ) - state["next_executable_mainline_state"] = ( - ( - "production_preflight_ready_wait_for_next_real_all_host_reboot_event_" - "or_separate_break_glass_reboot_drill_authorization" - ) - if p0_006_event_gated - else ( - "blocked_reboot_slo_scorecard_service_data_backup_readback_present_" - "blockers_open" - if service_data_backup_readback_present - else "blocked_reboot_slo_scorecard_requires_controlled_service_data_backup_readback" - ) + state["next_executable_mainline_state"] = _reboot_slo_next_mainline_state( + event_gated=p0_006_event_gated, + service_data_backup_readback_present=service_data_backup_readback_present, + service_data_backup_blocking_fields=service_data_backup_blocking_fields, + windows99_verify_collection_blockers=windows99_verify_collection_blockers, + active_blockers=reboot_active_blockers, ) runtime_sha = str( @@ -3408,27 +3467,58 @@ def _enrich_from_current_readbacks( ) workplan["status"] = "blocked_waiting_fresh_all_host_reboot_window" else: + service_backup_closed = _service_data_backup_blockers_closed( + service_data_backup_readback_present=service_data_backup_readback_present, + service_data_backup_blocking_fields=service_data_backup_blocking_fields, + ) professional_fix["action"] = ( - "Continue P0-006 from the reboot SLO scorecard truth: close the " - "service, product-data, backup, 188 service, host reachability, " - "disk, and Wazuh blockers in controlled lanes; keep the " - "controlled service/data/backup readback visible as the source " - "selector for these blockers. For Windows 99, use the " - "windows99_verify_collection packet to collect no-secret Verify " - "stdout, keep windows99_management_channel visible for the current " - "remote execution blocker, and rerun the scorecard. For StockPlatform " - "freshness/ingestion postgres_not_ready, use the production " - "migration/control-channel path with target selector, dry-run, " - "rollback, public API verifier, and KM/RAG/MCP/PlayBook writeback. " - "Do not reboot, restart services, write DB rows, fake freshness, " - "trigger workflows, or read secrets from this lane." + ( + "Continue P0-006 from the reboot SLO scorecard truth: " + "service/data/backup readback blockers are closed, so the " + "visible next blocker groups are host boot/reboot-event " + "detection and Windows99 verify collection. Run the " + "reboot-event detector and host probe in verifier-only mode, " + "collect the Windows99 no-secret Verify/local-console " + "readback through the visible windows99_verify_collection " + "packet, then rerun the scorecard. Do not reboot, restart " + "services, write DB rows, fake freshness, trigger workflows, " + "or read secrets from this lane." + ) + if service_backup_closed + else ( + "Continue P0-006 from the reboot SLO scorecard truth: close " + "the service, product-data, backup, 188 service, host " + "reachability, disk, and Wazuh blockers in controlled lanes; " + "keep the controlled service/data/backup readback visible as " + "the source selector for these blockers. For Windows 99, use " + "the windows99_verify_collection packet to collect no-secret " + "Verify stdout, keep windows99_management_channel visible for " + "the current remote execution blocker, and rerun the " + "scorecard. For StockPlatform freshness/ingestion " + "postgres_not_ready, use the production migration/control-" + "channel path with target selector, dry-run, rollback, public " + "API verifier, and KM/RAG/MCP/PlayBook writeback. Do not " + "reboot, restart services, write DB rows, fake freshness, " + "trigger workflows, or read secrets from this lane." + ) ) workplan["reason"] = ( - "The current reboot SLO scorecard is blocked, not merely waiting " - "for a fresh reboot window. Controlled service/product-data/backup " - "readback is present, but its blocker fields remain open and must " - "stay visible in the priority readback until controlled recovery " - "evidence closes." + ( + "The current reboot SLO scorecard is blocked, not merely " + "waiting for a fresh reboot window. Controlled " + "service/product-data/backup readback is present and its " + "blocker fields are closed, so the priority readback now " + "routes the next mainline state to host boot/reboot-event " + "detection and Windows99 verify collection." + ) + if service_backup_closed + else ( + "The current reboot SLO scorecard is blocked, not merely " + "waiting for a fresh reboot window. Controlled " + "service/product-data/backup readback is present, but its " + "blocker fields remain open and must stay visible in the " + "priority readback until controlled recovery evidence closes." + ) ) workplan["safe_next_step"] = str(reboot_slo.get("safe_next_step") or "") workplan["status"] = "blocked_reboot_auto_recovery_slo_not_ready" @@ -3471,13 +3561,27 @@ def _enrich_from_current_readbacks( ), ] else: + service_backup_closed = _service_data_backup_blockers_closed( + service_data_backup_readback_present=service_data_backup_readback_present, + service_data_backup_blocking_fields=service_data_backup_blocking_fields, + ) payload["next_execution_order"] = [ ( - "P0-006: reboot SLO scorecard is blocked; close the active " - "service/product-data/backup/host/Wazuh blockers in controlled " - "lanes, and keep StockPlatform postgres_not_ready on the " - "production migration/control-channel path with verifier and " - "metadata writeback." + ( + "P0-006: service/data/backup readback is green, but reboot " + "SLO is still blocked by host boot/reboot-event detection " + "and Windows99 verify collection; run verifier-only host " + "probe plus no-secret Windows99 collection, then rerun the " + "scorecard. Do not reboot/restart/DB-write/read secrets." + ) + if service_backup_closed + else ( + "P0-006: reboot SLO scorecard is blocked; close the active " + "service/product-data/backup/host/Wazuh blockers in " + "controlled lanes, and keep StockPlatform postgres_not_ready " + "on the production migration/control-channel path with " + "verifier and metadata writeback." + ) ), ( "P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: " diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 2d44ae1de..8864bc42d 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -11,6 +11,7 @@ from fastapi.testclient import TestClient from src.api.v1 import agents from src.api.v1.agents import router from src.services import awoooi_priority_work_order_readback as priority_service +from src.services import reboot_auto_recovery_slo_scorecard as reboot_slo_scorecard from src.services.ai_agent_log_controlled_writeback_executor_readback import ( load_latest_ai_agent_log_controlled_writeback_executor_readback, ) @@ -20,7 +21,6 @@ from src.services.ai_agent_log_post_write_verifier_dry_run import ( from src.services.awoooi_priority_work_order_readback import ( apply_ai_automation_live_closure_readbacks, apply_ai_loop_current_blocker_execution_queue, - apply_controlled_cd_lane_live_metric_readback, apply_harbor_registry_controlled_recovery_preflight, apply_stockplatform_public_api_controlled_recovery_preflight, apply_stockplatform_public_api_runtime_readback, @@ -30,12 +30,12 @@ from src.services.awoooi_priority_work_order_readback import ( from src.services.harbor_registry_controlled_recovery_preflight import ( load_latest_harbor_registry_controlled_recovery_preflight, ) -from src.services.telegram_alert_learning_context_post_apply_verifier import ( - build_telegram_alert_learning_context_post_apply_verifier, -) from src.services.stockplatform_public_api_controlled_recovery_preflight import ( load_latest_stockplatform_public_api_controlled_recovery_preflight, ) +from src.services.telegram_alert_learning_context_post_apply_verifier import ( + build_telegram_alert_learning_context_post_apply_verifier, +) _REPO_ROOT = Path(__file__).resolve().parents[3] _SNAPSHOT_PATH = ( @@ -547,6 +547,127 @@ def test_awoooi_priority_work_order_readback_surfaces_gitea_bundle_truth(): ] +def test_awoooi_priority_work_order_readback_routes_closed_backup_to_host_boot_and_windows99( + monkeypatch: pytest.MonkeyPatch, +): + base_loader = reboot_slo_scorecard.load_latest_reboot_auto_recovery_slo_scorecard + + def _reboot_slo_backup_closed_host_windows99_blocked() -> dict: + payload = base_loader() + active_blockers = [ + "all_required_hosts_not_in_10_minute_reboot_window", + "host_boot_observation_older_than_target_window", + "host_unreachable_after_reboot", + "host_uptime_unknown", + "reboot_event_required_host_unreachable", + "windows99_vmware_guest_power_not_ready", + "windows99_vmware_vmx_missing", + ] + payload["active_blockers"] = active_blockers + payload["primary_blocker"] = "reboot_event_required_host_unreachable" + payload["current_phase"] = "host_boot_detection_blocked" + payload["readiness_percent"] = 67 + payload["next_safe_action"] = ( + "rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot" + ) + payload["post_reboot_readiness"].update( + { + "service_green": True, + "product_data_green": True, + "backup_core_green": True, + "host_188_service_green": True, + "post_start_blocked": 0, + "wazuh_dashboard_degraded": False, + } + ) + payload["controlled_service_data_backup_readback"].update( + { + "status": "ready_service_data_backup_green", + "readback_present": True, + "service_green": True, + "product_data_green": True, + "backup_core_green": True, + "host_188_service_green": True, + "post_start_blocked": 0, + "wazuh_dashboard_degraded": False, + "blocking_fields": [], + "can_clear_service_data_backup_blockers": True, + "next_safe_action": "rerun_reboot_slo_scorecard_no_reboot", + } + ) + payload["windows99_verify_collection"].update( + { + "status": "blocked_windows99_verify_collection_not_ready", + "readback_present": True, + "collection_blockers": [ + "windows99_vmware_guest_power_not_ready", + "windows99_vmware_vmx_missing", + ], + "can_collect_no_secret_verify": True, + "host99_reachable": True, + "host99_uptime_known": False, + } + ) + payload["rollups"].update( + { + "active_blocker_count": len(active_blockers), + "service_green": True, + "product_data_green": True, + "backup_core_green": True, + "host_188_service_green": True, + "post_start_blocked": 0, + "controlled_service_data_backup_readback_present": True, + "controlled_service_data_backup_readback_status": ( + "ready_service_data_backup_green" + ), + "controlled_service_data_backup_blocker_count": 0, + "controlled_service_data_backup_can_clear_blockers": True, + "windows99_verify_collection_status": ( + "blocked_windows99_verify_collection_not_ready" + ), + "windows99_verify_collection_blocker_count": 2, + "windows99_verify_collection_can_collect_no_secret": True, + } + ) + payload["readback"]["active_blocker_count"] = len(active_blockers) + return payload + + monkeypatch.setattr( + reboot_slo_scorecard, + "load_latest_reboot_auto_recovery_slo_scorecard", + _reboot_slo_backup_closed_host_windows99_blocked, + ) + + payload = load_latest_awoooi_priority_work_order_readback() + + state = payload["mainline_execution_state"] + in_progress = payload["in_progress_or_blocked_in_priority_order"][0] + assert state["controlled_service_data_backup_blocker_count"] == 0 + assert state["next_executable_mainline_state"] == ( + "blocked_reboot_slo_scorecard_host_boot_and_windows99_verify_" + "collection_not_ready" + ) + assert state["windows99_verify_collection_blockers"] == [ + "windows99_vmware_guest_power_not_ready", + "windows99_vmware_vmx_missing", + ] + assert payload["summary"]["controlled_service_data_backup_blocker_count"] == 0 + assert payload["summary"]["next_executable_mainline_state"] == ( + "blocked_reboot_slo_scorecard_host_boot_and_windows99_verify_" + "collection_not_ready" + ) + assert "service/data/backup readback is green" in payload[ + "next_execution_order" + ][0] + assert "service/product-data/backup/host/Wazuh blockers" not in payload[ + "next_execution_order" + ][0] + assert "blocker fields are closed" in in_progress["reason"] + assert "Windows99 verify collection" in in_progress["professional_fix"][ + "action" + ] + + def test_ai_automation_live_closure_readbacks_close_node_receipts(): payload = load_latest_awoooi_priority_work_order_readback() executor = load_latest_ai_agent_log_controlled_writeback_executor_readback()