From f574885c8b925456851da88eedbd62eb165a2e5b Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Jun 2026 21:03:29 +0800 Subject: [PATCH] fix(api): align workbench reboot slo test --- .../test_delivery_closure_workbench_api.py | 157 ++++++++++-------- 1 file changed, 90 insertions(+), 67 deletions(-) diff --git a/apps/api/tests/test_delivery_closure_workbench_api.py b/apps/api/tests/test_delivery_closure_workbench_api.py index 569a83feb..4c0db75fe 100644 --- a/apps/api/tests/test_delivery_closure_workbench_api.py +++ b/apps/api/tests/test_delivery_closure_workbench_api.py @@ -6,6 +6,36 @@ from fastapi.testclient import TestClient from src.api.v1.agents import router from src.services.delivery_closure_workbench import load_delivery_closure_workbench +REBOOT_SAFE_NEXT_STEP = ( + "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_rerun_" + "scorecard_until_status_slo_ready" +) +REBOOT_ACTIVE_BLOCKERS = [ + "all_required_hosts_not_in_10_minute_reboot_window", + "backup_core_green_not_1", + "host_188_service_green_not_1", + "host_boot_observation_older_than_target_window", + "host_unreachable_after_reboot", + "host_uptime_unknown", + "local_disk_free_below_minimum", + "post_start_blocked_not_zero", + "product_data_green_not_1", + "reboot_event_required_host_unreachable", + "service_green_not_1", + "wazuh_dashboard_degraded", +] +DRILL_ACTIVE_BLOCKERS = [ + "service_green_not_ready", + "product_data_green_not_ready", + "backup_core_green_not_ready", + "host_188_service_green_not_ready", + "all_required_hosts_reachable_not_ready", + "latest_verify_only_metric_present_not_ready", + "stockplatform_freshness_ok_not_ready", + "stockplatform_ingestion_ok_not_ready", + "blocked_only_by_fresh_reboot_window_not_ready", +] + def test_delivery_closure_workbench_endpoint_returns_product_summary(): app = FastAPI() @@ -189,32 +219,32 @@ def test_delivery_closure_workbench_exposes_p0_006_reboot_slo_lane(): assert lane["source_id"] == "reboot_auto_recovery_slo_scorecard" assert lane["status"] == "blocked_reboot_auto_recovery_slo_not_ready" - assert lane["blocker_count"] == 1 - assert lane["completion_percent"] == 82 + assert lane["blocker_count"] == 12 + assert lane["completion_percent"] == 18 assert lane["metric"]["kind"] == "reboot_auto_recovery_slo" assert lane["metric"]["workplan_id"] == "P0-006" assert lane["metric"]["target_minutes"] == 10 assert lane["metric"]["can_claim_all_services_recovered_within_target"] is False - assert lane["metric"]["observed_host_count"] == 4 + assert lane["metric"]["observed_host_count"] == 7 assert lane["metric"]["missing_host_count"] == 0 - assert lane["metric"]["unreachable_host_count"] == 0 - assert lane["metric"]["stale_host_count"] == 4 - assert lane["metric"]["service_green"] is True - assert lane["metric"]["product_data_green"] is True - assert lane["metric"]["backup_core_green"] is True - assert lane["metric"]["stockplatform_freshness_status"] == "ok" - assert lane["metric"]["stockplatform_ingestion_status"] == "ok" - assert lane["metric"]["stockplatform_freshness_blocker_count"] == 0 - assert lane["metric"]["stockplatform_ingestion_blocker_count"] == 0 - assert lane["metric"]["stockplatform_final_retry_window_passed"] is True + assert lane["metric"]["unreachable_host_count"] == 1 + assert lane["metric"]["stale_host_count"] == 5 + assert lane["metric"]["service_green"] is False + assert lane["metric"]["product_data_green"] is False + assert lane["metric"]["backup_core_green"] is False + assert lane["metric"]["stockplatform_freshness_status"] == "not_configured" + assert lane["metric"]["stockplatform_ingestion_status"] == "not_configured" + assert lane["metric"]["stockplatform_freshness_blocker_count"] == 1 + assert lane["metric"]["stockplatform_ingestion_blocker_count"] == 1 + assert lane["metric"]["stockplatform_final_retry_window_passed"] is False assert lane["metric"]["stockplatform_controlled_recovery_gate_required"] is False assert lane["metric"]["drill_preflight_status"] == ( - "ready_for_break_glass_reboot_drill_authorization" + "blocked_reboot_drill_preflight_not_ready" ) - assert lane["metric"]["drill_preflight_ready"] is True - assert lane["metric"]["drill_preflight_ready_count"] == 1 - assert lane["metric"]["drill_preflight_blocker_count"] == 0 - assert lane["metric"]["drill_preflight_active_blockers"] == [] + assert lane["metric"]["drill_preflight_ready"] is False + assert lane["metric"]["drill_preflight_ready_count"] == 0 + assert lane["metric"]["drill_preflight_blocker_count"] == 9 + assert lane["metric"]["drill_preflight_active_blockers"] == DRILL_ACTIVE_BLOCKERS assert ( lane["metric"]["drill_preflight_break_glass_authorization_required"] is True @@ -228,11 +258,11 @@ def test_delivery_closure_workbench_exposes_p0_006_reboot_slo_lane(): is False ) assert lane["metric"]["drill_preflight_runtime_write_allowed"] is False - assert lane["metric"]["drill_preflight_target_required_host_count"] == 4 - assert lane["metric"]["drill_preflight_target_observed_host_count"] == 4 + assert lane["metric"]["drill_preflight_target_required_host_count"] == 7 + assert lane["metric"]["drill_preflight_target_observed_host_count"] == 7 assert lane["metric"]["drill_preflight_target_missing_host_count"] == 0 - assert lane["metric"]["drill_preflight_target_unreachable_host_count"] == 0 - assert lane["metric"]["drill_preflight_target_stale_host_count"] == 4 + assert lane["metric"]["drill_preflight_target_unreachable_host_count"] == 1 + assert lane["metric"]["drill_preflight_target_stale_host_count"] == 5 assert lane["metric"]["drill_preflight_verify_only_available"] is True assert lane["metric"]["drill_preflight_post_apply_verifier_endpoint"] == ( "/api/v1/agents/reboot-auto-recovery-slo-scorecard" @@ -246,24 +276,14 @@ def test_delivery_closure_workbench_exposes_p0_006_reboot_slo_lane(): assert lane["metric"]["service_restart_performed"] is False assert lane["metric"]["database_write_or_restore_performed"] is False assert lane["metric"]["secret_value_collection_allowed"] is False - assert lane["metric"]["active_blockers"] == [ - "host_boot_observation_older_than_target_window" - ] - assert lane["next_action"] == ( - "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_event_" - "or_approved_reboot_drill_to_prove_10_minute_slo" - ) + assert lane["metric"]["active_blockers"] == REBOOT_ACTIVE_BLOCKERS + assert lane["next_action"] == REBOOT_SAFE_NEXT_STEP def _assert_delivery_workbench_shape(data: dict): assert data["schema_version"] == "delivery_closure_workbench_v1" - assert data["safe_next_step"] == ( - "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_event_" - "or_approved_reboot_drill_to_prove_10_minute_slo" - ) - assert data["active_blockers"] == [ - "host_boot_observation_older_than_target_window" - ] + assert data["safe_next_step"] == REBOOT_SAFE_NEXT_STEP + assert data["active_blockers"] == REBOOT_ACTIVE_BLOCKERS assert data["readback"]["workbench_status"] == "blocked_delivery_actions_required" assert data["readback"]["current_p0_workplan_id"] == "P0-006" assert data["readback"]["current_p0_source_id"] == ( @@ -274,11 +294,11 @@ def _assert_delivery_workbench_shape(data: dict): ) assert data["readback"]["current_p0_safe_next_step"] == data["safe_next_step"] assert data["readback"]["current_p0_active_blockers"] == data["active_blockers"] - assert data["readback"]["current_p0_readiness_percent"] == 82 + assert data["readback"]["current_p0_readiness_percent"] == 18 assert data["readback"]["current_p0_drill_preflight_status"] == ( - "ready_for_break_glass_reboot_drill_authorization" + "blocked_reboot_drill_preflight_not_ready" ) - assert data["readback"]["current_p0_drill_preflight_ready"] is True + assert data["readback"]["current_p0_drill_preflight_ready"] is False assert ( data["readback"]["current_p0_drill_preflight_break_glass_required"] is True @@ -299,17 +319,23 @@ def _assert_delivery_workbench_shape(data: dict): assert data["rollups"]["high_risk_blocker_count"] == data["summary"][ "high_risk_blocker_count" ] - assert data["rollups"]["current_p0_blocker_count"] == 1 - assert data["rollups"]["current_p0_readiness_percent"] == 82 - assert data["rollups"]["current_p0_blocked_by_fresh_reboot_window_only"] is True - assert data["rollups"]["current_p0_service_green"] is True - assert data["rollups"]["current_p0_product_data_green"] is True - assert data["rollups"]["current_p0_backup_core_green"] is True - assert data["rollups"]["current_p0_stockplatform_freshness_status"] == "ok" - assert data["rollups"]["current_p0_stockplatform_ingestion_status"] == "ok" - assert data["rollups"]["current_p0_drill_preflight_ready"] is True - assert data["rollups"]["current_p0_drill_preflight_ready_count"] == 1 - assert data["rollups"]["current_p0_drill_preflight_blocker_count"] == 0 + assert data["rollups"]["current_p0_blocker_count"] == 12 + assert data["rollups"]["current_p0_readiness_percent"] == 18 + assert data["rollups"]["current_p0_blocked_by_fresh_reboot_window_only"] is False + assert data["rollups"]["current_p0_service_green"] is False + assert data["rollups"]["current_p0_product_data_green"] is False + assert data["rollups"]["current_p0_backup_core_green"] is False + assert ( + data["rollups"]["current_p0_stockplatform_freshness_status"] + == "not_configured" + ) + assert ( + data["rollups"]["current_p0_stockplatform_ingestion_status"] + == "not_configured" + ) + assert data["rollups"]["current_p0_drill_preflight_ready"] is False + assert data["rollups"]["current_p0_drill_preflight_ready_count"] == 0 + assert data["rollups"]["current_p0_drill_preflight_blocker_count"] == 9 assert data["rollups"]["current_p0_drill_preflight_break_glass_required"] is True assert ( data["rollups"][ @@ -366,26 +392,26 @@ def _assert_delivery_workbench_shape(data: dict): "blocked_reboot_auto_recovery_slo_not_ready" ) assert data["summary"]["reboot_auto_recovery_workplan_id"] == "P0-006" - assert data["summary"]["reboot_auto_recovery_readiness_percent"] == 82 - assert data["summary"]["reboot_auto_recovery_active_blocker_count"] == 1 + assert data["summary"]["reboot_auto_recovery_readiness_percent"] == 18 + assert data["summary"]["reboot_auto_recovery_active_blocker_count"] == 12 assert data["summary"]["reboot_auto_recovery_can_claim_slo"] is False - assert data["summary"]["reboot_auto_recovery_service_green"] is True - assert data["summary"]["reboot_auto_recovery_product_data_green"] is True - assert data["summary"]["reboot_auto_recovery_observed_host_count"] == 4 - assert data["summary"]["reboot_auto_recovery_stale_host_count"] == 4 + assert data["summary"]["reboot_auto_recovery_service_green"] is False + assert data["summary"]["reboot_auto_recovery_product_data_green"] is False + assert data["summary"]["reboot_auto_recovery_observed_host_count"] == 7 + assert data["summary"]["reboot_auto_recovery_stale_host_count"] == 5 assert ( data["summary"]["reboot_auto_recovery_stockplatform_freshness_status"] - == "ok" + == "not_configured" ) assert ( data["summary"]["reboot_auto_recovery_stockplatform_ingestion_status"] - == "ok" + == "not_configured" ) assert ( data["summary"][ "reboot_auto_recovery_stockplatform_final_retry_window_passed" ] - is True + is False ) assert ( data["summary"][ @@ -393,16 +419,13 @@ def _assert_delivery_workbench_shape(data: dict): ] is False ) - assert data["summary"]["reboot_auto_recovery_safe_next_step"] == ( - "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_event_" - "or_approved_reboot_drill_to_prove_10_minute_slo" - ) + assert data["summary"]["reboot_auto_recovery_safe_next_step"] == REBOOT_SAFE_NEXT_STEP assert data["summary"]["reboot_auto_recovery_drill_preflight_status"] == ( - "ready_for_break_glass_reboot_drill_authorization" + "blocked_reboot_drill_preflight_not_ready" ) - assert data["summary"]["reboot_auto_recovery_drill_preflight_ready"] is True - assert data["summary"]["reboot_auto_recovery_drill_preflight_ready_count"] == 1 - assert data["summary"]["reboot_auto_recovery_drill_preflight_blocker_count"] == 0 + assert data["summary"]["reboot_auto_recovery_drill_preflight_ready"] is False + assert data["summary"]["reboot_auto_recovery_drill_preflight_ready_count"] == 0 + assert data["summary"]["reboot_auto_recovery_drill_preflight_blocker_count"] == 9 assert ( data["summary"][ "reboot_auto_recovery_drill_preflight_break_glass_required"