fix(recovery): update reboot slo fail-closed readback

apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py

@@ -11,6 +11,33 @@ from src.services.reboot_auto_recovery_slo_scorecard import (
     load_latest_reboot_auto_recovery_slo_scorecard,
 )
 
+EXPECTED_REBOOT_SLO_BLOCKERS = [
+    "all_required_hosts_not_in_10_minute_reboot_window",
+    "backup_core_green_not_1",
+    "host_188_service_green_not_1",
+    "host_boot_observation_older_than_target_window",
+    "host_unreachable_after_reboot",
+    "host_uptime_unknown",
+    "local_disk_free_below_minimum",
+    "post_start_blocked_not_zero",
+    "product_data_green_not_1",
+    "reboot_event_required_host_unreachable",
+    "service_green_not_1",
+    "wazuh_dashboard_degraded",
+]
+
+EXPECTED_DRILL_PREFLIGHT_BLOCKERS = [
+    "service_green_not_ready",
+    "product_data_green_not_ready",
+    "backup_core_green_not_ready",
+    "host_188_service_green_not_ready",
+    "all_required_hosts_reachable_not_ready",
+    "latest_verify_only_metric_present_not_ready",
+    "stockplatform_freshness_ok_not_ready",
+    "stockplatform_ingestion_ok_not_ready",
+    "blocked_only_by_fresh_reboot_window_not_ready",
+]
+
 
 def test_reboot_auto_recovery_slo_scorecard_loader_exposes_stockplatform_gate():
     payload = load_latest_reboot_auto_recovery_slo_scorecard()
@@ -51,78 +78,75 @@ def _assert_reboot_slo_payload(payload: dict):
     assert payload["priority"] == "P0-006"
     assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready"
     assert payload["safe_next_step"] == (
-        "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_"
-        "event_or_approved_reboot_drill_to_prove_10_minute_slo"
+        "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_"
+        "rerun_scorecard_until_status_slo_ready"
     )
     assert payload["can_claim_all_services_recovered_within_target"] is False
-    assert payload["active_blocker_count"] == 1
-    assert payload["readiness_percent"] == 82
-    assert payload["service_green"] is True
-    assert payload["product_data_green"] is True
-    assert payload["backup_core_green"] is True
-    assert payload["host_188_service_green"] is True
-    assert payload["observed_host_count"] == 4
+    assert payload["active_blocker_count"] == 12
+    assert payload["readiness_percent"] == 18
+    assert payload["service_green"] is False
+    assert payload["product_data_green"] is False
+    assert payload["backup_core_green"] is False
+    assert payload["host_188_service_green"] is False
+    assert payload["observed_host_count"] == 7
     assert payload["missing_host_count"] == 0
-    assert payload["unreachable_host_count"] == 0
-    assert payload["stale_host_count"] == 4
-    assert payload["blocked_by_fresh_reboot_window_only"] is True
-    assert payload["latest_verify_only_metric_present"] is True
+    assert payload["unreachable_host_count"] == 1
+    assert payload["stale_host_count"] == 5
+    assert payload["blocked_by_fresh_reboot_window_only"] is False
+    assert payload["latest_verify_only_metric_present"] is False
     assert payload["latest_verify_only_metric_ready"] == 0
-    assert payload["latest_verify_only_metric_blocker_count"] == 1
-    assert payload["latest_verify_only_metric_max_host_uptime_seconds"] == 541538
-    assert payload["latest_verify_only_metric_last_run_timestamp"] > 0
-    assert payload["stockplatform_freshness_status"] == "ok"
-    assert payload["stockplatform_ingestion_status"] == "ok"
+    assert payload["latest_verify_only_metric_blocker_count"] == 0
+    assert payload["latest_verify_only_metric_max_host_uptime_seconds"] == 0
+    assert payload["latest_verify_only_metric_last_run_timestamp"] == 0
+    assert payload["stockplatform_freshness_status"] == "not_configured"
+    assert payload["stockplatform_ingestion_status"] == "not_configured"
     assert payload["readback"]["workplan_id"] == "P0-006"
     assert payload["readback"]["target_minutes"] == 10
     assert payload["readback"]["safe_next_step"] == (
-        "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_"
-        "event_or_approved_reboot_drill_to_prove_10_minute_slo"
+        "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_"
+        "rerun_scorecard_until_status_slo_ready"
     )
-    assert payload["readback"]["active_blocker_count"] == 1
-    assert payload["readback"]["readiness_percent"] == 82
-    assert payload["readback"]["blocked_by_fresh_reboot_window_only"] is True
-    assert payload["readback"]["latest_verify_only_metric_present"] is True
-    assert payload["rollups"]["active_blocker_count"] == 1
-    assert payload["rollups"]["readiness_percent"] == 82
-    assert payload["rollups"]["observed_host_count"] == 4
+    assert payload["readback"]["active_blocker_count"] == 12
+    assert payload["readback"]["readiness_percent"] == 18
+    assert payload["readback"]["blocked_by_fresh_reboot_window_only"] is False
+    assert payload["readback"]["latest_verify_only_metric_present"] is False
+    assert payload["rollups"]["active_blocker_count"] == 12
+    assert payload["rollups"]["readiness_percent"] == 18
+    assert payload["rollups"]["observed_host_count"] == 7
     assert payload["rollups"]["missing_host_count"] == 0
-    assert payload["rollups"]["unreachable_host_count"] == 0
-    assert payload["rollups"]["stale_host_count"] == 4
-    assert payload["rollups"]["service_green"] is True
-    assert payload["rollups"]["product_data_green"] is True
-    assert payload["rollups"]["backup_core_green"] is True
-    assert payload["rollups"]["host_188_service_green"] is True
-    assert payload["rollups"]["blocked_by_fresh_reboot_window_only"] is True
-    assert payload["rollups"]["latest_verify_only_metric_present"] is True
+    assert payload["rollups"]["unreachable_host_count"] == 1
+    assert payload["rollups"]["stale_host_count"] == 5
+    assert payload["rollups"]["service_green"] is False
+    assert payload["rollups"]["product_data_green"] is False
+    assert payload["rollups"]["backup_core_green"] is False
+    assert payload["rollups"]["host_188_service_green"] is False
+    assert payload["rollups"]["blocked_by_fresh_reboot_window_only"] is False
+    assert payload["rollups"]["latest_verify_only_metric_present"] is False
     assert payload["rollups"]["latest_verify_only_metric_ready"] == 0
-    assert payload["rollups"]["latest_verify_only_metric_blocker_count"] == 1
-    assert payload["rollups"]["stockplatform_freshness_status"] == "ok"
-    assert payload["rollups"]["stockplatform_ingestion_status"] == "ok"
-    assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 0
-    assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 0
-    assert payload["rollups"]["stockplatform_final_retry_window_passed"] is True
+    assert payload["rollups"]["latest_verify_only_metric_blocker_count"] == 0
+    assert payload["rollups"]["stockplatform_freshness_status"] == "not_configured"
+    assert payload["rollups"]["stockplatform_ingestion_status"] == "not_configured"
+    assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 1
+    assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 1
+    assert payload["rollups"]["stockplatform_final_retry_window_passed"] is False
     assert (
         payload["rollups"]["stockplatform_controlled_recovery_gate_required"]
         is False
     )
-    assert payload["active_blockers"] == ["host_boot_observation_older_than_target_window"]
+    assert payload["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS
     stockplatform = payload["stockplatform_data_freshness"]
     assert stockplatform["freshness_endpoint_readback_present"] is True
     assert stockplatform["ingestion_endpoint_readback_present"] is True
-    assert stockplatform["freshness_blockers"] == []
-    assert stockplatform["ingestion_blockers"] == []
-    assert stockplatform["margin_short_recovery"]["status"] == "recovered"
-    assert stockplatform["margin_short_recovery"]["successful_source_run_ids"] == [
-        3389,
-        3390,
-    ]
-    assert stockplatform["ai_recommendations_recovery"]["status"] == "recovered"
-    assert stockplatform["eod_window"]["final_retry_window_passed"] is True
+    assert stockplatform["freshness_blockers"] == ["postgres_not_ready"]
+    assert stockplatform["ingestion_blockers"] == ["postgres_not_ready"]
+    assert stockplatform["margin_short_recovery"]["status"] == "not_verified"
+    assert stockplatform["margin_short_recovery"]["successful_source_run_ids"] == []
+    assert stockplatform["ai_recommendations_recovery"]["status"] == "not_verified"
+    assert stockplatform["eod_window"]["final_retry_window_passed"] is False
     assert stockplatform["controlled_recovery_gate"]["required"] is False
     assert (
         stockplatform["controlled_recovery_gate"]["status"]
-        == "not_required_freshness_recovered"
+        == "not_required_yet"
     )
     assert "manual_db_update" in stockplatform["controlled_recovery_gate"][
         "forbidden_actions"
@@ -142,10 +166,10 @@ def _assert_drill_preflight_payload(payload: dict):
         == "reboot_auto_recovery_drill_preflight_readback_v1"
     )
     assert payload["priority"] == "P0-006"
-    assert payload["status"] == "ready_for_break_glass_reboot_drill_authorization"
-    assert payload["preflight_ready"] is True
-    assert payload["preflight_blocker_count"] == 0
-    assert payload["active_blockers"] == []
+    assert payload["status"] == "blocked_reboot_drill_preflight_not_ready"
+    assert payload["preflight_ready"] is False
+    assert payload["preflight_blocker_count"] == 9
+    assert payload["active_blockers"] == EXPECTED_DRILL_PREFLIGHT_BLOCKERS
     assert payload["break_glass_authorization_required"] is True
     assert payload["execution_authorized_by_this_endpoint"] is False
     assert payload["safe_next_step"] == (
@@ -153,12 +177,20 @@ def _assert_drill_preflight_payload(payload: dict):
         "real_all_host_reboot_event_then_rerun_verify_only"
     )
     target_selector = payload["target_selector"]
-    assert target_selector["required_host_aliases"] == ["110", "120", "121", "188"]
-    assert target_selector["required_host_count"] == 4
-    assert target_selector["observed_host_count"] == 4
+    assert target_selector["required_host_aliases"] == [
+        "110",
+        "111",
+        "112",
+        "120",
+        "121",
+        "188",
+        "99",
+    ]
+    assert target_selector["required_host_count"] == 7
+    assert target_selector["observed_host_count"] == 7
     assert target_selector["missing_host_count"] == 0
-    assert target_selector["unreachable_host_count"] == 0
-    assert target_selector["stale_host_count"] == 4
+    assert target_selector["unreachable_host_count"] == 1
+    assert target_selector["stale_host_count"] == 5
     assert set(payload["preconditions"]) == {
         "source_controls_present",
         "service_green",
@@ -172,18 +204,28 @@ def _assert_drill_preflight_payload(payload: dict):
         "stockplatform_ingestion_ok",
         "blocked_only_by_fresh_reboot_window",
     }
-    assert all(payload["preconditions"].values())
+    assert payload["preconditions"] == {
+        "source_controls_present": True,
+        "service_green": False,
+        "product_data_green": False,
+        "backup_core_green": False,
+        "host_188_service_green": False,
+        "all_required_hosts_observed": True,
+        "all_required_hosts_reachable": False,
+        "latest_verify_only_metric_present": False,
+        "stockplatform_freshness_ok": False,
+        "stockplatform_ingestion_ok": False,
+        "blocked_only_by_fresh_reboot_window": False,
+    }
     current = payload["current_readback"]
     assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready"
-    assert current["readiness_percent"] == 82
-    assert current["active_blocker_count"] == 1
-    assert current["active_blockers"] == [
-        "host_boot_observation_older_than_target_window"
-    ]
+    assert current["readiness_percent"] == 18
+    assert current["active_blocker_count"] == 12
+    assert current["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS
     assert current["latest_verify_only_metric_ready"] == 0
-    assert current["latest_verify_only_metric_blocker_count"] == 1
-    assert current["stockplatform_freshness_status"] == "ok"
-    assert current["stockplatform_ingestion_status"] == "ok"
+    assert current["latest_verify_only_metric_blocker_count"] == 0
+    assert current["stockplatform_freshness_status"] == "not_configured"
+    assert current["stockplatform_ingestion_status"] == "not_configured"
     check_mode = payload["check_mode"]
     assert check_mode["verify_only_available"] is True
     assert (
@@ -201,17 +243,18 @@ def _assert_drill_preflight_payload(payload: dict):
     assert rollback["rollback_required_for_this_endpoint"] is False
     readback = payload["readback"]
     assert readback["workplan_id"] == "P0-006"
-    assert readback["preflight_ready"] is True
-    assert readback["target_selector"]["required_host_count"] == 4
-    assert readback["current_readback"]["active_blocker_count"] == 1
+    assert readback["preflight_ready"] is False
+    assert readback["target_selector"]["required_host_count"] == 7
+    assert readback["current_readback"]["active_blocker_count"] == 12
     assert readback["safe_next_step"] == payload["safe_next_step"]
     rollups = payload["rollups"]
-    assert rollups["preflight_ready"] is True
-    assert rollups["preflight_ready_count"] == 1
-    assert rollups["preflight_blocker_count"] == 0
-    assert rollups["target_required_host_count"] == 4
-    assert rollups["target_observed_host_count"] == 4
-    assert rollups["blocked_only_by_fresh_reboot_window"] is True
+    assert rollups["preflight_ready"] is False
+    assert rollups["preflight_ready_count"] == 0
+    assert rollups["preflight_blocker_count"] == 9
+    assert rollups["target_required_host_count"] == 7
+    assert rollups["target_observed_host_count"] == 7
+    assert rollups["target_unreachable_host_count"] == 1
+    assert rollups["blocked_only_by_fresh_reboot_window"] is False
     assert rollups["host_reboot_authorized_by_this_endpoint"] is False
     boundaries = payload["operation_boundaries"]
     assert boundaries["host_reboot_authorized_by_this_endpoint"] is False