wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(reboot): expose windows99 management channel readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Failing after 41s
Details
CD Pipeline / build-and-deploy (push) Has been skipped
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details

Browse Source
This commit is contained in:
Your Name
2026-07-02 15:19:09 +08:00
parent 25148d0752
commit aaa617f00f
14 changed files with 983 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
apps/api/src/services/awoooi_priority_work_order_readback.py
View File

@@ -2564,6 +2564,8 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
windows99_verify_collection = _dict(
reboot_slo.get("windows99_verify_collection")
)
windows99_management = _dict(reboot_slo.get("windows99_management_channel"))
windows99_ssh_batch = _dict(windows99_management.get("ssh_batch"))
reboot_preflight_rollups = _dict(reboot_preflight.get("rollups"))
reboot_preflight_target_selector = _dict(reboot_preflight.get("target_selector"))
reboot_active_blockers = _strings(reboot_slo.get("active_blockers"))
@@ -2657,6 +2659,27 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
state["windows99_verify_collection_post_verifier"] = str(
windows99_verify_collection.get("post_verifier") or ""
)
state["windows99_management_readback_present"] = bool(
windows99_management.get("readback_present") is True
)
state["windows99_management_host_reachable"] = bool(
windows99_management.get("host_reachable") is True
)
state["windows99_remote_execution_channel_ready"] = bool(
windows99_management.get("remote_execution_channel_ready") is True
)
state["windows99_ssh_batch_status"] = str(
windows99_ssh_batch.get("status") or "unknown"
)
state["windows99_rdp_console_reachable"] = bool(
windows99_management.get("rdp_console_reachable") is True
)
state["windows99_winrm_http_open"] = bool(
windows99_management.get("winrm_http_open") is True
)
state["windows99_winrm_https_open"] = bool(
windows99_management.get("winrm_https_open") is True
)
state["stale_snapshot_or_old_cd_runs_must_not_reopen_closed_work"] = True
state["p0_004_template_copy_apply_gate_production_http_status"] = 200
state["p0_004_template_copy_apply_gate_runtime_readback_state"] = (
@@ -2829,6 +2852,23 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
evidence["windows99_verify_collection_host99_uptime_known"] = bool(
windows99_verify_collection.get("host99_uptime_known") is True
)
evidence["windows99_management_readback_present"] = state[
"windows99_management_readback_present"
]
evidence["windows99_management_host_reachable"] = state[
"windows99_management_host_reachable"
]
evidence["windows99_remote_execution_channel_ready"] = state[
"windows99_remote_execution_channel_ready"
]
evidence["windows99_ssh_batch_status"] = state[
"windows99_ssh_batch_status"
]
evidence["windows99_rdp_console_reachable"] = state[
"windows99_rdp_console_reachable"
]
evidence["windows99_winrm_http_open"] = state["windows99_winrm_http_open"]
evidence["windows99_winrm_https_open"] = state["windows99_winrm_https_open"]
evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "")
evidence["drill_preflight_ready"] = (
reboot_preflight_rollups.get("preflight_ready") is True
@@ -2892,7 +2932,8 @@ def _enrich_from_current_readbacks(payload: dict[str, Any]) -> None:
"controlled service/data/backup readback visible as the source "
"selector for these blockers. For Windows 99, use the "
"windows99_verify_collection packet to collect no-secret Verify "
"stdout and rerun the scorecard. For StockPlatform "
"stdout, keep windows99_management_channel visible for the current "
"remote execution blocker, and rerun the scorecard. For StockPlatform "
"freshness/ingestion postgres_not_ready, use the production "
"migration/control-channel path with target selector, dry-run, "
"rollback, public API verifier, and KM/RAG/MCP/PlayBook writeback. "
@@ -3092,6 +3133,18 @@ def _set_rollups_and_summary(
"windows99_verify_collection_host99_uptime_known": (
state.get("windows99_verify_collection_host99_uptime_known") is True
),
"windows99_management_readback_present": (
state.get("windows99_management_readback_present") is True
),
"windows99_remote_execution_channel_ready": (
state.get("windows99_remote_execution_channel_ready") is True
),
"windows99_ssh_batch_status": str(
state.get("windows99_ssh_batch_status") or "unknown"
),
"windows99_rdp_console_reachable": (
state.get("windows99_rdp_console_reachable") is True
),
"p0_004_runtime_readback_ready": p0_004_ready,
"reboot_drill_preflight_runtime_readback_ready": (
state.get("reboot_drill_preflight_runtime_readback_state") == "ready"
@@ -3148,6 +3201,27 @@ def _set_rollups_and_summary(
"windows99_verify_collection_post_verifier": str(
state.get("windows99_verify_collection_post_verifier") or ""
),
"windows99_management_readback_present": (
state.get("windows99_management_readback_present") is True
),
"windows99_management_host_reachable": (
state.get("windows99_management_host_reachable") is True
),
"windows99_remote_execution_channel_ready": (
state.get("windows99_remote_execution_channel_ready") is True
),
"windows99_ssh_batch_status": str(
state.get("windows99_ssh_batch_status") or "unknown"
),
"windows99_rdp_console_reachable": (
state.get("windows99_rdp_console_reachable") is True
),
"windows99_winrm_http_open": (
state.get("windows99_winrm_http_open") is True
),
"windows99_winrm_https_open": (
state.get("windows99_winrm_https_open") is True
),
"latest_successful_deployed_source_sha": latest_source_sha,
"latest_successful_deployed_source_short_sha": latest_source_sha[:10],
"latest_successful_deploy_marker": str(

42
apps/api/src/services/reboot_auto_recovery_slo_scorecard.py
View File

@@ -43,6 +43,8 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
post_reboot_readiness = _dict(scorecard.get("post_reboot_readiness"))
stockplatform = _dict(scorecard.get("stockplatform_data_freshness"))
windows99 = _dict(scorecard.get("windows99_vmware_autostart"))
windows99_management = _dict(scorecard.get("windows99_management_channel"))
windows99_ssh_batch = _dict(windows99_management.get("ssh_batch"))
source_controls = _dict(scorecard.get("source_controls"))
active_blockers = _strings(scorecard.get("active_blockers"))
required_checks = {
@@ -53,6 +55,10 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"windows_update_no_auto_reboot_ready": (
windows99.get("windows_update_no_auto_reboot_ready") is True
),
"windows99_management_channel_ready": (
windows99.get("verify_ready") is True
or windows99_management.get("remote_execution_channel_ready") is True
),
"service_green": post_reboot_readiness.get("service_green") is True,
"product_data_green": post_reboot_readiness.get("product_data_green") is True,
"backup_core_green": post_reboot_readiness.get("backup_core_green") is True,
@@ -197,6 +203,31 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"windows99_host99_uptime_known": (
windows99_verify_collection["host99_uptime_known"] is True
),
"windows99_management_readback_present": (
windows99_management.get("readback_present") is True
),
"windows99_host_reachable": (
windows99_management.get("host_reachable") is True
),
"windows99_remote_execution_channel_ready": (
windows99_management.get("remote_execution_channel_ready") is True
),
"windows99_can_collect_vmware_verify_without_secret": (
windows99_management.get("can_collect_vmware_verify_without_secret") is True
),
"windows99_ssh_batch_ready": windows99_ssh_batch.get("ready") is True,
"windows99_ssh_batch_status": str(
windows99_ssh_batch.get("status") or "unknown"
),
"windows99_winrm_http_open": (
windows99_management.get("winrm_http_open") is True
),
"windows99_winrm_https_open": (
windows99_management.get("winrm_https_open") is True
),
"windows99_rdp_console_reachable": (
windows99_management.get("rdp_console_reachable") is True
),
}
return {
"schema_version": _API_SCHEMA_VERSION,
@@ -238,6 +269,10 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"windows99_update_no_auto_reboot_ready": rollups[
"windows99_update_no_auto_reboot_ready"
],
"windows99_remote_execution_channel_ready": rollups[
"windows99_remote_execution_channel_ready"
],
"windows99_ssh_batch_status": rollups["windows99_ssh_batch_status"],
"readback": {
"workplan_id": "P0-006",
"workplan_title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO",
@@ -273,6 +308,10 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"windows99_verify_collection_can_collect_no_secret": rollups[
"windows99_verify_collection_can_collect_no_secret"
],
"windows99_remote_execution_channel_ready": rollups[
"windows99_remote_execution_channel_ready"
],
"windows99_ssh_batch_status": rollups["windows99_ssh_batch_status"],
},
"reboot_sop_progress": sop_progress,
"controlled_service_data_backup_readback": (
@@ -283,6 +322,7 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]:
"stockplatform_data_freshness": stockplatform,
"windows99_vmware_autostart": windows99,
"windows99_verify_collection": windows99_verify_collection,
"windows99_management_channel": windows99_management,
"source_controls": source_controls,
"active_blockers": active_blockers,
"required_checks": required_checks,
@@ -547,6 +587,7 @@ def _reboot_sop_current_phase(active_blockers: list[str], can_claim_slo: bool) -
"fresh_all_host_reboot_event_missing",
"all_required_hosts_not_in_10_minute_reboot_window",
"windows99_vmware_autostart_readback_missing",
"windows99_remote_execution_channel_unavailable",
"windows99_vmrun_missing",
"windows99_vmware_vmx_missing",
"windows99_vmware_autostart_config_not_ready",
@@ -593,6 +634,7 @@ def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str:
"host_boot_observation_older_than_target_window",
"host_uptime_unknown",
"windows99_vmware_autostart_readback_missing",
"windows99_remote_execution_channel_unavailable",
"windows99_vmrun_missing",
"windows99_vmware_vmx_missing",
"windows99_vmware_autostart_config_not_ready",

74
apps/api/tests/test_awoooi_priority_work_order_readback_api.py
View File

@@ -58,7 +58,7 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
"blockers_open"
)
assert payload["mainline_execution_state"]["active_p0_immediate_apply_gap_count"] == 0
assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 15
assert payload["mainline_execution_state"]["active_p0_readiness_percent"] == 43
assert (
payload["mainline_execution_state"][
"controlled_service_data_backup_readback_present"
@@ -75,19 +75,15 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
payload["mainline_execution_state"][
"controlled_service_data_backup_blocker_count"
]
== 8
== 4
)
assert payload["mainline_execution_state"][
"controlled_service_data_backup_blocking_fields"
] == [
"service_green",
"post_start_blocked",
"product_data_green",
"backup_core_green",
"host_188_service_green",
"wazuh_dashboard_degraded",
"stockplatform_freshness_status",
"stockplatform_ingestion_status",
]
assert (
payload["mainline_execution_state"][
@@ -128,20 +124,36 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
]
is False
)
assert (
payload["mainline_execution_state"]["windows99_management_readback_present"]
is True
)
assert (
payload["mainline_execution_state"][
"windows99_remote_execution_channel_ready"
]
is False
)
assert (
payload["mainline_execution_state"]["windows99_ssh_batch_status"]
== "permission_denied"
)
assert (
payload["mainline_execution_state"]["windows99_rdp_console_reachable"]
is True
)
assert payload["next_execution_order"][0].startswith("P0-006:")
in_progress = payload["in_progress_or_blocked_in_priority_order"][0]
assert in_progress["workplan_id"] == "P0-006"
assert in_progress["status"] == "blocked_reboot_auto_recovery_slo_not_ready"
assert in_progress["evidence"]["service_green"] is False
assert in_progress["evidence"]["product_data_green"] is False
assert in_progress["evidence"]["product_data_green"] is True
assert in_progress["evidence"]["backup_core_green"] is False
assert in_progress["evidence"]["host_188_service_green"] is False
assert in_progress["evidence"]["stock_freshness_status"] == "not_configured"
assert in_progress["evidence"]["stock_ingestion_status"] == "not_configured"
assert in_progress["evidence"]["stock_blockers"] == ["postgres_not_ready"]
assert in_progress["evidence"]["stock_ingestion_blockers"] == [
"postgres_not_ready"
]
assert in_progress["evidence"]["host_188_service_green"] is True
assert in_progress["evidence"]["stock_freshness_status"] == "ok"
assert in_progress["evidence"]["stock_ingestion_status"] == "ok"
assert in_progress["evidence"]["stock_blockers"] == []
assert in_progress["evidence"]["stock_ingestion_blockers"] == []
assert (
in_progress["evidence"][
"controlled_service_data_backup_readback_present"
@@ -153,18 +165,14 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
] == "blocked_service_data_backup_readback_not_green"
assert in_progress["evidence"][
"controlled_service_data_backup_blocker_count"
] == 8
] == 4
assert in_progress["evidence"][
"controlled_service_data_backup_blocking_fields"
] == [
"service_green",
"post_start_blocked",
"product_data_green",
"backup_core_green",
"host_188_service_green",
"wazuh_dashboard_degraded",
"stockplatform_freshness_status",
"stockplatform_ingestion_status",
]
assert in_progress["evidence"]["windows99_verify_collection_status"] == (
"blocked_windows99_verify_output_missing_host_reachable"
@@ -187,6 +195,16 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
in_progress["evidence"]["windows99_verify_collection_host99_uptime_known"]
is False
)
assert in_progress["evidence"]["windows99_management_readback_present"] is True
assert in_progress["evidence"]["windows99_management_host_reachable"] is True
assert (
in_progress["evidence"]["windows99_remote_execution_channel_ready"]
is False
)
assert in_progress["evidence"]["windows99_ssh_batch_status"] == (
"permission_denied"
)
assert in_progress["evidence"]["windows99_rdp_console_reachable"] is True
assert (
in_progress["evidence"][
"controlled_service_data_backup_can_clear_blockers"
@@ -194,7 +212,7 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
is False
)
assert in_progress["evidence"]["drill_preflight_ready"] is False
assert in_progress["evidence"]["drill_preflight_blocker_count"] == 9
assert in_progress["evidence"]["drill_preflight_blocker_count"] == 5
assert (
in_progress["evidence"][
"drill_preflight_execution_authorized_by_this_endpoint"
@@ -210,6 +228,9 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
assert "windows99_verify_collection packet" in in_progress[
"professional_fix"
]["action"]
assert "windows99_management_channel visible" in in_progress[
"professional_fix"
]["action"]
assert "production migration/control-channel" in in_progress[
"professional_fix"
]["action"]
@@ -222,12 +243,12 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
payload["rollups"]["controlled_service_data_backup_readback_present"]
is True
)
assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 8
assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 4
assert (
payload["summary"]["controlled_service_data_backup_readback_present"]
is True
)
assert payload["summary"]["controlled_service_data_backup_blocker_count"] == 8
assert payload["summary"]["controlled_service_data_backup_blocker_count"] == 4
assert payload["summary"][
"controlled_service_data_backup_next_safe_action"
] == (
@@ -251,6 +272,15 @@ def test_awoooi_priority_work_order_readback_loader_returns_mainline_order():
payload["rollups"]["windows99_verify_collection_host99_reachable"]
is True
)
assert payload["rollups"]["windows99_management_readback_present"] is True
assert payload["rollups"]["windows99_remote_execution_channel_ready"] is False
assert payload["rollups"]["windows99_ssh_batch_status"] == "permission_denied"
assert payload["rollups"]["windows99_rdp_console_reachable"] is True
assert payload["summary"]["windows99_management_readback_present"] is True
assert payload["summary"]["windows99_management_host_reachable"] is True
assert payload["summary"]["windows99_remote_execution_channel_ready"] is False
assert payload["summary"]["windows99_ssh_batch_status"] == "permission_denied"
assert payload["summary"]["windows99_rdp_console_reachable"] is True
assert payload["operation_boundaries"]["github_api_used"] is False
assert payload["operation_boundaries"]["github_cli_used"] is False
assert payload["operation_boundaries"]["secret_or_runner_token_read"] is False

125
apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py
View File

@@ -14,31 +14,30 @@ from src.services.reboot_auto_recovery_slo_scorecard import (
EXPECTED_REBOOT_SLO_BLOCKERS = [
"all_required_hosts_not_in_10_minute_reboot_window",
"backup_core_green_not_1",
"host_188_service_green_not_1",
"host_boot_observation_older_than_target_window",
"host_unreachable_after_reboot",
"host_uptime_unknown",
"local_disk_free_below_minimum",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"reboot_event_required_host_unreachable",
"service_green_not_1",
"wazuh_dashboard_degraded",
"windows99_remote_execution_channel_unavailable",
"windows99_vmware_autostart_readback_missing",
]
EXPECTED_DRILL_PREFLIGHT_BLOCKERS = [
"service_green_not_ready",
"product_data_green_not_ready",
"backup_core_green_not_ready",
"host_188_service_green_not_ready",
"all_required_hosts_reachable_not_ready",
"latest_verify_only_metric_present_not_ready",
"stockplatform_freshness_ok_not_ready",
"stockplatform_ingestion_ok_not_ready",
"blocked_only_by_fresh_reboot_window_not_ready",
]
EXPECTED_WINDOWS99_NEXT_STEP = (
"restore_windows99_no_secret_management_channel_or_collect_local_console_"
"verify_readback_then_rerun_reboot_scorecard_no_reboot"
)
def test_reboot_auto_recovery_slo_scorecard_loader_exposes_stockplatform_gate():
payload = load_latest_reboot_auto_recovery_slo_scorecard()
@@ -84,18 +83,15 @@ def _assert_reboot_slo_payload(payload: dict):
"event_and_probe"
)
assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
assert payload["safe_next_step"] == (
"collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_"
"reboot_scorecard_no_secret_no_reboot"
)
assert payload["safe_next_step"] == EXPECTED_WINDOWS99_NEXT_STEP
assert payload["next_safe_action"] == payload["safe_next_step"]
assert payload["can_claim_all_services_recovered_within_target"] is False
assert payload["active_blocker_count"] == 13
assert payload["readiness_percent"] == 15
assert payload["active_blocker_count"] == 11
assert payload["readiness_percent"] == 43
assert payload["service_green"] is False
assert payload["product_data_green"] is False
assert payload["product_data_green"] is True
assert payload["backup_core_green"] is False
assert payload["host_188_service_green"] is False
assert payload["host_188_service_green"] is True
assert payload["observed_host_count"] == 7
assert payload["missing_host_count"] == 0
assert payload["unreachable_host_count"] == 1
@@ -106,13 +102,18 @@ def _assert_reboot_slo_payload(payload: dict):
assert payload["latest_verify_only_metric_blocker_count"] == 0
assert payload["latest_verify_only_metric_max_host_uptime_seconds"] == 0
assert payload["latest_verify_only_metric_last_run_timestamp"] == 0
assert payload["stockplatform_freshness_status"] == "not_configured"
assert payload["stockplatform_ingestion_status"] == "not_configured"
assert payload["stockplatform_freshness_status"] == "ok"
assert payload["stockplatform_ingestion_status"] == "ok"
assert payload["windows99_vmware_verify_ready"] is False
assert payload["windows99_update_no_auto_reboot_ready"] is False
assert payload["windows99_remote_execution_channel_ready"] is False
assert payload["windows99_ssh_batch_status"] == "permission_denied"
assert payload["source_controls"][
"host_110_startup_controlled_drain_guarded_autostart_source_present"
] is True
assert payload["source_controls"][
"windows99_management_channel_probe_source_present"
] is True
assert payload["readback"]["workplan_id"] == "P0-006"
assert payload["readback"]["target_minutes"] == 10
assert payload["readback"]["current_phase"] == "host_boot_detection_blocked"
@@ -123,13 +124,10 @@ def _assert_reboot_slo_payload(payload: dict):
assert payload["readback"]["primary_blocker"] == (
"reboot_event_required_host_unreachable"
)
assert payload["readback"]["safe_next_step"] == (
"collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_"
"reboot_scorecard_no_secret_no_reboot"
)
assert payload["readback"]["safe_next_step"] == EXPECTED_WINDOWS99_NEXT_STEP
assert payload["readback"]["next_safe_action"] == payload["safe_next_step"]
assert payload["readback"]["active_blocker_count"] == 13
assert payload["readback"]["readiness_percent"] == 15
assert payload["readback"]["active_blocker_count"] == 11
assert payload["readback"]["readiness_percent"] == 43
assert payload["readback"]["blocked_by_fresh_reboot_window_only"] is False
assert payload["readback"]["latest_verify_only_metric_present"] is False
assert payload["readback"]["windows99_vmware_verify_ready"] is False
@@ -141,29 +139,31 @@ def _assert_reboot_slo_payload(payload: dict):
payload["readback"]["windows99_verify_collection_can_collect_no_secret"]
is True
)
assert payload["rollups"]["active_blocker_count"] == 13
assert payload["rollups"]["readiness_percent"] == 15
assert payload["readback"]["windows99_remote_execution_channel_ready"] is False
assert payload["readback"]["windows99_ssh_batch_status"] == "permission_denied"
assert payload["rollups"]["active_blocker_count"] == 11
assert payload["rollups"]["readiness_percent"] == 43
assert payload["rollups"]["observed_host_count"] == 7
assert payload["rollups"]["missing_host_count"] == 0
assert payload["rollups"]["unreachable_host_count"] == 1
assert payload["rollups"]["stale_host_count"] == 5
assert payload["rollups"]["service_green"] is False
assert payload["rollups"]["product_data_green"] is False
assert payload["rollups"]["product_data_green"] is True
assert payload["rollups"]["backup_core_green"] is False
assert payload["rollups"]["host_188_service_green"] is False
assert payload["rollups"]["host_188_service_green"] is True
assert payload["rollups"]["blocked_by_fresh_reboot_window_only"] is False
assert payload["rollups"]["latest_verify_only_metric_present"] is False
assert payload["rollups"]["latest_verify_only_metric_ready"] == 0
assert payload["rollups"]["latest_verify_only_metric_blocker_count"] == 0
assert payload["rollups"]["stockplatform_freshness_status"] == "not_configured"
assert payload["rollups"]["stockplatform_ingestion_status"] == "not_configured"
assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 1
assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 1
assert payload["rollups"]["stockplatform_freshness_status"] == "ok"
assert payload["rollups"]["stockplatform_ingestion_status"] == "ok"
assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 0
assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 0
assert (
payload["rollups"]["controlled_service_data_backup_readback_present"]
is True
)
assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 8
assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 4
assert (
payload["rollups"]["controlled_service_data_backup_can_clear_blockers"]
is False
@@ -184,6 +184,11 @@ def _assert_reboot_slo_payload(payload: dict):
assert payload["rollups"]["windows99_verify_collection_blocker_count"] == 2
assert payload["rollups"]["windows99_host99_reachable"] is True
assert payload["rollups"]["windows99_host99_uptime_known"] is False
assert payload["rollups"]["windows99_management_readback_present"] is True
assert payload["rollups"]["windows99_host_reachable"] is True
assert payload["rollups"]["windows99_remote_execution_channel_ready"] is False
assert payload["rollups"]["windows99_ssh_batch_status"] == "permission_denied"
assert payload["rollups"]["windows99_rdp_console_reachable"] is True
assert payload["rollups"]["stockplatform_final_retry_window_passed"] is False
assert (
payload["rollups"]["stockplatform_controlled_recovery_gate_required"]
@@ -197,8 +202,8 @@ def _assert_reboot_slo_payload(payload: dict):
"event_and_probe"
)
assert progress["primary_blocker"] == "reboot_event_required_host_unreachable"
assert progress["active_blocker_count"] == 13
assert progress["readiness_percent"] == 15
assert progress["active_blocker_count"] == 11
assert progress["readiness_percent"] == 43
assert progress["next_safe_action"] == payload["safe_next_step"]
assert progress["fixed_triage_order"][0] == "99_vmware_autostart_and_vm_power"
service_data_backup = payload["controlled_service_data_backup_readback"]
@@ -210,28 +215,22 @@ def _assert_reboot_slo_payload(payload: dict):
)
assert service_data_backup["readback_present"] is True
assert service_data_backup["service_green"] is False
assert service_data_backup["product_data_green"] is False
assert service_data_backup["product_data_green"] is True
assert service_data_backup["backup_core_green"] is False
assert service_data_backup["host_188_service_green"] is False
assert service_data_backup["post_start_blocked"] == 8
assert service_data_backup["host_188_service_green"] is True
assert service_data_backup["post_start_blocked"] == 1
assert service_data_backup["wazuh_dashboard_degraded"] is True
assert service_data_backup["stockplatform_freshness_status"] == "not_configured"
assert service_data_backup["stockplatform_ingestion_status"] == "not_configured"
assert service_data_backup["stockplatform_freshness_status"] == "ok"
assert service_data_backup["stockplatform_ingestion_status"] == "ok"
assert service_data_backup["blocking_fields"] == [
"service_green",
"post_start_blocked",
"product_data_green",
"backup_core_green",
"host_188_service_green",
"wazuh_dashboard_degraded",
"stockplatform_freshness_status",
"stockplatform_ingestion_status",
]
assert service_data_backup["related_active_blockers"] == [
"backup_core_green_not_1",
"host_188_service_green_not_1",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"service_green_not_1",
"wazuh_dashboard_degraded",
]
@@ -265,19 +264,25 @@ def _assert_reboot_slo_payload(payload: dict):
assert "-Mode Verify" in collection["no_secret_verify_command"]
assert "host_reboot" in collection["forbidden_actions"]
assert "windows_password_or_secret_collection" in collection["forbidden_actions"]
windows99_management = payload["windows99_management_channel"]
assert windows99_management["readback_present"] is True
assert windows99_management["host_reachable"] is True
assert windows99_management["remote_execution_channel_ready"] is False
assert windows99_management["ssh_batch"]["status"] == "permission_denied"
assert windows99_management["rdp_console_reachable"] is True
stockplatform = payload["stockplatform_data_freshness"]
assert stockplatform["freshness_endpoint_readback_present"] is True
assert stockplatform["ingestion_endpoint_readback_present"] is True
assert stockplatform["freshness_blockers"] == ["postgres_not_ready"]
assert stockplatform["ingestion_blockers"] == ["postgres_not_ready"]
assert stockplatform["freshness_blockers"] == []
assert stockplatform["ingestion_blockers"] == []
assert stockplatform["margin_short_recovery"]["status"] == "not_verified"
assert stockplatform["margin_short_recovery"]["successful_source_run_ids"] == []
assert stockplatform["ai_recommendations_recovery"]["status"] == "not_verified"
assert stockplatform["ai_recommendations_recovery"]["status"] == "recovered"
assert stockplatform["eod_window"]["final_retry_window_passed"] is False
assert stockplatform["controlled_recovery_gate"]["required"] is False
assert (
stockplatform["controlled_recovery_gate"]["status"]
== "not_required_yet"
== "not_required_freshness_recovered"
)
assert "manual_db_update" in stockplatform["controlled_recovery_gate"][
"forbidden_actions"
@@ -299,7 +304,7 @@ def _assert_drill_preflight_payload(payload: dict):
assert payload["priority"] == "P0-006"
assert payload["status"] == "blocked_reboot_drill_preflight_not_ready"
assert payload["preflight_ready"] is False
assert payload["preflight_blocker_count"] == 9
assert payload["preflight_blocker_count"] == 5
assert payload["active_blockers"] == EXPECTED_DRILL_PREFLIGHT_BLOCKERS
assert payload["break_glass_authorization_required"] is True
assert payload["execution_authorized_by_this_endpoint"] is False
@@ -338,25 +343,25 @@ def _assert_drill_preflight_payload(payload: dict):
assert payload["preconditions"] == {
"source_controls_present": True,
"service_green": False,
"product_data_green": False,
"product_data_green": True,
"backup_core_green": False,
"host_188_service_green": False,
"host_188_service_green": True,
"all_required_hosts_observed": True,
"all_required_hosts_reachable": False,
"latest_verify_only_metric_present": False,
"stockplatform_freshness_ok": False,
"stockplatform_ingestion_ok": False,
"stockplatform_freshness_ok": True,
"stockplatform_ingestion_ok": True,
"blocked_only_by_fresh_reboot_window": False,
}
current = payload["current_readback"]
assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready"
assert current["readiness_percent"] == 15
assert current["active_blocker_count"] == 13
assert current["readiness_percent"] == 43
assert current["active_blocker_count"] == 11
assert current["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS
assert current["latest_verify_only_metric_ready"] == 0
assert current["latest_verify_only_metric_blocker_count"] == 0
assert current["stockplatform_freshness_status"] == "not_configured"
assert current["stockplatform_ingestion_status"] == "not_configured"
assert current["stockplatform_freshness_status"] == "ok"
assert current["stockplatform_ingestion_status"] == "ok"
check_mode = payload["check_mode"]
assert check_mode["verify_only_available"] is True
assert (
@@ -376,12 +381,12 @@ def _assert_drill_preflight_payload(payload: dict):
assert readback["workplan_id"] == "P0-006"
assert readback["preflight_ready"] is False
assert readback["target_selector"]["required_host_count"] == 7
assert readback["current_readback"]["active_blocker_count"] == 13
assert readback["current_readback"]["active_blocker_count"] == 11
assert readback["safe_next_step"] == payload["safe_next_step"]
rollups = payload["rollups"]
assert rollups["preflight_ready"] is False
assert rollups["preflight_ready_count"] == 0
assert rollups["preflight_blocker_count"] == 9
assert rollups["preflight_blocker_count"] == 5
assert rollups["target_required_host_count"] == 7
assert rollups["target_observed_host_count"] == 7
assert rollups["target_unreachable_host_count"] == 1