fix(reboot): project runtime service green readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 1m27s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 1m27s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -685,6 +685,118 @@ def _add_runtime_readback_active_blockers_missing_from_metric(
|
||||
return merged
|
||||
|
||||
|
||||
def _apply_prometheus_service_backup_boolean_readback(
|
||||
payload: dict[str, Any],
|
||||
active_blockers: list[str],
|
||||
) -> None:
|
||||
active = set(active_blockers)
|
||||
service_green = not (
|
||||
{"service_green_not_1", "post_start_blocked_not_zero"} & active
|
||||
)
|
||||
product_data_green = "product_data_green_not_1" not in active
|
||||
backup_core_green = "backup_core_green_not_1" not in active
|
||||
host_188_service_green = "host_188_service_green_not_1" not in active
|
||||
post_start_blocked = 1 if "post_start_blocked_not_zero" in active else 0
|
||||
wazuh_dashboard_degraded = "wazuh_dashboard_degraded" in active
|
||||
|
||||
payload["service_green"] = service_green
|
||||
payload["product_data_green"] = product_data_green
|
||||
payload["backup_core_green"] = backup_core_green
|
||||
payload["host_188_service_green"] = host_188_service_green
|
||||
|
||||
post_reboot = _dict(payload.setdefault("post_reboot_readiness", {}))
|
||||
post_reboot.update(
|
||||
{
|
||||
"service_green": service_green,
|
||||
"product_data_green": product_data_green,
|
||||
"backup_core_green": backup_core_green,
|
||||
"host_188_service_green": host_188_service_green,
|
||||
"post_start_blocked": post_start_blocked,
|
||||
"wazuh_dashboard_degraded": wazuh_dashboard_degraded,
|
||||
}
|
||||
)
|
||||
|
||||
required_checks = _dict(payload.setdefault("required_checks", {}))
|
||||
required_checks["service_green"] = service_green
|
||||
required_checks["product_data_green"] = product_data_green
|
||||
required_checks["backup_core_green"] = backup_core_green
|
||||
required_checks["host_188_service_green"] = host_188_service_green
|
||||
|
||||
readback = _dict(payload.setdefault("readback", {}))
|
||||
readback["service_green"] = service_green
|
||||
readback["product_data_green"] = product_data_green
|
||||
readback["backup_core_green"] = backup_core_green
|
||||
readback["host_188_service_green"] = host_188_service_green
|
||||
readback["post_start_blocked"] = post_start_blocked
|
||||
readback["wazuh_dashboard_degraded"] = wazuh_dashboard_degraded
|
||||
readback["runtime_metric_service_backup_boolean_overlay_applied"] = True
|
||||
|
||||
blocking_fields: list[str] = []
|
||||
if not service_green:
|
||||
blocking_fields.append("service_green")
|
||||
if post_start_blocked != 0:
|
||||
blocking_fields.append("post_start_blocked")
|
||||
if not product_data_green:
|
||||
blocking_fields.append("product_data_green")
|
||||
if not backup_core_green:
|
||||
blocking_fields.append("backup_core_green")
|
||||
if not host_188_service_green:
|
||||
blocking_fields.append("host_188_service_green")
|
||||
if wazuh_dashboard_degraded:
|
||||
blocking_fields.append("wazuh_dashboard_degraded")
|
||||
|
||||
related_blockers = [
|
||||
blocker
|
||||
for blocker in active_blockers
|
||||
if blocker
|
||||
in {
|
||||
"backup_core_green_not_1",
|
||||
"host_188_service_green_not_1",
|
||||
"post_start_blocked_not_zero",
|
||||
"product_data_green_not_1",
|
||||
"service_green_not_1",
|
||||
"wazuh_dashboard_degraded",
|
||||
}
|
||||
]
|
||||
service_backup = _dict(
|
||||
payload.setdefault("controlled_service_data_backup_readback", {})
|
||||
)
|
||||
service_backup.update(
|
||||
{
|
||||
"status": (
|
||||
"ready_service_data_backup_green"
|
||||
if not blocking_fields
|
||||
else "blocked_service_data_backup_readback_not_green"
|
||||
),
|
||||
"readback_present": True,
|
||||
"service_green": service_green,
|
||||
"product_data_green": product_data_green,
|
||||
"backup_core_green": backup_core_green,
|
||||
"host_188_service_green": host_188_service_green,
|
||||
"post_start_blocked": post_start_blocked,
|
||||
"wazuh_dashboard_degraded": wazuh_dashboard_degraded,
|
||||
"blocking_fields": blocking_fields,
|
||||
"related_active_blockers": related_blockers,
|
||||
"can_clear_service_data_backup_blockers": not blocking_fields,
|
||||
"controlled_service_data_backup_blocker_count": len(blocking_fields),
|
||||
"runtime_metric_boolean_overlay_applied": True,
|
||||
}
|
||||
)
|
||||
|
||||
rollups = _dict(payload.setdefault("rollups", {}))
|
||||
rollups["service_green"] = service_green
|
||||
rollups["product_data_green"] = product_data_green
|
||||
rollups["backup_core_green"] = backup_core_green
|
||||
rollups["host_188_service_green"] = host_188_service_green
|
||||
rollups["post_start_blocked"] = post_start_blocked
|
||||
rollups["wazuh_dashboard_degraded"] = wazuh_dashboard_degraded
|
||||
rollups["controlled_service_data_backup_readback_present"] = True
|
||||
rollups["controlled_service_data_backup_blocker_count"] = len(blocking_fields)
|
||||
rollups["controlled_service_data_backup_can_clear_blockers"] = not blocking_fields
|
||||
rollups["controlled_service_data_backup_readback_status"] = service_backup["status"]
|
||||
rollups["runtime_metric_service_backup_boolean_overlay_applied"] = True
|
||||
|
||||
|
||||
def _apply_prometheus_metric_active_blockers(
|
||||
payload: dict[str, Any],
|
||||
metric_readback: dict[str, Any],
|
||||
@@ -702,6 +814,7 @@ def _apply_prometheus_metric_active_blockers(
|
||||
payload,
|
||||
active_blockers,
|
||||
)
|
||||
_apply_prometheus_service_backup_boolean_readback(payload, active_blockers)
|
||||
active_blockers = _add_runtime_readback_active_blockers_missing_from_metric(
|
||||
payload,
|
||||
active_blockers,
|
||||
|
||||
@@ -331,16 +331,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
|
||||
payload = load_latest_reboot_auto_recovery_slo_scorecard(
|
||||
prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK
|
||||
)
|
||||
expected_runtime_readback_added_blockers = [
|
||||
"service_green_not_1",
|
||||
"post_start_blocked_not_zero",
|
||||
"backup_core_green_not_1",
|
||||
"wazuh_dashboard_degraded",
|
||||
]
|
||||
expected_active_blockers = [
|
||||
*PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS,
|
||||
*expected_runtime_readback_added_blockers,
|
||||
]
|
||||
expected_active_blockers = PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS
|
||||
|
||||
assert payload["runtime_scorecard_readback_present"] is False
|
||||
assert payload["runtime_metric_readback_present"] is True
|
||||
@@ -352,20 +343,35 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
|
||||
)
|
||||
assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1
|
||||
assert payload["active_blockers"] == expected_active_blockers
|
||||
assert payload["active_blocker_count"] == 11
|
||||
assert payload["active_blocker_count"] == 7
|
||||
assert payload["readiness_percent"] == 47
|
||||
assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
|
||||
assert payload["next_safe_action"] == (
|
||||
"rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot"
|
||||
)
|
||||
assert "backup_core_green_not_1" in payload["active_blockers"]
|
||||
assert "service_green_not_1" in payload["active_blockers"]
|
||||
assert "backup_core_green_not_1" not in payload["active_blockers"]
|
||||
assert "service_green_not_1" not in payload["active_blockers"]
|
||||
assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"]
|
||||
assert payload["runtime_metric_runtime_readback_added_blockers"] == (
|
||||
expected_runtime_readback_added_blockers
|
||||
assert payload["runtime_metric_runtime_readback_added_blockers"] == []
|
||||
assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 0
|
||||
assert payload["service_green"] is True
|
||||
assert payload["backup_core_green"] is True
|
||||
assert payload["product_data_green"] is True
|
||||
assert payload["host_188_service_green"] is True
|
||||
assert payload["post_reboot_readiness"]["service_green"] is True
|
||||
assert payload["post_reboot_readiness"]["backup_core_green"] is True
|
||||
assert payload["post_reboot_readiness"]["post_start_blocked"] == 0
|
||||
assert payload["controlled_service_data_backup_readback"]["status"] == (
|
||||
"ready_service_data_backup_green"
|
||||
)
|
||||
assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 4
|
||||
assert payload["active_blocker_action_matrix"]["item_count"] == 11
|
||||
assert (
|
||||
payload["controlled_service_data_backup_readback"][
|
||||
"can_clear_service_data_backup_blockers"
|
||||
]
|
||||
is True
|
||||
)
|
||||
assert payload["controlled_service_data_backup_readback"]["blocking_fields"] == []
|
||||
assert payload["active_blocker_action_matrix"]["item_count"] == 7
|
||||
assert payload["windows99_vmware_autostart"]["readback_present"] is True
|
||||
assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"]
|
||||
assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [
|
||||
@@ -406,6 +412,19 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
|
||||
assert payload["rollups"]["windows99_verify_collection_status"] == (
|
||||
"blocked_windows99_verify_collection_not_ready"
|
||||
)
|
||||
assert payload["rollups"]["service_green"] is True
|
||||
assert payload["rollups"]["backup_core_green"] is True
|
||||
assert payload["rollups"]["product_data_green"] is True
|
||||
assert payload["rollups"]["host_188_service_green"] is True
|
||||
assert payload["rollups"]["post_start_blocked"] == 0
|
||||
assert payload["rollups"][
|
||||
"runtime_metric_service_backup_boolean_overlay_applied"
|
||||
] is True
|
||||
assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 0
|
||||
assert (
|
||||
payload["rollups"]["controlled_service_data_backup_can_clear_blockers"]
|
||||
is True
|
||||
)
|
||||
action_by_blocker = {
|
||||
item["blocker"]: item
|
||||
for item in payload["active_blocker_action_matrix"]["items"]
|
||||
@@ -416,33 +435,42 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
|
||||
0,
|
||||
) == 0
|
||||
assert payload["active_blocker_action_matrix"]["category_counts"][
|
||||
"post_reboot_service_readiness"
|
||||
"host_boot_detection"
|
||||
] == 5
|
||||
assert payload["active_blocker_action_matrix"]["category_counts"][
|
||||
"windows99_vmware_autostart"
|
||||
] == 2
|
||||
assert payload["active_blocker_action_matrix"]["category_counts"][
|
||||
"backup_observability"
|
||||
] == 1
|
||||
assert payload["active_blocker_action_matrix"]["category_counts"][
|
||||
"security_observability"
|
||||
] == 1
|
||||
assert payload["readback"]["active_blocker_count"] == 11
|
||||
assert payload["active_blocker_action_matrix"]["category_counts"].get(
|
||||
"post_reboot_service_readiness",
|
||||
0,
|
||||
) == 0
|
||||
assert payload["active_blocker_action_matrix"]["category_counts"].get(
|
||||
"backup_observability",
|
||||
0,
|
||||
) == 0
|
||||
assert payload["active_blocker_action_matrix"]["category_counts"].get(
|
||||
"security_observability",
|
||||
0,
|
||||
) == 0
|
||||
assert payload["readback"]["active_blocker_count"] == 7
|
||||
assert payload["readback"]["runtime_metric_readback_present"] is True
|
||||
assert payload["readback"][
|
||||
"runtime_metric_source_control_reconciled_blocker_count"
|
||||
] == 1
|
||||
assert payload["readback"][
|
||||
"runtime_metric_runtime_readback_added_blockers"
|
||||
] == expected_runtime_readback_added_blockers
|
||||
] == []
|
||||
assert payload["readback"][
|
||||
"runtime_metric_runtime_readback_added_blocker_count"
|
||||
] == 4
|
||||
assert payload["rollups"]["active_blocker_count"] == 11
|
||||
] == 0
|
||||
assert payload["rollups"]["active_blocker_count"] == 7
|
||||
assert payload["rollups"]["runtime_metric_readback_present"] is True
|
||||
assert payload["rollups"][
|
||||
"runtime_metric_source_control_reconciled_blocker_count"
|
||||
] == 1
|
||||
assert payload["rollups"][
|
||||
"runtime_metric_runtime_readback_added_blocker_count"
|
||||
] == 4
|
||||
] == 0
|
||||
assert payload["rollups"]["primary_blocker_owner_lane"] == (
|
||||
"reboot_event_detector_and_host_probe"
|
||||
)
|
||||
@@ -465,14 +493,8 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when
|
||||
prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK,
|
||||
)
|
||||
|
||||
assert payload["active_blockers"] == [
|
||||
*PROMETHEUS_RUNTIME_BLOCKERS,
|
||||
"service_green_not_1",
|
||||
"post_start_blocked_not_zero",
|
||||
"backup_core_green_not_1",
|
||||
"wazuh_dashboard_degraded",
|
||||
]
|
||||
assert payload["active_blocker_count"] == 12
|
||||
assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
|
||||
assert payload["active_blocker_count"] == 8
|
||||
assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0
|
||||
action_by_blocker = {
|
||||
item["blocker"]: item
|
||||
|
||||
Reference in New Issue
Block a user