fix(reboot): project runtime service green readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 1m27s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
Your Name
2026-07-03 03:16:38 +08:00
parent c36a65ecfb
commit 2bc73325a4
8 changed files with 211 additions and 53 deletions

View File

@@ -685,6 +685,118 @@ def _add_runtime_readback_active_blockers_missing_from_metric(
return merged
def _apply_prometheus_service_backup_boolean_readback(
payload: dict[str, Any],
active_blockers: list[str],
) -> None:
active = set(active_blockers)
service_green = not (
{"service_green_not_1", "post_start_blocked_not_zero"} & active
)
product_data_green = "product_data_green_not_1" not in active
backup_core_green = "backup_core_green_not_1" not in active
host_188_service_green = "host_188_service_green_not_1" not in active
post_start_blocked = 1 if "post_start_blocked_not_zero" in active else 0
wazuh_dashboard_degraded = "wazuh_dashboard_degraded" in active
payload["service_green"] = service_green
payload["product_data_green"] = product_data_green
payload["backup_core_green"] = backup_core_green
payload["host_188_service_green"] = host_188_service_green
post_reboot = _dict(payload.setdefault("post_reboot_readiness", {}))
post_reboot.update(
{
"service_green": service_green,
"product_data_green": product_data_green,
"backup_core_green": backup_core_green,
"host_188_service_green": host_188_service_green,
"post_start_blocked": post_start_blocked,
"wazuh_dashboard_degraded": wazuh_dashboard_degraded,
}
)
required_checks = _dict(payload.setdefault("required_checks", {}))
required_checks["service_green"] = service_green
required_checks["product_data_green"] = product_data_green
required_checks["backup_core_green"] = backup_core_green
required_checks["host_188_service_green"] = host_188_service_green
readback = _dict(payload.setdefault("readback", {}))
readback["service_green"] = service_green
readback["product_data_green"] = product_data_green
readback["backup_core_green"] = backup_core_green
readback["host_188_service_green"] = host_188_service_green
readback["post_start_blocked"] = post_start_blocked
readback["wazuh_dashboard_degraded"] = wazuh_dashboard_degraded
readback["runtime_metric_service_backup_boolean_overlay_applied"] = True
blocking_fields: list[str] = []
if not service_green:
blocking_fields.append("service_green")
if post_start_blocked != 0:
blocking_fields.append("post_start_blocked")
if not product_data_green:
blocking_fields.append("product_data_green")
if not backup_core_green:
blocking_fields.append("backup_core_green")
if not host_188_service_green:
blocking_fields.append("host_188_service_green")
if wazuh_dashboard_degraded:
blocking_fields.append("wazuh_dashboard_degraded")
related_blockers = [
blocker
for blocker in active_blockers
if blocker
in {
"backup_core_green_not_1",
"host_188_service_green_not_1",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"service_green_not_1",
"wazuh_dashboard_degraded",
}
]
service_backup = _dict(
payload.setdefault("controlled_service_data_backup_readback", {})
)
service_backup.update(
{
"status": (
"ready_service_data_backup_green"
if not blocking_fields
else "blocked_service_data_backup_readback_not_green"
),
"readback_present": True,
"service_green": service_green,
"product_data_green": product_data_green,
"backup_core_green": backup_core_green,
"host_188_service_green": host_188_service_green,
"post_start_blocked": post_start_blocked,
"wazuh_dashboard_degraded": wazuh_dashboard_degraded,
"blocking_fields": blocking_fields,
"related_active_blockers": related_blockers,
"can_clear_service_data_backup_blockers": not blocking_fields,
"controlled_service_data_backup_blocker_count": len(blocking_fields),
"runtime_metric_boolean_overlay_applied": True,
}
)
rollups = _dict(payload.setdefault("rollups", {}))
rollups["service_green"] = service_green
rollups["product_data_green"] = product_data_green
rollups["backup_core_green"] = backup_core_green
rollups["host_188_service_green"] = host_188_service_green
rollups["post_start_blocked"] = post_start_blocked
rollups["wazuh_dashboard_degraded"] = wazuh_dashboard_degraded
rollups["controlled_service_data_backup_readback_present"] = True
rollups["controlled_service_data_backup_blocker_count"] = len(blocking_fields)
rollups["controlled_service_data_backup_can_clear_blockers"] = not blocking_fields
rollups["controlled_service_data_backup_readback_status"] = service_backup["status"]
rollups["runtime_metric_service_backup_boolean_overlay_applied"] = True
def _apply_prometheus_metric_active_blockers(
payload: dict[str, Any],
metric_readback: dict[str, Any],
@@ -702,6 +814,7 @@ def _apply_prometheus_metric_active_blockers(
payload,
active_blockers,
)
_apply_prometheus_service_backup_boolean_readback(payload, active_blockers)
active_blockers = _add_runtime_readback_active_blockers_missing_from_metric(
payload,
active_blockers,

View File

@@ -331,16 +331,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
payload = load_latest_reboot_auto_recovery_slo_scorecard(
prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK
)
expected_runtime_readback_added_blockers = [
"service_green_not_1",
"post_start_blocked_not_zero",
"backup_core_green_not_1",
"wazuh_dashboard_degraded",
]
expected_active_blockers = [
*PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS,
*expected_runtime_readback_added_blockers,
]
expected_active_blockers = PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS
assert payload["runtime_scorecard_readback_present"] is False
assert payload["runtime_metric_readback_present"] is True
@@ -352,20 +343,35 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
)
assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1
assert payload["active_blockers"] == expected_active_blockers
assert payload["active_blocker_count"] == 11
assert payload["active_blocker_count"] == 7
assert payload["readiness_percent"] == 47
assert payload["primary_blocker"] == "reboot_event_required_host_unreachable"
assert payload["next_safe_action"] == (
"rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot"
)
assert "backup_core_green_not_1" in payload["active_blockers"]
assert "service_green_not_1" in payload["active_blockers"]
assert "backup_core_green_not_1" not in payload["active_blockers"]
assert "service_green_not_1" not in payload["active_blockers"]
assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"]
assert payload["runtime_metric_runtime_readback_added_blockers"] == (
expected_runtime_readback_added_blockers
assert payload["runtime_metric_runtime_readback_added_blockers"] == []
assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 0
assert payload["service_green"] is True
assert payload["backup_core_green"] is True
assert payload["product_data_green"] is True
assert payload["host_188_service_green"] is True
assert payload["post_reboot_readiness"]["service_green"] is True
assert payload["post_reboot_readiness"]["backup_core_green"] is True
assert payload["post_reboot_readiness"]["post_start_blocked"] == 0
assert payload["controlled_service_data_backup_readback"]["status"] == (
"ready_service_data_backup_green"
)
assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 4
assert payload["active_blocker_action_matrix"]["item_count"] == 11
assert (
payload["controlled_service_data_backup_readback"][
"can_clear_service_data_backup_blockers"
]
is True
)
assert payload["controlled_service_data_backup_readback"]["blocking_fields"] == []
assert payload["active_blocker_action_matrix"]["item_count"] == 7
assert payload["windows99_vmware_autostart"]["readback_present"] is True
assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"]
assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [
@@ -406,6 +412,19 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
assert payload["rollups"]["windows99_verify_collection_status"] == (
"blocked_windows99_verify_collection_not_ready"
)
assert payload["rollups"]["service_green"] is True
assert payload["rollups"]["backup_core_green"] is True
assert payload["rollups"]["product_data_green"] is True
assert payload["rollups"]["host_188_service_green"] is True
assert payload["rollups"]["post_start_blocked"] == 0
assert payload["rollups"][
"runtime_metric_service_backup_boolean_overlay_applied"
] is True
assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 0
assert (
payload["rollups"]["controlled_service_data_backup_can_clear_blockers"]
is True
)
action_by_blocker = {
item["blocker"]: item
for item in payload["active_blocker_action_matrix"]["items"]
@@ -416,33 +435,42 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics(
0,
) == 0
assert payload["active_blocker_action_matrix"]["category_counts"][
"post_reboot_service_readiness"
"host_boot_detection"
] == 5
assert payload["active_blocker_action_matrix"]["category_counts"][
"windows99_vmware_autostart"
] == 2
assert payload["active_blocker_action_matrix"]["category_counts"][
"backup_observability"
] == 1
assert payload["active_blocker_action_matrix"]["category_counts"][
"security_observability"
] == 1
assert payload["readback"]["active_blocker_count"] == 11
assert payload["active_blocker_action_matrix"]["category_counts"].get(
"post_reboot_service_readiness",
0,
) == 0
assert payload["active_blocker_action_matrix"]["category_counts"].get(
"backup_observability",
0,
) == 0
assert payload["active_blocker_action_matrix"]["category_counts"].get(
"security_observability",
0,
) == 0
assert payload["readback"]["active_blocker_count"] == 7
assert payload["readback"]["runtime_metric_readback_present"] is True
assert payload["readback"][
"runtime_metric_source_control_reconciled_blocker_count"
] == 1
assert payload["readback"][
"runtime_metric_runtime_readback_added_blockers"
] == expected_runtime_readback_added_blockers
] == []
assert payload["readback"][
"runtime_metric_runtime_readback_added_blocker_count"
] == 4
assert payload["rollups"]["active_blocker_count"] == 11
] == 0
assert payload["rollups"]["active_blocker_count"] == 7
assert payload["rollups"]["runtime_metric_readback_present"] is True
assert payload["rollups"][
"runtime_metric_source_control_reconciled_blocker_count"
] == 1
assert payload["rollups"][
"runtime_metric_runtime_readback_added_blocker_count"
] == 4
] == 0
assert payload["rollups"]["primary_blocker_owner_lane"] == (
"reboot_event_detector_and_host_probe"
)
@@ -465,14 +493,8 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when
prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK,
)
assert payload["active_blockers"] == [
*PROMETHEUS_RUNTIME_BLOCKERS,
"service_green_not_1",
"post_start_blocked_not_zero",
"backup_core_green_not_1",
"wazuh_dashboard_degraded",
]
assert payload["active_blocker_count"] == 12
assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS
assert payload["active_blocker_count"] == 8
assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0
action_by_blocker = {
item["blocker"]: item