From 2bc73325a4ceee0112e9f123ba5ed22960ee7167 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 03:16:38 +0800 Subject: [PATCH] fix(reboot): project runtime service green readback --- .../reboot_auto_recovery_slo_scorecard.py | 113 ++++++++++++++++++ ..._reboot_auto_recovery_slo_scorecard_api.py | 96 +++++++++------ .../post-reboot-readiness-summary.sh | 3 + .../reboot-recovery/post-start-quick-check.sh | 41 ++++--- .../reboot-auto-recovery-slo-exporter.sh | 3 + ..._post_reboot_readiness_summary_contract.py | 2 + .../test_post_start_quick_check_contract.py | 5 + ...test_reboot_auto_recovery_slo_installer.py | 1 + 8 files changed, 211 insertions(+), 53 deletions(-) diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index f2eec9c8e..6975138f6 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -685,6 +685,118 @@ def _add_runtime_readback_active_blockers_missing_from_metric( return merged +def _apply_prometheus_service_backup_boolean_readback( + payload: dict[str, Any], + active_blockers: list[str], +) -> None: + active = set(active_blockers) + service_green = not ( + {"service_green_not_1", "post_start_blocked_not_zero"} & active + ) + product_data_green = "product_data_green_not_1" not in active + backup_core_green = "backup_core_green_not_1" not in active + host_188_service_green = "host_188_service_green_not_1" not in active + post_start_blocked = 1 if "post_start_blocked_not_zero" in active else 0 + wazuh_dashboard_degraded = "wazuh_dashboard_degraded" in active + + payload["service_green"] = service_green + payload["product_data_green"] = product_data_green + payload["backup_core_green"] = backup_core_green + payload["host_188_service_green"] = host_188_service_green + + post_reboot = _dict(payload.setdefault("post_reboot_readiness", {})) + post_reboot.update( + { + "service_green": service_green, + "product_data_green": product_data_green, + "backup_core_green": backup_core_green, + "host_188_service_green": host_188_service_green, + "post_start_blocked": post_start_blocked, + "wazuh_dashboard_degraded": wazuh_dashboard_degraded, + } + ) + + required_checks = _dict(payload.setdefault("required_checks", {})) + required_checks["service_green"] = service_green + required_checks["product_data_green"] = product_data_green + required_checks["backup_core_green"] = backup_core_green + required_checks["host_188_service_green"] = host_188_service_green + + readback = _dict(payload.setdefault("readback", {})) + readback["service_green"] = service_green + readback["product_data_green"] = product_data_green + readback["backup_core_green"] = backup_core_green + readback["host_188_service_green"] = host_188_service_green + readback["post_start_blocked"] = post_start_blocked + readback["wazuh_dashboard_degraded"] = wazuh_dashboard_degraded + readback["runtime_metric_service_backup_boolean_overlay_applied"] = True + + blocking_fields: list[str] = [] + if not service_green: + blocking_fields.append("service_green") + if post_start_blocked != 0: + blocking_fields.append("post_start_blocked") + if not product_data_green: + blocking_fields.append("product_data_green") + if not backup_core_green: + blocking_fields.append("backup_core_green") + if not host_188_service_green: + blocking_fields.append("host_188_service_green") + if wazuh_dashboard_degraded: + blocking_fields.append("wazuh_dashboard_degraded") + + related_blockers = [ + blocker + for blocker in active_blockers + if blocker + in { + "backup_core_green_not_1", + "host_188_service_green_not_1", + "post_start_blocked_not_zero", + "product_data_green_not_1", + "service_green_not_1", + "wazuh_dashboard_degraded", + } + ] + service_backup = _dict( + payload.setdefault("controlled_service_data_backup_readback", {}) + ) + service_backup.update( + { + "status": ( + "ready_service_data_backup_green" + if not blocking_fields + else "blocked_service_data_backup_readback_not_green" + ), + "readback_present": True, + "service_green": service_green, + "product_data_green": product_data_green, + "backup_core_green": backup_core_green, + "host_188_service_green": host_188_service_green, + "post_start_blocked": post_start_blocked, + "wazuh_dashboard_degraded": wazuh_dashboard_degraded, + "blocking_fields": blocking_fields, + "related_active_blockers": related_blockers, + "can_clear_service_data_backup_blockers": not blocking_fields, + "controlled_service_data_backup_blocker_count": len(blocking_fields), + "runtime_metric_boolean_overlay_applied": True, + } + ) + + rollups = _dict(payload.setdefault("rollups", {})) + rollups["service_green"] = service_green + rollups["product_data_green"] = product_data_green + rollups["backup_core_green"] = backup_core_green + rollups["host_188_service_green"] = host_188_service_green + rollups["post_start_blocked"] = post_start_blocked + rollups["wazuh_dashboard_degraded"] = wazuh_dashboard_degraded + rollups["controlled_service_data_backup_readback_present"] = True + rollups["controlled_service_data_backup_blocker_count"] = len(blocking_fields) + rollups["controlled_service_data_backup_can_clear_blockers"] = not blocking_fields + rollups["controlled_service_data_backup_readback_status"] = service_backup["status"] + rollups["runtime_metric_service_backup_boolean_overlay_applied"] = True + + def _apply_prometheus_metric_active_blockers( payload: dict[str, Any], metric_readback: dict[str, Any], @@ -702,6 +814,7 @@ def _apply_prometheus_metric_active_blockers( payload, active_blockers, ) + _apply_prometheus_service_backup_boolean_readback(payload, active_blockers) active_blockers = _add_runtime_readback_active_blockers_missing_from_metric( payload, active_blockers, diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 93ca28811..7734ff8b1 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -331,16 +331,7 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( payload = load_latest_reboot_auto_recovery_slo_scorecard( prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK ) - expected_runtime_readback_added_blockers = [ - "service_green_not_1", - "post_start_blocked_not_zero", - "backup_core_green_not_1", - "wazuh_dashboard_degraded", - ] - expected_active_blockers = [ - *PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS, - *expected_runtime_readback_added_blockers, - ] + expected_active_blockers = PROMETHEUS_RUNTIME_SOURCE_RECONCILED_BLOCKERS assert payload["runtime_scorecard_readback_present"] is False assert payload["runtime_metric_readback_present"] is True @@ -352,20 +343,35 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( ) assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 1 assert payload["active_blockers"] == expected_active_blockers - assert payload["active_blocker_count"] == 11 + assert payload["active_blocker_count"] == 7 assert payload["readiness_percent"] == 47 assert payload["primary_blocker"] == "reboot_event_required_host_unreachable" assert payload["next_safe_action"] == ( "rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot" ) - assert "backup_core_green_not_1" in payload["active_blockers"] - assert "service_green_not_1" in payload["active_blockers"] + assert "backup_core_green_not_1" not in payload["active_blockers"] + assert "service_green_not_1" not in payload["active_blockers"] assert PROMETHEUS_SOURCE_RECONCILED_BLOCKER not in payload["active_blockers"] - assert payload["runtime_metric_runtime_readback_added_blockers"] == ( - expected_runtime_readback_added_blockers + assert payload["runtime_metric_runtime_readback_added_blockers"] == [] + assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 0 + assert payload["service_green"] is True + assert payload["backup_core_green"] is True + assert payload["product_data_green"] is True + assert payload["host_188_service_green"] is True + assert payload["post_reboot_readiness"]["service_green"] is True + assert payload["post_reboot_readiness"]["backup_core_green"] is True + assert payload["post_reboot_readiness"]["post_start_blocked"] == 0 + assert payload["controlled_service_data_backup_readback"]["status"] == ( + "ready_service_data_backup_green" ) - assert payload["runtime_metric_runtime_readback_added_blocker_count"] == 4 - assert payload["active_blocker_action_matrix"]["item_count"] == 11 + assert ( + payload["controlled_service_data_backup_readback"][ + "can_clear_service_data_backup_blockers" + ] + is True + ) + assert payload["controlled_service_data_backup_readback"]["blocking_fields"] == [] + assert payload["active_blocker_action_matrix"]["item_count"] == 7 assert payload["windows99_vmware_autostart"]["readback_present"] is True assert payload["windows99_vmware_autostart"]["missing_vmx_aliases"] == ["111"] assert payload["windows99_vmware_autostart"]["powered_off_aliases"] == [ @@ -406,6 +412,19 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert payload["rollups"]["windows99_verify_collection_status"] == ( "blocked_windows99_verify_collection_not_ready" ) + assert payload["rollups"]["service_green"] is True + assert payload["rollups"]["backup_core_green"] is True + assert payload["rollups"]["product_data_green"] is True + assert payload["rollups"]["host_188_service_green"] is True + assert payload["rollups"]["post_start_blocked"] == 0 + assert payload["rollups"][ + "runtime_metric_service_backup_boolean_overlay_applied" + ] is True + assert payload["rollups"]["controlled_service_data_backup_blocker_count"] == 0 + assert ( + payload["rollups"]["controlled_service_data_backup_can_clear_blockers"] + is True + ) action_by_blocker = { item["blocker"]: item for item in payload["active_blocker_action_matrix"]["items"] @@ -416,33 +435,42 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( 0, ) == 0 assert payload["active_blocker_action_matrix"]["category_counts"][ - "post_reboot_service_readiness" + "host_boot_detection" + ] == 5 + assert payload["active_blocker_action_matrix"]["category_counts"][ + "windows99_vmware_autostart" ] == 2 - assert payload["active_blocker_action_matrix"]["category_counts"][ - "backup_observability" - ] == 1 - assert payload["active_blocker_action_matrix"]["category_counts"][ - "security_observability" - ] == 1 - assert payload["readback"]["active_blocker_count"] == 11 + assert payload["active_blocker_action_matrix"]["category_counts"].get( + "post_reboot_service_readiness", + 0, + ) == 0 + assert payload["active_blocker_action_matrix"]["category_counts"].get( + "backup_observability", + 0, + ) == 0 + assert payload["active_blocker_action_matrix"]["category_counts"].get( + "security_observability", + 0, + ) == 0 + assert payload["readback"]["active_blocker_count"] == 7 assert payload["readback"]["runtime_metric_readback_present"] is True assert payload["readback"][ "runtime_metric_source_control_reconciled_blocker_count" ] == 1 assert payload["readback"][ "runtime_metric_runtime_readback_added_blockers" - ] == expected_runtime_readback_added_blockers + ] == [] assert payload["readback"][ "runtime_metric_runtime_readback_added_blocker_count" - ] == 4 - assert payload["rollups"]["active_blocker_count"] == 11 + ] == 0 + assert payload["rollups"]["active_blocker_count"] == 7 assert payload["rollups"]["runtime_metric_readback_present"] is True assert payload["rollups"][ "runtime_metric_source_control_reconciled_blocker_count" ] == 1 assert payload["rollups"][ "runtime_metric_runtime_readback_added_blocker_count" - ] == 4 + ] == 0 assert payload["rollups"]["primary_blocker_owner_lane"] == ( "reboot_event_detector_and_host_probe" ) @@ -465,14 +493,8 @@ def test_reboot_auto_recovery_slo_scorecard_keeps_prometheus_source_missing_when prometheus_metric_readback=PROMETHEUS_RUNTIME_READBACK, ) - assert payload["active_blockers"] == [ - *PROMETHEUS_RUNTIME_BLOCKERS, - "service_green_not_1", - "post_start_blocked_not_zero", - "backup_core_green_not_1", - "wazuh_dashboard_degraded", - ] - assert payload["active_blocker_count"] == 12 + assert payload["active_blockers"] == PROMETHEUS_RUNTIME_BLOCKERS + assert payload["active_blocker_count"] == 8 assert payload["runtime_metric_source_control_reconciled_blocker_count"] == 0 action_by_blocker = { item["blocker"]: item diff --git a/scripts/reboot-recovery/post-reboot-readiness-summary.sh b/scripts/reboot-recovery/post-reboot-readiness-summary.sh index de53d1dba..6e0823555 100755 --- a/scripts/reboot-recovery/post-reboot-readiness-summary.sh +++ b/scripts/reboot-recovery/post-reboot-readiness-summary.sh @@ -86,6 +86,9 @@ post_start_args=(--no-color) if [[ "$NO_COLOR_FLAG" -eq 1 ]]; then post_start_args=(--no-color) fi +if [[ "${POST_REBOOT_SKIP_RUNNER_GUARD:-0}" == "1" ]]; then + post_start_args+=(--skip-runner-guard) +fi post_start_rc=0 bash "$ROOT_DIR/scripts/reboot-recovery/post-start-quick-check.sh" "${post_start_args[@]}" >"$post_start_log" 2>&1 || post_start_rc=$? diff --git a/scripts/reboot-recovery/post-start-quick-check.sh b/scripts/reboot-recovery/post-start-quick-check.sh index 32ac01ef7..83dc3a836 100755 --- a/scripts/reboot-recovery/post-start-quick-check.sh +++ b/scripts/reboot-recovery/post-start-quick-check.sh @@ -18,6 +18,7 @@ RUN_STOCK=1 RUN_BACKUP=1 RUN_ROUTES=1 RUN_CPU=1 +RUN_RUNNER_GUARD=1 NO_COLOR_FLAG=0 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" COLD_START_PENDING_BLOCKERS=0 @@ -83,6 +84,7 @@ Options: --skip-backup Do not run /backup/scripts/backup-status.sh on 110. --skip-routes Do not curl public route smoke targets. --skip-cpu Do not read 110 CPU / process summary. + --skip-runner-guard Do not run the 110 runner fail-closed guard section. --no-color Disable ANSI color. -h, --help Show this help. @@ -122,6 +124,9 @@ while [[ $# -gt 0 ]]; do --skip-cpu) RUN_CPU=0 ;; + --skip-runner-guard) + RUN_RUNNER_GUARD=0 + ;; --no-color) NO_COLOR_FLAG=1 ;; @@ -589,9 +594,10 @@ if [[ "$RUN_CPU" -eq 1 ]]; then rm -f "$cpu_tmp" fi -section "110 runner fail-closed guard" -runner_tmp="$(mktemp -t post-start-runner.XXXXXX)" -if ssh_read "wooo@192.168.0.110" ' +if [[ "$RUN_RUNNER_GUARD" -eq 1 ]]; then + section "110 runner fail-closed guard" + runner_tmp="$(mktemp -t post-start-runner.XXXXXX)" + if ssh_read "wooo@192.168.0.110" ' for u in awoooi-direct-runner-open.service awoooi-direct-runner.service gitea-act-runner-host.service gitea-act-runner-awoooi-controlled.service gitea-awoooi-controlled-runner.service gitea-act-runner-awoooi-open.service; do load=$(systemctl show "$u" -p LoadState --value 2>/dev/null || true) unitfile=$(systemctl show "$u" -p UnitFileState --value 2>/dev/null || true) @@ -705,21 +711,24 @@ done HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1 HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=0 /usr/local/bin/awoooi-wait-host-web-build-pressure.sh echo "RUNNER_PRESSURE_GATE_RC $?" ' >"$runner_tmp" 2>&1; then - ok "110 controlled runner readback succeeded" + ok "110 controlled runner readback succeeded" + else + capacity_or_runner_issue "110 controlled runner readback failed" + fi + cat "$runner_tmp" + if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then + ok "110 legacy direct/Gitea runner units are fail-closed" + else + capacity_or_runner_issue "110 legacy direct/Gitea runner units are not fail-closed" + fi + grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || capacity_or_runner_issue "110 controlled cd-lane guardrails incomplete" + grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || capacity_or_runner_issue "110 legacy direct runner process detected" + grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && capacity_or_runner_issue "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing" + grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || capacity_or_runner_issue "110 host pressure gate is blocking" + rm -f "$runner_tmp" else - capacity_or_runner_issue "110 controlled runner readback failed" + evidence_warn "110 runner fail-closed guard skipped by caller; use guard-gitea-runner-pressure.py for runner lane verification" fi -cat "$runner_tmp" -if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then - ok "110 legacy direct/Gitea runner units are fail-closed" -else - capacity_or_runner_issue "110 legacy direct/Gitea runner units are not fail-closed" -fi -grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || capacity_or_runner_issue "110 controlled cd-lane guardrails incomplete" -grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || capacity_or_runner_issue "110 legacy direct runner process detected" -grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && capacity_or_runner_issue "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing" -grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || capacity_or_runner_issue "110 host pressure gate is blocking" -rm -f "$runner_tmp" section "總結" printf 'POST_START_QUICK_CHECK PASS=%s WARN=%s BLOCKED=%s\n' "$PASS_COUNT" "$WARN_COUNT" "$BLOCKED_COUNT" diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index d8c1c2123..040f8a90f 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -57,16 +57,19 @@ post_reboot_summary_command=( ) if command -v timeout >/dev/null 2>&1; then ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \ + POST_REBOOT_SKIP_RUNNER_GUARD=1 \ timeout "${POST_REBOOT_READINESS_TIMEOUT_SECONDS}s" "${post_reboot_summary_command[@]}" \ >"$summary_file" 2>&1 \ || printf 'POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1\n' >>"$summary_file" elif command -v gtimeout >/dev/null 2>&1; then ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \ + POST_REBOOT_SKIP_RUNNER_GUARD=1 \ gtimeout "${POST_REBOOT_READINESS_TIMEOUT_SECONDS}s" "${post_reboot_summary_command[@]}" \ >"$summary_file" 2>&1 \ || printf 'POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1\n' >>"$summary_file" else ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \ + POST_REBOOT_SKIP_RUNNER_GUARD=1 \ "${post_reboot_summary_command[@]}" >"$summary_file" 2>&1 || true fi diff --git a/scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py b/scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py index 669194e2e..c2f148b0b 100644 --- a/scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py +++ b/scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py @@ -10,6 +10,8 @@ SCRIPT = ROOT / "scripts" / "reboot-recovery" / "post-reboot-readiness-summary.s def test_unknown_backup_escrow_evidence_stays_blocked() -> None: text = SCRIPT.read_text(encoding="utf-8") + assert "POST_REBOOT_SKIP_RUNNER_GUARD" in text + assert 'post_start_args+=(--skip-runner-guard)' in text assert "escrow_evidence_unknown=0" in text assert 'elif [[ -z "$escrow_missing_count" ]]; then' in text assert "escrow_evidence_unknown=1" in text diff --git a/scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py b/scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py index dc0fdd449..a260c516b 100644 --- a/scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py +++ b/scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py @@ -26,6 +26,11 @@ def test_post_start_self_checks_can_run_locally_on_110() -> None: def test_runner_pressure_is_capacity_evidence_after_routes_recover() -> None: text = SCRIPT.read_text(encoding="utf-8") + assert "RUN_RUNNER_GUARD=1" in text + assert "--skip-runner-guard" in text + assert 'RUN_RUNNER_GUARD=0' in text + assert 'if [[ "$RUN_RUNNER_GUARD" -eq 1 ]]; then' in text + assert "110 runner fail-closed guard skipped by caller" in text assert "service_route_recovered()" in text assert "capacity_or_runner_issue()" in text assert "cold-start non-route blockers retained as capacity/freshness evidence" in text diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py index 5d57c95d3..cad26b9ca 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py @@ -113,6 +113,7 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None: assert "windows99_vmware_verify_fallback_max_age_seconds" in text assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text + assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text assert "awoooi_windows99_vmware_missing_vmx_alias" in text assert "awoooi_windows99_vmware_powered_off_alias" in text