fix(awooop): route reboot blockers after backup clear
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m20s
CD Pipeline / build-and-deploy (push) Successful in 5m13s
CD Pipeline / post-deploy-checks (push) Successful in 2m25s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m20s
CD Pipeline / build-and-deploy (push) Successful in 5m13s
CD Pipeline / post-deploy-checks (push) Successful in 2m25s
This commit is contained in:
@@ -62,6 +62,14 @@ _PARALLEL_MAINLINE_FORBIDDEN_ACTIONS = [
|
||||
"db_write_or_restore_from_this_wait_lane",
|
||||
"docker_or_host_restart_for_source_pending",
|
||||
]
|
||||
_REBOOT_HOST_BOOT_DETECTION_BLOCKERS = {
|
||||
"all_required_hosts_not_in_10_minute_reboot_window",
|
||||
"fresh_all_host_reboot_event_missing",
|
||||
"host_boot_observation_older_than_target_window",
|
||||
"host_unreachable_after_reboot",
|
||||
"host_uptime_unknown",
|
||||
"reboot_event_required_host_unreachable",
|
||||
}
|
||||
_AI_AUTOMATION_NODE_RECEIPT_SCHEMA_VERSION = "ai_automation_node_receipt_v1"
|
||||
_AI_AUTOMATION_NODE_RECEIPT_REQUIRED_FIELDS = [
|
||||
"trace_id",
|
||||
@@ -2519,6 +2527,63 @@ def _harbor_registry_next_state(*, status: str, candidate_packaged: bool) -> str
|
||||
return "controlled_recovery_waiting_registry_v2_evidence_and_executor_contract"
|
||||
|
||||
|
||||
def _reboot_slo_next_mainline_state(
|
||||
*,
|
||||
event_gated: bool,
|
||||
service_data_backup_readback_present: bool,
|
||||
service_data_backup_blocking_fields: list[str],
|
||||
windows99_verify_collection_blockers: list[str],
|
||||
active_blockers: list[str],
|
||||
) -> str:
|
||||
if event_gated:
|
||||
return (
|
||||
"production_preflight_ready_wait_for_next_real_all_host_reboot_event_"
|
||||
"or_separate_break_glass_reboot_drill_authorization"
|
||||
)
|
||||
if not service_data_backup_readback_present:
|
||||
return (
|
||||
"blocked_reboot_slo_scorecard_requires_controlled_service_data_"
|
||||
"backup_readback"
|
||||
)
|
||||
if service_data_backup_blocking_fields:
|
||||
return (
|
||||
"blocked_reboot_slo_scorecard_service_data_backup_readback_present_"
|
||||
"blockers_open"
|
||||
)
|
||||
|
||||
has_windows99_blockers = bool(
|
||||
windows99_verify_collection_blockers
|
||||
or [blocker for blocker in active_blockers if blocker.startswith("windows99_")]
|
||||
)
|
||||
has_host_boot_blockers = bool(
|
||||
[
|
||||
blocker
|
||||
for blocker in active_blockers
|
||||
if blocker in _REBOOT_HOST_BOOT_DETECTION_BLOCKERS
|
||||
]
|
||||
)
|
||||
if has_host_boot_blockers and has_windows99_blockers:
|
||||
return (
|
||||
"blocked_reboot_slo_scorecard_host_boot_and_windows99_verify_"
|
||||
"collection_not_ready"
|
||||
)
|
||||
if has_windows99_blockers:
|
||||
return "blocked_reboot_slo_scorecard_windows99_verify_collection_not_ready"
|
||||
if has_host_boot_blockers:
|
||||
return "blocked_reboot_slo_scorecard_host_boot_detection_not_ready"
|
||||
if active_blockers:
|
||||
return "blocked_reboot_slo_scorecard_active_blockers_open"
|
||||
return "blocked_reboot_slo_scorecard_waiting_post_verifier_readback"
|
||||
|
||||
|
||||
def _service_data_backup_blockers_closed(
|
||||
*,
|
||||
service_data_backup_readback_present: bool,
|
||||
service_data_backup_blocking_fields: list[str],
|
||||
) -> bool:
|
||||
return service_data_backup_readback_present and not service_data_backup_blocking_fields
|
||||
|
||||
|
||||
def _harbor_registry_reason(*, status: str, candidate_packaged: bool) -> str:
|
||||
if candidate_packaged and "upstream_502" in status:
|
||||
return (
|
||||
@@ -3150,18 +3215,12 @@ def _enrich_from_current_readbacks(
|
||||
if p0_006_event_gated
|
||||
else "P0-006-REBOOT-AUTO-RECOVERY-SLO-SCORECARD"
|
||||
)
|
||||
state["next_executable_mainline_state"] = (
|
||||
(
|
||||
"production_preflight_ready_wait_for_next_real_all_host_reboot_event_"
|
||||
"or_separate_break_glass_reboot_drill_authorization"
|
||||
)
|
||||
if p0_006_event_gated
|
||||
else (
|
||||
"blocked_reboot_slo_scorecard_service_data_backup_readback_present_"
|
||||
"blockers_open"
|
||||
if service_data_backup_readback_present
|
||||
else "blocked_reboot_slo_scorecard_requires_controlled_service_data_backup_readback"
|
||||
)
|
||||
state["next_executable_mainline_state"] = _reboot_slo_next_mainline_state(
|
||||
event_gated=p0_006_event_gated,
|
||||
service_data_backup_readback_present=service_data_backup_readback_present,
|
||||
service_data_backup_blocking_fields=service_data_backup_blocking_fields,
|
||||
windows99_verify_collection_blockers=windows99_verify_collection_blockers,
|
||||
active_blockers=reboot_active_blockers,
|
||||
)
|
||||
|
||||
runtime_sha = str(
|
||||
@@ -3408,27 +3467,58 @@ def _enrich_from_current_readbacks(
|
||||
)
|
||||
workplan["status"] = "blocked_waiting_fresh_all_host_reboot_window"
|
||||
else:
|
||||
service_backup_closed = _service_data_backup_blockers_closed(
|
||||
service_data_backup_readback_present=service_data_backup_readback_present,
|
||||
service_data_backup_blocking_fields=service_data_backup_blocking_fields,
|
||||
)
|
||||
professional_fix["action"] = (
|
||||
"Continue P0-006 from the reboot SLO scorecard truth: close the "
|
||||
"service, product-data, backup, 188 service, host reachability, "
|
||||
"disk, and Wazuh blockers in controlled lanes; keep the "
|
||||
"controlled service/data/backup readback visible as the source "
|
||||
"selector for these blockers. For Windows 99, use the "
|
||||
"windows99_verify_collection packet to collect no-secret Verify "
|
||||
"stdout, keep windows99_management_channel visible for the current "
|
||||
"remote execution blocker, and rerun the scorecard. For StockPlatform "
|
||||
"freshness/ingestion postgres_not_ready, use the production "
|
||||
"migration/control-channel path with target selector, dry-run, "
|
||||
"rollback, public API verifier, and KM/RAG/MCP/PlayBook writeback. "
|
||||
"Do not reboot, restart services, write DB rows, fake freshness, "
|
||||
"trigger workflows, or read secrets from this lane."
|
||||
(
|
||||
"Continue P0-006 from the reboot SLO scorecard truth: "
|
||||
"service/data/backup readback blockers are closed, so the "
|
||||
"visible next blocker groups are host boot/reboot-event "
|
||||
"detection and Windows99 verify collection. Run the "
|
||||
"reboot-event detector and host probe in verifier-only mode, "
|
||||
"collect the Windows99 no-secret Verify/local-console "
|
||||
"readback through the visible windows99_verify_collection "
|
||||
"packet, then rerun the scorecard. Do not reboot, restart "
|
||||
"services, write DB rows, fake freshness, trigger workflows, "
|
||||
"or read secrets from this lane."
|
||||
)
|
||||
if service_backup_closed
|
||||
else (
|
||||
"Continue P0-006 from the reboot SLO scorecard truth: close "
|
||||
"the service, product-data, backup, 188 service, host "
|
||||
"reachability, disk, and Wazuh blockers in controlled lanes; "
|
||||
"keep the controlled service/data/backup readback visible as "
|
||||
"the source selector for these blockers. For Windows 99, use "
|
||||
"the windows99_verify_collection packet to collect no-secret "
|
||||
"Verify stdout, keep windows99_management_channel visible for "
|
||||
"the current remote execution blocker, and rerun the "
|
||||
"scorecard. For StockPlatform freshness/ingestion "
|
||||
"postgres_not_ready, use the production migration/control-"
|
||||
"channel path with target selector, dry-run, rollback, public "
|
||||
"API verifier, and KM/RAG/MCP/PlayBook writeback. Do not "
|
||||
"reboot, restart services, write DB rows, fake freshness, "
|
||||
"trigger workflows, or read secrets from this lane."
|
||||
)
|
||||
)
|
||||
workplan["reason"] = (
|
||||
"The current reboot SLO scorecard is blocked, not merely waiting "
|
||||
"for a fresh reboot window. Controlled service/product-data/backup "
|
||||
"readback is present, but its blocker fields remain open and must "
|
||||
"stay visible in the priority readback until controlled recovery "
|
||||
"evidence closes."
|
||||
(
|
||||
"The current reboot SLO scorecard is blocked, not merely "
|
||||
"waiting for a fresh reboot window. Controlled "
|
||||
"service/product-data/backup readback is present and its "
|
||||
"blocker fields are closed, so the priority readback now "
|
||||
"routes the next mainline state to host boot/reboot-event "
|
||||
"detection and Windows99 verify collection."
|
||||
)
|
||||
if service_backup_closed
|
||||
else (
|
||||
"The current reboot SLO scorecard is blocked, not merely "
|
||||
"waiting for a fresh reboot window. Controlled "
|
||||
"service/product-data/backup readback is present, but its "
|
||||
"blocker fields remain open and must stay visible in the "
|
||||
"priority readback until controlled recovery evidence closes."
|
||||
)
|
||||
)
|
||||
workplan["safe_next_step"] = str(reboot_slo.get("safe_next_step") or "")
|
||||
workplan["status"] = "blocked_reboot_auto_recovery_slo_not_ready"
|
||||
@@ -3471,13 +3561,27 @@ def _enrich_from_current_readbacks(
|
||||
),
|
||||
]
|
||||
else:
|
||||
service_backup_closed = _service_data_backup_blockers_closed(
|
||||
service_data_backup_readback_present=service_data_backup_readback_present,
|
||||
service_data_backup_blocking_fields=service_data_backup_blocking_fields,
|
||||
)
|
||||
payload["next_execution_order"] = [
|
||||
(
|
||||
"P0-006: reboot SLO scorecard is blocked; close the active "
|
||||
"service/product-data/backup/host/Wazuh blockers in controlled "
|
||||
"lanes, and keep StockPlatform postgres_not_ready on the "
|
||||
"production migration/control-channel path with verifier and "
|
||||
"metadata writeback."
|
||||
(
|
||||
"P0-006: service/data/backup readback is green, but reboot "
|
||||
"SLO is still blocked by host boot/reboot-event detection "
|
||||
"and Windows99 verify collection; run verifier-only host "
|
||||
"probe plus no-secret Windows99 collection, then rerun the "
|
||||
"scorecard. Do not reboot/restart/DB-write/read secrets."
|
||||
)
|
||||
if service_backup_closed
|
||||
else (
|
||||
"P0-006: reboot SLO scorecard is blocked; close the active "
|
||||
"service/product-data/backup/host/Wazuh blockers in "
|
||||
"controlled lanes, and keep StockPlatform postgres_not_ready "
|
||||
"on the production migration/control-channel path with "
|
||||
"verifier and metadata writeback."
|
||||
)
|
||||
),
|
||||
(
|
||||
"P0-006-STOCKPLATFORM-PUBLIC-API-CONTROLLED-RECOVERY-PREFLIGHT: "
|
||||
|
||||
Reference in New Issue
Block a user