fix(reboot): expose windows99 vmx repair action chain
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m25s
CD Pipeline / build-and-deploy (push) Successful in 4m43s
CD Pipeline / post-deploy-checks (push) Successful in 1m49s

This commit is contained in:
Your Name
2026-07-03 11:44:13 +08:00
parent a484a8f896
commit a8893d4fa7
9 changed files with 517 additions and 2 deletions

View File

@@ -3029,6 +3029,26 @@ def _enrich_from_current_readbacks(
windows99_vmx_source_repair_package = (
build_windows99_vmx_source_repair_package(reboot_slo)
)
windows99_vmx_source_repair_action_chain = _dict(
windows99_vmx_source_repair_package.get("action_chain")
)
windows99_vmx_source_repair_target_selector = _dict(
windows99_vmx_source_repair_action_chain.get("target_selector")
)
windows99_vmx_source_repair_check_mode = _dict(
windows99_vmx_source_repair_action_chain.get("check_mode")
)
windows99_vmx_source_repair_rollback = _dict(
windows99_vmx_source_repair_action_chain.get("rollback")
)
windows99_vmx_source_repair_diff = (
windows99_vmx_source_repair_action_chain.get("source_of_truth_diff")
if isinstance(
windows99_vmx_source_repair_action_chain.get("source_of_truth_diff"),
list,
)
else []
)
p0_006_event_gated = bool(
reboot_slo_rollups.get("blocked_by_fresh_reboot_window_only") is True
)
@@ -3269,6 +3289,46 @@ def _enrich_from_current_readbacks(
windows99_vmx_source_repair_package.get("apply_allowed_by_this_package")
is True
)
state["windows99_vmx_source_repair_work_order_id"] = str(
windows99_vmx_source_repair_package.get("work_order_id")
or windows99_vmx_source_repair_action_chain.get("work_order_id")
or ""
)
state["windows99_vmx_source_repair_action_stage"] = str(
windows99_vmx_source_repair_package.get("action_stage")
or windows99_vmx_source_repair_action_chain.get("stage")
or ""
)
state["windows99_vmx_source_repair_next_executable_step"] = str(
windows99_vmx_source_repair_package.get("next_executable_step")
or windows99_vmx_source_repair_action_chain.get("next_executable_step")
or ""
)
state["windows99_vmx_source_repair_check_mode_allowed"] = bool(
windows99_vmx_source_repair_package.get("check_mode_allowed_by_this_package")
is True
or windows99_vmx_source_repair_check_mode.get("allowed_by_this_package")
is True
)
state["windows99_vmx_source_repair_rollback_required"] = bool(
windows99_vmx_source_repair_package.get("rollback_required") is True
or windows99_vmx_source_repair_rollback.get("required") is True
)
state["windows99_vmx_source_repair_vm_power_change_allowed"] = bool(
windows99_vmx_source_repair_package.get("vm_power_change_allowed") is True
or windows99_vmx_source_repair_target_selector.get("vm_power_change_allowed")
is True
)
state["windows99_vmx_source_repair_destructive_restore_allowed"] = bool(
windows99_vmx_source_repair_package.get("destructive_restore_allowed") is True
or windows99_vmx_source_repair_target_selector.get(
"destructive_restore_allowed"
)
is True
)
state["windows99_vmx_source_repair_source_diff_alias_count"] = len(
windows99_vmx_source_repair_diff
)
state["stale_snapshot_or_old_cd_runs_must_not_reopen_closed_work"] = True
state["p0_004_template_copy_apply_gate_production_http_status"] = 200
state["p0_004_template_copy_apply_gate_runtime_readback_state"] = (
@@ -3545,6 +3605,30 @@ def _enrich_from_current_readbacks(
evidence["windows99_vmx_source_repair_apply_allowed_by_this_package"] = (
state["windows99_vmx_source_repair_apply_allowed_by_this_package"]
)
evidence["windows99_vmx_source_repair_work_order_id"] = state[
"windows99_vmx_source_repair_work_order_id"
]
evidence["windows99_vmx_source_repair_action_stage"] = state[
"windows99_vmx_source_repair_action_stage"
]
evidence["windows99_vmx_source_repair_next_executable_step"] = state[
"windows99_vmx_source_repair_next_executable_step"
]
evidence["windows99_vmx_source_repair_check_mode_allowed"] = state[
"windows99_vmx_source_repair_check_mode_allowed"
]
evidence["windows99_vmx_source_repair_rollback_required"] = state[
"windows99_vmx_source_repair_rollback_required"
]
evidence["windows99_vmx_source_repair_vm_power_change_allowed"] = state[
"windows99_vmx_source_repair_vm_power_change_allowed"
]
evidence["windows99_vmx_source_repair_destructive_restore_allowed"] = state[
"windows99_vmx_source_repair_destructive_restore_allowed"
]
evidence["windows99_vmx_source_repair_source_diff_alias_count"] = state[
"windows99_vmx_source_repair_source_diff_alias_count"
]
evidence["drill_preflight_status"] = str(reboot_preflight.get("status") or "")
evidence["drill_preflight_ready"] = (
reboot_preflight_rollups.get("preflight_ready") is True
@@ -4194,6 +4278,31 @@ def _set_rollups_and_summary(
state.get("windows99_vmx_source_repair_apply_allowed_by_this_package")
is True
),
"windows99_vmx_source_repair_work_order_id": str(
state.get("windows99_vmx_source_repair_work_order_id") or ""
),
"windows99_vmx_source_repair_action_stage": str(
state.get("windows99_vmx_source_repair_action_stage") or ""
),
"windows99_vmx_source_repair_next_executable_step": str(
state.get("windows99_vmx_source_repair_next_executable_step") or ""
),
"windows99_vmx_source_repair_check_mode_allowed": (
state.get("windows99_vmx_source_repair_check_mode_allowed") is True
),
"windows99_vmx_source_repair_rollback_required": (
state.get("windows99_vmx_source_repair_rollback_required") is True
),
"windows99_vmx_source_repair_vm_power_change_allowed": (
state.get("windows99_vmx_source_repair_vm_power_change_allowed") is True
),
"windows99_vmx_source_repair_destructive_restore_allowed": (
state.get("windows99_vmx_source_repair_destructive_restore_allowed")
is True
),
"windows99_vmx_source_repair_source_diff_alias_count": _int(
state.get("windows99_vmx_source_repair_source_diff_alias_count")
),
"p0_004_runtime_readback_ready": p0_004_ready,
"reboot_drill_preflight_runtime_readback_ready": (
state.get("reboot_drill_preflight_runtime_readback_state") == "ready"
@@ -4369,6 +4478,31 @@ def _set_rollups_and_summary(
state.get("windows99_vmx_source_repair_apply_allowed_by_this_package")
is True
),
"windows99_vmx_source_repair_work_order_id": str(
state.get("windows99_vmx_source_repair_work_order_id") or ""
),
"windows99_vmx_source_repair_action_stage": str(
state.get("windows99_vmx_source_repair_action_stage") or ""
),
"windows99_vmx_source_repair_next_executable_step": str(
state.get("windows99_vmx_source_repair_next_executable_step") or ""
),
"windows99_vmx_source_repair_check_mode_allowed": (
state.get("windows99_vmx_source_repair_check_mode_allowed") is True
),
"windows99_vmx_source_repair_rollback_required": (
state.get("windows99_vmx_source_repair_rollback_required") is True
),
"windows99_vmx_source_repair_vm_power_change_allowed": (
state.get("windows99_vmx_source_repair_vm_power_change_allowed") is True
),
"windows99_vmx_source_repair_destructive_restore_allowed": (
state.get("windows99_vmx_source_repair_destructive_restore_allowed")
is True
),
"windows99_vmx_source_repair_source_diff_alias_count": _int(
state.get("windows99_vmx_source_repair_source_diff_alias_count")
),
"windows99_winrm_http_open": (
state.get("windows99_winrm_http_open") is True
),

View File

@@ -2434,6 +2434,112 @@ def _powershell_test_path(path: str) -> str:
return f"Test-Path -LiteralPath '{escaped}'"
def _build_windows99_vmx_source_repair_action_chain(
*,
missing_vmx_aliases: list[str],
powered_off_aliases: list[str],
) -> dict[str, Any]:
repair_aliases = missing_vmx_aliases or powered_off_aliases
primary_alias = repair_aliases[0] if repair_aliases else ""
if missing_vmx_aliases:
stage = "source_repair_check_mode_ready"
next_executable_step = "locate_or_relink_missing_vmx_source_check_mode"
candidate_status = "candidate_waiting_existing_vmx_source_or_backup_path"
elif powered_off_aliases:
stage = "power_readback_check_mode_ready"
next_executable_step = "build_vmware_autostart_power_readback_check_mode"
candidate_status = "candidate_waiting_power_policy_check_mode"
else:
stage = "verify_only_ready"
next_executable_step = "rerun_no_secret_collector_and_scorecard"
candidate_status = "no_repair_required"
source_diffs = []
for alias in repair_aliases:
candidates = _WINDOWS99_EXPECTED_VMX_CANDIDATES.get(alias, [])
source_diffs.append(
{
"alias": alias,
"current_state": (
"expected_vmx_source_missing"
if alias in missing_vmx_aliases
else "guest_power_readback_not_ready"
),
"desired_state": (
"expected_vmx_source_present_at_known_path"
if alias in missing_vmx_aliases
else "guest_power_state_observable_without_start_or_stop"
),
"candidate_paths": candidates,
"diff_mode": "path_existence_only_no_file_content_read",
"destructive_restore_allowed": False,
}
)
return {
"work_order_id": (
f"windows99-vmx-source-repair-{primary_alias}"
if primary_alias
else "windows99-vmx-source-repair-verify"
),
"stage": stage,
"next_executable_step": next_executable_step,
"target_selector": {
"host_alias": "99",
"vm_aliases": repair_aliases,
"path_scope": "windows99_vmware_expected_vmx_candidates",
"secret_collection_allowed": False,
"vm_power_change_allowed": False,
"destructive_restore_allowed": False,
},
"source_of_truth_diff": source_diffs,
"check_mode": {
"allowed_by_this_package": True,
"probe": "test_expected_vmx_candidate_paths_without_file_content_read",
"probe_command_count": sum(
len(_WINDOWS99_EXPECTED_VMX_CANDIDATES.get(alias, []))
for alias in repair_aliases
),
},
"controlled_apply_candidate": {
"status": candidate_status,
"executor": "windows99_no_secret_management_channel_or_local_console",
"command_id": "windows99_vmx_source_relink_or_restore_scoped",
"apply_allowed_by_this_package": False,
"allowed_when": [
"existing_vmx_source_or_backup_artifact_is_identified",
"pre_apply_backup_or_copy_plan_is_recorded",
"dry_run_path_check_passed",
"post_apply_no_secret_verifier_is_available",
],
},
"rollback": {
"required": True,
"mode": "revert_scoped_vmx_source_link_or_file_copy",
"backup_required_before_apply": True,
},
"post_apply_verifier": [
"bash scripts/reboot-recovery/collect-windows99-vmware-verify.sh --collect",
"bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh --verify-only",
"GET /api/v1/agents/reboot-auto-recovery-slo-scorecard",
"GET /api/v1/agents/windows99-vmx-source-repair-package",
],
"learning_writeback": [
"awooop_work_item_receipt",
"reboot_slo_scorecard_rollups",
"km_playbook_trust_writeback_after_verifier",
],
"manager_summary": {
"primary_alias": primary_alias or "none",
"missing_vmx_count": len(missing_vmx_aliases),
"powered_off_count": len(powered_off_aliases),
"check_mode_allowed": True,
"apply_allowed": False,
"vm_power_change_allowed": False,
},
}
def _build_windows99_vmx_source_repair_package(
windows99: dict[str, Any],
*,
@@ -2481,6 +2587,11 @@ def _build_windows99_vmx_source_repair_package(
}
)
action_chain = _build_windows99_vmx_source_repair_action_chain(
missing_vmx_aliases=missing_vmx_aliases,
powered_off_aliases=powered_off_aliases,
)
return {
"schema_version": "windows99_vmx_source_repair_check_mode_package_v1",
"generated_at": generated_at,
@@ -2491,6 +2602,20 @@ def _build_windows99_vmx_source_repair_package(
"powered_off_aliases": powered_off_aliases,
"required_vm_aliases": required_aliases,
"vm_rows": rows,
"action_chain": action_chain,
"work_order_id": action_chain["work_order_id"],
"action_stage": action_chain["stage"],
"next_executable_step": action_chain["next_executable_step"],
"check_mode_allowed_by_this_package": action_chain["check_mode"][
"allowed_by_this_package"
],
"rollback_required": action_chain["rollback"]["required"],
"vm_power_change_allowed": action_chain["target_selector"][
"vm_power_change_allowed"
],
"destructive_restore_allowed": action_chain["target_selector"][
"destructive_restore_allowed"
],
"safe_next_step": safe_next_step,
"controlled_apply_mode": "check_mode_package_only_no_windows_or_vm_write",
"apply_allowed_by_this_package": False,

View File

@@ -72,6 +72,30 @@ def _mark_package_public_redaction_boundary(
else:
redacted_rows.append(row)
payload["vm_rows"] = redacted_rows
action_chain = payload.get("action_chain")
if isinstance(action_chain, dict):
redacted_action_chain = dict(action_chain)
redacted_action_chain["public_lan_topology_redacted"] = True
redacted_action_chain["redacted_public_display_only"] = True
check_mode = redacted_action_chain.get("check_mode")
if isinstance(check_mode, dict):
redacted_check_mode = dict(check_mode)
redacted_check_mode["probe_commands_executable"] = False
redacted_action_chain["check_mode"] = redacted_check_mode
source_diffs = redacted_action_chain.get("source_of_truth_diff")
if isinstance(source_diffs, list):
redacted_diffs: list[Any] = []
for diff in source_diffs:
if isinstance(diff, dict):
redacted_diff = dict(diff)
redacted_diff["public_lan_topology_redacted"] = True
redacted_diff["redacted_public_display_only"] = True
redacted_diffs.append(redacted_diff)
else:
redacted_diffs.append(diff)
redacted_action_chain["source_of_truth_diff"] = redacted_diffs
payload["action_chain"] = redacted_action_chain
return payload
@@ -102,6 +126,17 @@ def _alias_lists_match(package: dict[str, Any], windows99: dict[str, Any]) -> bo
)
def _package_has_action_chain(package: dict[str, Any]) -> bool:
action_chain = _dict(package.get("action_chain"))
target_selector = _dict(action_chain.get("target_selector"))
return bool(
action_chain.get("work_order_id")
and action_chain.get("stage")
and action_chain.get("next_executable_step")
and target_selector.get("host_alias")
)
def build_windows99_vmx_source_repair_package(
scorecard: dict[str, Any],
*,
@@ -114,6 +149,7 @@ def build_windows99_vmx_source_repair_package(
if existing_package
and generated_at is None
and _alias_lists_match(existing_package, windows99)
and _package_has_action_chain(existing_package)
else _build_windows99_vmx_source_repair_package(
windows99,
generated_at=generated_at