fix(reboot): surface windows99 console artifact blockers
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m13s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
Your Name
2026-07-03 09:25:20 +08:00
parent 43620a977f
commit e28ebd5b3e
10 changed files with 374 additions and 9 deletions

View File

@@ -299,6 +299,10 @@ def parse_windows99_management_readback(path: Path | None) -> dict[str, Any]:
"rdp_console_reachable": False,
"local_console_channel_reachable": False,
"console_collection_channels": [],
"console_artifact_status": "unknown",
"console_artifact_reliable": False,
"console_artifact_blockers": [],
"console_artifact_safe_next_step": "",
"remote_execution_channel_ready": False,
"can_collect_vmware_verify_without_secret": False,
"blockers": [],
@@ -345,6 +349,16 @@ def parse_windows99_management_readback(path: Path | None) -> dict[str, Any]:
"console_collection_channels": strings(
payload.get("console_collection_channels")
),
"console_artifact_status": str(
payload.get("console_artifact_status") or "unknown"
),
"console_artifact_reliable": payload.get("console_artifact_reliable") is True,
"console_artifact_blockers": strings(
payload.get("console_artifact_blockers")
),
"console_artifact_safe_next_step": str(
payload.get("console_artifact_safe_next_step") or ""
),
"remote_execution_channel_ready": (
payload.get("remote_execution_channel_ready") is True
),
@@ -1024,6 +1038,15 @@ def build_windows99_verify_collection_packet(
windows99_management.get("local_console_channel_reachable") is True
)
console_channels = strings(windows99_management.get("console_collection_channels"))
console_artifact_status = str(
windows99_management.get("console_artifact_status") or "unknown"
)
console_artifact_reliable = (
windows99_management.get("console_artifact_reliable") is True
)
console_artifact_blockers = strings(
windows99_management.get("console_artifact_blockers")
)
collector_present = windows99_collector.get("readback_present") is True
collector_status = str(windows99_collector.get("status") or "unknown")
collector_ssh_ready = (
@@ -1044,6 +1067,11 @@ def build_windows99_verify_collection_packet(
for blocker in strings(windows99_collector.get("blockers"))
if blocker not in collection_blockers
)
collection_blockers.extend(
blocker
for blocker in console_artifact_blockers
if blocker not in collection_blockers
)
if not host99_uptime_known:
collection_blockers.append("windows99_uptime_unknown")
@@ -1081,6 +1109,12 @@ def build_windows99_verify_collection_packet(
)
),
"available_collection_channels": available_channels,
"console_artifact_status": console_artifact_status,
"console_artifact_reliable": console_artifact_reliable,
"console_artifact_blockers": console_artifact_blockers,
"console_artifact_safe_next_step": str(
windows99_management.get("console_artifact_safe_next_step") or ""
),
"no_secret_collector_readback_present": collector_present,
"no_secret_collector_status": collector_status,
"no_secret_collector_safe_next_step": str(
@@ -1325,6 +1359,9 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str
"windows99_vmware_autostart_config_not_ready",
"windows99_vmware_guest_power_not_ready",
"windows99_update_no_auto_reboot_policy_not_ready",
"windows99_console_clipboard_unreliable",
"windows99_console_focus_unreliable",
"windows99_console_verify_output_truncated",
}
if any(blocker in host_boot_blockers for blocker in active_blockers):
return "host_boot_detection_blocked"
@@ -1373,6 +1410,9 @@ def reboot_sop_primary_blocker(active_blockers: list[str]) -> str:
"windows99_vmware_autostart_config_not_ready",
"windows99_vmware_guest_power_not_ready",
"windows99_update_no_auto_reboot_policy_not_ready",
"windows99_console_clipboard_unreliable",
"windows99_console_focus_unreliable",
"windows99_console_verify_output_truncated",
"public_route_raw_5xx_without_maintenance_fallback",
"public_route_unreachable_without_external_l1_fallback",
"public_maintenance_fallback_runtime_readback_missing",
@@ -1519,6 +1559,15 @@ def active_blocker_action_row(
"restore_windows99_no_secret_management_channel_or_collect_local_"
"console_verify_readback_then_rerun_reboot_scorecard_no_reboot"
)
elif blocker in {
"windows99_console_clipboard_unreliable",
"windows99_console_focus_unreliable",
"windows99_console_verify_output_truncated",
}:
next_safe_action = (
"stop_unreliable_rdp_clipboard_path_and_use_authorized_no_secret_"
"management_channel_or_validated_console_stdout_artifact"
)
post_verifier = (
"bash scripts/reboot-recovery/collect-windows99-vmware-verify.sh "
"--check && rerun_reboot_auto_recovery_slo_scorecard"
@@ -1883,6 +1932,18 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
"windows99_available_collection_channels": strings(
windows99_verify_collection.get("available_collection_channels")
),
"windows99_console_artifact_status": str(
windows99_verify_collection.get("console_artifact_status") or "unknown"
),
"windows99_console_artifact_reliable": (
windows99_verify_collection.get("console_artifact_reliable") is True
),
"windows99_console_artifact_blockers": strings(
windows99_verify_collection.get("console_artifact_blockers")
),
"windows99_console_artifact_safe_next_step": str(
windows99_verify_collection.get("console_artifact_safe_next_step") or ""
),
"windows99_host99_reachable": (
windows99_verify_collection["host99_reachable"] is True
),
@@ -1968,6 +2029,18 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]:
"windows99_available_collection_channels": rollups[
"windows99_available_collection_channels"
],
"windows99_console_artifact_status": rollups[
"windows99_console_artifact_status"
],
"windows99_console_artifact_reliable": rollups[
"windows99_console_artifact_reliable"
],
"windows99_console_artifact_blockers": rollups[
"windows99_console_artifact_blockers"
],
"windows99_console_artifact_safe_next_step": rollups[
"windows99_console_artifact_safe_next_step"
],
"windows99_remote_execution_channel_ready": rollups[
"windows99_remote_execution_channel_ready"
],
@@ -2173,6 +2246,7 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]:
blockers.extend(strings(host_pressure.get("blockers")))
blockers.extend(strings(public_maintenance.get("blockers")))
blockers.extend(strings(windows99.get("blockers")))
blockers.extend(strings(windows99_management.get("console_artifact_blockers")))
if (
windows99.get("readback_present") is False
and windows99_management.get("readback_present") is True

View File

@@ -598,6 +598,56 @@ def test_windows99_management_channel_unavailable_is_visible(tmp_path: Path) ->
assert payload["readback"]["windows99_remote_execution_channel_ready"] is False
def test_windows99_console_artifact_blocker_is_visible(tmp_path: Path) -> None:
management = {
**WINDOWS99_MANAGEMENT_BLOCKED,
"console_artifact_status": "blocked_clipboard_unreliable",
"console_artifact_reliable": False,
"console_artifact_blockers": ["windows99_console_clipboard_unreliable"],
"console_artifact_safe_next_step": (
"use_authorized_no_secret_management_channel_or_manual_console_stdout_capture"
),
"blockers": [
*WINDOWS99_MANAGEMENT_BLOCKED["blockers"],
"windows99_console_clipboard_unreliable",
],
}
payload = run_scorecard(
tmp_path,
GREEN_SUMMARY,
windows99="",
windows99_management=json.dumps(management),
)
assert "windows99_console_clipboard_unreliable" in payload["active_blockers"]
collection = payload["windows99_verify_collection"]
assert collection["console_artifact_status"] == "blocked_clipboard_unreliable"
assert collection["console_artifact_reliable"] is False
assert collection["console_artifact_blockers"] == [
"windows99_console_clipboard_unreliable"
]
assert "windows99_console_clipboard_unreliable" in collection[
"collection_blockers"
]
assert payload["rollups"]["windows99_console_artifact_status"] == (
"blocked_clipboard_unreliable"
)
assert payload["rollups"]["windows99_console_artifact_reliable"] is False
assert payload["readback"]["windows99_console_artifact_blockers"] == [
"windows99_console_clipboard_unreliable"
]
action_by_blocker = {
item["blocker"]: item
for item in payload["active_blocker_action_matrix"]["items"]
}
assert action_by_blocker["windows99_console_clipboard_unreliable"][
"next_safe_action"
] == (
"stop_unreliable_rdp_clipboard_path_and_use_authorized_no_secret_"
"management_channel_or_validated_console_stdout_artifact"
)
def test_windows99_no_secret_collector_publickey_blocker_is_visible(
tmp_path: Path,
) -> None:

View File

@@ -51,6 +51,8 @@ def test_management_probe_surfaces_no_secret_console_channels(monkeypatch):
ssh_timeout=1,
ports=None,
skip_ssh=False,
console_artifact_status="not_attempted",
console_artifact_blockers=None,
generated_at="2026-07-02T16:00:00+08:00",
output=None,
)
@@ -65,6 +67,9 @@ def test_management_probe_surfaces_no_secret_console_channels(monkeypatch):
"rdp_console",
"hyperv_vmconnect",
]
assert payload["console_artifact_status"] == "not_attempted"
assert payload["console_artifact_reliable"] is False
assert payload["console_artifact_blockers"] == []
assert payload["remote_execution_channel_ready"] is False
assert payload["can_collect_vmware_verify_without_secret"] is False
assert "windows99_remote_execution_channel_unavailable" in payload["blockers"]
@@ -105,6 +110,8 @@ def test_management_probe_surfaces_multiple_ssh_candidates(monkeypatch):
ssh_timeout=1,
ports=None,
skip_ssh=False,
console_artifact_status="not_attempted",
console_artifact_blockers=None,
generated_at="2026-07-02T16:00:00+08:00",
output=None,
)
@@ -129,3 +136,51 @@ def test_management_probe_surfaces_multiple_ssh_candidates(monkeypatch):
"status": "ready",
},
]
def test_management_probe_surfaces_console_artifact_clipboard_blocker(monkeypatch):
module = _load_module()
def fake_tcp_status(_host: str, port: int, _timeout: float) -> str:
return "open" if port in {22, 2179, 3389} else "timeout"
monkeypatch.setattr(module, "tcp_status", fake_tcp_status)
monkeypatch.setattr(
module,
"ping_status",
lambda _host: {"checked": True, "ok": True, "status": "ok"},
)
monkeypatch.setattr(
module,
"ssh_batch_status",
lambda *_args, **_kwargs: {
"checked": True,
"ready": False,
"status": "permission_denied",
},
)
payload = module.build_payload(
argparse.Namespace(
host="192.168.0.99",
ssh_users=["administrator"],
tcp_timeout=0.01,
ssh_timeout=1,
ports=None,
skip_ssh=False,
console_artifact_status="blocked_clipboard_unreliable",
console_artifact_blockers=None,
generated_at="2026-07-03T09:20:00+08:00",
output=None,
)
)
assert payload["console_artifact_status"] == "blocked_clipboard_unreliable"
assert payload["console_artifact_reliable"] is False
assert payload["console_artifact_blockers"] == [
"windows99_console_clipboard_unreliable"
]
assert "windows99_console_clipboard_unreliable" in payload["blockers"]
assert payload["console_artifact_safe_next_step"] == (
"use_authorized_no_secret_management_channel_or_manual_console_stdout_capture"
)

View File

@@ -45,6 +45,28 @@ def parse_args() -> argparse.Namespace:
help="TCP port to probe. May be passed more than once.",
)
parser.add_argument("--skip-ssh", action="store_true")
parser.add_argument(
"--console-artifact-status",
default="not_attempted",
choices=[
"not_attempted",
"blocked_clipboard_unreliable",
"blocked_focus_unreliable",
"blocked_truncated_output",
"collected_stdout",
"validated_artifact",
],
help=(
"Optional no-secret console artifact collection status. Use a blocked "
"value only after an attempted console stdout capture path fails."
),
)
parser.add_argument(
"--console-artifact-blocker",
action="append",
dest="console_artifact_blockers",
help="Optional machine-readable console artifact blocker. May be repeated.",
)
parser.add_argument("--generated-at", help="Override generated_at.")
parser.add_argument("--output", type=Path, help="Write JSON to this path.")
args = parser.parse_args()
@@ -204,6 +226,24 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
ssh_probe = dict(ready_candidate or ssh_batch_candidates[0])
ssh_probe.pop("user", None)
remote_execution_ready = ssh_probe["ready"] is True
console_artifact_status = str(
getattr(args, "console_artifact_status", "not_attempted")
or "not_attempted"
)
console_artifact_blockers = list(
getattr(args, "console_artifact_blockers", None) or []
)
if console_artifact_status == "blocked_clipboard_unreliable":
console_artifact_blockers.append("windows99_console_clipboard_unreliable")
elif console_artifact_status == "blocked_focus_unreliable":
console_artifact_blockers.append("windows99_console_focus_unreliable")
elif console_artifact_status == "blocked_truncated_output":
console_artifact_blockers.append("windows99_console_verify_output_truncated")
console_artifact_blockers = list(dict.fromkeys(console_artifact_blockers))
console_artifact_reliable = console_artifact_status in {
"collected_stdout",
"validated_artifact",
}
blockers: list[str] = []
if not host_reachable:
blockers.append("windows99_host_unreachable_from_management_probe")
@@ -213,6 +253,7 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
blockers.append("windows99_winrm_unavailable")
if tcp_ports.get("22") == "open" and ssh_probe["status"] == "permission_denied":
blockers.append("windows99_ssh_batch_denied")
blockers.extend(console_artifact_blockers)
return {
"schema_version": SCHEMA_VERSION,
@@ -243,6 +284,18 @@ def build_payload(args: argparse.Namespace) -> dict[str, Any]:
"rdp_console_reachable": rdp_console_reachable,
"local_console_channel_reachable": local_console_channel_reachable,
"console_collection_channels": console_collection_channels,
"console_artifact_status": console_artifact_status,
"console_artifact_reliable": console_artifact_reliable,
"console_artifact_blockers": console_artifact_blockers,
"console_artifact_safe_next_step": (
"validate_collected_console_stdout_then_rerun_reboot_scorecard"
if console_artifact_reliable
else (
"use_authorized_no_secret_management_channel_or_manual_console_stdout_capture"
if console_artifact_blockers
else "attempt_console_stdout_capture_only_if_focus_and_clipboard_are_reliable"
)
),
"remote_execution_channel_ready": remote_execution_ready,
"can_collect_vmware_verify_without_secret": remote_execution_ready,
"blockers": blockers,