feat(recovery): expose 110 control path readiness
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 37s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 37s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -292,6 +292,81 @@ BLOCKER_COUNT=1
|
||||
assert payload["rollups"]["controlled_cd_lane_blocker_count"] == 1
|
||||
|
||||
|
||||
def test_harbor_recovery_receipt_surfaces_control_path_readiness_blocker() -> None:
|
||||
diagnosis_output = _ssh_publickey_diagnosis_output().replace(
|
||||
"rc=124 classification=server_accepts_key_then_timeout",
|
||||
"rc=255 classification=publickey_offer_timeout",
|
||||
).replace(
|
||||
"NODE_LOAD1_PER_CPU=0.93",
|
||||
"NODE_LOAD1_PER_CPU=4.53",
|
||||
).replace(
|
||||
"NODE_LOAD_CLASSIFIER=load_not_high",
|
||||
"NODE_LOAD_CLASSIFIER=high_load",
|
||||
)
|
||||
|
||||
payload = validate_harbor_registry_controlled_recovery_receipt(
|
||||
{
|
||||
"ssh_publickey_diagnosis_output": diagnosis_output,
|
||||
"public_registry_v2_http_status": 502,
|
||||
"internal_registry_v2_http_status": 502,
|
||||
"gitea_actions_queue_readback": _gitea_queue_cd_jobs_head_sha_mismatch(),
|
||||
}
|
||||
)
|
||||
|
||||
readiness = payload["readback"]["control_path_readiness"]
|
||||
assert readiness["status"] == (
|
||||
"blocked_110_high_load_and_awoooi_host_control_path_unavailable"
|
||||
)
|
||||
assert readiness["primary_blocker"] == "ssh_publickey_node_high_load_on_110"
|
||||
assert readiness["safe_next_action"] == (
|
||||
"hold_110_capacity_protection_then_rerun_readonly_awoooi_host_and_registry_verifiers"
|
||||
)
|
||||
assert readiness["node_high_load"] is True
|
||||
assert readiness["node_load_classifier"] == "high_load"
|
||||
assert readiness["node_load1_per_cpu"] == 4.53
|
||||
assert readiness["ssh_publickey_offer_timeout"] is True
|
||||
assert readiness["runner_systemctl_show_timeout"] is True
|
||||
assert readiness["awoooi_host_runner_unavailable"] is True
|
||||
assert readiness["harbor_110_repair_no_matching_runner"] is True
|
||||
assert readiness["harbor_110_repair_no_matching_runner_label"] == "awoooi-host"
|
||||
assert readiness["harbor_110_repair_jobs_stale_or_mismatched"] is True
|
||||
assert readiness["cd_jobs_head_sha_mismatch"] is True
|
||||
assert readiness["registry_v2_public_http_status"] == 502
|
||||
assert readiness["registry_v2_internal_http_status"] == 502
|
||||
assert readiness["registry_v2_ready"] is False
|
||||
assert readiness["metadata_only"] is True
|
||||
assert readiness["raw_output_returned"] is False
|
||||
assert readiness["signal_ids"] == [
|
||||
"ssh_publickey_node_high_load_on_110",
|
||||
"ssh_publickey_offer_timeout_on_wooo",
|
||||
"runner_systemctl_show_timeout_on_110",
|
||||
"gitea_queue_harbor_110_repair_no_matching_runner",
|
||||
"gitea_queue_harbor_110_repair_jobs_stale_or_mismatched",
|
||||
"gitea_queue_cd_jobs_head_sha_mismatch",
|
||||
"gitea_queue_cd_jobs_stale_or_mismatched",
|
||||
"public_registry_v2_verifier_not_green",
|
||||
"internal_registry_v2_verifier_not_green",
|
||||
]
|
||||
assert payload["rollups"]["control_path_readiness_status"] == (
|
||||
"blocked_110_high_load_and_awoooi_host_control_path_unavailable"
|
||||
)
|
||||
assert payload["rollups"]["control_path_readiness_blocker_count"] == 9
|
||||
assert payload["rollups"]["control_path_readiness_node_high_load"] is True
|
||||
assert (
|
||||
payload["rollups"]["control_path_readiness_runner_systemctl_timeout"]
|
||||
is True
|
||||
)
|
||||
assert payload["rollups"]["control_path_readiness_awoooi_host_unavailable"] is True
|
||||
assert (
|
||||
payload["rollups"]["control_path_readiness_registry_v2_public_ready"]
|
||||
is False
|
||||
)
|
||||
assert (
|
||||
payload["rollups"]["control_path_readiness_registry_v2_internal_ready"]
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_harbor_recovery_receipt_surfaces_gitea_queue_blockers() -> None:
|
||||
payload = validate_harbor_registry_controlled_recovery_receipt(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user