wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(ops): classify 110 recovery receipt blockers
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
CD Pipeline / tests (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Your Name
2026-07-02 00:59:03 +08:00
parent 54f70ee011
commit f4ca49c516
6 changed files with 1042 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
apps/api/src/services/harbor_registry_controlled_recovery_receipt.py
View File

@@ -435,22 +435,32 @@ def _control_path_readiness(
) -> dict[str, Any]:
public_registry_ready = bool(verifier["public_registry_v2_ready"])
internal_registry_ready = bool(verifier["internal_registry_v2_ready"])
command_path_ready = bool(ssh_diagnosis["wooo_command_path_ready"])
harbor_110_repair_queue_superseded = (
command_path_ready and public_registry_ready and internal_registry_ready
)
queue_no_matching_runner = bool(
gitea_queue["harbor_110_repair_no_matching_runner"]
)
queue_jobs_stale = bool(gitea_queue["harbor_110_repair_jobs_stale_or_mismatched"])
) and not harbor_110_repair_queue_superseded
queue_jobs_stale = bool(
gitea_queue["harbor_110_repair_jobs_stale_or_mismatched"]
) and not harbor_110_repair_queue_superseded
queue_jobs_cross_workflow = bool(
gitea_queue["harbor_110_repair_jobs_cross_workflow_mismatch"]
)
) and not harbor_110_repair_queue_superseded
queue_harbor_running_jobs_api_stale = bool(
gitea_queue["harbor_110_repair_visible_running_jobs_api_stale"]
)
) and not harbor_110_repair_queue_superseded
current_cd_waiting_behind_harbor_running = bool(
gitea_queue["current_cd_waiting_behind_harbor_110_repair_running"]
)
queue_remote_control_unavailable = bool(
) and not harbor_110_repair_queue_superseded
queue_remote_control_unavailable_raw = bool(
gitea_queue["harbor_110_repair_remote_control_channel_unavailable"]
)
queue_remote_control_unavailable = (
queue_remote_control_unavailable_raw
and not harbor_110_repair_queue_superseded
)
cd_harbor_repair_requires_110_lane = bool(
gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"]
)
@@ -469,7 +479,6 @@ def _control_path_readiness(
non110_runner_not_ready = bool(
non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]
)
command_path_ready = bool(ssh_diagnosis["wooo_command_path_ready"])
runner_timeout = bool(
ssh_diagnosis["runner_systemctl_show_timeout_effective_seen"]
)
@@ -590,6 +599,9 @@ def _control_path_readiness(
"harbor_110_repair_remote_control_channel_unavailable": (
queue_remote_control_unavailable
),
"harbor_110_repair_remote_control_channel_unavailable_raw": (
queue_remote_control_unavailable_raw
),
"harbor_110_repair_failure_classifier": gitea_queue[
"harbor_110_repair_failure_classifier"
],
@@ -719,6 +731,8 @@ def _control_path_readiness_status(
return "blocked_awoooi_host_runner_queue_unavailable"
if node_high_load:
return "blocked_110_node_high_load"
if any(item.startswith("gitea_queue_cd_jobs_") for item in signal_ids):
return "blocked_gitea_cd_jobs_readback_stale_or_mismatched"
if not registry_v2_ready:
return "blocked_registry_v2_verifier_not_green"
return "blocked_control_path_evidence_not_clear"
@@ -739,6 +753,8 @@ def _control_path_safe_next_action(*, status: str) -> str:
return "rerun_non_secret_110_ssh_and_runner_control_path_diagnosis"
if status == "blocked_registry_v2_verifier_not_green":
return "rerun_public_and_internal_registry_v2_verifier_before_cd_retry"
if status == "blocked_gitea_cd_jobs_readback_stale_or_mismatched":
return "rerun_public_gitea_queue_and_deploy_marker_readback_before_cd_retry"
return "normalize_control_path_evidence_then_retry_readback"
@@ -1240,6 +1256,25 @@ def _parse_controlled_cd_lane_readiness_output(output: str) -> dict[str, Any]:
def _parse_non110_runner_readiness_output(output: str) -> dict[str, Any]:
fields = _parse_key_values(output)
marker_seen = "AWOOOI_NON110_RUNNER_READY=" in output
if not marker_seen:
return {
"receipt_seen": False,
"non110_runner_ready": False,
"ready_config_count": 0,
"ready_binary_count": 0,
"ready_registration_count": 0,
"ready_service_count": 0,
"ready_active_service_count": 0,
"ready_autostart_path_count": 0,
"warning_count": 0,
"blocker_count": 0,
"blockers": [],
"safe_next_step": "",
"raw_runner_registration_read": False,
"runner_token_read": False,
"metadata_only": True,
"raw_output_returned": False,
}
blockers = _prefixed_blockers(
output,
prefix="non110_runner_readiness:",
@@ -2086,11 +2121,85 @@ def _active_blockers(
blockers.append("public_registry_v2_verifier_not_green")
if not verifier["internal_registry_v2_ready"]:
blockers.append("internal_registry_v2_verifier_not_green")
blockers.extend(_strings(gitea_queue.get("blockers")))
blockers.extend(
_effective_gitea_queue_blockers(
ssh_diagnosis=ssh_diagnosis,
ssh_local=ssh_local,
watchdog_check=watchdog_check,
controlled_cd_lane=controlled_cd_lane,
verifier=verifier,
gitea_queue=gitea_queue,
)
)
blockers.extend(_strings(deploy_marker.get("blockers")))
return _unique_strings(blockers)
def _effective_gitea_queue_blockers(
*,
ssh_diagnosis: dict[str, Any],
ssh_local: dict[str, Any],
watchdog_check: dict[str, Any],
controlled_cd_lane: dict[str, Any],
verifier: dict[str, Any],
gitea_queue: dict[str, Any],
) -> list[str]:
blockers = _strings(gitea_queue.get("blockers"))
superseded = _gitea_queue_blockers_superseded_by_local_receipt(
ssh_diagnosis=ssh_diagnosis,
ssh_local=ssh_local,
watchdog_check=watchdog_check,
controlled_cd_lane=controlled_cd_lane,
verifier=verifier,
)
if not superseded:
return blockers
return [blocker for blocker in blockers if blocker not in superseded]
def _gitea_queue_blockers_superseded_by_local_receipt(
*,
ssh_diagnosis: dict[str, Any],
ssh_local: dict[str, Any],
watchdog_check: dict[str, Any],
controlled_cd_lane: dict[str, Any],
verifier: dict[str, Any],
) -> set[str]:
control_channel_ready = bool(
ssh_diagnosis["wooo_command_path_ready"]
or ssh_local["control_channel_metadata_ready"]
)
harbor_ready = bool(watchdog_check["harbor_ready"] or verifier["registry_v2_ready"])
if not control_channel_ready or not harbor_ready:
return set()
superseded = {
"gitea_queue_harbor_110_remote_control_channel_unavailable",
"gitea_queue_harbor_110_repair_blocked",
"gitea_queue_harbor_110_repair_visible_running_jobs_api_stale",
"gitea_queue_current_cd_waiting_behind_harbor_110_repair_running",
}
if verifier["registry_v2_ready"]:
superseded.update(
{
"gitea_queue_current_cd_harbor_retrying_unavailable",
"gitea_queue_harbor_110_remote_local_registry_v2_unavailable",
"gitea_queue_harbor_public_registry_v2_unavailable_after_remote_repair",
}
)
if controlled_cd_lane["receipt_seen"]:
superseded.update(
{
"gitea_queue_harbor_110_repair_no_matching_runner",
"gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch",
"gitea_queue_harbor_110_repair_jobs_stale_or_mismatched",
"gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host",
"gitea_queue_cd_harbor_repair_requires_110_controlled_lane",
}
)
return superseded
def _ssh_local_metadata_blockers(ssh_local: dict[str, Any]) -> list[str]:
blockers: list[str] = []
if not ssh_local["sshd_config_syntax_ok"]:
@@ -2129,10 +2238,16 @@ def _status(
) -> str:
if verifier["registry_v2_ready"] and not active_blockers:
return "harbor_registry_recovery_receipt_verified"
if controlled_cd_lane["receipt_seen"] and not controlled_cd_lane[
"controlled_cd_lane_ready"
]:
return "controlled_cd_lane_readiness_receipt_blocked"
if non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]:
return "non110_runner_readiness_receipt_blocked"
if (
verifier["registry_v2_ready"]
and gitea_queue["receipt_seen"]
and gitea_queue["blocker_count"] > 0
and any(blocker.startswith("gitea_queue_") for blocker in active_blockers)
):
return "harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance"
if (
@@ -2141,12 +2256,6 @@ def _status(
and deploy_marker["blocker_count"] > 0
):
return "harbor_registry_recovery_receipt_verified_waiting_deploy_marker_readback"
if controlled_cd_lane["receipt_seen"] and not controlled_cd_lane[
"controlled_cd_lane_ready"
]:
return "controlled_cd_lane_readiness_receipt_blocked"
if non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]:
return "non110_runner_readiness_receipt_blocked"
if watchdog_repair["receipt_seen"]:
return "harbor_registry_repair_receipt_waiting_registry_v2_verifier"
if watchdog_check["receipt_seen"] and watchdog_check["harbor_ready"]:

130
apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py
View File

@@ -297,19 +297,9 @@ def test_harbor_recovery_receipt_accepts_ssh_publickey_diagnosis() -> None:
def test_harbor_recovery_receipt_command_path_ready_overrides_stale_timeout() -> None:
diagnosis_output = _ssh_publickey_diagnosis_output().replace(
"SSH_AUTH user=wooo mode=password_disabled rc=255 classification=permission_denied",
(
"SSH_COMMAND_PATH user=wooo rc=0 classification=command_path_ready "
"marker_seen=true remote_user_match=true\n"
"SSH_AUTH user=wooo mode=password_disabled rc=255 "
"classification=permission_denied"
),
)
payload = validate_harbor_registry_controlled_recovery_receipt(
{
"ssh_publickey_diagnosis_output": diagnosis_output,
"ssh_publickey_diagnosis_output": _ssh_publickey_command_path_ready_output(),
"watchdog_check_output": _watchdog_check_output(ready=True, status=401),
"public_registry_v2_http_status": 401,
"internal_registry_v2_http_status": 401,
@@ -348,6 +338,66 @@ def test_harbor_recovery_receipt_command_path_ready_overrides_stale_timeout() ->
)
def test_harbor_recovery_receipt_local_receipt_supersedes_stale_gitea_queue() -> None:
payload = validate_harbor_registry_controlled_recovery_receipt(
{
"ssh_publickey_diagnosis_output": _ssh_publickey_command_path_ready_output(),
"local_console_recovery_output": "\n".join(
[
_watchdog_check_output(ready=True, status=401),
_controlled_cd_lane_blocked_output(),
]
),
"public_registry_v2_http_status": 401,
"internal_registry_v2_http_status": 401,
"gitea_actions_queue_readback": _gitea_queue_remote_control_timeout(),
}
)
assert payload["status"] == "controlled_cd_lane_readiness_receipt_blocked"
assert payload["safe_next_step"] == (
"fix_controlled_cd_lane_guardrail_blockers_then_rerun_readiness_verifier"
)
assert "controlled_cd_lane_readiness:controlled_cd_lane_registration_missing" in (
payload["active_blockers"]
)
assert "controlled_cd_lane_readiness:controlled_cd_lane_service_not_active" in (
payload["active_blockers"]
)
assert not any(
blocker.startswith("gitea_queue_harbor_110")
for blocker in payload["active_blockers"]
)
assert not any(
blocker.startswith("non110_runner_readiness:")
for blocker in payload["active_blockers"]
)
assert payload["readback"]["non110_runner_readiness"]["receipt_seen"] is False
queue = payload["readback"]["gitea_actions_queue"]
assert queue["harbor_110_repair_remote_control_channel_unavailable"] is True
assert queue["blocker_count"] == 2
readiness = payload["readback"]["control_path_readiness"]
assert readiness["status"] == "ready"
assert readiness["ssh_command_path_ready"] is True
assert readiness["harbor_110_repair_remote_control_channel_unavailable"] is False
assert (
readiness["harbor_110_repair_remote_control_channel_unavailable_raw"]
is True
)
assert readiness["harbor_110_repair_jobs_stale_or_mismatched"] is False
assert not any(
signal.startswith("gitea_queue_harbor_110")
for signal in readiness["signal_ids"]
)
assert payload["rollups"]["gitea_queue_blocker_count"] == 2
assert (
payload["rollups"][
"gitea_queue_harbor_110_remote_control_channel_unavailable"
]
is True
)
def test_harbor_recovery_receipt_classifies_publickey_offer_timeout() -> None:
diagnosis_output = _ssh_publickey_diagnosis_output().replace(
"rc=124 classification=server_accepts_key_then_timeout",
@@ -893,6 +943,32 @@ def test_harbor_recovery_receipt_surfaces_cd_jobs_payload_classifier() -> None:
]
def test_harbor_recovery_receipt_classifies_cd_jobs_only_readback_mismatch() -> None:
payload = validate_harbor_registry_controlled_recovery_receipt(
{
"watchdog_check_output": _watchdog_check_output(
ready=True,
status=401,
),
"public_registry_v2_http_status": 401,
"internal_registry_v2_http_status": 401,
"gitea_actions_queue_readback": _gitea_queue_cd_jobs_head_sha_mismatch_only(),
}
)
readiness = payload["readback"]["control_path_readiness"]
assert readiness["status"] == (
"blocked_gitea_cd_jobs_readback_stale_or_mismatched"
)
assert readiness["safe_next_action"] == (
"rerun_public_gitea_queue_and_deploy_marker_readback_before_cd_retry"
)
assert readiness["signal_ids"] == [
"gitea_queue_cd_jobs_head_sha_mismatch",
"gitea_queue_cd_jobs_stale_or_mismatched",
]
def test_harbor_recovery_receipt_surfaces_current_cd_runner_readiness() -> None:
payload = validate_harbor_registry_controlled_recovery_receipt(
{
@@ -1157,6 +1233,18 @@ INTERPRETATION=server_accepts_key_then_timeout_on_wooo_means_check_110_session_p
"""
def _ssh_publickey_command_path_ready_output() -> str:
return _ssh_publickey_diagnosis_output().replace(
"SSH_AUTH user=wooo mode=password_disabled rc=255 classification=permission_denied",
(
"SSH_COMMAND_PATH user=wooo rc=0 classification=command_path_ready "
"marker_seen=true remote_user_match=true\n"
"SSH_AUTH user=wooo mode=password_disabled rc=255 "
"classification=permission_denied"
),
)
def _watchdog_check_output(*, ready: bool, status: int) -> str:
ready_text = "true" if ready else "false"
return f"""
@@ -1412,6 +1500,26 @@ def _gitea_queue_cd_jobs_head_sha_mismatch() -> dict:
return payload
def _gitea_queue_cd_jobs_head_sha_mismatch_only() -> dict:
payload = _gitea_queue_cd_jobs_head_sha_mismatch()
payload["readback"].update(
{
"latest_visible_harbor_110_repair_no_matching_runner_label": "",
"harbor_110_repair_jobs_stale_or_mismatched": False,
"harbor_110_repair_jobs_unexpected_names": [],
"harbor_110_repair_jobs_labels": [],
"harbor_110_repair_jobs_runner_names": [],
}
)
payload["rollups"].update(
{
"harbor_110_repair_no_matching_runner_label": "",
"harbor_110_repair_jobs_stale_or_mismatched": False,
}
)
return payload
def _gitea_queue_current_cd_waiting_non110() -> dict:
payload = _gitea_queue_cd_jobs_head_sha_mismatch()
payload["status"] = "blocked_current_cd_workflow_waiting_for_runner_or_queue"