fix(ops): classify 110 recovery receipt blockers

apps/api/src/services/harbor_registry_controlled_recovery_receipt.py

@@ -435,22 +435,32 @@ def _control_path_readiness(
 ) -> dict[str, Any]:
     public_registry_ready = bool(verifier["public_registry_v2_ready"])
     internal_registry_ready = bool(verifier["internal_registry_v2_ready"])
+    command_path_ready = bool(ssh_diagnosis["wooo_command_path_ready"])
+    harbor_110_repair_queue_superseded = (
+        command_path_ready and public_registry_ready and internal_registry_ready
+    )
     queue_no_matching_runner = bool(
         gitea_queue["harbor_110_repair_no_matching_runner"]
-    )
-    queue_jobs_stale = bool(gitea_queue["harbor_110_repair_jobs_stale_or_mismatched"])
+    ) and not harbor_110_repair_queue_superseded
+    queue_jobs_stale = bool(
+        gitea_queue["harbor_110_repair_jobs_stale_or_mismatched"]
+    ) and not harbor_110_repair_queue_superseded
     queue_jobs_cross_workflow = bool(
         gitea_queue["harbor_110_repair_jobs_cross_workflow_mismatch"]
-    )
+    ) and not harbor_110_repair_queue_superseded
     queue_harbor_running_jobs_api_stale = bool(
         gitea_queue["harbor_110_repair_visible_running_jobs_api_stale"]
-    )
+    ) and not harbor_110_repair_queue_superseded
     current_cd_waiting_behind_harbor_running = bool(
         gitea_queue["current_cd_waiting_behind_harbor_110_repair_running"]
-    )
-    queue_remote_control_unavailable = bool(
+    ) and not harbor_110_repair_queue_superseded
+    queue_remote_control_unavailable_raw = bool(
         gitea_queue["harbor_110_repair_remote_control_channel_unavailable"]
     )
+    queue_remote_control_unavailable = (
+        queue_remote_control_unavailable_raw
+        and not harbor_110_repair_queue_superseded
+    )
     cd_harbor_repair_requires_110_lane = bool(
         gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"]
     )
@@ -469,7 +479,6 @@ def _control_path_readiness(
     non110_runner_not_ready = bool(
         non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]
     )
-    command_path_ready = bool(ssh_diagnosis["wooo_command_path_ready"])
     runner_timeout = bool(
         ssh_diagnosis["runner_systemctl_show_timeout_effective_seen"]
     )
@@ -590,6 +599,9 @@ def _control_path_readiness(
         "harbor_110_repair_remote_control_channel_unavailable": (
             queue_remote_control_unavailable
         ),
+        "harbor_110_repair_remote_control_channel_unavailable_raw": (
+            queue_remote_control_unavailable_raw
+        ),
         "harbor_110_repair_failure_classifier": gitea_queue[
             "harbor_110_repair_failure_classifier"
         ],
@@ -719,6 +731,8 @@ def _control_path_readiness_status(
         return "blocked_awoooi_host_runner_queue_unavailable"
     if node_high_load:
         return "blocked_110_node_high_load"
+    if any(item.startswith("gitea_queue_cd_jobs_") for item in signal_ids):
+        return "blocked_gitea_cd_jobs_readback_stale_or_mismatched"
     if not registry_v2_ready:
         return "blocked_registry_v2_verifier_not_green"
     return "blocked_control_path_evidence_not_clear"
@@ -739,6 +753,8 @@ def _control_path_safe_next_action(*, status: str) -> str:
         return "rerun_non_secret_110_ssh_and_runner_control_path_diagnosis"
     if status == "blocked_registry_v2_verifier_not_green":
         return "rerun_public_and_internal_registry_v2_verifier_before_cd_retry"
+    if status == "blocked_gitea_cd_jobs_readback_stale_or_mismatched":
+        return "rerun_public_gitea_queue_and_deploy_marker_readback_before_cd_retry"
     return "normalize_control_path_evidence_then_retry_readback"
 
 
@@ -1240,6 +1256,25 @@ def _parse_controlled_cd_lane_readiness_output(output: str) -> dict[str, Any]:
 def _parse_non110_runner_readiness_output(output: str) -> dict[str, Any]:
     fields = _parse_key_values(output)
     marker_seen = "AWOOOI_NON110_RUNNER_READY=" in output
+    if not marker_seen:
+        return {
+            "receipt_seen": False,
+            "non110_runner_ready": False,
+            "ready_config_count": 0,
+            "ready_binary_count": 0,
+            "ready_registration_count": 0,
+            "ready_service_count": 0,
+            "ready_active_service_count": 0,
+            "ready_autostart_path_count": 0,
+            "warning_count": 0,
+            "blocker_count": 0,
+            "blockers": [],
+            "safe_next_step": "",
+            "raw_runner_registration_read": False,
+            "runner_token_read": False,
+            "metadata_only": True,
+            "raw_output_returned": False,
+        }
     blockers = _prefixed_blockers(
         output,
         prefix="non110_runner_readiness:",
@@ -2086,11 +2121,85 @@ def _active_blockers(
         blockers.append("public_registry_v2_verifier_not_green")
     if not verifier["internal_registry_v2_ready"]:
         blockers.append("internal_registry_v2_verifier_not_green")
-    blockers.extend(_strings(gitea_queue.get("blockers")))
+    blockers.extend(
+        _effective_gitea_queue_blockers(
+            ssh_diagnosis=ssh_diagnosis,
+            ssh_local=ssh_local,
+            watchdog_check=watchdog_check,
+            controlled_cd_lane=controlled_cd_lane,
+            verifier=verifier,
+            gitea_queue=gitea_queue,
+        )
+    )
     blockers.extend(_strings(deploy_marker.get("blockers")))
     return _unique_strings(blockers)
 
 
+def _effective_gitea_queue_blockers(
+    *,
+    ssh_diagnosis: dict[str, Any],
+    ssh_local: dict[str, Any],
+    watchdog_check: dict[str, Any],
+    controlled_cd_lane: dict[str, Any],
+    verifier: dict[str, Any],
+    gitea_queue: dict[str, Any],
+) -> list[str]:
+    blockers = _strings(gitea_queue.get("blockers"))
+    superseded = _gitea_queue_blockers_superseded_by_local_receipt(
+        ssh_diagnosis=ssh_diagnosis,
+        ssh_local=ssh_local,
+        watchdog_check=watchdog_check,
+        controlled_cd_lane=controlled_cd_lane,
+        verifier=verifier,
+    )
+    if not superseded:
+        return blockers
+    return [blocker for blocker in blockers if blocker not in superseded]
+
+
+def _gitea_queue_blockers_superseded_by_local_receipt(
+    *,
+    ssh_diagnosis: dict[str, Any],
+    ssh_local: dict[str, Any],
+    watchdog_check: dict[str, Any],
+    controlled_cd_lane: dict[str, Any],
+    verifier: dict[str, Any],
+) -> set[str]:
+    control_channel_ready = bool(
+        ssh_diagnosis["wooo_command_path_ready"]
+        or ssh_local["control_channel_metadata_ready"]
+    )
+    harbor_ready = bool(watchdog_check["harbor_ready"] or verifier["registry_v2_ready"])
+    if not control_channel_ready or not harbor_ready:
+        return set()
+
+    superseded = {
+        "gitea_queue_harbor_110_remote_control_channel_unavailable",
+        "gitea_queue_harbor_110_repair_blocked",
+        "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale",
+        "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running",
+    }
+    if verifier["registry_v2_ready"]:
+        superseded.update(
+            {
+                "gitea_queue_current_cd_harbor_retrying_unavailable",
+                "gitea_queue_harbor_110_remote_local_registry_v2_unavailable",
+                "gitea_queue_harbor_public_registry_v2_unavailable_after_remote_repair",
+            }
+        )
+    if controlled_cd_lane["receipt_seen"]:
+        superseded.update(
+            {
+                "gitea_queue_harbor_110_repair_no_matching_runner",
+                "gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch",
+                "gitea_queue_harbor_110_repair_jobs_stale_or_mismatched",
+                "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host",
+                "gitea_queue_cd_harbor_repair_requires_110_controlled_lane",
+            }
+        )
+    return superseded
+
+
 def _ssh_local_metadata_blockers(ssh_local: dict[str, Any]) -> list[str]:
     blockers: list[str] = []
     if not ssh_local["sshd_config_syntax_ok"]:
@@ -2129,10 +2238,16 @@ def _status(
 ) -> str:
     if verifier["registry_v2_ready"] and not active_blockers:
         return "harbor_registry_recovery_receipt_verified"
+    if controlled_cd_lane["receipt_seen"] and not controlled_cd_lane[
+        "controlled_cd_lane_ready"
+    ]:
+        return "controlled_cd_lane_readiness_receipt_blocked"
+    if non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]:
+        return "non110_runner_readiness_receipt_blocked"
     if (
         verifier["registry_v2_ready"]
         and gitea_queue["receipt_seen"]
-        and gitea_queue["blocker_count"] > 0
+        and any(blocker.startswith("gitea_queue_") for blocker in active_blockers)
     ):
         return "harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance"
     if (
@@ -2141,12 +2256,6 @@ def _status(
         and deploy_marker["blocker_count"] > 0
     ):
         return "harbor_registry_recovery_receipt_verified_waiting_deploy_marker_readback"
-    if controlled_cd_lane["receipt_seen"] and not controlled_cd_lane[
-        "controlled_cd_lane_ready"
-    ]:
-        return "controlled_cd_lane_readiness_receipt_blocked"
-    if non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]:
-        return "non110_runner_readiness_receipt_blocked"
     if watchdog_repair["receipt_seen"]:
         return "harbor_registry_repair_receipt_waiting_registry_v2_verifier"
     if watchdog_check["receipt_seen"] and watchdog_check["harbor_ready"]: