fix(runner): classify harbor remote control blocker

apps/api/src/services/harbor_registry_controlled_recovery_receipt.py

@@ -268,6 +268,15 @@ def validate_harbor_registry_controlled_recovery_receipt(
             "gitea_queue_harbor_110_repair_run_status": gitea_queue[
                 "latest_visible_harbor_110_repair_run_status"
             ],
+            "gitea_queue_harbor_110_repair_failure_classifier": gitea_queue[
+                "harbor_110_repair_failure_classifier"
+            ],
+            "gitea_queue_harbor_110_remote_control_channel_unavailable": (
+                gitea_queue["harbor_110_repair_remote_control_channel_unavailable"]
+            ),
+            "gitea_queue_harbor_110_bounded_ssh_timeout_seen": gitea_queue[
+                "harbor_110_repair_bounded_ssh_timeout_seen"
+            ],
             "gitea_queue_current_cd_inflight_classifier": gitea_queue[
                 "current_cd_inflight_classifier"
             ],
@@ -383,6 +392,9 @@ def _control_path_readiness(
     queue_jobs_cross_workflow = bool(
         gitea_queue["harbor_110_repair_jobs_cross_workflow_mismatch"]
     )
+    queue_remote_control_unavailable = bool(
+        gitea_queue["harbor_110_repair_remote_control_channel_unavailable"]
+    )
     cd_harbor_repair_requires_110_lane = bool(
         gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"]
     )
@@ -427,6 +439,7 @@ def _control_path_readiness(
         queue_no_matching_runner=queue_no_matching_runner,
         queue_jobs_stale=queue_jobs_stale,
         queue_jobs_cross_workflow=queue_jobs_cross_workflow,
+        queue_remote_control_unavailable=queue_remote_control_unavailable,
         cd_harbor_repair_requires_110_lane=cd_harbor_repair_requires_110_lane,
         cd_harbor_repair_blocked_by_awoooi_host=(
             cd_harbor_repair_blocked_by_awoooi_host
@@ -443,7 +456,11 @@ def _control_path_readiness(
     status = _control_path_readiness_status(
         node_high_load=node_high_load,
         runner_timeout=runner_timeout,
-        ssh_timeout=publickey_offer_timeout or server_accepts_key_then_timeout,
+        ssh_timeout=(
+            publickey_offer_timeout
+            or server_accepts_key_then_timeout
+            or queue_remote_control_unavailable
+        ),
         awoooi_host_unavailable=awoooi_host_unavailable,
         non110_runner_unavailable=non110_runner_unavailable,
         registry_v2_ready=registry_v2_ready,
@@ -484,6 +501,18 @@ def _control_path_readiness(
         ],
         "harbor_110_repair_jobs_stale_or_mismatched": queue_jobs_stale,
         "harbor_110_repair_jobs_cross_workflow_mismatch": queue_jobs_cross_workflow,
+        "harbor_110_repair_remote_control_channel_unavailable": (
+            queue_remote_control_unavailable
+        ),
+        "harbor_110_repair_failure_classifier": gitea_queue[
+            "harbor_110_repair_failure_classifier"
+        ],
+        "harbor_110_repair_remote_ssh_reachable": gitea_queue[
+            "harbor_110_repair_remote_ssh_reachable"
+        ],
+        "harbor_110_repair_bounded_ssh_timeout_seen": gitea_queue[
+            "harbor_110_repair_bounded_ssh_timeout_seen"
+        ],
         "cd_harbor_repair_requires_110_controlled_lane": (
             cd_harbor_repair_requires_110_lane
         ),
@@ -518,6 +547,7 @@ def _control_path_signal_ids(
     queue_no_matching_runner: bool,
     queue_jobs_stale: bool,
     queue_jobs_cross_workflow: bool,
+    queue_remote_control_unavailable: bool,
     cd_harbor_repair_requires_110_lane: bool,
     cd_harbor_repair_blocked_by_awoooi_host: bool,
     current_cd_waiting: bool,
@@ -544,6 +574,8 @@ def _control_path_signal_ids(
         signal_ids.append("gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch")
     if queue_jobs_stale:
         signal_ids.append("gitea_queue_harbor_110_repair_jobs_stale_or_mismatched")
+    if queue_remote_control_unavailable:
+        signal_ids.append("gitea_queue_harbor_110_remote_control_channel_unavailable")
     if cd_harbor_repair_blocked_by_awoooi_host:
         signal_ids.append(
             "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host"
@@ -585,12 +617,12 @@ def _control_path_readiness_status(
         return "blocked_110_high_load_and_awoooi_host_control_path_unavailable"
     if non110_runner_unavailable:
         return "blocked_non110_controlled_runner_unavailable"
+    if runner_timeout or ssh_timeout:
+        return "blocked_110_ssh_or_runner_control_path_timeout"
     if awoooi_host_unavailable:
         return "blocked_awoooi_host_runner_queue_unavailable"
     if node_high_load:
         return "blocked_110_node_high_load"
-    if runner_timeout or ssh_timeout:
-        return "blocked_110_ssh_or_runner_control_path_timeout"
     if not registry_v2_ready:
         return "blocked_registry_v2_verifier_not_green"
     return "blocked_control_path_evidence_not_clear"
@@ -1095,6 +1127,13 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
             "cd_run_jobs_run_ids": [],
             "latest_visible_harbor_110_repair_run_id": "",
             "latest_visible_harbor_110_repair_run_status": "",
+            "harbor_110_repair_failure_classifier": "",
+            "harbor_110_repair_remote_control_channel_unavailable": False,
+            "harbor_110_repair_remote_ssh_reachable": None,
+            "harbor_110_repair_bounded_ssh_timeout_seen": False,
+            "harbor_110_repair_local_registry_v2_status": "",
+            "harbor_110_repair_public_registry_v2_status": "",
+            "harbor_110_repair_verified": False,
             "harbor_110_repair_no_matching_runner_label": "",
             "harbor_110_repair_no_matching_runner": False,
             "harbor_110_repair_waiting": False,
@@ -1141,6 +1180,51 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
         rollups.get("harbor_110_repair_blocked") is True
         or readback.get("latest_visible_harbor_110_repair_blocked") is True
     )
+    harbor_repair_failure_classifier = str(
+        rollups.get("harbor_110_repair_failure_classifier")
+        or readback.get("latest_visible_harbor_110_repair_failure_classifier")
+        or ""
+    )
+    harbor_repair_remote_control_unavailable = bool(
+        rollups.get("harbor_110_repair_remote_control_channel_unavailable") is True
+        or readback.get(
+            "latest_visible_harbor_110_repair_remote_control_channel_unavailable"
+        )
+        is True
+        or harbor_repair_failure_classifier
+        == "harbor_110_remote_control_channel_unavailable"
+    )
+    harbor_repair_bounded_ssh_timeout_seen = bool(
+        rollups.get("harbor_110_repair_bounded_ssh_timeout_seen") is True
+        or readback.get("latest_visible_harbor_110_repair_bounded_ssh_timeout_seen")
+        is True
+    )
+    harbor_repair_remote_ssh_reachable = (
+        readback.get("latest_visible_harbor_110_repair_remote_ssh_reachable")
+        if "latest_visible_harbor_110_repair_remote_ssh_reachable" in readback
+        else rollups.get("harbor_110_repair_remote_ssh_reachable")
+    )
+    harbor_repair_local_registry_status = str(
+        rollups.get("harbor_110_repair_local_registry_v2_status")
+        or readback.get("latest_visible_harbor_110_repair_local_registry_v2_status")
+        or ""
+    )
+    harbor_repair_public_registry_status = str(
+        rollups.get("harbor_110_repair_public_registry_v2_status")
+        or readback.get("latest_visible_harbor_110_repair_public_registry_v2_status")
+        or ""
+    )
+    harbor_repair_verified = bool(
+        rollups.get("harbor_110_repair_verified") is True
+        or readback.get("latest_visible_harbor_110_repair_verified") is True
+    )
+    harbor_repair_local_v2_unavailable = (
+        harbor_repair_failure_classifier
+        == "harbor_110_remote_local_registry_v2_unavailable"
+    )
+    harbor_repair_public_v2_unavailable = (
+        harbor_repair_failure_classifier == "harbor_public_registry_v2_unavailable"
+    )
     current_cd_inflight_classifier = str(
         rollups.get("current_main_cd_inflight_classifier")
         or readback.get("latest_visible_cd_inflight_classifier")
@@ -1246,12 +1330,18 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
         cd_jobs_stale=cd_jobs_stale,
         cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch,
         cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch,
+        harbor_repair_remote_control_unavailable=(
+            harbor_repair_remote_control_unavailable
+        ),
+        harbor_repair_local_v2_unavailable=harbor_repair_local_v2_unavailable,
+        harbor_repair_public_v2_unavailable=harbor_repair_public_v2_unavailable,
         blocked=blocked,
         boundary_violation=boundary_violation,
     )
     normalized_classifier_fields = _gitea_queue_normalized_classifier_fields(
         cd_jobs_payload_classifier=cd_jobs_payload_classifier,
         harbor_jobs_payload_classifier=harbor_jobs_payload_classifier,
+        harbor_repair_failure_classifier=harbor_repair_failure_classifier,
         no_matching_label=no_matching_label,
         current_cd_no_matching_label=current_cd_no_matching_label,
         current_cd_waiting=current_cd_waiting,
@@ -1310,6 +1400,23 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
         "latest_visible_harbor_110_repair_run_status": str(
             readback.get("latest_visible_harbor_110_repair_run_status") or ""
         ),
+        "harbor_110_repair_failure_classifier": harbor_repair_failure_classifier,
+        "harbor_110_repair_remote_control_channel_unavailable": (
+            harbor_repair_remote_control_unavailable
+        ),
+        "harbor_110_repair_remote_ssh_reachable": (
+            harbor_repair_remote_ssh_reachable
+        ),
+        "harbor_110_repair_bounded_ssh_timeout_seen": (
+            harbor_repair_bounded_ssh_timeout_seen
+        ),
+        "harbor_110_repair_local_registry_v2_status": (
+            harbor_repair_local_registry_status
+        ),
+        "harbor_110_repair_public_registry_v2_status": (
+            harbor_repair_public_registry_status
+        ),
+        "harbor_110_repair_verified": harbor_repair_verified,
         "harbor_110_repair_no_matching_runner_label": no_matching_label,
         "harbor_110_repair_no_matching_runner": bool(no_matching_label),
         "harbor_110_repair_waiting": waiting,
@@ -1347,6 +1454,7 @@ def _gitea_queue_normalized_classifier_fields(
     *,
     cd_jobs_payload_classifier: str,
     harbor_jobs_payload_classifier: str,
+    harbor_repair_failure_classifier: str,
     no_matching_label: str,
     current_cd_no_matching_label: str,
     current_cd_waiting: bool,
@@ -1386,6 +1494,31 @@ def _gitea_queue_normalized_classifier_fields(
             "raw_output_returned": False,
         },
     ]
+    if harbor_repair_failure_classifier:
+        fields.append(
+            {
+                "field_id": "harbor_110_repair_failure_classifier",
+                "value": harbor_repair_failure_classifier,
+                "blockers": [
+                    item
+                    for item in blockers
+                    if item
+                    in {
+                        "gitea_queue_harbor_110_remote_control_channel_unavailable",
+                        (
+                            "gitea_queue_harbor_110_remote_local_registry_v2_"
+                            "unavailable"
+                        ),
+                        (
+                            "gitea_queue_harbor_public_registry_v2_unavailable_"
+                            "after_remote_repair"
+                        ),
+                    }
+                ],
+                "metadata_only": True,
+                "raw_output_returned": False,
+            }
+        )
     if current_cd_no_matching_label or current_cd_waiting:
         fields.append(
             {
@@ -1432,6 +1565,9 @@ def _gitea_queue_blockers(
     cd_jobs_stale: bool,
     cd_jobs_head_sha_mismatch: bool,
     cd_jobs_run_id_mismatch: bool,
+    harbor_repair_remote_control_unavailable: bool,
+    harbor_repair_local_v2_unavailable: bool,
+    harbor_repair_public_v2_unavailable: bool,
     blocked: bool,
     boundary_violation: bool,
 ) -> list[str]:
@@ -1458,7 +1594,21 @@ def _gitea_queue_blockers(
         blockers.append("gitea_queue_cd_jobs_stale_or_mismatched")
     if no_matching_runner:
         blockers.append("gitea_queue_harbor_110_repair_no_matching_runner")
-    elif blocked:
+    if harbor_repair_remote_control_unavailable:
+        blockers.append("gitea_queue_harbor_110_remote_control_channel_unavailable")
+    if harbor_repair_local_v2_unavailable:
+        blockers.append("gitea_queue_harbor_110_remote_local_registry_v2_unavailable")
+    if harbor_repair_public_v2_unavailable:
+        blockers.append(
+            "gitea_queue_harbor_public_registry_v2_unavailable_after_remote_repair"
+        )
+    if (
+        blocked
+        and not no_matching_runner
+        and not harbor_repair_remote_control_unavailable
+        and not harbor_repair_local_v2_unavailable
+        and not harbor_repair_public_v2_unavailable
+    ):
         blockers.append("gitea_queue_harbor_110_repair_blocked")
     if jobs_cross_workflow:
         blockers.append("gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch")