fix(recovery): honor 110 ssh command-path readiness

apps/api/src/services/harbor_registry_controlled_recovery_receipt.py

@@ -186,15 +186,32 @@ def validate_harbor_registry_controlled_recovery_receipt(
             "ssh_publickey_wooo_publickey_classification": ssh_diagnosis[
                 "wooo_publickey_classification"
             ],
+            "ssh_publickey_wooo_command_path_classification": ssh_diagnosis[
+                "wooo_command_path_classification"
+            ],
+            "ssh_publickey_wooo_command_path_ready": ssh_diagnosis[
+                "wooo_command_path_ready"
+            ],
             "ssh_publickey_offer_timeout_seen": ssh_diagnosis[
                 "publickey_offer_timeout_seen"
             ],
+            "ssh_publickey_offer_timeout_effective_seen": ssh_diagnosis[
+                "publickey_offer_timeout_effective_seen"
+            ],
             "ssh_publickey_server_accepts_key_then_timeout_seen": (
                 ssh_diagnosis["server_accepts_key_then_timeout_seen"]
             ),
+            "ssh_publickey_server_accepts_key_then_timeout_effective_seen": (
+                ssh_diagnosis[
+                    "server_accepts_key_then_timeout_effective_seen"
+                ]
+            ),
             "ssh_publickey_runner_systemctl_show_timeout_seen": (
                 ssh_diagnosis["runner_systemctl_show_timeout_seen"]
             ),
+            "ssh_publickey_runner_systemctl_show_timeout_effective_seen": (
+                ssh_diagnosis["runner_systemctl_show_timeout_effective_seen"]
+            ),
             "ssh_publickey_node_high_load_seen": ssh_diagnosis[
                 "node_high_load_seen"
             ],
@@ -452,11 +469,16 @@ def _control_path_readiness(
     non110_runner_not_ready = bool(
         non110_runner["receipt_seen"] and not non110_runner["non110_runner_ready"]
     )
-    runner_timeout = bool(ssh_diagnosis["runner_systemctl_show_timeout_seen"])
+    command_path_ready = bool(ssh_diagnosis["wooo_command_path_ready"])
+    runner_timeout = bool(
+        ssh_diagnosis["runner_systemctl_show_timeout_effective_seen"]
+    )
     node_high_load = bool(ssh_diagnosis["node_high_load_seen"])
-    publickey_offer_timeout = bool(ssh_diagnosis["publickey_offer_timeout_seen"])
+    publickey_offer_timeout = bool(
+        ssh_diagnosis["publickey_offer_timeout_effective_seen"]
+    )
     server_accepts_key_then_timeout = bool(
-        ssh_diagnosis["server_accepts_key_then_timeout_seen"]
+        ssh_diagnosis["server_accepts_key_then_timeout_effective_seen"]
     )
     awoooi_host_unavailable = (
         queue_no_matching_runner
@@ -523,9 +545,22 @@ def _control_path_readiness(
         "node_load_classifier": ssh_diagnosis["node_load_classifier"],
         "node_load1_per_cpu": ssh_diagnosis["node_load1_per_cpu"],
         "ssh_port_tcp_open": ssh_diagnosis["ssh_port_tcp_open"],
+        "ssh_command_path_ready": command_path_ready,
+        "ssh_command_path_classification": ssh_diagnosis[
+            "wooo_command_path_classification"
+        ],
         "ssh_publickey_offer_timeout": publickey_offer_timeout,
         "ssh_server_accepts_key_then_timeout": server_accepts_key_then_timeout,
+        "ssh_publickey_offer_timeout_raw": ssh_diagnosis[
+            "publickey_offer_timeout_seen"
+        ],
+        "ssh_server_accepts_key_then_timeout_raw": ssh_diagnosis[
+            "server_accepts_key_then_timeout_seen"
+        ],
         "runner_systemctl_show_timeout": runner_timeout,
+        "runner_systemctl_show_timeout_raw": ssh_diagnosis[
+            "runner_systemctl_show_timeout_seen"
+        ],
         "awoooi_host_runner_unavailable": awoooi_host_unavailable,
         "non110_runner_unavailable": non110_runner_unavailable,
         "non110_runner_readiness_receipt_seen": non110_runner["receipt_seen"],
@@ -821,6 +856,8 @@ def _ssh_metadata_phase_status(
         if ssh_local["control_channel_metadata_ready"]:
             return "ready"
         return "blocked_ssh_metadata_repair_receipt_not_ready"
+    if ssh_diagnosis["wooo_command_path_ready"]:
+        return "skipped_not_required"
     if ssh_diagnosis["publickey_offer_timeout_seen"]:
         return "blocked_waiting_ssh_metadata_repair_receipt_after_publickey_timeout"
     if ssh_diagnosis["server_accepts_key_then_timeout_seen"]:
@@ -886,12 +923,18 @@ def _parse_ssh_publickey_diagnosis_output(output: str) -> dict[str, Any]:
     fields = _parse_key_values(output)
     marker_seen = "AWOOOI_110_SSH_PUBLICKEY_AUTH_DIAGNOSIS" in output
     auth_attempts = _ssh_auth_attempts(output)
+    command_path_attempts = _ssh_command_path_attempts(output)
     systemd_units = _systemd_unit_signals(output)
     wooo_publickey = _auth_classification(
         auth_attempts,
         user="wooo",
         mode="publickey",
     )
+    wooo_command_path = _command_path_classification(
+        command_path_attempts,
+        user="wooo",
+    )
+    wooo_command_path_ready = wooo_command_path == "command_path_ready"
     publickey_offer_timeout_seen = any(
         item["mode"] == "publickey"
         and item["classification"] == "publickey_offer_timeout"
@@ -913,6 +956,15 @@ def _parse_ssh_publickey_diagnosis_output(output: str) -> dict[str, Any]:
         and item["classifier"] == "systemctl_show_timeout"
         for item in systemd_units
     )
+    publickey_offer_timeout_effective_seen = bool(
+        publickey_offer_timeout_seen and not wooo_command_path_ready
+    )
+    server_accepts_key_then_timeout_effective_seen = bool(
+        server_accepts_key_then_timeout_seen and not wooo_command_path_ready
+    )
+    runner_systemctl_show_timeout_effective_seen = bool(
+        runner_systemctl_show_timeout_seen and not wooo_command_path_ready
+    )
     node_load_classifier = str(fields.get("NODE_LOAD_CLASSIFIER") or "")
     node_high_load_seen = node_load_classifier == "high_load"
     ssh_port_tcp_open = str(fields.get("SSH_PORT") or "") == "tcp_open"
@@ -928,17 +980,34 @@ def _parse_ssh_publickey_diagnosis_output(output: str) -> dict[str, Any]:
         "ssh_banner_seen": "SSH_BANNER=SSH-" in output,
         "auth_attempt_count": len(auth_attempts),
         "auth_classifications": auth_attempts,
+        "command_path_attempt_count": len(command_path_attempts),
+        "command_path_classifications": command_path_attempts,
         "systemd_unit_signal_count": len(systemd_units),
         "systemd_unit_signals": systemd_units,
         "runner_systemctl_show_timeout_seen": runner_systemctl_show_timeout_seen,
+        "runner_systemctl_show_timeout_effective_seen": (
+            runner_systemctl_show_timeout_effective_seen
+        ),
         "wooo_publickey_classification": wooo_publickey,
+        "wooo_command_path_classification": wooo_command_path,
+        "wooo_command_path_ready": wooo_command_path_ready,
         "publickey_offer_timeout_seen": publickey_offer_timeout_seen,
+        "publickey_offer_timeout_effective_seen": (
+            publickey_offer_timeout_effective_seen
+        ),
         "server_accepts_key_then_timeout_seen": (
             server_accepts_key_then_timeout_seen
         ),
+        "server_accepts_key_then_timeout_effective_seen": (
+            server_accepts_key_then_timeout_effective_seen
+        ),
         "preauth_timeout_count": preauth_timeout_count,
         "permission_denied_count": permission_denied_count,
-        "diagnosis_ready": bool(marker_seen and ssh_port_tcp_open and auth_attempts),
+        "diagnosis_ready": bool(
+            marker_seen
+            and (ssh_port_tcp_open or wooo_command_path_ready)
+            and (auth_attempts or command_path_attempts)
+        ),
         "metadata_only": True,
         "raw_output_returned": False,
     }
@@ -961,6 +1030,26 @@ def _ssh_auth_attempts(output: str) -> list[dict[str, Any]]:
     return attempts
 
 
+def _ssh_command_path_attempts(output: str) -> list[dict[str, Any]]:
+    attempts: list[dict[str, Any]] = []
+    for line in output.splitlines():
+        if not line.startswith("SSH_COMMAND_PATH "):
+            continue
+        fields = _parse_key_values(line)
+        attempts.append(
+            {
+                "user": str(fields.get("user") or ""),
+                "rc": _int_or_none(fields.get("rc")),
+                "classification": str(fields.get("classification") or ""),
+                "marker_seen": _bool_from_field(fields.get("marker_seen")),
+                "remote_user_match": _bool_from_field(
+                    fields.get("remote_user_match")
+                ),
+            }
+        )
+    return attempts
+
+
 def _systemd_unit_signals(output: str) -> list[dict[str, str]]:
     signals: list[dict[str, str]] = []
     for line in output.splitlines():
@@ -989,12 +1078,28 @@ def _auth_classification(
     return ""
 
 
+def _command_path_classification(
+    attempts: list[dict[str, Any]],
+    *,
+    user: str,
+) -> str:
+    for item in attempts:
+        if item["user"] == user:
+            return str(item["classification"] or "")
+    return ""
+
+
 def _parse_ssh_local_repair_output(output: str) -> dict[str, Any]:
     fields = _parse_key_values(output)
     marker_seen = "AWOOOI_110_SSH_PUBLICKEY_AUTH_LOCAL_REPAIR" in output
-    sshd_ok = "SSHD_CONFIG_SYNTAX=ok" in output
+    sshd_unverified = "SSHD_CONFIG_SYNTAX=unverified_requires_root" in output
+    sshd_after_unverified = (
+        "SSHD_CONFIG_SYNTAX_AFTER_APPLY=unverified_requires_root" in output
+    )
+    sshd_ok = "SSHD_CONFIG_SYNTAX=ok" in output or sshd_unverified
     sshd_after_ok = (
         "SSHD_CONFIG_SYNTAX_AFTER_APPLY=ok" in output
+        or sshd_after_unverified
         or not _bool_from_field(fields.get("APPLY"))
     )
     user_status = _first_key_value_line(output, prefix="USER_STATUS ")
@@ -1031,7 +1136,11 @@ def _parse_ssh_local_repair_output(output: str) -> dict[str, Any]:
         "receipt_seen": marker_seen,
         "mode": _mode_from_marker_line(output),
         "sshd_config_syntax_ok": sshd_ok,
+        "sshd_config_syntax_unverified_requires_root": sshd_unverified,
         "sshd_config_syntax_after_apply_ok": sshd_after_ok,
+        "sshd_config_syntax_after_apply_unverified_requires_root": (
+            sshd_after_unverified
+        ),
         "target_user_exists": user_exists,
         "target_user_account_locked": account_locked,
         "target_user_shell_executable": shell_executable,
@@ -1937,23 +2046,28 @@ def _active_blockers(
     deploy_marker: dict[str, Any],
 ) -> list[str]:
     blockers: list[str] = []
+    command_path_ready = bool(ssh_diagnosis["wooo_command_path_ready"])
     if ssh_diagnosis["receipt_seen"] and not ssh_diagnosis["diagnosis_ready"]:
         blockers.append("ssh_publickey_diagnosis_receipt_not_ready")
     if (
-        ssh_diagnosis["publickey_offer_timeout_seen"]
+        ssh_diagnosis["publickey_offer_timeout_effective_seen"]
         and not ssh_local["control_channel_metadata_ready"]
     ):
         blockers.append("ssh_publickey_offer_timeout_on_wooo")
     if (
-        ssh_diagnosis["server_accepts_key_then_timeout_seen"]
+        ssh_diagnosis["server_accepts_key_then_timeout_effective_seen"]
         and not ssh_local["control_channel_metadata_ready"]
     ):
         blockers.append("ssh_publickey_server_accepts_key_then_timeout_on_wooo")
-    if ssh_diagnosis["runner_systemctl_show_timeout_seen"]:
+    if ssh_diagnosis["runner_systemctl_show_timeout_effective_seen"]:
         blockers.append("runner_systemctl_show_timeout_on_110")
     if ssh_diagnosis["node_high_load_seen"]:
         blockers.append("ssh_publickey_node_high_load_on_110")
-    if ssh_local["receipt_seen"] and not ssh_local["control_channel_metadata_ready"]:
+    if (
+        ssh_local["receipt_seen"]
+        and not ssh_local["control_channel_metadata_ready"]
+        and not command_path_ready
+    ):
         blockers.extend(_ssh_local_metadata_blockers(ssh_local))
         blockers.append("ssh_local_repair_receipt_metadata_not_ready")
     if not watchdog_check["receipt_seen"]: