diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py index 8159e8005..1eaa38820 100644 --- a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py @@ -268,6 +268,15 @@ def validate_harbor_registry_controlled_recovery_receipt( "gitea_queue_harbor_110_repair_run_status": gitea_queue[ "latest_visible_harbor_110_repair_run_status" ], + "gitea_queue_harbor_110_repair_failure_classifier": gitea_queue[ + "harbor_110_repair_failure_classifier" + ], + "gitea_queue_harbor_110_remote_control_channel_unavailable": ( + gitea_queue["harbor_110_repair_remote_control_channel_unavailable"] + ), + "gitea_queue_harbor_110_bounded_ssh_timeout_seen": gitea_queue[ + "harbor_110_repair_bounded_ssh_timeout_seen" + ], "gitea_queue_current_cd_inflight_classifier": gitea_queue[ "current_cd_inflight_classifier" ], @@ -383,6 +392,9 @@ def _control_path_readiness( queue_jobs_cross_workflow = bool( gitea_queue["harbor_110_repair_jobs_cross_workflow_mismatch"] ) + queue_remote_control_unavailable = bool( + gitea_queue["harbor_110_repair_remote_control_channel_unavailable"] + ) cd_harbor_repair_requires_110_lane = bool( gitea_queue["current_cd_harbor_repair_requires_110_controlled_lane"] ) @@ -427,6 +439,7 @@ def _control_path_readiness( queue_no_matching_runner=queue_no_matching_runner, queue_jobs_stale=queue_jobs_stale, queue_jobs_cross_workflow=queue_jobs_cross_workflow, + queue_remote_control_unavailable=queue_remote_control_unavailable, cd_harbor_repair_requires_110_lane=cd_harbor_repair_requires_110_lane, cd_harbor_repair_blocked_by_awoooi_host=( cd_harbor_repair_blocked_by_awoooi_host @@ -443,7 +456,11 @@ def _control_path_readiness( status = _control_path_readiness_status( node_high_load=node_high_load, runner_timeout=runner_timeout, - ssh_timeout=publickey_offer_timeout or server_accepts_key_then_timeout, + ssh_timeout=( + publickey_offer_timeout + or server_accepts_key_then_timeout + or queue_remote_control_unavailable + ), awoooi_host_unavailable=awoooi_host_unavailable, non110_runner_unavailable=non110_runner_unavailable, registry_v2_ready=registry_v2_ready, @@ -484,6 +501,18 @@ def _control_path_readiness( ], "harbor_110_repair_jobs_stale_or_mismatched": queue_jobs_stale, "harbor_110_repair_jobs_cross_workflow_mismatch": queue_jobs_cross_workflow, + "harbor_110_repair_remote_control_channel_unavailable": ( + queue_remote_control_unavailable + ), + "harbor_110_repair_failure_classifier": gitea_queue[ + "harbor_110_repair_failure_classifier" + ], + "harbor_110_repair_remote_ssh_reachable": gitea_queue[ + "harbor_110_repair_remote_ssh_reachable" + ], + "harbor_110_repair_bounded_ssh_timeout_seen": gitea_queue[ + "harbor_110_repair_bounded_ssh_timeout_seen" + ], "cd_harbor_repair_requires_110_controlled_lane": ( cd_harbor_repair_requires_110_lane ), @@ -518,6 +547,7 @@ def _control_path_signal_ids( queue_no_matching_runner: bool, queue_jobs_stale: bool, queue_jobs_cross_workflow: bool, + queue_remote_control_unavailable: bool, cd_harbor_repair_requires_110_lane: bool, cd_harbor_repair_blocked_by_awoooi_host: bool, current_cd_waiting: bool, @@ -544,6 +574,8 @@ def _control_path_signal_ids( signal_ids.append("gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch") if queue_jobs_stale: signal_ids.append("gitea_queue_harbor_110_repair_jobs_stale_or_mismatched") + if queue_remote_control_unavailable: + signal_ids.append("gitea_queue_harbor_110_remote_control_channel_unavailable") if cd_harbor_repair_blocked_by_awoooi_host: signal_ids.append( "gitea_queue_cd_harbor_repair_blocked_by_no_matching_awoooi_host" @@ -585,12 +617,12 @@ def _control_path_readiness_status( return "blocked_110_high_load_and_awoooi_host_control_path_unavailable" if non110_runner_unavailable: return "blocked_non110_controlled_runner_unavailable" + if runner_timeout or ssh_timeout: + return "blocked_110_ssh_or_runner_control_path_timeout" if awoooi_host_unavailable: return "blocked_awoooi_host_runner_queue_unavailable" if node_high_load: return "blocked_110_node_high_load" - if runner_timeout or ssh_timeout: - return "blocked_110_ssh_or_runner_control_path_timeout" if not registry_v2_ready: return "blocked_registry_v2_verifier_not_green" return "blocked_control_path_evidence_not_clear" @@ -1095,6 +1127,13 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: "cd_run_jobs_run_ids": [], "latest_visible_harbor_110_repair_run_id": "", "latest_visible_harbor_110_repair_run_status": "", + "harbor_110_repair_failure_classifier": "", + "harbor_110_repair_remote_control_channel_unavailable": False, + "harbor_110_repair_remote_ssh_reachable": None, + "harbor_110_repair_bounded_ssh_timeout_seen": False, + "harbor_110_repair_local_registry_v2_status": "", + "harbor_110_repair_public_registry_v2_status": "", + "harbor_110_repair_verified": False, "harbor_110_repair_no_matching_runner_label": "", "harbor_110_repair_no_matching_runner": False, "harbor_110_repair_waiting": False, @@ -1141,6 +1180,51 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: rollups.get("harbor_110_repair_blocked") is True or readback.get("latest_visible_harbor_110_repair_blocked") is True ) + harbor_repair_failure_classifier = str( + rollups.get("harbor_110_repair_failure_classifier") + or readback.get("latest_visible_harbor_110_repair_failure_classifier") + or "" + ) + harbor_repair_remote_control_unavailable = bool( + rollups.get("harbor_110_repair_remote_control_channel_unavailable") is True + or readback.get( + "latest_visible_harbor_110_repair_remote_control_channel_unavailable" + ) + is True + or harbor_repair_failure_classifier + == "harbor_110_remote_control_channel_unavailable" + ) + harbor_repair_bounded_ssh_timeout_seen = bool( + rollups.get("harbor_110_repair_bounded_ssh_timeout_seen") is True + or readback.get("latest_visible_harbor_110_repair_bounded_ssh_timeout_seen") + is True + ) + harbor_repair_remote_ssh_reachable = ( + readback.get("latest_visible_harbor_110_repair_remote_ssh_reachable") + if "latest_visible_harbor_110_repair_remote_ssh_reachable" in readback + else rollups.get("harbor_110_repair_remote_ssh_reachable") + ) + harbor_repair_local_registry_status = str( + rollups.get("harbor_110_repair_local_registry_v2_status") + or readback.get("latest_visible_harbor_110_repair_local_registry_v2_status") + or "" + ) + harbor_repair_public_registry_status = str( + rollups.get("harbor_110_repair_public_registry_v2_status") + or readback.get("latest_visible_harbor_110_repair_public_registry_v2_status") + or "" + ) + harbor_repair_verified = bool( + rollups.get("harbor_110_repair_verified") is True + or readback.get("latest_visible_harbor_110_repair_verified") is True + ) + harbor_repair_local_v2_unavailable = ( + harbor_repair_failure_classifier + == "harbor_110_remote_local_registry_v2_unavailable" + ) + harbor_repair_public_v2_unavailable = ( + harbor_repair_failure_classifier == "harbor_public_registry_v2_unavailable" + ) current_cd_inflight_classifier = str( rollups.get("current_main_cd_inflight_classifier") or readback.get("latest_visible_cd_inflight_classifier") @@ -1246,12 +1330,18 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: cd_jobs_stale=cd_jobs_stale, cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch, cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch, + harbor_repair_remote_control_unavailable=( + harbor_repair_remote_control_unavailable + ), + harbor_repair_local_v2_unavailable=harbor_repair_local_v2_unavailable, + harbor_repair_public_v2_unavailable=harbor_repair_public_v2_unavailable, blocked=blocked, boundary_violation=boundary_violation, ) normalized_classifier_fields = _gitea_queue_normalized_classifier_fields( cd_jobs_payload_classifier=cd_jobs_payload_classifier, harbor_jobs_payload_classifier=harbor_jobs_payload_classifier, + harbor_repair_failure_classifier=harbor_repair_failure_classifier, no_matching_label=no_matching_label, current_cd_no_matching_label=current_cd_no_matching_label, current_cd_waiting=current_cd_waiting, @@ -1310,6 +1400,23 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: "latest_visible_harbor_110_repair_run_status": str( readback.get("latest_visible_harbor_110_repair_run_status") or "" ), + "harbor_110_repair_failure_classifier": harbor_repair_failure_classifier, + "harbor_110_repair_remote_control_channel_unavailable": ( + harbor_repair_remote_control_unavailable + ), + "harbor_110_repair_remote_ssh_reachable": ( + harbor_repair_remote_ssh_reachable + ), + "harbor_110_repair_bounded_ssh_timeout_seen": ( + harbor_repair_bounded_ssh_timeout_seen + ), + "harbor_110_repair_local_registry_v2_status": ( + harbor_repair_local_registry_status + ), + "harbor_110_repair_public_registry_v2_status": ( + harbor_repair_public_registry_status + ), + "harbor_110_repair_verified": harbor_repair_verified, "harbor_110_repair_no_matching_runner_label": no_matching_label, "harbor_110_repair_no_matching_runner": bool(no_matching_label), "harbor_110_repair_waiting": waiting, @@ -1347,6 +1454,7 @@ def _gitea_queue_normalized_classifier_fields( *, cd_jobs_payload_classifier: str, harbor_jobs_payload_classifier: str, + harbor_repair_failure_classifier: str, no_matching_label: str, current_cd_no_matching_label: str, current_cd_waiting: bool, @@ -1386,6 +1494,31 @@ def _gitea_queue_normalized_classifier_fields( "raw_output_returned": False, }, ] + if harbor_repair_failure_classifier: + fields.append( + { + "field_id": "harbor_110_repair_failure_classifier", + "value": harbor_repair_failure_classifier, + "blockers": [ + item + for item in blockers + if item + in { + "gitea_queue_harbor_110_remote_control_channel_unavailable", + ( + "gitea_queue_harbor_110_remote_local_registry_v2_" + "unavailable" + ), + ( + "gitea_queue_harbor_public_registry_v2_unavailable_" + "after_remote_repair" + ), + } + ], + "metadata_only": True, + "raw_output_returned": False, + } + ) if current_cd_no_matching_label or current_cd_waiting: fields.append( { @@ -1432,6 +1565,9 @@ def _gitea_queue_blockers( cd_jobs_stale: bool, cd_jobs_head_sha_mismatch: bool, cd_jobs_run_id_mismatch: bool, + harbor_repair_remote_control_unavailable: bool, + harbor_repair_local_v2_unavailable: bool, + harbor_repair_public_v2_unavailable: bool, blocked: bool, boundary_violation: bool, ) -> list[str]: @@ -1458,7 +1594,21 @@ def _gitea_queue_blockers( blockers.append("gitea_queue_cd_jobs_stale_or_mismatched") if no_matching_runner: blockers.append("gitea_queue_harbor_110_repair_no_matching_runner") - elif blocked: + if harbor_repair_remote_control_unavailable: + blockers.append("gitea_queue_harbor_110_remote_control_channel_unavailable") + if harbor_repair_local_v2_unavailable: + blockers.append("gitea_queue_harbor_110_remote_local_registry_v2_unavailable") + if harbor_repair_public_v2_unavailable: + blockers.append( + "gitea_queue_harbor_public_registry_v2_unavailable_after_remote_repair" + ) + if ( + blocked + and not no_matching_runner + and not harbor_repair_remote_control_unavailable + and not harbor_repair_local_v2_unavailable + and not harbor_repair_public_v2_unavailable + ): blockers.append("gitea_queue_harbor_110_repair_blocked") if jobs_cross_workflow: blockers.append("gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch") diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py index f415f912b..a87b588e3 100644 --- a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py @@ -539,6 +539,60 @@ def test_harbor_recovery_receipt_surfaces_cross_workflow_queue_payload() -> None ] +def test_harbor_recovery_receipt_surfaces_remote_control_timeout_queue_payload() -> None: + payload = validate_harbor_registry_controlled_recovery_receipt( + { + "watchdog_check_output": _watchdog_check_output( + ready=True, + status=401, + ), + "public_registry_v2_http_status": 401, + "internal_registry_v2_http_status": 401, + "gitea_actions_queue_readback": _gitea_queue_remote_control_timeout(), + } + ) + + assert payload["status"] == ( + "harbor_registry_recovery_receipt_verified_waiting_gitea_queue_clearance" + ) + assert payload["active_blockers"] == [ + "gitea_queue_harbor_110_remote_control_channel_unavailable", + "gitea_queue_harbor_110_repair_jobs_stale_or_mismatched", + ] + queue = payload["readback"]["gitea_actions_queue"] + assert queue["status"] == "blocked_harbor_110_remote_control_channel_unavailable" + assert queue["harbor_110_repair_failure_classifier"] == ( + "harbor_110_remote_control_channel_unavailable" + ) + assert queue["harbor_110_repair_remote_control_channel_unavailable"] is True + assert queue["harbor_110_repair_remote_ssh_reachable"] is False + assert queue["harbor_110_repair_bounded_ssh_timeout_seen"] is True + assert payload["rollups"][ + "gitea_queue_harbor_110_remote_control_channel_unavailable" + ] is True + assert payload["rollups"]["gitea_queue_harbor_110_repair_failure_classifier"] == ( + "harbor_110_remote_control_channel_unavailable" + ) + normalized = { + item["field_id"]: item for item in queue["normalized_classifier_fields"] + } + assert normalized["harbor_110_repair_failure_classifier"]["value"] == ( + "harbor_110_remote_control_channel_unavailable" + ) + assert normalized["harbor_110_repair_failure_classifier"]["blockers"] == [ + "gitea_queue_harbor_110_remote_control_channel_unavailable" + ] + readiness = payload["readback"]["control_path_readiness"] + assert readiness["status"] == "blocked_110_ssh_or_runner_control_path_timeout" + assert ( + "gitea_queue_harbor_110_remote_control_channel_unavailable" + in readiness["signal_ids"] + ) + assert ( + readiness["harbor_110_repair_remote_control_channel_unavailable"] is True + ) + + def test_harbor_recovery_receipt_surfaces_cd_jobs_payload_classifier() -> None: payload = validate_harbor_registry_controlled_recovery_receipt( { @@ -1001,6 +1055,41 @@ def _gitea_queue_cross_workflow_jobs() -> dict: return payload +def _gitea_queue_remote_control_timeout() -> dict: + payload = _gitea_queue_no_matching_runner() + payload["status"] = "blocked_harbor_110_remote_control_channel_unavailable" + payload["readback"].update( + { + "latest_visible_harbor_110_repair_run_status": "Failure", + "latest_visible_harbor_110_repair_no_matching_runner_label": "", + "latest_visible_harbor_110_repair_waiting": False, + "latest_visible_harbor_110_repair_blocked": True, + "latest_visible_harbor_110_repair_failure_classifier": ( + "harbor_110_remote_control_channel_unavailable" + ), + "latest_visible_harbor_110_repair_remote_control_channel_unavailable": ( + True + ), + "latest_visible_harbor_110_repair_remote_ssh_reachable": False, + "latest_visible_harbor_110_repair_bounded_ssh_timeout_seen": True, + } + ) + payload["rollups"].update( + { + "harbor_110_repair_blocked": True, + "harbor_110_repair_waiting": False, + "harbor_110_repair_no_matching_runner_label": "", + "harbor_110_repair_failure_classifier": ( + "harbor_110_remote_control_channel_unavailable" + ), + "harbor_110_repair_remote_control_channel_unavailable": True, + "harbor_110_repair_remote_ssh_reachable": False, + "harbor_110_repair_bounded_ssh_timeout_seen": True, + } + ) + return payload + + def _gitea_queue_cd_jobs_head_sha_mismatch() -> dict: payload = _gitea_queue_no_matching_runner() payload["readback"].update( diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index b5d7b20ab..8b3ffbb85 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -14,6 +14,21 @@ **邊界**：只改 CD fail-fast source guard / tests / LOGBOOK；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未讀 `.runner` 內容；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未重啟主機、未 restart Docker / Nginx / K3s / DB / firewall。 +## 2026-07-01 — 09:33 Harbor repair remote-control classifier / receipt propagation + +**照主線修正的問題**： +- Harbor repair `#4212` 的 public job log 可讀，實際 failure 不是 runner no-matching，而是 bounded SSH 到 `wooo@192.168.0.110:22` timeout：`BLOCKED harbor_110_remote_control_channel_unavailable`、`harbor_110_remote_ssh_reachable=false`。 +- `ops/runner/read-public-gitea-actions-queue.py` 補 Harbor repair job log fetch / classifier，輸出 `latest_visible_harbor_110_repair_failure_classifier=harbor_110_remote_control_channel_unavailable`、`remote_control_channel_unavailable=true`、`bounded_ssh_timeout_seen=true`，並把 live status 精準化為 `blocked_harbor_110_remote_control_channel_unavailable`。 +- `apps/api/src/services/harbor_registry_controlled_recovery_receipt.py` 同步吃進 Gitea queue 的 Harbor repair failure classifier，將 remote control channel unavailable / bounded SSH timeout 納入 control-path readiness、normalized classifier fields 與 safe blocker evidence，讓 AwoooP / AI Agent 不再只看到泛化 Harbor 502。 + +**驗證**： +- live reader 回 `status=blocked_harbor_110_remote_control_channel_unavailable`、`harbor_log_http_status=200`、`remote_ssh_reachable=false`、`ssh_timeout=true`、`controlled_profile_no_matching_runner_label_count=0`。 +- `python3.11 -m pytest ops/runner/test_read_public_gitea_actions_queue.py -q` 通過（32 passed）。 +- `DATABASE_URL='postgresql+asyncpg://awoooi:awoooi_test_2026@127.0.0.1:5432/awoooi_test' python3.11 -m pytest apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py ops/runner/test_verify_awoooi_non110_cd_closure.py -q` 通過（26 passed）。 +- `python3.11 -m py_compile ops/runner/read-public-gitea-actions-queue.py apps/api/src/services/harbor_registry_controlled_recovery_receipt.py` 與 `git diff --check` 通過。 + +**邊界**：只讀 public Gitea HTML / jobs API / job log 與 188 runner metadata；未讀 token / secret / `.env` / raw sessions / SQLite / auth；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未新增 host write、未重啟主機、未 restart Docker / Nginx / K3s / DB / firewall。 + ## 2026-07-01 — 09:24 Harbor repair waiting truth / public route blocker **照主線修正的問題**： diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index c0f6611cd..1161fc895 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -28,6 +28,9 @@ DEFAULT_CD_BUILD_JOB_LOG_URL_TEMPLATE = ( DEFAULT_CD_TESTS_JOB_LOG_URL_TEMPLATE = ( "https://gitea.wooo.work/wooo/awoooi/actions/runs/{run_id}/jobs/2/logs" ) +DEFAULT_HARBOR_110_REPAIR_JOB_LOG_URL_TEMPLATE = ( + "https://gitea.wooo.work/wooo/awoooi/actions/runs/{run_id}/jobs/1/logs" +) SCHEMA_VERSION = "awoooi_public_gitea_actions_queue_readback_v1" EXPECTED_HARBOR_110_REPAIR_JOB_NAMES = { "workflow-shape", @@ -75,6 +78,26 @@ _HARBOR_CONTROLLED_REPAIR_SKIP_RE = re.compile( _HARBOR_CONTROLLED_REPAIR_STATUS_RE = re.compile( r"harbor_controlled_repair_public_registry_v2_status=(?P\d{3})" ) +_HARBOR_110_REMOTE_SSH_REACHABLE_RE = re.compile( + r"harbor_110_remote_ssh_reachable=(?Ptrue|false)" +) +_HARBOR_110_REMOTE_SSH_TIMEOUT_RE = re.compile( + r"(Connection to 192\.168\.0\.110 port 22 timed out|" + r"ssh: connect to host 192\.168\.0\.110 port 22: Operation timed out)" +) +_HARBOR_110_REMOTE_LOCAL_V2_STATUS_RE = re.compile( + r"harbor_110_remote_local_v2_http_status=(?P\d{3})" +) +_HARBOR_110_PUBLIC_V2_STATUS_RE = re.compile( + r"harbor_public_registry_v2_http_status=(?P\d{3})" +) +_HARBOR_110_REMOTE_LOCAL_V2_BLOCKER_RE = re.compile( + r"BLOCKED harbor_110_remote_local_registry_v2_unavailable " + r"status=(?P\d{3})" +) +_HARBOR_110_PUBLIC_V2_BLOCKER_RE = re.compile( + r"BLOCKED harbor_public_registry_v2_unavailable status=(?P\d{3})" +) _HOST_PRESSURE_ATTEMPT_RE = re.compile( r"host web/build/smoke pressure detected " r"\(attempt (?P\d+)/(?P\d+)\)" @@ -231,6 +254,8 @@ def build_readback( latest_cd_build_log_text: str = "", latest_cd_tests_log_http_status: int = 0, latest_cd_tests_log_text: str = "", + latest_harbor_110_repair_log_http_status: int = 0, + latest_harbor_110_repair_log_text: str = "", ) -> dict[str, Any]: visible_runs = parse_visible_runs(actions_html) cd_workflow_visible_runs = parse_visible_runs( @@ -368,6 +393,9 @@ def build_readback( ) build_log_classifier = classify_cd_build_log(latest_cd_build_log_text) tests_log_classifier = classify_cd_tests_log(latest_cd_tests_log_text) + harbor_110_repair_log_classifier = classify_harbor_110_repair_log( + latest_harbor_110_repair_log_text + ) latest_cd_status = latest_cd_run.get("status", "") latest_cd_visible_blocked = latest_cd_status == "Blocked" latest_cd_waiting = latest_cd_status == "Waiting" @@ -472,6 +500,7 @@ def build_readback( or harbor_110_repair_failed or bool(harbor_110_repair_no_matching_runner_label) or harbor_110_repair_waiting_after_cd_harbor_blocker + or bool(harbor_110_repair_log_classifier["failure_classifier"]) ) readback = { @@ -586,6 +615,30 @@ def build_readback( harbor_110_repair_status_blocked ), "latest_visible_harbor_110_repair_blocked": harbor_110_repair_blocked, + "latest_visible_harbor_110_repair_log_http_status": ( + latest_harbor_110_repair_log_http_status + ), + "latest_visible_harbor_110_repair_failure_classifier": ( + harbor_110_repair_log_classifier["failure_classifier"] + ), + "latest_visible_harbor_110_repair_remote_control_channel_unavailable": ( + harbor_110_repair_log_classifier["remote_control_channel_unavailable"] + ), + "latest_visible_harbor_110_repair_remote_ssh_reachable": ( + harbor_110_repair_log_classifier["remote_ssh_reachable"] + ), + "latest_visible_harbor_110_repair_bounded_ssh_timeout_seen": ( + harbor_110_repair_log_classifier["bounded_ssh_timeout_seen"] + ), + "latest_visible_harbor_110_repair_local_registry_v2_status": ( + harbor_110_repair_log_classifier["local_registry_v2_status"] + ), + "latest_visible_harbor_110_repair_public_registry_v2_status": ( + harbor_110_repair_log_classifier["public_registry_v2_status"] + ), + "latest_visible_harbor_110_repair_verified": ( + harbor_110_repair_log_classifier["repair_verified"] + ), "harbor_110_repair_waiting_after_cd_harbor_blocker": ( harbor_110_repair_waiting_after_cd_harbor_blocker ), @@ -674,6 +727,12 @@ def build_readback( if latest_cd_visible_blocked else "blocked_current_cd_workflow_waiting_for_runner_or_queue" if latest_cd_waiting + else "blocked_harbor_110_remote_control_channel_unavailable" + if harbor_110_repair_log_classifier["remote_control_channel_unavailable"] + else "blocked_harbor_110_remote_local_registry_v2_unavailable" + if harbor_110_repair_log_classifier["local_registry_v2_unavailable"] + else "blocked_harbor_public_registry_v2_unavailable_after_remote_repair" + if harbor_110_repair_log_classifier["public_registry_v2_unavailable"] else "blocked_harbor_110_repair_failed" if ( build_log_classifier["harbor_public_route_blocked_or_retrying"] @@ -803,6 +862,27 @@ def build_readback( "harbor_110_repair_waiting_after_cd_harbor_blocker": ( harbor_110_repair_waiting_after_cd_harbor_blocker ), + "harbor_110_repair_failure_classifier": ( + harbor_110_repair_log_classifier["failure_classifier"] + ), + "harbor_110_repair_remote_control_channel_unavailable": ( + harbor_110_repair_log_classifier["remote_control_channel_unavailable"] + ), + "harbor_110_repair_remote_ssh_reachable": ( + harbor_110_repair_log_classifier["remote_ssh_reachable"] + ), + "harbor_110_repair_bounded_ssh_timeout_seen": ( + harbor_110_repair_log_classifier["bounded_ssh_timeout_seen"] + ), + "harbor_110_repair_local_registry_v2_status": ( + harbor_110_repair_log_classifier["local_registry_v2_status"] + ), + "harbor_110_repair_public_registry_v2_status": ( + harbor_110_repair_log_classifier["public_registry_v2_status"] + ), + "harbor_110_repair_verified": harbor_110_repair_log_classifier[ + "repair_verified" + ], "harbor_110_repair_no_matching_runner_label": ( harbor_110_repair_no_matching_runner_label ), @@ -903,6 +983,66 @@ def classify_cd_build_log(text: str) -> dict[str, Any]: } +def classify_harbor_110_repair_log(text: str) -> dict[str, Any]: + ssh_reachable_matches = list(_HARBOR_110_REMOTE_SSH_REACHABLE_RE.finditer(text)) + remote_ssh_reachable: bool | None = None + if ssh_reachable_matches: + remote_ssh_reachable = ( + ssh_reachable_matches[-1].group("reachable") == "true" + ) + + bounded_ssh_timeout_seen = _HARBOR_110_REMOTE_SSH_TIMEOUT_RE.search(text) is not None + remote_control_channel_unavailable = ( + "harbor_110_remote_control_channel_unavailable" in text + or (bounded_ssh_timeout_seen and remote_ssh_reachable is False) + ) + local_registry_v2_unavailable = ( + _HARBOR_110_REMOTE_LOCAL_V2_BLOCKER_RE.search(text) is not None + ) + public_registry_v2_unavailable = ( + _HARBOR_110_PUBLIC_V2_BLOCKER_RE.search(text) is not None + ) + + local_status = _last_named_match_group( + _HARBOR_110_REMOTE_LOCAL_V2_STATUS_RE, + text, + "status", + ) or _last_named_match_group( + _HARBOR_110_REMOTE_LOCAL_V2_BLOCKER_RE, + text, + "status", + ) + public_status = _last_named_match_group( + _HARBOR_110_PUBLIC_V2_STATUS_RE, + text, + "status", + ) or _last_named_match_group( + _HARBOR_110_PUBLIC_V2_BLOCKER_RE, + text, + "status", + ) + + return { + "failure_classifier": ( + "harbor_110_remote_control_channel_unavailable" + if remote_control_channel_unavailable + else "harbor_110_remote_local_registry_v2_unavailable" + if local_registry_v2_unavailable + else "harbor_public_registry_v2_unavailable" + if public_registry_v2_unavailable + else "" + ), + "remote_control_channel_unavailable": remote_control_channel_unavailable, + "remote_ssh_reachable": remote_ssh_reachable, + "bounded_ssh_timeout_seen": bounded_ssh_timeout_seen, + "local_registry_v2_status": local_status, + "public_registry_v2_status": public_status, + "local_registry_v2_unavailable": local_registry_v2_unavailable, + "public_registry_v2_unavailable": public_registry_v2_unavailable, + "repair_verified": "harbor_110_remote_repair_verified=true" in text, + } + + def classify_cd_tests_log(text: str) -> dict[str, Any]: attempt_numbers: list[int] = [] attempt_limits: list[int] = [] @@ -982,6 +1122,11 @@ def _int(value: Any) -> int: return 0 +def _last_named_match_group(pattern: re.Pattern[str], text: str, group: str) -> str: + matches = list(pattern.finditer(text)) + return matches[-1].group(group) if matches else "" + + def _read_text_file(path: Path) -> str: return path.read_text(encoding="utf-8") @@ -1037,6 +1182,18 @@ def _human_summary(payload: dict[str, Any]) -> str: "LATEST_VISIBLE_HARBOR_110_REPAIR_NO_MATCHING_RUNNER_LABEL=" f"{readback['latest_visible_harbor_110_repair_no_matching_runner_label']}" ), + ( + "LATEST_VISIBLE_HARBOR_110_REPAIR_FAILURE_CLASSIFIER=" + f"{readback['latest_visible_harbor_110_repair_failure_classifier']}" + ), + ( + "LATEST_VISIBLE_HARBOR_110_REPAIR_REMOTE_CONTROL_CHANNEL_UNAVAILABLE=" + f"{int(readback['latest_visible_harbor_110_repair_remote_control_channel_unavailable'])}" + ), + ( + "LATEST_VISIBLE_HARBOR_110_REPAIR_REMOTE_SSH_REACHABLE=" + f"{readback['latest_visible_harbor_110_repair_remote_ssh_reachable']}" + ), ( "HARBOR_110_REPAIR_WAITING_AFTER_CD_HARBOR_BLOCKER=" f"{int(readback['harbor_110_repair_waiting_after_cd_harbor_blocker'])}" @@ -1081,6 +1238,10 @@ def main(argv: list[str] | None = None) -> int: "--cd-tests-job-log-url-template", default=DEFAULT_CD_TESTS_JOB_LOG_URL_TEMPLATE, ) + parser.add_argument( + "--harbor-110-repair-job-log-url-template", + default=DEFAULT_HARBOR_110_REPAIR_JOB_LOG_URL_TEMPLATE, + ) parser.add_argument("--timeout-seconds", type=float, default=10.0) parser.add_argument("--actions-html-file", type=Path) parser.add_argument("--cd-workflow-actions-html-file", type=Path) @@ -1094,8 +1255,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--cd-build-job-log-http-status", type=int) parser.add_argument("--cd-tests-job-log-file", type=Path) parser.add_argument("--cd-tests-job-log-http-status", type=int) + parser.add_argument("--harbor-110-repair-job-log-file", type=Path) + parser.add_argument("--harbor-110-repair-job-log-http-status", type=int) parser.add_argument("--skip-cd-build-job-log-read", action="store_true") parser.add_argument("--skip-cd-tests-job-log-read", action="store_true") + parser.add_argument("--skip-harbor-110-repair-job-log-read", action="store_true") parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) @@ -1250,6 +1414,45 @@ def main(argv: list[str] | None = None) -> int: cd_tests_job_log_http_status = 0 cd_tests_job_log_text = "" + if args.harbor_110_repair_job_log_file: + harbor_110_repair_job_log_http_status = ( + args.harbor_110_repair_job_log_http_status or 0 + ) + harbor_110_repair_job_log_text = _read_text_file( + args.harbor_110_repair_job_log_file + ) + elif args.skip_harbor_110_repair_job_log_read: + harbor_110_repair_job_log_http_status = 0 + harbor_110_repair_job_log_text = "" + else: + latest_harbor_110_repair_run = next( + ( + run + for run in actions_lookup_runs + if run.get("workflow") == "harbor-110-local-repair.yaml" + ), + {}, + ) + latest_harbor_110_repair_run_id = latest_harbor_110_repair_run.get( + "run_id", + "", + ) + if latest_harbor_110_repair_run_id: + repair_log_url = args.harbor_110_repair_job_log_url_template.format( + run_id=latest_harbor_110_repair_run_id, + ) + harbor_110_repair_job_log_read = fetch_public_url( + repair_log_url, + args.timeout_seconds, + ) + harbor_110_repair_job_log_http_status = ( + harbor_110_repair_job_log_read.http_status + ) + harbor_110_repair_job_log_text = harbor_110_repair_job_log_read.text + else: + harbor_110_repair_job_log_http_status = 0 + harbor_110_repair_job_log_text = "" + payload = build_readback( actions_html=actions_html, cd_workflow_actions_html=cd_workflow_actions_html, @@ -1263,6 +1466,10 @@ def main(argv: list[str] | None = None) -> int: latest_cd_build_log_text=cd_build_job_log_text, latest_cd_tests_log_http_status=cd_tests_job_log_http_status, latest_cd_tests_log_text=cd_tests_job_log_text, + latest_harbor_110_repair_log_http_status=( + harbor_110_repair_job_log_http_status + ), + latest_harbor_110_repair_log_text=harbor_110_repair_job_log_text, ) if args.json: json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True) diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 6bf29de98..0a1922bc0 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -269,6 +269,19 @@ def _harbor_retrying_unavailable_log() -> str: """ +def _harbor_110_repair_remote_control_unavailable_log() -> str: + return """ +operation_boundary_secret_value_read=false +operation_boundary_docker_daemon_restart_performed=false +operation_boundary_host_reboot_performed=false +operation_boundary_node_drain_performed=false +operation_boundary_remote_ssh_bounded=true +Connection to 192.168.0.110 port 22 timed out +BLOCKED harbor_110_remote_control_channel_unavailable target=wooo@192.168.0.110 +harbor_110_remote_ssh_reachable=false +""" + + def _harbor_110_repair_success_jobs() -> dict: return { "total_count": 2, @@ -635,6 +648,55 @@ def test_build_readback_keeps_visible_harbor_waiting_before_stale_jobs_status() assert payload["readback"]["latest_visible_harbor_110_repair_waiting"] is True +def test_build_readback_classifies_harbor_repair_remote_control_unavailable() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_cd_running_harbor_repair_waiting().replace( + 'data-tooltip-content="Waiting"', + 'data-tooltip-content="Failure"', + 1, + ), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + harbor_110_repair_jobs_http_status=200, + harbor_110_repair_jobs_payload=_harbor_110_repair_stale_code_review_jobs(), + latest_cd_build_log_http_status=200, + latest_cd_build_log_text=_harbor_blocked_log(), + latest_harbor_110_repair_log_http_status=200, + latest_harbor_110_repair_log_text=( + _harbor_110_repair_remote_control_unavailable_log() + ), + ) + + assert payload["status"] == "blocked_harbor_110_remote_control_channel_unavailable" + assert payload["readback"]["latest_visible_harbor_110_repair_log_http_status"] == 200 + assert payload["readback"]["latest_visible_harbor_110_repair_failure_classifier"] == ( + "harbor_110_remote_control_channel_unavailable" + ) + assert ( + payload["readback"][ + "latest_visible_harbor_110_repair_remote_control_channel_unavailable" + ] + is True + ) + assert ( + payload["readback"]["latest_visible_harbor_110_repair_remote_ssh_reachable"] + is False + ) + assert ( + payload["readback"]["latest_visible_harbor_110_repair_bounded_ssh_timeout_seen"] + is True + ) + assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True + assert ( + payload["rollups"]["harbor_110_repair_remote_control_channel_unavailable"] + is True + ) + assert payload["operation_boundaries"]["host_write_performed"] is False + + def test_build_readback_surfaces_harbor_110_repair_no_matching_runner() -> None: module = _load_module() payload = module.build_readback( @@ -1209,6 +1271,7 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None: jobs_path = tmp_path / "jobs.json" log_path = tmp_path / "build.log" tests_log_path = tmp_path / "tests.log" + repair_log_path = tmp_path / "harbor-repair.log" html_path.write_text(_actions_html(), encoding="utf-8") list_path.write_text( json.dumps( @@ -1222,6 +1285,7 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None: jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8") log_path.write_text("", encoding="utf-8") tests_log_path.write_text("", encoding="utf-8") + repair_log_path.write_text("", encoding="utf-8") result = subprocess.run( [ @@ -1245,6 +1309,10 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None: str(tests_log_path), "--cd-tests-job-log-http-status", "200", + "--harbor-110-repair-job-log-file", + str(repair_log_path), + "--harbor-110-repair-job-log-http-status", + "200", "--json", ], check=False,