diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py index 86ab5318f..0676806ee 100644 --- a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py @@ -268,6 +268,12 @@ def validate_harbor_registry_controlled_recovery_receipt( "gitea_queue_harbor_110_jobs_payload_classifier": gitea_queue[ "harbor_110_repair_jobs_payload_classifier" ], + "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale": ( + gitea_queue["harbor_110_repair_visible_running_jobs_api_stale"] + ), + "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running": ( + gitea_queue["current_cd_waiting_behind_harbor_110_repair_running"] + ), "gitea_queue_latest_cd_run_status": gitea_queue[ "latest_visible_cd_run_status" ], @@ -407,6 +413,12 @@ def _control_path_readiness( queue_jobs_cross_workflow = bool( gitea_queue["harbor_110_repair_jobs_cross_workflow_mismatch"] ) + queue_harbor_running_jobs_api_stale = bool( + gitea_queue["harbor_110_repair_visible_running_jobs_api_stale"] + ) + current_cd_waiting_behind_harbor_running = bool( + gitea_queue["current_cd_waiting_behind_harbor_110_repair_running"] + ) queue_remote_control_unavailable = bool( gitea_queue["harbor_110_repair_remote_control_channel_unavailable"] ) @@ -437,6 +449,8 @@ def _control_path_readiness( awoooi_host_unavailable = ( queue_no_matching_runner or queue_jobs_stale + or queue_harbor_running_jobs_api_stale + or current_cd_waiting_behind_harbor_running or current_cd_no_matching_runner or controlled_profile_no_matching_runner ) @@ -454,6 +468,10 @@ def _control_path_readiness( queue_no_matching_runner=queue_no_matching_runner, queue_jobs_stale=queue_jobs_stale, queue_jobs_cross_workflow=queue_jobs_cross_workflow, + queue_harbor_running_jobs_api_stale=queue_harbor_running_jobs_api_stale, + current_cd_waiting_behind_harbor_running=( + current_cd_waiting_behind_harbor_running + ), queue_remote_control_unavailable=queue_remote_control_unavailable, cd_harbor_repair_requires_110_lane=cd_harbor_repair_requires_110_lane, cd_harbor_repair_blocked_by_awoooi_host=( @@ -516,6 +534,12 @@ def _control_path_readiness( ], "harbor_110_repair_jobs_stale_or_mismatched": queue_jobs_stale, "harbor_110_repair_jobs_cross_workflow_mismatch": queue_jobs_cross_workflow, + "harbor_110_repair_visible_running_jobs_api_stale": ( + queue_harbor_running_jobs_api_stale + ), + "current_cd_waiting_behind_harbor_110_repair_running": ( + current_cd_waiting_behind_harbor_running + ), "harbor_110_repair_remote_control_channel_unavailable": ( queue_remote_control_unavailable ), @@ -562,6 +586,8 @@ def _control_path_signal_ids( queue_no_matching_runner: bool, queue_jobs_stale: bool, queue_jobs_cross_workflow: bool, + queue_harbor_running_jobs_api_stale: bool, + current_cd_waiting_behind_harbor_running: bool, queue_remote_control_unavailable: bool, cd_harbor_repair_requires_110_lane: bool, cd_harbor_repair_blocked_by_awoooi_host: bool, @@ -589,6 +615,14 @@ def _control_path_signal_ids( signal_ids.append("gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch") if queue_jobs_stale: signal_ids.append("gitea_queue_harbor_110_repair_jobs_stale_or_mismatched") + if queue_harbor_running_jobs_api_stale: + signal_ids.append( + "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale" + ) + if current_cd_waiting_behind_harbor_running: + signal_ids.append( + "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running" + ) if queue_remote_control_unavailable: signal_ids.append("gitea_queue_harbor_110_remote_control_channel_unavailable") if cd_harbor_repair_blocked_by_awoooi_host: @@ -1201,6 +1235,8 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: "harbor_110_repair_jobs_stale_or_mismatched": False, "harbor_110_repair_jobs_cross_workflow_mismatch": False, "harbor_110_repair_jobs_payload_classifier": "", + "harbor_110_repair_visible_running_jobs_api_stale": False, + "current_cd_waiting_behind_harbor_110_repair_running": False, "harbor_110_repair_jobs_expected_names": [], "harbor_110_repair_jobs_unexpected_names": [], "harbor_110_repair_jobs_labels": [], @@ -1362,6 +1398,14 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: or readback.get("harbor_110_repair_jobs_payload_classifier") or "" ) + harbor_running_jobs_api_stale = bool( + rollups.get("harbor_110_repair_visible_running_jobs_api_stale") is True + or readback.get("harbor_110_repair_visible_running_jobs_api_stale") is True + ) + current_cd_waiting_behind_harbor_running = bool( + rollups.get("current_cd_waiting_behind_harbor_110_repair_running") is True + or readback.get("current_cd_waiting_behind_harbor_110_repair_running") is True + ) boundary_violation = any( operation_boundaries.get(flag) is True for flag in ( @@ -1387,6 +1431,10 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: current_cd_harbor_repair_blocked_by_awoooi_host ), controlled_profile_no_matching=controlled_profile_no_matching, + harbor_running_jobs_api_stale=harbor_running_jobs_api_stale, + current_cd_waiting_behind_harbor_running=( + current_cd_waiting_behind_harbor_running + ), cd_jobs_stale=cd_jobs_stale, cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch, cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch, @@ -1406,6 +1454,10 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: current_cd_no_matching_label=current_cd_no_matching_label, current_cd_waiting=current_cd_waiting, controlled_profile_no_matching_labels=controlled_profile_no_matching_labels, + harbor_running_jobs_api_stale=harbor_running_jobs_api_stale, + current_cd_waiting_behind_harbor_running=( + current_cd_waiting_behind_harbor_running + ), blockers=blockers, ) return { @@ -1484,6 +1536,12 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: "harbor_110_repair_jobs_stale_or_mismatched": jobs_stale, "harbor_110_repair_jobs_cross_workflow_mismatch": jobs_cross_workflow, "harbor_110_repair_jobs_payload_classifier": harbor_jobs_payload_classifier, + "harbor_110_repair_visible_running_jobs_api_stale": ( + harbor_running_jobs_api_stale + ), + "current_cd_waiting_behind_harbor_110_repair_running": ( + current_cd_waiting_behind_harbor_running + ), "harbor_110_repair_jobs_expected_names": _strings( readback.get("harbor_110_repair_jobs_expected_names") ), @@ -1519,6 +1577,8 @@ def _gitea_queue_normalized_classifier_fields( current_cd_no_matching_label: str, current_cd_waiting: bool, controlled_profile_no_matching_labels: dict[str, str], + harbor_running_jobs_api_stale: bool, + current_cd_waiting_behind_harbor_running: bool, blockers: list[str], ) -> list[dict[str, Any]]: fields = [ @@ -1594,6 +1654,39 @@ def _gitea_queue_normalized_classifier_fields( "raw_output_returned": False, } ) + if harbor_running_jobs_api_stale: + fields.append( + { + "field_id": "harbor_110_repair_visible_running_jobs_api_stale", + "value": True, + "blockers": [ + item + for item in blockers + if item + == "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale" + ], + "metadata_only": True, + "raw_output_returned": False, + } + ) + if current_cd_waiting_behind_harbor_running: + fields.append( + { + "field_id": "current_cd_waiting_behind_harbor_110_repair_running", + "value": True, + "blockers": [ + item + for item in blockers + if item + == ( + "gitea_queue_current_cd_waiting_behind_harbor_110_" + "repair_running" + ) + ], + "metadata_only": True, + "raw_output_returned": False, + } + ) if controlled_profile_no_matching_labels: fields.append( { @@ -1622,6 +1715,8 @@ def _gitea_queue_blockers( current_cd_harbor_repair_requires_110_lane: bool, current_cd_harbor_repair_blocked_by_awoooi_host: bool, controlled_profile_no_matching: bool, + harbor_running_jobs_api_stale: bool, + current_cd_waiting_behind_harbor_running: bool, cd_jobs_stale: bool, cd_jobs_head_sha_mismatch: bool, cd_jobs_run_id_mismatch: bool, @@ -1668,12 +1763,22 @@ def _gitea_queue_blockers( and not harbor_repair_remote_control_unavailable and not harbor_repair_local_v2_unavailable and not harbor_repair_public_v2_unavailable + and not harbor_running_jobs_api_stale + and not current_cd_waiting_behind_harbor_running ): blockers.append("gitea_queue_harbor_110_repair_blocked") if jobs_cross_workflow: blockers.append("gitea_queue_harbor_110_repair_jobs_cross_workflow_mismatch") if jobs_stale: blockers.append("gitea_queue_harbor_110_repair_jobs_stale_or_mismatched") + if harbor_running_jobs_api_stale: + blockers.append( + "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale" + ) + if current_cd_waiting_behind_harbor_running: + blockers.append( + "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running" + ) if boundary_violation: blockers.append("gitea_queue_readback_boundary_violation") return blockers diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py index 6507556e8..4a1fd21e2 100644 --- a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py @@ -621,6 +621,60 @@ def test_harbor_recovery_receipt_surfaces_cross_workflow_queue_payload() -> None ] +def test_harbor_recovery_receipt_surfaces_stale_harbor_running_blocker() -> None: + payload = validate_harbor_registry_controlled_recovery_receipt( + { + "public_registry_v2_http_status": 502, + "internal_registry_v2_http_status": 502, + "gitea_actions_queue_readback": ( + _gitea_queue_cd_waiting_behind_stale_harbor_running() + ), + } + ) + + assert ( + "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale" + in payload["active_blockers"] + ) + assert ( + "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running" + in payload["active_blockers"] + ) + queue = payload["readback"]["gitea_actions_queue"] + assert queue["harbor_110_repair_visible_running_jobs_api_stale"] is True + assert queue["current_cd_waiting_behind_harbor_110_repair_running"] is True + readiness = payload["readback"]["control_path_readiness"] + assert ( + "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale" + in readiness["signal_ids"] + ) + assert ( + "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running" + in readiness["signal_ids"] + ) + assert ( + payload["rollups"][ + "gitea_queue_harbor_110_repair_visible_running_jobs_api_stale" + ] + is True + ) + assert ( + payload["rollups"][ + "gitea_queue_current_cd_waiting_behind_harbor_110_repair_running" + ] + is True + ) + normalized = { + item["field_id"]: item for item in queue["normalized_classifier_fields"] + } + assert normalized["harbor_110_repair_visible_running_jobs_api_stale"][ + "blockers" + ] == ["gitea_queue_harbor_110_repair_visible_running_jobs_api_stale"] + assert normalized["current_cd_waiting_behind_harbor_110_repair_running"][ + "blockers" + ] == ["gitea_queue_current_cd_waiting_behind_harbor_110_repair_running"] + + def test_harbor_recovery_receipt_surfaces_remote_control_timeout_queue_payload() -> None: payload = validate_harbor_registry_controlled_recovery_receipt( { @@ -1139,6 +1193,39 @@ def _gitea_queue_cross_workflow_jobs() -> dict: return payload +def _gitea_queue_cd_waiting_behind_stale_harbor_running() -> dict: + payload = _gitea_queue_cross_workflow_jobs() + payload["status"] = ( + "blocked_current_cd_waiting_behind_stale_harbor_110_repair_readback" + ) + payload["readback"].update( + { + "latest_visible_cd_run_id": "4243", + "latest_visible_cd_run_status": "Waiting", + "latest_visible_cd_run_waiting": True, + "latest_visible_harbor_110_repair_run_id": "4237", + "latest_visible_harbor_110_repair_run_status": "Running", + "latest_visible_harbor_110_repair_no_matching_runner_label": "", + "latest_visible_harbor_110_repair_waiting": False, + "latest_visible_harbor_110_repair_running": True, + "latest_visible_harbor_110_repair_blocked": True, + "harbor_110_repair_visible_running_jobs_api_stale": True, + "current_cd_waiting_behind_harbor_110_repair_running": True, + } + ) + payload["rollups"].update( + { + "current_main_cd_run_waiting": True, + "harbor_110_repair_waiting": False, + "harbor_110_repair_running": True, + "harbor_110_repair_no_matching_runner_label": "", + "harbor_110_repair_visible_running_jobs_api_stale": True, + "current_cd_waiting_behind_harbor_110_repair_running": True, + } + ) + return payload + + def _gitea_queue_remote_control_timeout() -> dict: payload = _gitea_queue_no_matching_runner() payload["status"] = "blocked_harbor_110_remote_control_channel_unavailable" diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 7dc7014ca..b0e2827dc 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,16 @@ +## 2026-07-01 — 14:15 Harbor receipt 納入 stale repair running blocker + +**照主線修正的問題**： +- Harbor controlled recovery receipt 先前能讀 `harbor_110_repair_jobs_payload_classifier` 與 current CD waiting，但尚未把 `harbor_110_repair_visible_running_jobs_api_stale` / `current_cd_waiting_behind_harbor_110_repair_running` 轉成 receipt blocker。 +- 本次把這兩個欄位接入 `gitea_actions_queue` readback、active blockers、control_path_readiness signal、rollups 與 normalized classifier fields。 +- 目前 live queue 形狀 CD `Waiting` + Harbor repair `Running` + jobs API stale/cross-workflow 會被 receipt 層保留為 AI Agent / verifier / KM / RAG / PlayBook 可學習的 blocker。 + +**驗證**： +- `DATABASE_URL=... PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py apps/api/tests/test_ai_agent_log_controlled_writeback_executor_readback_api.py apps/api/tests/test_awoooi_priority_work_order_readback_api.py apps/api/tests/test_ai_agent_autonomous_runtime_control.py ops/runner/test_read_public_gitea_actions_queue.py ops/runner/test_verify_awoooi_non110_cd_closure.py -q`：`89 passed`。 +- `ruff check`、`py_compile`、`git diff --check`：通過。 + +**邊界**：只改 metadata-only receipt normalizer / tests / LOGBOOK；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未讀 authorized_keys 內容或 `.runner` 內容；未執行 110 runtime apply。 + ## 2026-07-01 — 14:05 AI Agent normalizer 納入 stale Harbor repair readback **照主線修正的問題**：