From 75b67a3a771c1ac620a099f1632162637f7b732e Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 1 Jul 2026 07:45:39 +0800 Subject: [PATCH] fix(recovery): classify stale cd jobs payload --- ...or_registry_controlled_recovery_receipt.py | 60 ++++++++++++++++ ...or_registry_controlled_recovery_receipt.py | 71 +++++++++++++++++++ ops/runner/read-public-gitea-actions-queue.py | 35 +++++++++ .../test_read_public_gitea_actions_queue.py | 42 +++++++++++ 4 files changed, 208 insertions(+) diff --git a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py index 7eb4014dc..a789fdb23 100644 --- a/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/src/services/harbor_registry_controlled_recovery_receipt.py @@ -200,6 +200,18 @@ def validate_harbor_registry_controlled_recovery_receipt( "gitea_queue_current_cd_harbor_retrying_unavailable": gitea_queue[ "current_cd_harbor_retrying_unavailable" ], + "gitea_queue_cd_jobs_stale_or_mismatched": gitea_queue[ + "cd_run_jobs_stale_or_mismatched" + ], + "gitea_queue_cd_jobs_payload_classifier": gitea_queue[ + "cd_run_jobs_payload_classifier" + ], + "gitea_queue_cd_jobs_head_sha_mismatch": gitea_queue[ + "cd_run_jobs_head_sha_mismatch" + ], + "gitea_queue_cd_jobs_run_id_mismatch": gitea_queue[ + "cd_run_jobs_run_id_mismatch" + ], "deploy_marker_readback_seen": deploy_marker["receipt_seen"], "deploy_marker_verified": deploy_marker["deploy_marker_verified"], "deploy_marker_blocker_count": deploy_marker["blocker_count"], @@ -545,6 +557,14 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: "current_cd_inflight_classifier": "", "current_cd_harbor_latest_registry_v2_status": "", "current_cd_harbor_retrying_unavailable": False, + "cd_run_jobs_stale_or_mismatched": False, + "cd_run_jobs_payload_classifier": "", + "cd_run_jobs_head_sha_mismatch": False, + "cd_run_jobs_run_id_mismatch": False, + "cd_run_jobs_expected_run_id": "", + "cd_run_jobs_expected_head_sha": "", + "cd_run_jobs_head_shas": [], + "cd_run_jobs_run_ids": [], "latest_visible_harbor_110_repair_run_id": "", "latest_visible_harbor_110_repair_run_status": "", "harbor_110_repair_no_matching_runner_label": "", @@ -608,6 +628,18 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: or current_cd_inflight_classifier == "harbor_registry_public_route_unavailable_pending_retry" ) + cd_jobs_stale = bool( + rollups.get("cd_run_jobs_stale_or_mismatched") is True + or readback.get("cd_run_jobs_stale_or_mismatched") is True + ) + cd_jobs_head_sha_mismatch = bool( + rollups.get("cd_run_jobs_head_sha_mismatch") is True + or readback.get("cd_run_jobs_head_sha_mismatch") is True + ) + cd_jobs_run_id_mismatch = bool( + rollups.get("cd_run_jobs_run_id_mismatch") is True + or readback.get("cd_run_jobs_run_id_mismatch") is True + ) boundary_violation = any( operation_boundaries.get(flag) is True for flag in ( @@ -624,6 +656,9 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: jobs_stale=jobs_stale, jobs_cross_workflow=jobs_cross_workflow, current_cd_harbor_retrying=current_cd_harbor_retrying, + cd_jobs_stale=cd_jobs_stale, + cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch, + cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch, blocked=blocked, boundary_violation=boundary_violation, ) @@ -643,6 +678,22 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]: current_cd_latest_registry_status ), "current_cd_harbor_retrying_unavailable": current_cd_harbor_retrying, + "cd_run_jobs_stale_or_mismatched": cd_jobs_stale, + "cd_run_jobs_payload_classifier": str( + rollups.get("cd_run_jobs_payload_classifier") + or readback.get("cd_run_jobs_payload_classifier") + or "" + ), + "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch, + "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch, + "cd_run_jobs_expected_run_id": str( + readback.get("cd_run_jobs_expected_run_id") or "" + ), + "cd_run_jobs_expected_head_sha": str( + readback.get("cd_run_jobs_expected_head_sha") or "" + ), + "cd_run_jobs_head_shas": _strings(readback.get("cd_run_jobs_head_shas")), + "cd_run_jobs_run_ids": _strings(readback.get("cd_run_jobs_run_ids")), "latest_visible_harbor_110_repair_run_id": str( readback.get("latest_visible_harbor_110_repair_run_id") or "" ), @@ -687,12 +738,21 @@ def _gitea_queue_blockers( jobs_stale: bool, jobs_cross_workflow: bool, current_cd_harbor_retrying: bool, + cd_jobs_stale: bool, + cd_jobs_head_sha_mismatch: bool, + cd_jobs_run_id_mismatch: bool, blocked: bool, boundary_violation: bool, ) -> list[str]: blockers: list[str] = [] if current_cd_harbor_retrying: blockers.append("gitea_queue_current_cd_harbor_retrying_unavailable") + if cd_jobs_head_sha_mismatch: + blockers.append("gitea_queue_cd_jobs_head_sha_mismatch") + if cd_jobs_run_id_mismatch: + blockers.append("gitea_queue_cd_jobs_run_id_mismatch") + if cd_jobs_stale: + blockers.append("gitea_queue_cd_jobs_stale_or_mismatched") if no_matching_runner: blockers.append("gitea_queue_harbor_110_repair_no_matching_runner") elif blocked: diff --git a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py index 903421e17..2ff63c6be 100644 --- a/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py +++ b/apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py @@ -279,6 +279,40 @@ def test_harbor_recovery_receipt_surfaces_cross_workflow_queue_payload() -> None ) +def test_harbor_recovery_receipt_surfaces_cd_jobs_payload_classifier() -> None: + payload = validate_harbor_registry_controlled_recovery_receipt( + { + "watchdog_check_output": _watchdog_check_output( + ready=True, + status=401, + ), + "public_registry_v2_http_status": 401, + "internal_registry_v2_http_status": 401, + "gitea_actions_queue_readback": _gitea_queue_cd_jobs_head_sha_mismatch(), + } + ) + + assert "gitea_queue_cd_jobs_head_sha_mismatch" in payload["active_blockers"] + assert "gitea_queue_cd_jobs_stale_or_mismatched" in payload["active_blockers"] + queue = payload["readback"]["gitea_actions_queue"] + assert queue["cd_run_jobs_stale_or_mismatched"] is True + assert queue["cd_run_jobs_head_sha_mismatch"] is True + assert queue["cd_run_jobs_run_id_mismatch"] is False + assert queue["cd_run_jobs_payload_classifier"] == ( + "cd_jobs_api_head_sha_mismatch_for_visible_cd_run" + ) + assert queue["cd_run_jobs_expected_run_id"] == "4175" + assert queue["cd_run_jobs_head_shas"] == [ + "26b67d11f7b7de4f9c9d95c01bb1dacf4000e887" + ] + assert payload["rollups"]["gitea_queue_cd_jobs_stale_or_mismatched"] is True + assert payload["rollups"]["gitea_queue_cd_jobs_head_sha_mismatch"] is True + assert payload["rollups"]["gitea_queue_cd_jobs_run_id_mismatch"] is False + assert payload["rollups"]["gitea_queue_cd_jobs_payload_classifier"] == ( + "cd_jobs_api_head_sha_mismatch_for_visible_cd_run" + ) + + def test_harbor_recovery_receipt_waits_for_deploy_marker_readback() -> None: payload = validate_harbor_registry_controlled_recovery_receipt( { @@ -549,6 +583,43 @@ def _gitea_queue_cross_workflow_jobs() -> dict: return payload +def _gitea_queue_cd_jobs_head_sha_mismatch() -> dict: + payload = _gitea_queue_no_matching_runner() + payload["readback"].update( + { + "latest_visible_cd_run_id": "4175", + "latest_visible_cd_run_commit_sha": ( + "77f9bb0417543507a7540ace90bee9b33e9a8e31" + ), + "cd_run_jobs_expected_run_id": "4175", + "cd_run_jobs_expected_head_sha": ( + "77f9bb0417543507a7540ace90bee9b33e9a8e31" + ), + "cd_run_jobs_head_shas": [ + "26b67d11f7b7de4f9c9d95c01bb1dacf4000e887" + ], + "cd_run_jobs_run_ids": ["4175"], + "cd_run_jobs_head_sha_mismatch": True, + "cd_run_jobs_run_id_mismatch": False, + "cd_run_jobs_stale_or_mismatched": True, + "cd_run_jobs_payload_classifier": ( + "cd_jobs_api_head_sha_mismatch_for_visible_cd_run" + ), + } + ) + payload["rollups"].update( + { + "cd_run_jobs_head_sha_mismatch": True, + "cd_run_jobs_run_id_mismatch": False, + "cd_run_jobs_stale_or_mismatched": True, + "cd_run_jobs_payload_classifier": ( + "cd_jobs_api_head_sha_mismatch_for_visible_cd_run" + ), + } + ) + return payload + + def _deploy_marker_verified() -> dict: return { "schema_version": "awoooi_production_deploy_readback_blocker_v1", diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index fba05c36b..697729e2f 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -338,6 +338,29 @@ def build_readback( or not cd_jobs_run_id_matches_visible ) ) + cd_jobs_head_sha_mismatch = ( + cd_jobs_http_status == 200 + and jobs_total_count > 0 + and bool(latest_cd_commit_sha) + and not cd_jobs_head_sha_matches_visible + ) + cd_jobs_run_id_mismatch = ( + cd_jobs_http_status == 200 + and jobs_total_count > 0 + and bool(latest_cd_run_id) + and not cd_jobs_run_id_matches_visible + ) + cd_jobs_payload_classifier = ( + "cd_jobs_api_head_sha_and_run_id_mismatch_for_visible_cd_run" + if cd_jobs_head_sha_mismatch and cd_jobs_run_id_mismatch + else "cd_jobs_api_head_sha_mismatch_for_visible_cd_run" + if cd_jobs_head_sha_mismatch + else "cd_jobs_api_run_id_mismatch_for_visible_cd_run" + if cd_jobs_run_id_mismatch + else "cd_jobs_api_stale_or_mismatched_payload" + if cd_jobs_stale_or_mismatched + else "" + ) build_log_classifier = classify_cd_build_log(latest_cd_build_log_text) tests_log_classifier = classify_cd_tests_log(latest_cd_tests_log_text) latest_cd_visible_blocked = latest_cd_run.get("status", "") == "Blocked" @@ -406,12 +429,17 @@ def build_readback( "actions_list_without_token_message": actions_list_message, "cd_run_jobs_http_status": cd_jobs_http_status, "cd_run_jobs_total_count": jobs_total_count, + "cd_run_jobs_expected_run_id": latest_cd_run_id, + "cd_run_jobs_expected_head_sha": latest_cd_commit_sha, "cd_run_jobs_head_shas": job_head_shas, "cd_run_jobs_run_ids": job_run_ids, "cd_run_jobs_conclusion_counts": job_conclusion_counts, "cd_run_jobs_head_sha_matches_visible": cd_jobs_head_sha_matches_visible, "cd_run_jobs_run_id_matches_visible": cd_jobs_run_id_matches_visible, + "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch, + "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch, "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched, + "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier, "latest_visible_no_matching_runner_run_id": no_matching.get("run_id", ""), "latest_visible_no_matching_runner_workflow": no_matching.get( "workflow", workflow_no_matching.get("workflow", "") @@ -601,7 +629,10 @@ def build_readback( "cd_workflow_fallback_used": cd_workflow_fallback_used, "actions_list_requires_token": actions_list_http_status == 401, "cd_run_jobs_total_count": jobs_total_count, + "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch, + "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch, "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched, + "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier, "current_main_cd_run_visible": bool(latest_cd_run), "current_main_cd_run_status": latest_cd_run.get("status", ""), "current_main_cd_run_blocked": latest_cd_visible_blocked, @@ -819,6 +850,10 @@ def _human_summary(payload: dict[str, Any]) -> str: f"{readback['actions_list_without_token_http_status']}" ), f"CD_RUN_JOBS_TOTAL_COUNT={readback['cd_run_jobs_total_count']}", + ( + "CD_RUN_JOBS_PAYLOAD_CLASSIFIER=" + f"{readback['cd_run_jobs_payload_classifier']}" + ), ( "NO_MATCHING_ONLINE_RUNNER_VISIBLE=" f"{int(readback['no_matching_online_runner_visible'])}" diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 90dbda113..f29f967a5 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -739,10 +739,52 @@ def test_build_readback_flags_stale_cd_jobs_api_payload() -> None: assert payload["readback"]["cd_run_jobs_head_shas"] == [ "b17a28c29375afd207a90d05b4898daaa0f3f730" ] + assert payload["readback"]["cd_run_jobs_expected_run_id"] == "3863" + assert payload["readback"]["cd_run_jobs_expected_head_sha"] == ( + "a70c6756d9e76c33143676eef82bab7a49ac1839" + ) assert payload["readback"]["cd_run_jobs_head_sha_matches_visible"] is False assert payload["readback"]["cd_run_jobs_run_id_matches_visible"] is True + assert payload["readback"]["cd_run_jobs_head_sha_mismatch"] is True + assert payload["readback"]["cd_run_jobs_run_id_mismatch"] is False assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True + assert payload["readback"]["cd_run_jobs_payload_classifier"] == ( + "cd_jobs_api_head_sha_mismatch_for_visible_cd_run" + ) assert payload["rollups"]["cd_run_jobs_stale_or_mismatched"] is True + assert payload["rollups"]["cd_run_jobs_payload_classifier"] == ( + "cd_jobs_api_head_sha_mismatch_for_visible_cd_run" + ) + + +def test_build_readback_classifies_cross_run_cd_jobs_api_payload() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_single_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={ + "jobs": [ + { + "run_id": 4172, + "name": "build-and-deploy", + "head_sha": "26b67d11f7b7de4f9c9d95c01bb1dacf4000e887", + "conclusion": "success", + } + ], + "total_count": 1, + }, + ) + + assert payload["status"] == "cd_jobs_stale_or_mismatched" + assert payload["readback"]["cd_run_jobs_head_sha_mismatch"] is True + assert payload["readback"]["cd_run_jobs_run_id_mismatch"] is True + assert payload["readback"]["cd_run_jobs_payload_classifier"] == ( + "cd_jobs_api_head_sha_and_run_id_mismatch_for_visible_cd_run" + ) + assert payload["rollups"]["cd_run_jobs_head_sha_mismatch"] is True + assert payload["rollups"]["cd_run_jobs_run_id_mismatch"] is True def test_harbor_blocker_takes_status_precedence_over_stale_jobs_payload() -> None: