fix(recovery): classify stale cd jobs payload

apps/api/src/services/harbor_registry_controlled_recovery_receipt.py

@@ -200,6 +200,18 @@ def validate_harbor_registry_controlled_recovery_receipt(
             "gitea_queue_current_cd_harbor_retrying_unavailable": gitea_queue[
                 "current_cd_harbor_retrying_unavailable"
             ],
+            "gitea_queue_cd_jobs_stale_or_mismatched": gitea_queue[
+                "cd_run_jobs_stale_or_mismatched"
+            ],
+            "gitea_queue_cd_jobs_payload_classifier": gitea_queue[
+                "cd_run_jobs_payload_classifier"
+            ],
+            "gitea_queue_cd_jobs_head_sha_mismatch": gitea_queue[
+                "cd_run_jobs_head_sha_mismatch"
+            ],
+            "gitea_queue_cd_jobs_run_id_mismatch": gitea_queue[
+                "cd_run_jobs_run_id_mismatch"
+            ],
             "deploy_marker_readback_seen": deploy_marker["receipt_seen"],
             "deploy_marker_verified": deploy_marker["deploy_marker_verified"],
             "deploy_marker_blocker_count": deploy_marker["blocker_count"],
@@ -545,6 +557,14 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
             "current_cd_inflight_classifier": "",
             "current_cd_harbor_latest_registry_v2_status": "",
             "current_cd_harbor_retrying_unavailable": False,
+            "cd_run_jobs_stale_or_mismatched": False,
+            "cd_run_jobs_payload_classifier": "",
+            "cd_run_jobs_head_sha_mismatch": False,
+            "cd_run_jobs_run_id_mismatch": False,
+            "cd_run_jobs_expected_run_id": "",
+            "cd_run_jobs_expected_head_sha": "",
+            "cd_run_jobs_head_shas": [],
+            "cd_run_jobs_run_ids": [],
             "latest_visible_harbor_110_repair_run_id": "",
             "latest_visible_harbor_110_repair_run_status": "",
             "harbor_110_repair_no_matching_runner_label": "",
@@ -608,6 +628,18 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
         or current_cd_inflight_classifier
         == "harbor_registry_public_route_unavailable_pending_retry"
     )
+    cd_jobs_stale = bool(
+        rollups.get("cd_run_jobs_stale_or_mismatched") is True
+        or readback.get("cd_run_jobs_stale_or_mismatched") is True
+    )
+    cd_jobs_head_sha_mismatch = bool(
+        rollups.get("cd_run_jobs_head_sha_mismatch") is True
+        or readback.get("cd_run_jobs_head_sha_mismatch") is True
+    )
+    cd_jobs_run_id_mismatch = bool(
+        rollups.get("cd_run_jobs_run_id_mismatch") is True
+        or readback.get("cd_run_jobs_run_id_mismatch") is True
+    )
     boundary_violation = any(
         operation_boundaries.get(flag) is True
         for flag in (
@@ -624,6 +656,9 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
         jobs_stale=jobs_stale,
         jobs_cross_workflow=jobs_cross_workflow,
         current_cd_harbor_retrying=current_cd_harbor_retrying,
+        cd_jobs_stale=cd_jobs_stale,
+        cd_jobs_head_sha_mismatch=cd_jobs_head_sha_mismatch,
+        cd_jobs_run_id_mismatch=cd_jobs_run_id_mismatch,
         blocked=blocked,
         boundary_violation=boundary_violation,
     )
@@ -643,6 +678,22 @@ def _gitea_queue_readback(value: Any) -> dict[str, Any]:
             current_cd_latest_registry_status
         ),
         "current_cd_harbor_retrying_unavailable": current_cd_harbor_retrying,
+        "cd_run_jobs_stale_or_mismatched": cd_jobs_stale,
+        "cd_run_jobs_payload_classifier": str(
+            rollups.get("cd_run_jobs_payload_classifier")
+            or readback.get("cd_run_jobs_payload_classifier")
+            or ""
+        ),
+        "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch,
+        "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch,
+        "cd_run_jobs_expected_run_id": str(
+            readback.get("cd_run_jobs_expected_run_id") or ""
+        ),
+        "cd_run_jobs_expected_head_sha": str(
+            readback.get("cd_run_jobs_expected_head_sha") or ""
+        ),
+        "cd_run_jobs_head_shas": _strings(readback.get("cd_run_jobs_head_shas")),
+        "cd_run_jobs_run_ids": _strings(readback.get("cd_run_jobs_run_ids")),
         "latest_visible_harbor_110_repair_run_id": str(
             readback.get("latest_visible_harbor_110_repair_run_id") or ""
         ),
@@ -687,12 +738,21 @@ def _gitea_queue_blockers(
     jobs_stale: bool,
     jobs_cross_workflow: bool,
     current_cd_harbor_retrying: bool,
+    cd_jobs_stale: bool,
+    cd_jobs_head_sha_mismatch: bool,
+    cd_jobs_run_id_mismatch: bool,
     blocked: bool,
     boundary_violation: bool,
 ) -> list[str]:
     blockers: list[str] = []
     if current_cd_harbor_retrying:
         blockers.append("gitea_queue_current_cd_harbor_retrying_unavailable")
+    if cd_jobs_head_sha_mismatch:
+        blockers.append("gitea_queue_cd_jobs_head_sha_mismatch")
+    if cd_jobs_run_id_mismatch:
+        blockers.append("gitea_queue_cd_jobs_run_id_mismatch")
+    if cd_jobs_stale:
+        blockers.append("gitea_queue_cd_jobs_stale_or_mismatched")
     if no_matching_runner:
         blockers.append("gitea_queue_harbor_110_repair_no_matching_runner")
     elif blocked:

apps/api/tests/test_harbor_registry_controlled_recovery_receipt.py

@@ -279,6 +279,40 @@ def test_harbor_recovery_receipt_surfaces_cross_workflow_queue_payload() -> None
     )
 
 
+def test_harbor_recovery_receipt_surfaces_cd_jobs_payload_classifier() -> None:
+    payload = validate_harbor_registry_controlled_recovery_receipt(
+        {
+            "watchdog_check_output": _watchdog_check_output(
+                ready=True,
+                status=401,
+            ),
+            "public_registry_v2_http_status": 401,
+            "internal_registry_v2_http_status": 401,
+            "gitea_actions_queue_readback": _gitea_queue_cd_jobs_head_sha_mismatch(),
+        }
+    )
+
+    assert "gitea_queue_cd_jobs_head_sha_mismatch" in payload["active_blockers"]
+    assert "gitea_queue_cd_jobs_stale_or_mismatched" in payload["active_blockers"]
+    queue = payload["readback"]["gitea_actions_queue"]
+    assert queue["cd_run_jobs_stale_or_mismatched"] is True
+    assert queue["cd_run_jobs_head_sha_mismatch"] is True
+    assert queue["cd_run_jobs_run_id_mismatch"] is False
+    assert queue["cd_run_jobs_payload_classifier"] == (
+        "cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
+    )
+    assert queue["cd_run_jobs_expected_run_id"] == "4175"
+    assert queue["cd_run_jobs_head_shas"] == [
+        "26b67d11f7b7de4f9c9d95c01bb1dacf4000e887"
+    ]
+    assert payload["rollups"]["gitea_queue_cd_jobs_stale_or_mismatched"] is True
+    assert payload["rollups"]["gitea_queue_cd_jobs_head_sha_mismatch"] is True
+    assert payload["rollups"]["gitea_queue_cd_jobs_run_id_mismatch"] is False
+    assert payload["rollups"]["gitea_queue_cd_jobs_payload_classifier"] == (
+        "cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
+    )
+
+
 def test_harbor_recovery_receipt_waits_for_deploy_marker_readback() -> None:
     payload = validate_harbor_registry_controlled_recovery_receipt(
         {
@@ -549,6 +583,43 @@ def _gitea_queue_cross_workflow_jobs() -> dict:
     return payload
 
 
+def _gitea_queue_cd_jobs_head_sha_mismatch() -> dict:
+    payload = _gitea_queue_no_matching_runner()
+    payload["readback"].update(
+        {
+            "latest_visible_cd_run_id": "4175",
+            "latest_visible_cd_run_commit_sha": (
+                "77f9bb0417543507a7540ace90bee9b33e9a8e31"
+            ),
+            "cd_run_jobs_expected_run_id": "4175",
+            "cd_run_jobs_expected_head_sha": (
+                "77f9bb0417543507a7540ace90bee9b33e9a8e31"
+            ),
+            "cd_run_jobs_head_shas": [
+                "26b67d11f7b7de4f9c9d95c01bb1dacf4000e887"
+            ],
+            "cd_run_jobs_run_ids": ["4175"],
+            "cd_run_jobs_head_sha_mismatch": True,
+            "cd_run_jobs_run_id_mismatch": False,
+            "cd_run_jobs_stale_or_mismatched": True,
+            "cd_run_jobs_payload_classifier": (
+                "cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
+            ),
+        }
+    )
+    payload["rollups"].update(
+        {
+            "cd_run_jobs_head_sha_mismatch": True,
+            "cd_run_jobs_run_id_mismatch": False,
+            "cd_run_jobs_stale_or_mismatched": True,
+            "cd_run_jobs_payload_classifier": (
+                "cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
+            ),
+        }
+    )
+    return payload
+
+
 def _deploy_marker_verified() -> dict:
     return {
         "schema_version": "awoooi_production_deploy_readback_blocker_v1",

ops/runner/read-public-gitea-actions-queue.py

@@ -338,6 +338,29 @@ def build_readback(
             or not cd_jobs_run_id_matches_visible
         )
     )
+    cd_jobs_head_sha_mismatch = (
+        cd_jobs_http_status == 200
+        and jobs_total_count > 0
+        and bool(latest_cd_commit_sha)
+        and not cd_jobs_head_sha_matches_visible
+    )
+    cd_jobs_run_id_mismatch = (
+        cd_jobs_http_status == 200
+        and jobs_total_count > 0
+        and bool(latest_cd_run_id)
+        and not cd_jobs_run_id_matches_visible
+    )
+    cd_jobs_payload_classifier = (
+        "cd_jobs_api_head_sha_and_run_id_mismatch_for_visible_cd_run"
+        if cd_jobs_head_sha_mismatch and cd_jobs_run_id_mismatch
+        else "cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
+        if cd_jobs_head_sha_mismatch
+        else "cd_jobs_api_run_id_mismatch_for_visible_cd_run"
+        if cd_jobs_run_id_mismatch
+        else "cd_jobs_api_stale_or_mismatched_payload"
+        if cd_jobs_stale_or_mismatched
+        else ""
+    )
     build_log_classifier = classify_cd_build_log(latest_cd_build_log_text)
     tests_log_classifier = classify_cd_tests_log(latest_cd_tests_log_text)
     latest_cd_visible_blocked = latest_cd_run.get("status", "") == "Blocked"
@@ -406,12 +429,17 @@ def build_readback(
         "actions_list_without_token_message": actions_list_message,
         "cd_run_jobs_http_status": cd_jobs_http_status,
         "cd_run_jobs_total_count": jobs_total_count,
+        "cd_run_jobs_expected_run_id": latest_cd_run_id,
+        "cd_run_jobs_expected_head_sha": latest_cd_commit_sha,
         "cd_run_jobs_head_shas": job_head_shas,
         "cd_run_jobs_run_ids": job_run_ids,
         "cd_run_jobs_conclusion_counts": job_conclusion_counts,
         "cd_run_jobs_head_sha_matches_visible": cd_jobs_head_sha_matches_visible,
         "cd_run_jobs_run_id_matches_visible": cd_jobs_run_id_matches_visible,
+        "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch,
+        "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch,
         "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched,
+        "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier,
         "latest_visible_no_matching_runner_run_id": no_matching.get("run_id", ""),
         "latest_visible_no_matching_runner_workflow": no_matching.get(
             "workflow", workflow_no_matching.get("workflow", "")
@@ -601,7 +629,10 @@ def build_readback(
             "cd_workflow_fallback_used": cd_workflow_fallback_used,
             "actions_list_requires_token": actions_list_http_status == 401,
             "cd_run_jobs_total_count": jobs_total_count,
+            "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch,
+            "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch,
             "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched,
+            "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier,
             "current_main_cd_run_visible": bool(latest_cd_run),
             "current_main_cd_run_status": latest_cd_run.get("status", ""),
             "current_main_cd_run_blocked": latest_cd_visible_blocked,
@@ -819,6 +850,10 @@ def _human_summary(payload: dict[str, Any]) -> str:
             f"{readback['actions_list_without_token_http_status']}"
         ),
         f"CD_RUN_JOBS_TOTAL_COUNT={readback['cd_run_jobs_total_count']}",
+        (
+            "CD_RUN_JOBS_PAYLOAD_CLASSIFIER="
+            f"{readback['cd_run_jobs_payload_classifier']}"
+        ),
         (
             "NO_MATCHING_ONLINE_RUNNER_VISIBLE="
             f"{int(readback['no_matching_online_runner_visible'])}"

ops/runner/test_read_public_gitea_actions_queue.py

@@ -739,10 +739,52 @@ def test_build_readback_flags_stale_cd_jobs_api_payload() -> None:
     assert payload["readback"]["cd_run_jobs_head_shas"] == [
         "b17a28c29375afd207a90d05b4898daaa0f3f730"
     ]
+    assert payload["readback"]["cd_run_jobs_expected_run_id"] == "3863"
+    assert payload["readback"]["cd_run_jobs_expected_head_sha"] == (
+        "a70c6756d9e76c33143676eef82bab7a49ac1839"
+    )
     assert payload["readback"]["cd_run_jobs_head_sha_matches_visible"] is False
     assert payload["readback"]["cd_run_jobs_run_id_matches_visible"] is True
+    assert payload["readback"]["cd_run_jobs_head_sha_mismatch"] is True
+    assert payload["readback"]["cd_run_jobs_run_id_mismatch"] is False
     assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
+    assert payload["readback"]["cd_run_jobs_payload_classifier"] == (
+        "cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
+    )
     assert payload["rollups"]["cd_run_jobs_stale_or_mismatched"] is True
+    assert payload["rollups"]["cd_run_jobs_payload_classifier"] == (
+        "cd_jobs_api_head_sha_mismatch_for_visible_cd_run"
+    )
+
+
+def test_build_readback_classifies_cross_run_cd_jobs_api_payload() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_single_cd_run(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={
+            "jobs": [
+                {
+                    "run_id": 4172,
+                    "name": "build-and-deploy",
+                    "head_sha": "26b67d11f7b7de4f9c9d95c01bb1dacf4000e887",
+                    "conclusion": "success",
+                }
+            ],
+            "total_count": 1,
+        },
+    )
+
+    assert payload["status"] == "cd_jobs_stale_or_mismatched"
+    assert payload["readback"]["cd_run_jobs_head_sha_mismatch"] is True
+    assert payload["readback"]["cd_run_jobs_run_id_mismatch"] is True
+    assert payload["readback"]["cd_run_jobs_payload_classifier"] == (
+        "cd_jobs_api_head_sha_and_run_id_mismatch_for_visible_cd_run"
+    )
+    assert payload["rollups"]["cd_run_jobs_head_sha_mismatch"] is True
+    assert payload["rollups"]["cd_run_jobs_run_id_mismatch"] is True
 
 
 def test_harbor_blocker_takes_status_precedence_over_stale_jobs_payload() -> None: