From c53e9f610f7533fa0526c209258991e3f97355c6 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 18:23:58 +0800 Subject: [PATCH] fix(cd): keep older repair failures behind running cd --- ops/runner/read-public-gitea-actions-queue.py | 69 ++++++++-- .../test_read_public_gitea_actions_queue.py | 125 ++++++++++++++++++ 2 files changed, 180 insertions(+), 14 deletions(-) diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index 49ed66a1b..f6b7b25ca 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -448,10 +448,15 @@ def build_readback( ) host_pressure_waiting_from_stale_jobs = ( cd_jobs_stale_or_mismatched + and not latest_cd_deploy_evidence_succeeded and latest_cd_status in {"Blocked", "Canceled", "Failure"} and tests_log_classifier["host_pressure_waiting"] and not tests_log_classifier["host_pressure_refused"] ) + host_pressure_ignored_after_deploy_success = bool( + latest_cd_deploy_evidence_succeeded + and tests_log_classifier["host_pressure_blocked_or_waiting"] + ) effective_tests_log_classifier = dict(tests_log_classifier) if host_pressure_waiting_from_stale_jobs: effective_tests_log_classifier["host_pressure_classifier"] = ( @@ -459,6 +464,18 @@ def build_readback( ) effective_tests_log_classifier["host_pressure_waiting"] = False effective_tests_log_classifier["host_pressure_blocked_or_waiting"] = False + elif host_pressure_ignored_after_deploy_success: + effective_tests_log_classifier["host_pressure_classifier"] = ( + ( + tests_log_classifier["host_pressure_classifier"] + or "host_web_build_pressure" + ) + + "_ignored_after_deploy_success" + ) + effective_tests_log_classifier["host_pressure_waiting"] = False + effective_tests_log_classifier["host_pressure_refused"] = False + effective_tests_log_classifier["host_pressure_interrupted"] = False + effective_tests_log_classifier["host_pressure_blocked_or_waiting"] = False effective_cd_failure_classifier = build_log_classifier["failure_classifier"] if ( not effective_cd_failure_classifier @@ -575,41 +592,53 @@ def build_readback( and harbor_110_repair_run_id and harbor_110_repair_run_id != latest_cd_run_id ) + harbor_110_repair_historical_before_current_cd_running = bool( + latest_cd_running + and latest_cd_run_id + and harbor_110_repair_run_id + and harbor_110_repair_failed + and _int(harbor_110_repair_run_id) < _int(latest_cd_run_id) + and not build_log_classifier["harbor_public_route_blocked_or_retrying"] + ) + harbor_110_repair_ignored_for_current_cd_context = bool( + harbor_110_repair_historical_after_latest_cd_success + or harbor_110_repair_historical_before_current_cd_running + ) effective_harbor_110_repair_waiting = bool( harbor_110_repair_waiting - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) effective_harbor_110_repair_running = bool( harbor_110_repair_running - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) effective_remote_ssh_publickey_auth_stalled = bool( harbor_110_repair_log_classifier["remote_ssh_publickey_auth_stalled"] - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) effective_remote_control_channel_unavailable = bool( harbor_110_repair_log_classifier["remote_control_channel_unavailable"] - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) effective_remote_local_registry_v2_unavailable = bool( harbor_110_repair_log_classifier["local_registry_v2_unavailable"] - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) effective_remote_public_registry_v2_unavailable = bool( harbor_110_repair_log_classifier["public_registry_v2_unavailable"] - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) effective_harbor_110_repair_failed = bool( harbor_110_repair_failed - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) effective_harbor_110_repair_jobs_stale_or_mismatched = bool( harbor_110_repair_jobs_stale_or_mismatched - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) effective_harbor_110_repair_visible_failure_jobs_api_stale = bool( harbor_110_repair_visible_failure_jobs_api_stale - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) harbor_110_repair_blocked_raw = ( harbor_110_repair_status_blocked @@ -621,14 +650,14 @@ def build_readback( ) harbor_110_repair_blocked = bool( harbor_110_repair_blocked_raw - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ) safe_next_action = _queue_safe_next_action( latest_cd_waiting=latest_cd_waiting, latest_cd_status=latest_cd_status, latest_cd_no_matching_runner_label=latest_cd_no_matching_runner_label, cd_jobs_stale_or_mismatched=( - cd_jobs_stale_or_mismatched and not latest_cd_success + cd_jobs_stale_or_mismatched and not latest_cd_effective_success ), cd_jobs_payload_classifier=cd_jobs_payload_classifier, effective_host_pressure_classifier=effective_tests_log_classifier[ @@ -649,7 +678,7 @@ def build_readback( ], harbor_110_repair_no_matching_runner_label=( harbor_110_repair_no_matching_runner_label - if not harbor_110_repair_historical_after_latest_cd_success + if not harbor_110_repair_ignored_for_current_cd_context else "" ), harbor_110_repair_waiting=effective_harbor_110_repair_waiting, @@ -657,7 +686,7 @@ def build_readback( harbor_110_repair_failed=effective_harbor_110_repair_failed, harbor_110_repair_waiting_after_cd_harbor_blocker=( harbor_110_repair_waiting_after_cd_harbor_blocker - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ), harbor_110_repair_jobs_stale_or_mismatched=( effective_harbor_110_repair_jobs_stale_or_mismatched @@ -668,7 +697,7 @@ def build_readback( ), harbor_110_repair_visible_running_jobs_api_stale=( harbor_110_repair_visible_running_jobs_api_stale - and not harbor_110_repair_historical_after_latest_cd_success + and not harbor_110_repair_ignored_for_current_cd_context ), harbor_110_repair_visible_failure_jobs_api_stale=( effective_harbor_110_repair_visible_failure_jobs_api_stale @@ -837,6 +866,9 @@ def build_readback( "latest_visible_harbor_110_repair_historical_after_latest_cd_success": ( harbor_110_repair_historical_after_latest_cd_success ), + "latest_visible_harbor_110_repair_historical_before_current_cd_running": ( + harbor_110_repair_historical_before_current_cd_running + ), "latest_visible_harbor_110_repair_status_blocked": ( harbor_110_repair_status_blocked ), @@ -1015,6 +1047,9 @@ def build_readback( "latest_visible_cd_host_pressure_log_stale_or_mismatched": ( host_pressure_waiting_from_stale_jobs ), + "latest_visible_cd_host_pressure_ignored_after_deploy_success": ( + host_pressure_ignored_after_deploy_success + ), "no_matching_online_runner_visible": bool(no_matching) or bool(workflow_no_matching_labels), "top_visible_runs": combined_visible_runs[:10], @@ -1186,6 +1221,9 @@ def build_readback( "current_main_cd_host_pressure_log_stale_or_mismatched": ( host_pressure_waiting_from_stale_jobs ), + "current_main_cd_host_pressure_ignored_after_deploy_success": ( + host_pressure_ignored_after_deploy_success + ), "no_matching_online_runner_visible": bool(no_matching) or bool(workflow_no_matching_labels), "harbor_110_repair_run_visible": bool(latest_harbor_110_repair_run), @@ -1196,6 +1234,9 @@ def build_readback( "harbor_110_repair_historical_after_latest_cd_success": ( harbor_110_repair_historical_after_latest_cd_success ), + "harbor_110_repair_historical_before_current_cd_running": ( + harbor_110_repair_historical_before_current_cd_running + ), "harbor_110_repair_blocked": harbor_110_repair_blocked, "harbor_110_repair_blocked_raw": harbor_110_repair_blocked_raw, "harbor_110_repair_waiting_after_cd_harbor_blocker": ( diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 4a791f4e6..706fb20f1 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -125,6 +125,40 @@ def _actions_html_running_cd_after_harbor_repair_success() -> str: """ +def _actions_html_running_cd_after_harbor_repair_failure() -> str: + return """ +
+
+
+ +
+
+ + fix(cd): classify nonfatal deploy notification lag + +
+ cd.yaml #4660:Commit + 709724bd89 +
+
+
+
+
+ +
+
+ + fix(km): expose degraded readback reason + +
+ harbor-110-local-repair.yaml #4659:Scheduled +
+
+
+
+""" + + def _cd_workflow_actions_html_single_cd_run() -> str: return _actions_html_single_cd_run().replace( "cd.yaml #3863:Commit", @@ -1445,6 +1479,55 @@ def test_build_readback_downgrades_running_lag_after_deploy_success() -> None: ) +def test_build_readback_ignores_host_pressure_after_deploy_success_lag() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_single_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={ + "jobs": [ + { + "run_id": 3863, + "head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923", + "conclusion": "success", + } + ], + "total_count": 1, + }, + latest_cd_build_log_http_status=200, + latest_cd_build_log_text=_cd_deploy_success_log(), + latest_cd_tests_log_http_status=200, + latest_cd_tests_log_text=_host_pressure_waiting_log(), + ) + + assert payload["status"] == "cd_deploy_success_with_run_level_lag" + assert payload["readback"]["latest_visible_cd_run_status"] == "Running" + assert ( + payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"] + is True + ) + assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True + assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == ( + "host_web_build_pressure_waiting_ignored_after_deploy_success" + ) + assert ( + payload["readback"]["latest_visible_cd_host_pressure_ignored_after_deploy_success"] + is True + ) + assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is False + assert ( + payload["readback"]["safe_next_action_id"] + == "continue_public_gitea_queue_readback" + ) + assert ( + payload["rollups"]["current_main_cd_host_pressure_ignored_after_deploy_success"] + is True + ) + assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is False + + def test_build_readback_treats_notification_failure_as_nonfatal_after_deploy_success() -> None: module = _load_module() payload = module.build_readback( @@ -1558,6 +1641,48 @@ def test_build_readback_keeps_historical_harbor_running_behind_deploy_success_la ) +def test_build_readback_keeps_older_harbor_failure_behind_current_cd_running() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_running_cd_after_harbor_repair_failure(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={ + "jobs": [ + { + "run_id": 4660, + "name": "tests", + "head_sha": "ac3258524f50735cd350c3de8b4deab4bd56cac2", + "conclusion": "success", + } + ], + "total_count": 1, + }, + harbor_110_repair_jobs_http_status=200, + harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(), + latest_cd_build_log_http_status=404, + latest_cd_build_log_text="", + latest_harbor_110_repair_log_http_status=200, + latest_harbor_110_repair_log_text=( + _harbor_110_repair_publickey_auth_stalled_log() + ), + ) + + assert payload["status"] == "cd_jobs_stale_or_mismatched" + assert ( + payload["readback"][ + "latest_visible_harbor_110_repair_historical_before_current_cd_running" + ] + is True + ) + assert payload["readback"]["latest_visible_harbor_110_repair_blocked"] is False + assert ( + payload["readback"]["safe_next_action_id"] + == "ignore_stale_cd_jobs_api_payload_and_poll_visible_cd_or_marker" + ) + + def test_build_readback_classifies_host_pressure_waiting() -> None: module = _load_module() payload = module.build_readback(