From 709724bd89c8d9dbedd8817983fd1d9c29a28480 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 18:17:19 +0800 Subject: [PATCH] fix(cd): classify nonfatal deploy notification lag --- ops/runner/read-public-gitea-actions-queue.py | 41 ++++- .../test_read_public_gitea_actions_queue.py | 157 ++++++++++++++++++ 2 files changed, 193 insertions(+), 5 deletions(-) diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index fe9668d33..49ed66a1b 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -424,7 +424,6 @@ def build_readback( build_log_classifier["job_succeeded"] and build_log_classifier["deploy_marker_pushed"] and build_log_classifier["production_deploy_readback_matched"] - and build_log_classifier["build_deploy_success_notification_sent"] ) latest_cd_run_ui_blocked = latest_cd_status == "Blocked" latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool( @@ -444,6 +443,9 @@ def build_readback( latest_cd_waiting = bool( latest_cd_status == "Waiting" and not latest_cd_deploy_evidence_succeeded ) + latest_cd_running = bool( + latest_cd_status == "Running" and not latest_cd_deploy_evidence_succeeded + ) host_pressure_waiting_from_stale_jobs = ( cd_jobs_stale_or_mismatched and latest_cd_status in {"Blocked", "Canceled", "Failure"} @@ -554,12 +556,33 @@ def build_readback( current_cd_waiting_behind_harbor_110_repair_running = ( latest_cd_waiting and harbor_110_repair_running ) + current_cd_running_without_blocker = bool( + latest_cd_running + and not latest_cd_visible_blocked + and not cd_jobs_stale_or_mismatched + and not latest_cd_no_matching_runner_label + and not workflow_no_matching + and not harbor_110_repair_waiting + and not harbor_110_repair_running + and not effective_cd_failure_classifier + and not effective_tests_log_classifier["host_pressure_blocked_or_waiting"] + and not build_log_classifier["inflight_classifier"] + and not build_log_classifier["harbor_public_route_blocked_or_retrying"] + ) harbor_110_repair_historical_after_latest_cd_success = bool( latest_cd_effective_success and latest_cd_run_id and harbor_110_repair_run_id and harbor_110_repair_run_id != latest_cd_run_id ) + effective_harbor_110_repair_waiting = bool( + harbor_110_repair_waiting + and not harbor_110_repair_historical_after_latest_cd_success + ) + effective_harbor_110_repair_running = bool( + harbor_110_repair_running + and not harbor_110_repair_historical_after_latest_cd_success + ) effective_remote_ssh_publickey_auth_stalled = bool( harbor_110_repair_log_classifier["remote_ssh_publickey_auth_stalled"] and not harbor_110_repair_historical_after_latest_cd_success @@ -629,8 +652,8 @@ def build_readback( if not harbor_110_repair_historical_after_latest_cd_success else "" ), - harbor_110_repair_waiting=harbor_110_repair_waiting, - harbor_110_repair_running=harbor_110_repair_running, + harbor_110_repair_waiting=effective_harbor_110_repair_waiting, + harbor_110_repair_running=effective_harbor_110_repair_running, harbor_110_repair_failed=effective_harbor_110_repair_failed, harbor_110_repair_waiting_after_cd_harbor_blocker=( harbor_110_repair_waiting_after_cd_harbor_blocker @@ -638,6 +661,7 @@ def build_readback( ), harbor_110_repair_jobs_stale_or_mismatched=( effective_harbor_110_repair_jobs_stale_or_mismatched + and not current_cd_running_without_blocker ), harbor_110_repair_jobs_payload_classifier=( harbor_110_repair_jobs_payload_classifier @@ -699,6 +723,7 @@ def build_readback( "latest_visible_cd_run_id": latest_cd_run.get("run_id", ""), "latest_visible_cd_run_status": latest_cd_status, "latest_visible_cd_run_waiting": latest_cd_waiting, + "latest_visible_cd_run_running": latest_cd_running, "latest_visible_cd_run_blocked": latest_cd_visible_blocked, "latest_visible_cd_run_ui_blocked": latest_cd_run_ui_blocked, "latest_visible_cd_deploy_evidence_succeeded_from_log": ( @@ -1053,10 +1078,12 @@ def build_readback( if harbor_110_repair_no_matching_runner_label else "blocked_no_matching_online_runner" if workflow_no_matching + else "current_main_cd_running" + if current_cd_running_without_blocker else "harbor_110_repair_waiting_for_runner_or_queue" - if harbor_110_repair_waiting + if effective_harbor_110_repair_waiting else "harbor_110_repair_running" - if harbor_110_repair_running + if effective_harbor_110_repair_running else "blocked_harbor_110_repair_failed" if effective_harbor_110_repair_failed else "blocked_harbor_110_repair_run" @@ -1083,6 +1110,10 @@ def build_readback( "current_main_cd_run_visible": bool(latest_cd_run), "current_main_cd_run_status": latest_cd_status, "current_main_cd_run_waiting": latest_cd_waiting, + "current_main_cd_run_running": latest_cd_running, + "current_main_cd_running_without_blocker": ( + current_cd_running_without_blocker + ), "current_cd_waiting_behind_harbor_110_repair_running": ( current_cd_waiting_behind_harbor_110_repair_running ), diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index d37768057..4a791f4e6 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -91,6 +91,40 @@ def _actions_html_single_cd_run() -> str: """ +def _actions_html_running_cd_after_harbor_repair_success() -> str: + return """ +
+
+
+ +
+
+ + fix(km): normalize legacy timestamps + +
+ cd.yaml #4657:Commit + b257f9e98f +
+
+
+
+
+ +
+
+ + fix(reboot): automate windows99 backup search receipt + +
+ harbor-110-local-repair.yaml #4652:Scheduled +
+
+
+
+""" + + def _cd_workflow_actions_html_single_cd_run() -> str: return _actions_html_single_cd_run().replace( "cd.yaml #3863:Commit", @@ -328,6 +362,16 @@ def _cd_deploy_success_log() -> str: """ +def _cd_deploy_success_with_notification_failure_log() -> str: + return """ +2026-07-09T19:33:19.8265526Z [main 57046c9] chore(cd): deploy b257f9e [skip ci] +2026-07-09T19:33:22.3412803Z b257f9e..57046c9 main -> main +2026-07-09T19:34:01.6866643Z Production deploy readback matches this build and GitOps desired image tag (b257f9e98f) on attempt 1/36;matches_main=True +2026-07-09T19:34:11.9558448Z CI/CD build-deploy success notification failed (non-fatal) +2026-07-09T19:34:12.5523925Z Job succeeded +""" + + def _harbor_retrying_unavailable_log() -> str: return """ 2026-06-30T14:49:17.1327272Z harbor_login_attempt=1 registry_v2_status=502 @@ -1401,6 +1445,119 @@ def test_build_readback_downgrades_running_lag_after_deploy_success() -> None: ) +def test_build_readback_treats_notification_failure_as_nonfatal_after_deploy_success() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_running_cd_after_harbor_repair_success(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + harbor_110_repair_jobs_http_status=200, + harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(), + latest_cd_build_log_http_status=200, + latest_cd_build_log_text=_cd_deploy_success_with_notification_failure_log(), + ) + + assert payload["status"] == "cd_deploy_success_with_run_level_lag" + assert ( + payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"] + is True + ) + assert ( + payload["readback"][ + "latest_visible_cd_build_deploy_success_notification_from_log" + ] + is False + ) + assert payload["readback"]["latest_visible_cd_job_succeeded_from_log"] is True + assert payload["readback"]["latest_visible_cd_deploy_marker_pushed_from_log"] is True + assert ( + payload["readback"][ + "latest_visible_cd_production_deploy_readback_matched_from_log" + ] + is True + ) + assert ( + payload["readback"]["latest_visible_harbor_110_repair_historical_after_latest_cd_success"] + is True + ) + assert ( + payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] + is True + ) + assert ( + payload["rollups"]["harbor_110_repair_historical_after_latest_cd_success"] + is True + ) + + +def test_build_readback_keeps_current_cd_running_ahead_of_stale_harbor_jobs_payload() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_running_cd_after_harbor_repair_success(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + harbor_110_repair_jobs_http_status=200, + harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(), + latest_cd_build_log_http_status=200, + latest_cd_build_log_text="", + ) + + assert payload["status"] == "current_main_cd_running" + assert payload["readback"]["latest_visible_cd_run_running"] is True + assert ( + payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] + is True + ) + assert payload["rollups"]["current_main_cd_run_running"] is True + assert payload["rollups"]["current_main_cd_running_without_blocker"] is True + assert ( + payload["rollups"]["harbor_110_repair_historical_after_latest_cd_success"] + is False + ) + assert ( + payload["readback"]["safe_next_action_id"] + == "continue_public_gitea_queue_readback" + ) + + +def test_build_readback_keeps_historical_harbor_running_behind_deploy_success_lag() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_cd_waiting_harbor_repair_running(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + harbor_110_repair_jobs_http_status=200, + harbor_110_repair_jobs_payload=_harbor_110_repair_cross_workflow_jobs(), + latest_cd_build_log_http_status=200, + latest_cd_build_log_text=_cd_deploy_success_log(), + ) + + assert payload["status"] == "cd_deploy_success_with_run_level_lag" + assert ( + payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"] + is True + ) + assert ( + payload["readback"]["latest_visible_cd_run_level_lag_after_deploy_success"] + is True + ) + assert payload["readback"]["latest_visible_harbor_110_repair_running"] is True + assert ( + payload["readback"]["latest_visible_harbor_110_repair_historical_after_latest_cd_success"] + is True + ) + assert ( + payload["readback"]["safe_next_action_id"] + == "continue_public_gitea_queue_readback" + ) + + def test_build_readback_classifies_host_pressure_waiting() -> None: module = _load_module() payload = module.build_readback(