From 06b913cf049e5dc165a3e356632b8a6326f125ad Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 18:45:28 +0800 Subject: [PATCH] fix(cd): classify running queue lag as nonblocking --- ops/runner/read-public-gitea-actions-queue.py | 52 +++++++++- .../test_read_public_gitea_actions_queue.py | 94 +++++++++++++++++++ 2 files changed, 144 insertions(+), 2 deletions(-) diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index f6b7b25ca..d6ef704e3 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -123,6 +123,7 @@ _HOST_PRESSURE_POSTGRES_RE = re.compile( _HOST_PRESSURE_REFUSAL_RE = re.compile( r"refusing to start AWOOI image build while host web/build/smoke pressure is still active" ) +_HOST_PRESSURE_CLEAR_RE = re.compile(r"no host web/build/smoke pressure detected") _HOST_PRESSURE_INTERRUPTED_RE = re.compile(r"signal:\s*interrupt") _CD_DEPLOY_MARKER_RE = re.compile( r"Z\s+\[main\s+(?P[0-9a-f]{7,40})\]\s+" @@ -573,10 +574,22 @@ def build_readback( current_cd_waiting_behind_harbor_110_repair_running = ( latest_cd_waiting and harbor_110_repair_running ) + cd_jobs_head_sha_lag_for_current_cd_running = bool( + latest_cd_running + and cd_jobs_stale_or_mismatched + and cd_jobs_run_id_matches_visible + and cd_jobs_head_sha_mismatch + and not cd_jobs_run_id_mismatch + and latest_cd_build_log_http_status in {200, 404} + and latest_cd_tests_log_http_status == 200 + ) current_cd_running_without_blocker = bool( latest_cd_running and not latest_cd_visible_blocked - and not cd_jobs_stale_or_mismatched + and ( + not cd_jobs_stale_or_mismatched + or cd_jobs_head_sha_lag_for_current_cd_running + ) and not latest_cd_no_matching_runner_label and not workflow_no_matching and not harbor_110_repair_waiting @@ -586,6 +599,10 @@ def build_readback( and not build_log_classifier["inflight_classifier"] and not build_log_classifier["harbor_public_route_blocked_or_retrying"] ) + cd_jobs_ignored_for_current_cd_running = bool( + current_cd_running_without_blocker + and cd_jobs_head_sha_lag_for_current_cd_running + ) harbor_110_repair_historical_after_latest_cd_success = bool( latest_cd_effective_success and latest_cd_run_id @@ -658,6 +675,7 @@ def build_readback( latest_cd_no_matching_runner_label=latest_cd_no_matching_runner_label, cd_jobs_stale_or_mismatched=( cd_jobs_stale_or_mismatched and not latest_cd_effective_success + and not cd_jobs_ignored_for_current_cd_running ), cd_jobs_payload_classifier=cd_jobs_payload_classifier, effective_host_pressure_classifier=effective_tests_log_classifier[ @@ -737,6 +755,9 @@ def build_readback( "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch, "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch, "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched, + "cd_run_jobs_ignored_for_current_cd_running": ( + cd_jobs_ignored_for_current_cd_running + ), "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier, "latest_visible_no_matching_runner_run_id": no_matching.get("run_id", ""), "latest_visible_no_matching_runner_workflow": no_matching.get( @@ -1035,6 +1056,9 @@ def build_readback( "latest_visible_cd_host_pressure_interrupted": effective_tests_log_classifier[ "host_pressure_interrupted" ], + "latest_visible_cd_host_pressure_cleared_after_wait": ( + effective_tests_log_classifier["host_pressure_cleared_after_wait"] + ), "latest_visible_cd_postgres_recovery_cpu_pressure": ( effective_tests_log_classifier["postgres_recovery_cpu_pressure"] ), @@ -1141,6 +1165,9 @@ def build_readback( "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch, "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch, "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched, + "cd_run_jobs_ignored_for_current_cd_running": ( + cd_jobs_ignored_for_current_cd_running + ), "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier, "current_main_cd_run_visible": bool(latest_cd_run), "current_main_cd_run_status": latest_cd_status, @@ -1215,6 +1242,9 @@ def build_readback( "current_main_cd_host_pressure_interrupted": ( effective_tests_log_classifier["host_pressure_interrupted"] ), + "current_main_cd_host_pressure_cleared_after_wait": ( + effective_tests_log_classifier["host_pressure_cleared_after_wait"] + ), "current_main_cd_postgres_recovery_cpu_pressure": ( effective_tests_log_classifier["postgres_recovery_cpu_pressure"] ), @@ -1939,9 +1969,11 @@ def classify_harbor_110_repair_log(text: str) -> dict[str, Any]: def classify_cd_tests_log(text: str) -> dict[str, Any]: attempt_numbers: list[int] = [] attempt_limits: list[int] = [] + latest_attempt_end = 0 for match in _HOST_PRESSURE_ATTEMPT_RE.finditer(text): attempt_numbers.append(_int(match.group("attempt"))) attempt_limits.append(_int(match.group("limit"))) + latest_attempt_end = match.end() latest_load = "" latest_threshold = "" @@ -1959,7 +1991,20 @@ def classify_cd_tests_log(text: str) -> dict[str, Any]: host_pressure_refused = _HOST_PRESSURE_REFUSAL_RE.search(text) is not None host_pressure_interrupted = _HOST_PRESSURE_INTERRUPTED_RE.search(text) is not None - host_pressure_waiting = bool(attempt_numbers) and not host_pressure_refused + latest_clear_end = 0 + for match in _HOST_PRESSURE_CLEAR_RE.finditer(text): + latest_clear_end = match.end() + host_pressure_cleared_after_wait = bool( + attempt_numbers + and latest_clear_end + and latest_clear_end > latest_attempt_end + and not host_pressure_refused + ) + host_pressure_waiting = ( + bool(attempt_numbers) + and not host_pressure_refused + and not host_pressure_cleared_after_wait + ) host_pressure_blocked_or_waiting = host_pressure_waiting or host_pressure_refused return { "host_pressure_classifier": ( @@ -1967,6 +2012,8 @@ def classify_cd_tests_log(text: str) -> dict[str, Any]: if host_pressure_refused else "host_web_build_pressure_interrupted" if host_pressure_interrupted + else "host_web_build_pressure_cleared" + if host_pressure_cleared_after_wait else "host_web_build_pressure_waiting" if host_pressure_waiting else "" @@ -1980,6 +2027,7 @@ def classify_cd_tests_log(text: str) -> dict[str, Any]: "host_pressure_waiting": host_pressure_waiting, "host_pressure_refused": host_pressure_refused, "host_pressure_interrupted": host_pressure_interrupted, + "host_pressure_cleared_after_wait": host_pressure_cleared_after_wait, "postgres_recovery_cpu_pressure": postgres_recovery_cpu_pressure, "postgres_recovery_cpu_cores": latest_postgres_cores, "postgres_recovery_cpu_threshold": latest_postgres_threshold, diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 706fb20f1..2c9542deb 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -567,6 +567,15 @@ def _host_pressure_waiting_log() -> str: """ +def _host_pressure_cleared_after_wait_log() -> str: + return """ +2026-07-09T10:39:27.3917245Z ⏳ host web/build/smoke pressure detected (attempt 1/60); waiting 10s +2026-07-09T10:40:58.5937673Z ⏳ host web/build/smoke pressure detected (attempt 10/60); waiting 10s +2026-07-09T10:41:08.7190656Z ✅ no host web/build/smoke pressure detected +2026-07-09T10:41:19.8182846Z tests/test_runtime_bootstrap_guards.py::test_init_db_skips_bootstrap_when_postgres_statement_timeout PASSED +""" + + def _host_pressure_refused_log() -> str: return """ 2026-06-30T11:58:31.0000000Z ⏳ host web/build/smoke pressure detected (attempt 60/60); waiting 10s @@ -1607,6 +1616,91 @@ def test_build_readback_keeps_current_cd_running_ahead_of_stale_harbor_jobs_payl ) +def test_build_readback_keeps_current_cd_running_with_only_stale_cd_jobs_api() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_single_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={ + "jobs": [ + { + "run_id": 3863, + "head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923", + "conclusion": "success", + } + ], + "total_count": 1, + }, + latest_cd_build_log_http_status=404, + latest_cd_build_log_text="404 page not found", + latest_cd_tests_log_http_status=200, + latest_cd_tests_log_text="", + ) + + assert payload["status"] == "current_main_cd_running" + assert payload["readback"]["latest_visible_cd_run_running"] is True + assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True + assert ( + payload["readback"]["cd_run_jobs_ignored_for_current_cd_running"] + is True + ) + assert payload["rollups"]["current_main_cd_running_without_blocker"] is True + assert ( + payload["rollups"]["cd_run_jobs_ignored_for_current_cd_running"] + is True + ) + assert ( + payload["readback"]["safe_next_action_id"] + == "continue_public_gitea_queue_readback" + ) + + +def test_build_readback_keeps_current_cd_running_after_host_pressure_clears() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_single_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={ + "jobs": [ + { + "run_id": 3863, + "head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923", + "conclusion": "success", + } + ], + "total_count": 1, + }, + latest_cd_build_log_http_status=200, + latest_cd_build_log_text="", + latest_cd_tests_log_http_status=200, + latest_cd_tests_log_text=_host_pressure_cleared_after_wait_log(), + ) + + assert payload["status"] == "current_main_cd_running" + assert payload["readback"]["latest_visible_cd_run_running"] is True + assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == ( + "host_web_build_pressure_cleared" + ) + assert ( + payload["readback"]["latest_visible_cd_host_pressure_cleared_after_wait"] + is True + ) + assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 10 + assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is False + assert ( + payload["rollups"]["current_main_cd_host_pressure_cleared_after_wait"] + is True + ) + assert ( + payload["readback"]["safe_next_action_id"] + == "continue_public_gitea_queue_readback" + ) + + def test_build_readback_keeps_historical_harbor_running_behind_deploy_success_lag() -> None: module = _load_module() payload = module.build_readback(