diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index eaa4fb872..81b1eb8f7 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -363,8 +363,22 @@ def build_readback( ) build_log_classifier = classify_cd_build_log(latest_cd_build_log_text) tests_log_classifier = classify_cd_tests_log(latest_cd_tests_log_text) - latest_cd_visible_blocked = latest_cd_run.get("status", "") == "Blocked" - latest_cd_waiting = latest_cd_run.get("status", "") == "Waiting" + latest_cd_status = latest_cd_run.get("status", "") + latest_cd_visible_blocked = latest_cd_status == "Blocked" + latest_cd_waiting = latest_cd_status == "Waiting" + host_pressure_waiting_from_stale_jobs = ( + cd_jobs_stale_or_mismatched + and latest_cd_status in {"Blocked", "Canceled", "Failure"} + and tests_log_classifier["host_pressure_waiting"] + and not tests_log_classifier["host_pressure_refused"] + ) + effective_tests_log_classifier = dict(tests_log_classifier) + if host_pressure_waiting_from_stale_jobs: + effective_tests_log_classifier["host_pressure_classifier"] = ( + "host_web_build_pressure_waiting_from_stale_jobs_payload" + ) + effective_tests_log_classifier["host_pressure_waiting"] = False + effective_tests_log_classifier["host_pressure_blocked_or_waiting"] = False latest_cd_no_matching_runner_label = ( latest_cd_run.get("no_matching_runner_label", "") or workflow_no_matching_labels.get("cd.yaml", "") @@ -481,7 +495,7 @@ def build_readback( ), "workflow_no_matching_runner_labels": workflow_no_matching_labels, "latest_visible_cd_run_id": latest_cd_run.get("run_id", ""), - "latest_visible_cd_run_status": latest_cd_run.get("status", ""), + "latest_visible_cd_run_status": latest_cd_status, "latest_visible_cd_run_waiting": latest_cd_waiting, "latest_visible_cd_run_blocked": latest_cd_visible_blocked, "latest_visible_cd_run_kind": latest_cd_run.get("kind", ""), @@ -596,27 +610,30 @@ def build_readback( "harbor_110_repair_visible_waiting_stale": ( harbor_110_repair_visible_waiting_stale ), - "latest_visible_cd_host_pressure_classifier": tests_log_classifier[ + "latest_visible_cd_host_pressure_classifier": effective_tests_log_classifier[ "host_pressure_classifier" ], - "latest_visible_cd_host_pressure_attempt_count": tests_log_classifier[ + "latest_visible_cd_host_pressure_attempt_count": effective_tests_log_classifier[ "host_pressure_attempt_count" ], - "latest_visible_cd_host_pressure_attempt_limit": tests_log_classifier[ + "latest_visible_cd_host_pressure_attempt_limit": effective_tests_log_classifier[ "host_pressure_attempt_limit" ], - "latest_visible_cd_host_pressure_latest_load5_per_core": tests_log_classifier[ - "latest_load5_per_core" - ], - "latest_visible_cd_host_pressure_load5_threshold": tests_log_classifier[ + "latest_visible_cd_host_pressure_latest_load5_per_core": ( + effective_tests_log_classifier["latest_load5_per_core"] + ), + "latest_visible_cd_host_pressure_load5_threshold": effective_tests_log_classifier[ "load5_per_core_threshold" ], - "latest_visible_cd_host_pressure_waiting": tests_log_classifier[ + "latest_visible_cd_host_pressure_waiting": effective_tests_log_classifier[ "host_pressure_waiting" ], - "latest_visible_cd_host_pressure_refused": tests_log_classifier[ + "latest_visible_cd_host_pressure_refused": effective_tests_log_classifier[ "host_pressure_refused" ], + "latest_visible_cd_host_pressure_log_stale_or_mismatched": ( + host_pressure_waiting_from_stale_jobs + ), "no_matching_online_runner_visible": bool(no_matching) or bool(workflow_no_matching_labels), "top_visible_runs": combined_visible_runs[:10], @@ -655,7 +672,7 @@ def build_readback( else "blocked_harbor_public_route_unavailable_pending_retry" if build_log_classifier["harbor_public_route_retrying_unavailable"] else "blocked_host_web_build_pressure" - if tests_log_classifier["host_pressure_blocked_or_waiting"] + if effective_tests_log_classifier["host_pressure_blocked_or_waiting"] else "blocked_harbor_110_repair_no_matching_runner" if harbor_110_repair_no_matching_runner_label else "blocked_no_matching_online_runner" @@ -684,7 +701,7 @@ def build_readback( "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched, "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier, "current_main_cd_run_visible": bool(latest_cd_run), - "current_main_cd_run_status": latest_cd_run.get("status", ""), + "current_main_cd_run_status": latest_cd_status, "current_main_cd_run_waiting": latest_cd_waiting, "current_main_cd_run_blocked": latest_cd_visible_blocked, "current_main_cd_no_matching_runner_label": ( @@ -726,15 +743,18 @@ def build_readback( "current_main_cd_harbor_repair_lane_classifier": ( cd_harbor_repair_lane_classifier ), - "current_main_cd_host_pressure_classifier": tests_log_classifier[ + "current_main_cd_host_pressure_classifier": effective_tests_log_classifier[ "host_pressure_classifier" ], - "current_main_cd_host_pressure_waiting": tests_log_classifier[ + "current_main_cd_host_pressure_waiting": effective_tests_log_classifier[ "host_pressure_waiting" ], - "current_main_cd_host_pressure_refused": tests_log_classifier[ + "current_main_cd_host_pressure_refused": effective_tests_log_classifier[ "host_pressure_refused" ], + "current_main_cd_host_pressure_log_stale_or_mismatched": ( + host_pressure_waiting_from_stale_jobs + ), "no_matching_online_runner_visible": bool(no_matching) or bool(workflow_no_matching_labels), "harbor_110_repair_run_visible": bool(latest_harbor_110_repair_run), diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 0dbbe5813..f57b9f9c4 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -858,6 +858,44 @@ def test_host_pressure_refusal_takes_status_precedence_over_stale_jobs() -> None assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True +def test_stale_waiting_host_pressure_log_does_not_hide_failed_cd_jobs_payload() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_failed_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={ + "jobs": [ + { + "run_id": 4043, + "head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923", + "conclusion": "success", + } + ], + "total_count": 1, + }, + latest_cd_tests_log_http_status=200, + latest_cd_tests_log_text=_host_pressure_waiting_log(), + ) + assert payload["status"] == "cd_jobs_stale_or_mismatched" + assert payload["readback"]["latest_visible_cd_run_status"] == "Failure" + assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True + assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == ( + "host_web_build_pressure_waiting_from_stale_jobs_payload" + ) + assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is False + assert ( + payload["readback"]["latest_visible_cd_host_pressure_log_stale_or_mismatched"] + is True + ) + assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is False + assert ( + payload["rollups"]["current_main_cd_host_pressure_log_stale_or_mismatched"] + is True + ) + + def test_build_readback_flags_stale_cd_jobs_api_payload() -> None: module = _load_module() payload = module.build_readback(