fix(runner): ignore stale waiting pressure logs

ops/runner/read-public-gitea-actions-queue.py

@@ -363,8 +363,22 @@ def build_readback(
     )
     build_log_classifier = classify_cd_build_log(latest_cd_build_log_text)
     tests_log_classifier = classify_cd_tests_log(latest_cd_tests_log_text)
-    latest_cd_visible_blocked = latest_cd_run.get("status", "") == "Blocked"
-    latest_cd_waiting = latest_cd_run.get("status", "") == "Waiting"
+    latest_cd_status = latest_cd_run.get("status", "")
+    latest_cd_visible_blocked = latest_cd_status == "Blocked"
+    latest_cd_waiting = latest_cd_status == "Waiting"
+    host_pressure_waiting_from_stale_jobs = (
+        cd_jobs_stale_or_mismatched
+        and latest_cd_status in {"Blocked", "Canceled", "Failure"}
+        and tests_log_classifier["host_pressure_waiting"]
+        and not tests_log_classifier["host_pressure_refused"]
+    )
+    effective_tests_log_classifier = dict(tests_log_classifier)
+    if host_pressure_waiting_from_stale_jobs:
+        effective_tests_log_classifier["host_pressure_classifier"] = (
+            "host_web_build_pressure_waiting_from_stale_jobs_payload"
+        )
+        effective_tests_log_classifier["host_pressure_waiting"] = False
+        effective_tests_log_classifier["host_pressure_blocked_or_waiting"] = False
     latest_cd_no_matching_runner_label = (
         latest_cd_run.get("no_matching_runner_label", "")
         or workflow_no_matching_labels.get("cd.yaml", "")
@@ -481,7 +495,7 @@ def build_readback(
         ),
         "workflow_no_matching_runner_labels": workflow_no_matching_labels,
         "latest_visible_cd_run_id": latest_cd_run.get("run_id", ""),
-        "latest_visible_cd_run_status": latest_cd_run.get("status", ""),
+        "latest_visible_cd_run_status": latest_cd_status,
         "latest_visible_cd_run_waiting": latest_cd_waiting,
         "latest_visible_cd_run_blocked": latest_cd_visible_blocked,
         "latest_visible_cd_run_kind": latest_cd_run.get("kind", ""),
@@ -596,27 +610,30 @@ def build_readback(
         "harbor_110_repair_visible_waiting_stale": (
             harbor_110_repair_visible_waiting_stale
         ),
-        "latest_visible_cd_host_pressure_classifier": tests_log_classifier[
+        "latest_visible_cd_host_pressure_classifier": effective_tests_log_classifier[
             "host_pressure_classifier"
         ],
-        "latest_visible_cd_host_pressure_attempt_count": tests_log_classifier[
+        "latest_visible_cd_host_pressure_attempt_count": effective_tests_log_classifier[
             "host_pressure_attempt_count"
         ],
-        "latest_visible_cd_host_pressure_attempt_limit": tests_log_classifier[
+        "latest_visible_cd_host_pressure_attempt_limit": effective_tests_log_classifier[
             "host_pressure_attempt_limit"
         ],
-        "latest_visible_cd_host_pressure_latest_load5_per_core": tests_log_classifier[
-            "latest_load5_per_core"
-        ],
-        "latest_visible_cd_host_pressure_load5_threshold": tests_log_classifier[
+        "latest_visible_cd_host_pressure_latest_load5_per_core": (
+            effective_tests_log_classifier["latest_load5_per_core"]
+        ),
+        "latest_visible_cd_host_pressure_load5_threshold": effective_tests_log_classifier[
             "load5_per_core_threshold"
         ],
-        "latest_visible_cd_host_pressure_waiting": tests_log_classifier[
+        "latest_visible_cd_host_pressure_waiting": effective_tests_log_classifier[
             "host_pressure_waiting"
         ],
-        "latest_visible_cd_host_pressure_refused": tests_log_classifier[
+        "latest_visible_cd_host_pressure_refused": effective_tests_log_classifier[
             "host_pressure_refused"
         ],
+        "latest_visible_cd_host_pressure_log_stale_or_mismatched": (
+            host_pressure_waiting_from_stale_jobs
+        ),
         "no_matching_online_runner_visible": bool(no_matching)
         or bool(workflow_no_matching_labels),
         "top_visible_runs": combined_visible_runs[:10],
@@ -655,7 +672,7 @@ def build_readback(
             else "blocked_harbor_public_route_unavailable_pending_retry"
             if build_log_classifier["harbor_public_route_retrying_unavailable"]
             else "blocked_host_web_build_pressure"
-            if tests_log_classifier["host_pressure_blocked_or_waiting"]
+            if effective_tests_log_classifier["host_pressure_blocked_or_waiting"]
             else "blocked_harbor_110_repair_no_matching_runner"
             if harbor_110_repair_no_matching_runner_label
             else "blocked_no_matching_online_runner"
@@ -684,7 +701,7 @@ def build_readback(
             "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched,
             "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier,
             "current_main_cd_run_visible": bool(latest_cd_run),
-            "current_main_cd_run_status": latest_cd_run.get("status", ""),
+            "current_main_cd_run_status": latest_cd_status,
             "current_main_cd_run_waiting": latest_cd_waiting,
             "current_main_cd_run_blocked": latest_cd_visible_blocked,
             "current_main_cd_no_matching_runner_label": (
@@ -726,15 +743,18 @@ def build_readback(
             "current_main_cd_harbor_repair_lane_classifier": (
                 cd_harbor_repair_lane_classifier
             ),
-            "current_main_cd_host_pressure_classifier": tests_log_classifier[
+            "current_main_cd_host_pressure_classifier": effective_tests_log_classifier[
                 "host_pressure_classifier"
             ],
-            "current_main_cd_host_pressure_waiting": tests_log_classifier[
+            "current_main_cd_host_pressure_waiting": effective_tests_log_classifier[
                 "host_pressure_waiting"
             ],
-            "current_main_cd_host_pressure_refused": tests_log_classifier[
+            "current_main_cd_host_pressure_refused": effective_tests_log_classifier[
                 "host_pressure_refused"
             ],
+            "current_main_cd_host_pressure_log_stale_or_mismatched": (
+                host_pressure_waiting_from_stale_jobs
+            ),
             "no_matching_online_runner_visible": bool(no_matching)
             or bool(workflow_no_matching_labels),
             "harbor_110_repair_run_visible": bool(latest_harbor_110_repair_run),

ops/runner/test_read_public_gitea_actions_queue.py

@@ -858,6 +858,44 @@ def test_host_pressure_refusal_takes_status_precedence_over_stale_jobs() -> None
     assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
 
 
+def test_stale_waiting_host_pressure_log_does_not_hide_failed_cd_jobs_payload() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_failed_cd_run(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={
+            "jobs": [
+                {
+                    "run_id": 4043,
+                    "head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923",
+                    "conclusion": "success",
+                }
+            ],
+            "total_count": 1,
+        },
+        latest_cd_tests_log_http_status=200,
+        latest_cd_tests_log_text=_host_pressure_waiting_log(),
+    )
+    assert payload["status"] == "cd_jobs_stale_or_mismatched"
+    assert payload["readback"]["latest_visible_cd_run_status"] == "Failure"
+    assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
+    assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
+        "host_web_build_pressure_waiting_from_stale_jobs_payload"
+    )
+    assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is False
+    assert (
+        payload["readback"]["latest_visible_cd_host_pressure_log_stale_or_mismatched"]
+        is True
+    )
+    assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is False
+    assert (
+        payload["rollups"]["current_main_cd_host_pressure_log_stale_or_mismatched"]
+        is True
+    )
+
+
 def test_build_readback_flags_stale_cd_jobs_api_payload() -> None:
     module = _load_module()
     payload = module.build_readback(