fix(cd): classify running queue lag as nonblocking

ops/runner/read-public-gitea-actions-queue.py

@@ -123,6 +123,7 @@ _HOST_PRESSURE_POSTGRES_RE = re.compile(
 _HOST_PRESSURE_REFUSAL_RE = re.compile(
     r"refusing to start AWOOI image build while host web/build/smoke pressure is still active"
 )
+_HOST_PRESSURE_CLEAR_RE = re.compile(r"no host web/build/smoke pressure detected")
 _HOST_PRESSURE_INTERRUPTED_RE = re.compile(r"signal:\s*interrupt")
 _CD_DEPLOY_MARKER_RE = re.compile(
     r"Z\s+\[main\s+(?P<marker_sha>[0-9a-f]{7,40})\]\s+"
@@ -573,10 +574,22 @@ def build_readback(
     current_cd_waiting_behind_harbor_110_repair_running = (
         latest_cd_waiting and harbor_110_repair_running
     )
+    cd_jobs_head_sha_lag_for_current_cd_running = bool(
+        latest_cd_running
+        and cd_jobs_stale_or_mismatched
+        and cd_jobs_run_id_matches_visible
+        and cd_jobs_head_sha_mismatch
+        and not cd_jobs_run_id_mismatch
+        and latest_cd_build_log_http_status in {200, 404}
+        and latest_cd_tests_log_http_status == 200
+    )
     current_cd_running_without_blocker = bool(
         latest_cd_running
         and not latest_cd_visible_blocked
-        and not cd_jobs_stale_or_mismatched
+        and (
+            not cd_jobs_stale_or_mismatched
+            or cd_jobs_head_sha_lag_for_current_cd_running
+        )
         and not latest_cd_no_matching_runner_label
         and not workflow_no_matching
         and not harbor_110_repair_waiting
@@ -586,6 +599,10 @@ def build_readback(
         and not build_log_classifier["inflight_classifier"]
         and not build_log_classifier["harbor_public_route_blocked_or_retrying"]
     )
+    cd_jobs_ignored_for_current_cd_running = bool(
+        current_cd_running_without_blocker
+        and cd_jobs_head_sha_lag_for_current_cd_running
+    )
     harbor_110_repair_historical_after_latest_cd_success = bool(
         latest_cd_effective_success
         and latest_cd_run_id
@@ -658,6 +675,7 @@ def build_readback(
         latest_cd_no_matching_runner_label=latest_cd_no_matching_runner_label,
         cd_jobs_stale_or_mismatched=(
             cd_jobs_stale_or_mismatched and not latest_cd_effective_success
+            and not cd_jobs_ignored_for_current_cd_running
         ),
         cd_jobs_payload_classifier=cd_jobs_payload_classifier,
         effective_host_pressure_classifier=effective_tests_log_classifier[
@@ -737,6 +755,9 @@ def build_readback(
         "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch,
         "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch,
         "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched,
+        "cd_run_jobs_ignored_for_current_cd_running": (
+            cd_jobs_ignored_for_current_cd_running
+        ),
         "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier,
         "latest_visible_no_matching_runner_run_id": no_matching.get("run_id", ""),
         "latest_visible_no_matching_runner_workflow": no_matching.get(
@@ -1035,6 +1056,9 @@ def build_readback(
         "latest_visible_cd_host_pressure_interrupted": effective_tests_log_classifier[
             "host_pressure_interrupted"
         ],
+        "latest_visible_cd_host_pressure_cleared_after_wait": (
+            effective_tests_log_classifier["host_pressure_cleared_after_wait"]
+        ),
         "latest_visible_cd_postgres_recovery_cpu_pressure": (
             effective_tests_log_classifier["postgres_recovery_cpu_pressure"]
         ),
@@ -1141,6 +1165,9 @@ def build_readback(
             "cd_run_jobs_head_sha_mismatch": cd_jobs_head_sha_mismatch,
             "cd_run_jobs_run_id_mismatch": cd_jobs_run_id_mismatch,
             "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched,
+            "cd_run_jobs_ignored_for_current_cd_running": (
+                cd_jobs_ignored_for_current_cd_running
+            ),
             "cd_run_jobs_payload_classifier": cd_jobs_payload_classifier,
             "current_main_cd_run_visible": bool(latest_cd_run),
             "current_main_cd_run_status": latest_cd_status,
@@ -1215,6 +1242,9 @@ def build_readback(
             "current_main_cd_host_pressure_interrupted": (
                 effective_tests_log_classifier["host_pressure_interrupted"]
             ),
+            "current_main_cd_host_pressure_cleared_after_wait": (
+                effective_tests_log_classifier["host_pressure_cleared_after_wait"]
+            ),
             "current_main_cd_postgres_recovery_cpu_pressure": (
                 effective_tests_log_classifier["postgres_recovery_cpu_pressure"]
             ),
@@ -1939,9 +1969,11 @@ def classify_harbor_110_repair_log(text: str) -> dict[str, Any]:
 def classify_cd_tests_log(text: str) -> dict[str, Any]:
     attempt_numbers: list[int] = []
     attempt_limits: list[int] = []
+    latest_attempt_end = 0
     for match in _HOST_PRESSURE_ATTEMPT_RE.finditer(text):
         attempt_numbers.append(_int(match.group("attempt")))
         attempt_limits.append(_int(match.group("limit")))
+        latest_attempt_end = match.end()
 
     latest_load = ""
     latest_threshold = ""
@@ -1959,7 +1991,20 @@ def classify_cd_tests_log(text: str) -> dict[str, Any]:
 
     host_pressure_refused = _HOST_PRESSURE_REFUSAL_RE.search(text) is not None
     host_pressure_interrupted = _HOST_PRESSURE_INTERRUPTED_RE.search(text) is not None
-    host_pressure_waiting = bool(attempt_numbers) and not host_pressure_refused
+    latest_clear_end = 0
+    for match in _HOST_PRESSURE_CLEAR_RE.finditer(text):
+        latest_clear_end = match.end()
+    host_pressure_cleared_after_wait = bool(
+        attempt_numbers
+        and latest_clear_end
+        and latest_clear_end > latest_attempt_end
+        and not host_pressure_refused
+    )
+    host_pressure_waiting = (
+        bool(attempt_numbers)
+        and not host_pressure_refused
+        and not host_pressure_cleared_after_wait
+    )
     host_pressure_blocked_or_waiting = host_pressure_waiting or host_pressure_refused
     return {
         "host_pressure_classifier": (
@@ -1967,6 +2012,8 @@ def classify_cd_tests_log(text: str) -> dict[str, Any]:
             if host_pressure_refused
             else "host_web_build_pressure_interrupted"
             if host_pressure_interrupted
+            else "host_web_build_pressure_cleared"
+            if host_pressure_cleared_after_wait
             else "host_web_build_pressure_waiting"
             if host_pressure_waiting
             else ""
@@ -1980,6 +2027,7 @@ def classify_cd_tests_log(text: str) -> dict[str, Any]:
         "host_pressure_waiting": host_pressure_waiting,
         "host_pressure_refused": host_pressure_refused,
         "host_pressure_interrupted": host_pressure_interrupted,
+        "host_pressure_cleared_after_wait": host_pressure_cleared_after_wait,
         "postgres_recovery_cpu_pressure": postgres_recovery_cpu_pressure,
         "postgres_recovery_cpu_cores": latest_postgres_cores,
         "postgres_recovery_cpu_threshold": latest_postgres_threshold,

ops/runner/test_read_public_gitea_actions_queue.py

@@ -567,6 +567,15 @@ def _host_pressure_waiting_log() -> str:
 """
 
 
+def _host_pressure_cleared_after_wait_log() -> str:
+    return """
+2026-07-09T10:39:27.3917245Z ⏳ host web/build/smoke pressure detected (attempt 1/60); waiting 10s
+2026-07-09T10:40:58.5937673Z ⏳ host web/build/smoke pressure detected (attempt 10/60); waiting 10s
+2026-07-09T10:41:08.7190656Z ✅ no host web/build/smoke pressure detected
+2026-07-09T10:41:19.8182846Z tests/test_runtime_bootstrap_guards.py::test_init_db_skips_bootstrap_when_postgres_statement_timeout PASSED
+"""
+
+
 def _host_pressure_refused_log() -> str:
     return """
 2026-06-30T11:58:31.0000000Z ⏳ host web/build/smoke pressure detected (attempt 60/60); waiting 10s
@@ -1607,6 +1616,91 @@ def test_build_readback_keeps_current_cd_running_ahead_of_stale_harbor_jobs_payl
     )
 
 
+def test_build_readback_keeps_current_cd_running_with_only_stale_cd_jobs_api() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_single_cd_run(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={
+            "jobs": [
+                {
+                    "run_id": 3863,
+                    "head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923",
+                    "conclusion": "success",
+                }
+            ],
+            "total_count": 1,
+        },
+        latest_cd_build_log_http_status=404,
+        latest_cd_build_log_text="404 page not found",
+        latest_cd_tests_log_http_status=200,
+        latest_cd_tests_log_text="",
+    )
+
+    assert payload["status"] == "current_main_cd_running"
+    assert payload["readback"]["latest_visible_cd_run_running"] is True
+    assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
+    assert (
+        payload["readback"]["cd_run_jobs_ignored_for_current_cd_running"]
+        is True
+    )
+    assert payload["rollups"]["current_main_cd_running_without_blocker"] is True
+    assert (
+        payload["rollups"]["cd_run_jobs_ignored_for_current_cd_running"]
+        is True
+    )
+    assert (
+        payload["readback"]["safe_next_action_id"]
+        == "continue_public_gitea_queue_readback"
+    )
+
+
+def test_build_readback_keeps_current_cd_running_after_host_pressure_clears() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_single_cd_run(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={
+            "jobs": [
+                {
+                    "run_id": 3863,
+                    "head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923",
+                    "conclusion": "success",
+                }
+            ],
+            "total_count": 1,
+        },
+        latest_cd_build_log_http_status=200,
+        latest_cd_build_log_text="",
+        latest_cd_tests_log_http_status=200,
+        latest_cd_tests_log_text=_host_pressure_cleared_after_wait_log(),
+    )
+
+    assert payload["status"] == "current_main_cd_running"
+    assert payload["readback"]["latest_visible_cd_run_running"] is True
+    assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
+        "host_web_build_pressure_cleared"
+    )
+    assert (
+        payload["readback"]["latest_visible_cd_host_pressure_cleared_after_wait"]
+        is True
+    )
+    assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 10
+    assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is False
+    assert (
+        payload["rollups"]["current_main_cd_host_pressure_cleared_after_wait"]
+        is True
+    )
+    assert (
+        payload["readback"]["safe_next_action_id"]
+        == "continue_public_gitea_queue_readback"
+    )
+
+
 def test_build_readback_keeps_historical_harbor_running_behind_deploy_success_lag() -> None:
     module = _load_module()
     payload = module.build_readback(