From d0e4933c59044b363580103ef882ab580b8ecb48 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Jun 2026 19:55:22 +0800 Subject: [PATCH] fix(runner): classify host pressure cd waits --- ops/runner/read-public-gitea-actions-queue.py | 128 +++++++++++++++++ .../test_read_public_gitea_actions_queue.py | 135 ++++++++++++++++++ 2 files changed, 263 insertions(+) diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index a8ccd2067..993ceb7e6 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -21,6 +21,9 @@ DEFAULT_CD_RUN_JOBS_API_URL = "" DEFAULT_CD_BUILD_JOB_LOG_URL_TEMPLATE = ( "https://gitea.wooo.work/wooo/awoooi/actions/runs/{run_id}/jobs/3/logs" ) +DEFAULT_CD_TESTS_JOB_LOG_URL_TEMPLATE = ( + "https://gitea.wooo.work/wooo/awoooi/actions/runs/{run_id}/jobs/2/logs" +) SCHEMA_VERSION = "awoooi_public_gitea_actions_queue_readback_v1" _RUN_ROW_RE = re.compile( @@ -48,6 +51,16 @@ _HARBOR_BLOCKER_RE = re.compile( r"BLOCKER harbor_registry_public_route_unavailable " r"registry_v2_status=(?P\d{3})" ) +_HOST_PRESSURE_ATTEMPT_RE = re.compile( + r"host web/build/smoke pressure detected " + r"\(attempt (?P\d+)/(?P\d+)\)" +) +_HOST_PRESSURE_LOAD_RE = re.compile( + r"host load5/core (?P[0-9.]+) > (?P[0-9.]+)" +) +_HOST_PRESSURE_REFUSAL_RE = re.compile( + r"refusing to start AWOOI image build while host web/build/smoke pressure is still active" +) @dataclass(frozen=True) @@ -151,6 +164,8 @@ def build_readback( cd_jobs_payload: Any, latest_cd_build_log_http_status: int = 0, latest_cd_build_log_text: str = "", + latest_cd_tests_log_http_status: int = 0, + latest_cd_tests_log_text: str = "", ) -> dict[str, Any]: visible_runs = parse_visible_runs(actions_html) no_matching = next( @@ -205,6 +220,7 @@ def build_readback( ) ) build_log_classifier = classify_cd_build_log(latest_cd_build_log_text) + tests_log_classifier = classify_cd_tests_log(latest_cd_tests_log_text) readback = { "actions_page_visible_run_count": len(visible_runs), @@ -245,6 +261,28 @@ def build_readback( "latest_visible_cd_harbor_public_route_blocked": build_log_classifier[ "harbor_public_route_blocked" ], + "latest_visible_cd_tests_log_http_status": latest_cd_tests_log_http_status, + "latest_visible_cd_host_pressure_classifier": tests_log_classifier[ + "host_pressure_classifier" + ], + "latest_visible_cd_host_pressure_attempt_count": tests_log_classifier[ + "host_pressure_attempt_count" + ], + "latest_visible_cd_host_pressure_attempt_limit": tests_log_classifier[ + "host_pressure_attempt_limit" + ], + "latest_visible_cd_host_pressure_latest_load5_per_core": tests_log_classifier[ + "latest_load5_per_core" + ], + "latest_visible_cd_host_pressure_load5_threshold": tests_log_classifier[ + "load5_per_core_threshold" + ], + "latest_visible_cd_host_pressure_waiting": tests_log_classifier[ + "host_pressure_waiting" + ], + "latest_visible_cd_host_pressure_refused": tests_log_classifier[ + "host_pressure_refused" + ], "no_matching_online_runner_visible": bool(no_matching), "top_visible_runs": visible_runs[:10], } @@ -255,6 +293,8 @@ def build_readback( if no_matching else "blocked_harbor_public_route_unavailable" if build_log_classifier["harbor_public_route_blocked"] + else "blocked_host_web_build_pressure" + if tests_log_classifier["host_pressure_blocked_or_waiting"] else "cd_jobs_stale_or_mismatched" if cd_jobs_stale_or_mismatched else "no_matching_runner_not_visible" @@ -273,6 +313,15 @@ def build_readback( "current_main_cd_harbor_public_route_blocked": build_log_classifier[ "harbor_public_route_blocked" ], + "current_main_cd_host_pressure_classifier": tests_log_classifier[ + "host_pressure_classifier" + ], + "current_main_cd_host_pressure_waiting": tests_log_classifier[ + "host_pressure_waiting" + ], + "current_main_cd_host_pressure_refused": tests_log_classifier[ + "host_pressure_refused" + ], "no_matching_online_runner_visible": bool(no_matching), }, "operation_boundaries": { @@ -312,6 +361,42 @@ def classify_cd_build_log(text: str) -> dict[str, Any]: } +def classify_cd_tests_log(text: str) -> dict[str, Any]: + attempt_numbers: list[int] = [] + attempt_limits: list[int] = [] + for match in _HOST_PRESSURE_ATTEMPT_RE.finditer(text): + attempt_numbers.append(_int(match.group("attempt"))) + attempt_limits.append(_int(match.group("limit"))) + + latest_load = "" + latest_threshold = "" + for match in _HOST_PRESSURE_LOAD_RE.finditer(text): + latest_load = match.group("load") + latest_threshold = match.group("threshold") + + host_pressure_refused = _HOST_PRESSURE_REFUSAL_RE.search(text) is not None + host_pressure_waiting = bool(attempt_numbers) and not host_pressure_refused + host_pressure_blocked_or_waiting = host_pressure_waiting or host_pressure_refused + return { + "host_pressure_classifier": ( + "host_web_build_pressure_refused" + if host_pressure_refused + else "host_web_build_pressure_waiting" + if host_pressure_waiting + else "" + ), + "host_pressure_attempt_count": max(attempt_numbers) + if attempt_numbers + else 0, + "host_pressure_attempt_limit": max(attempt_limits) if attempt_limits else 0, + "latest_load5_per_core": latest_load, + "load5_per_core_threshold": latest_threshold, + "host_pressure_waiting": host_pressure_waiting, + "host_pressure_refused": host_pressure_refused, + "host_pressure_blocked_or_waiting": host_pressure_blocked_or_waiting, + } + + def load_json_text(text: str) -> Any: try: return json.loads(text) @@ -367,6 +452,10 @@ def _human_summary(payload: dict[str, Any]) -> str: "LATEST_VISIBLE_CD_FAILURE_CLASSIFIER=" f"{readback['latest_visible_cd_failure_classifier']}" ), + ( + "LATEST_VISIBLE_CD_HOST_PRESSURE_CLASSIFIER=" + f"{readback['latest_visible_cd_host_pressure_classifier']}" + ), "WRITE_PERFORMED=false", "TOKEN_COLLECTED=false", ] @@ -387,6 +476,10 @@ def main(argv: list[str] | None = None) -> int: "--cd-build-job-log-url-template", default=DEFAULT_CD_BUILD_JOB_LOG_URL_TEMPLATE, ) + parser.add_argument( + "--cd-tests-job-log-url-template", + default=DEFAULT_CD_TESTS_JOB_LOG_URL_TEMPLATE, + ) parser.add_argument("--timeout-seconds", type=float, default=10.0) parser.add_argument("--actions-html-file", type=Path) parser.add_argument("--actions-list-json-file", type=Path) @@ -395,7 +488,10 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--cd-run-jobs-http-status", type=int) parser.add_argument("--cd-build-job-log-file", type=Path) parser.add_argument("--cd-build-job-log-http-status", type=int) + parser.add_argument("--cd-tests-job-log-file", type=Path) + parser.add_argument("--cd-tests-job-log-http-status", type=int) parser.add_argument("--skip-cd-build-job-log-read", action="store_true") + parser.add_argument("--skip-cd-tests-job-log-read", action="store_true") parser.add_argument("--json", action="store_true") args = parser.parse_args(argv) @@ -469,6 +565,36 @@ def main(argv: list[str] | None = None) -> int: cd_build_job_log_http_status = 0 cd_build_job_log_text = "" + if args.cd_tests_job_log_file: + cd_tests_job_log_http_status = args.cd_tests_job_log_http_status or 0 + cd_tests_job_log_text = _read_text_file(args.cd_tests_job_log_file) + elif args.skip_cd_tests_job_log_read: + cd_tests_job_log_http_status = 0 + cd_tests_job_log_text = "" + else: + latest_cd_run = next( + ( + run + for run in parse_visible_runs(actions_html) + if run.get("workflow") == "cd.yaml" + ), + {}, + ) + latest_cd_run_id = latest_cd_run.get("run_id", "") + if latest_cd_run_id: + tests_log_url = args.cd_tests_job_log_url_template.format( + run_id=latest_cd_run_id, + ) + cd_tests_job_log_read = fetch_public_url( + tests_log_url, + args.timeout_seconds, + ) + cd_tests_job_log_http_status = cd_tests_job_log_read.http_status + cd_tests_job_log_text = cd_tests_job_log_read.text + else: + cd_tests_job_log_http_status = 0 + cd_tests_job_log_text = "" + payload = build_readback( actions_html=actions_html, actions_list_http_status=actions_list_http_status, @@ -477,6 +603,8 @@ def main(argv: list[str] | None = None) -> int: cd_jobs_payload=cd_jobs_payload, latest_cd_build_log_http_status=cd_build_job_log_http_status, latest_cd_build_log_text=cd_build_job_log_text, + latest_cd_tests_log_http_status=cd_tests_job_log_http_status, + latest_cd_tests_log_text=cd_tests_job_log_text, ) if args.json: json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True) diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 06d2df968..8044f0cf9 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -122,6 +122,23 @@ def _harbor_blocked_log() -> str: """ +def _host_pressure_waiting_log() -> str: + return """ +2026-06-30T11:48:41.7864172Z ⏳ host web/build/smoke pressure detected (attempt 1/60); waiting 10s +2026-06-30T11:48:41.7918276Z host load5/core 0.935000 > 0.85 +2026-06-30T11:52:35.1754675Z ⏳ host web/build/smoke pressure detected (attempt 24/60); waiting 10s +2026-06-30T11:52:35.1827409Z host load5/core 0.931667 > 0.85 +""" + + +def _host_pressure_refused_log() -> str: + return """ +2026-06-30T11:58:31.0000000Z ⏳ host web/build/smoke pressure detected (attempt 60/60); waiting 10s +2026-06-30T11:58:31.0000000Z host load5/core 0.920000 > 0.85 +2026-06-30T11:58:41.0000000Z ❌ refusing to start AWOOI image build while host web/build/smoke pressure is still active +""" + + def test_parse_visible_runs_extracts_no_matching_runner_label() -> None: module = _load_module() runs = module.parse_visible_runs(_actions_html()) @@ -188,6 +205,64 @@ def test_build_readback_classifies_harbor_public_route_blocker() -> None: assert payload["rollups"]["current_main_cd_harbor_public_route_blocked"] is True +def test_build_readback_classifies_host_pressure_waiting() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_single_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + latest_cd_tests_log_http_status=200, + latest_cd_tests_log_text=_host_pressure_waiting_log(), + ) + assert payload["status"] == "blocked_host_web_build_pressure" + assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == ( + "host_web_build_pressure_waiting" + ) + assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 24 + assert payload["readback"]["latest_visible_cd_host_pressure_attempt_limit"] == 60 + assert ( + payload["readback"]["latest_visible_cd_host_pressure_latest_load5_per_core"] + == "0.931667" + ) + assert payload["readback"]["latest_visible_cd_host_pressure_load5_threshold"] == ( + "0.85" + ) + assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is True + assert payload["readback"]["latest_visible_cd_host_pressure_refused"] is False + assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is True + + +def test_host_pressure_refusal_takes_status_precedence_over_stale_jobs() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_failed_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={ + "jobs": [ + { + "run_id": 4047, + "head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923", + "conclusion": "success", + } + ], + "total_count": 1, + }, + latest_cd_tests_log_http_status=200, + latest_cd_tests_log_text=_host_pressure_refused_log(), + ) + assert payload["status"] == "blocked_host_web_build_pressure" + assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == ( + "host_web_build_pressure_refused" + ) + assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 60 + assert payload["readback"]["latest_visible_cd_host_pressure_refused"] is True + assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True + + def test_build_readback_flags_stale_cd_jobs_api_payload() -> None: module = _load_module() payload = module.build_readback( @@ -289,6 +364,7 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None: list_path = tmp_path / "actions-list.json" jobs_path = tmp_path / "jobs.json" log_path = tmp_path / "build.log" + tests_log_path = tmp_path / "tests.log" html_path.write_text(_actions_html(), encoding="utf-8") list_path.write_text( json.dumps( @@ -301,6 +377,7 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None: ) jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8") log_path.write_text("", encoding="utf-8") + tests_log_path.write_text("", encoding="utf-8") result = subprocess.run( [ @@ -320,6 +397,10 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None: str(log_path), "--cd-build-job-log-http-status", "200", + "--cd-tests-job-log-file", + str(tests_log_path), + "--cd-tests-job-log-http-status", + "200", "--json", ], check=False, @@ -342,10 +423,12 @@ def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) -> list_path = tmp_path / "actions-list.json" jobs_path = tmp_path / "jobs.json" log_path = tmp_path / "build.log" + tests_log_path = tmp_path / "tests.log" html_path.write_text(_actions_html_failed_cd_run(), encoding="utf-8") list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8") jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8") log_path.write_text(_harbor_blocked_log(), encoding="utf-8") + tests_log_path.write_text("", encoding="utf-8") result = subprocess.run( [ @@ -365,6 +448,10 @@ def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) -> str(log_path), "--cd-build-job-log-http-status", "200", + "--cd-tests-job-log-file", + str(tests_log_path), + "--cd-tests-job-log-http-status", + "200", "--json", ], check=False, @@ -377,3 +464,51 @@ def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) -> assert payload["status"] == "blocked_harbor_public_route_unavailable" assert payload["readback"]["latest_visible_cd_failure_status_code"] == "502" assert "HARBOR_PASSWORD" not in result.stdout + + +def test_cli_json_classifies_host_pressure_from_fixture_log(tmp_path: Path) -> None: + html_path = tmp_path / "actions.html" + list_path = tmp_path / "actions-list.json" + jobs_path = tmp_path / "jobs.json" + build_log_path = tmp_path / "build.log" + tests_log_path = tmp_path / "tests.log" + html_path.write_text(_actions_html_single_cd_run(), encoding="utf-8") + list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8") + jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8") + build_log_path.write_text("", encoding="utf-8") + tests_log_path.write_text(_host_pressure_waiting_log(), encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--actions-html-file", + str(html_path), + "--actions-list-json-file", + str(list_path), + "--actions-list-http-status", + "401", + "--cd-run-jobs-json-file", + str(jobs_path), + "--cd-run-jobs-http-status", + "200", + "--cd-build-job-log-file", + str(build_log_path), + "--cd-build-job-log-http-status", + "200", + "--cd-tests-job-log-file", + str(tests_log_path), + "--cd-tests-job-log-http-status", + "200", + "--json", + ], + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert result.returncode == 0, result.stdout + result.stderr + payload = json.loads(result.stdout) + assert payload["status"] == "blocked_host_web_build_pressure" + assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 24 + assert "HARBOR_PASSWORD" not in result.stdout