wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(runner): classify host pressure cd waits

Browse Source
This commit is contained in:
Your Name
2026-06-30 19:55:22 +08:00
parent f429999acf
commit d0e4933c59
2 changed files with 263 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
ops/runner/read-public-gitea-actions-queue.py
View File

@@ -21,6 +21,9 @@ DEFAULT_CD_RUN_JOBS_API_URL = ""
DEFAULT_CD_BUILD_JOB_LOG_URL_TEMPLATE = (
"https://gitea.wooo.work/wooo/awoooi/actions/runs/{run_id}/jobs/3/logs"
)
DEFAULT_CD_TESTS_JOB_LOG_URL_TEMPLATE = (
"https://gitea.wooo.work/wooo/awoooi/actions/runs/{run_id}/jobs/2/logs"
)
SCHEMA_VERSION = "awoooi_public_gitea_actions_queue_readback_v1"
_RUN_ROW_RE = re.compile(
@@ -48,6 +51,16 @@ _HARBOR_BLOCKER_RE = re.compile(
r"BLOCKER harbor_registry_public_route_unavailable "
r"registry_v2_status=(?P<status>\d{3})"
)
_HOST_PRESSURE_ATTEMPT_RE = re.compile(
r"host web/build/smoke pressure detected "
r"\(attempt (?P<attempt>\d+)/(?P<limit>\d+)\)"
)
_HOST_PRESSURE_LOAD_RE = re.compile(
r"host load5/core (?P<load>[0-9.]+) > (?P<threshold>[0-9.]+)"
)
_HOST_PRESSURE_REFUSAL_RE = re.compile(
r"refusing to start AWOOI image build while host web/build/smoke pressure is still active"
)
@dataclass(frozen=True)
@@ -151,6 +164,8 @@ def build_readback(
cd_jobs_payload: Any,
latest_cd_build_log_http_status: int = 0,
latest_cd_build_log_text: str = "",
latest_cd_tests_log_http_status: int = 0,
latest_cd_tests_log_text: str = "",
) -> dict[str, Any]:
visible_runs = parse_visible_runs(actions_html)
no_matching = next(
@@ -205,6 +220,7 @@ def build_readback(
)
)
build_log_classifier = classify_cd_build_log(latest_cd_build_log_text)
tests_log_classifier = classify_cd_tests_log(latest_cd_tests_log_text)
readback = {
"actions_page_visible_run_count": len(visible_runs),
@@ -245,6 +261,28 @@ def build_readback(
"latest_visible_cd_harbor_public_route_blocked": build_log_classifier[
"harbor_public_route_blocked"
],
"latest_visible_cd_tests_log_http_status": latest_cd_tests_log_http_status,
"latest_visible_cd_host_pressure_classifier": tests_log_classifier[
"host_pressure_classifier"
],
"latest_visible_cd_host_pressure_attempt_count": tests_log_classifier[
"host_pressure_attempt_count"
],
"latest_visible_cd_host_pressure_attempt_limit": tests_log_classifier[
"host_pressure_attempt_limit"
],
"latest_visible_cd_host_pressure_latest_load5_per_core": tests_log_classifier[
"latest_load5_per_core"
],
"latest_visible_cd_host_pressure_load5_threshold": tests_log_classifier[
"load5_per_core_threshold"
],
"latest_visible_cd_host_pressure_waiting": tests_log_classifier[
"host_pressure_waiting"
],
"latest_visible_cd_host_pressure_refused": tests_log_classifier[
"host_pressure_refused"
],
"no_matching_online_runner_visible": bool(no_matching),
"top_visible_runs": visible_runs[:10],
}
@@ -255,6 +293,8 @@ def build_readback(
if no_matching
else "blocked_harbor_public_route_unavailable"
if build_log_classifier["harbor_public_route_blocked"]
else "blocked_host_web_build_pressure"
if tests_log_classifier["host_pressure_blocked_or_waiting"]
else "cd_jobs_stale_or_mismatched"
if cd_jobs_stale_or_mismatched
else "no_matching_runner_not_visible"
@@ -273,6 +313,15 @@ def build_readback(
"current_main_cd_harbor_public_route_blocked": build_log_classifier[
"harbor_public_route_blocked"
],
"current_main_cd_host_pressure_classifier": tests_log_classifier[
"host_pressure_classifier"
],
"current_main_cd_host_pressure_waiting": tests_log_classifier[
"host_pressure_waiting"
],
"current_main_cd_host_pressure_refused": tests_log_classifier[
"host_pressure_refused"
],
"no_matching_online_runner_visible": bool(no_matching),
},
"operation_boundaries": {
@@ -312,6 +361,42 @@ def classify_cd_build_log(text: str) -> dict[str, Any]:
}
def classify_cd_tests_log(text: str) -> dict[str, Any]:
attempt_numbers: list[int] = []
attempt_limits: list[int] = []
for match in _HOST_PRESSURE_ATTEMPT_RE.finditer(text):
attempt_numbers.append(_int(match.group("attempt")))
attempt_limits.append(_int(match.group("limit")))
latest_load = ""
latest_threshold = ""
for match in _HOST_PRESSURE_LOAD_RE.finditer(text):
latest_load = match.group("load")
latest_threshold = match.group("threshold")
host_pressure_refused = _HOST_PRESSURE_REFUSAL_RE.search(text) is not None
host_pressure_waiting = bool(attempt_numbers) and not host_pressure_refused
host_pressure_blocked_or_waiting = host_pressure_waiting or host_pressure_refused
return {
"host_pressure_classifier": (
"host_web_build_pressure_refused"
if host_pressure_refused
else "host_web_build_pressure_waiting"
if host_pressure_waiting
else ""
),
"host_pressure_attempt_count": max(attempt_numbers)
if attempt_numbers
else 0,
"host_pressure_attempt_limit": max(attempt_limits) if attempt_limits else 0,
"latest_load5_per_core": latest_load,
"load5_per_core_threshold": latest_threshold,
"host_pressure_waiting": host_pressure_waiting,
"host_pressure_refused": host_pressure_refused,
"host_pressure_blocked_or_waiting": host_pressure_blocked_or_waiting,
}
def load_json_text(text: str) -> Any:
try:
return json.loads(text)
@@ -367,6 +452,10 @@ def _human_summary(payload: dict[str, Any]) -> str:
"LATEST_VISIBLE_CD_FAILURE_CLASSIFIER="
f"{readback['latest_visible_cd_failure_classifier']}"
),
(
"LATEST_VISIBLE_CD_HOST_PRESSURE_CLASSIFIER="
f"{readback['latest_visible_cd_host_pressure_classifier']}"
),
"WRITE_PERFORMED=false",
"TOKEN_COLLECTED=false",
]
@@ -387,6 +476,10 @@ def main(argv: list[str] | None = None) -> int:
"--cd-build-job-log-url-template",
default=DEFAULT_CD_BUILD_JOB_LOG_URL_TEMPLATE,
)
parser.add_argument(
"--cd-tests-job-log-url-template",
default=DEFAULT_CD_TESTS_JOB_LOG_URL_TEMPLATE,
)
parser.add_argument("--timeout-seconds", type=float, default=10.0)
parser.add_argument("--actions-html-file", type=Path)
parser.add_argument("--actions-list-json-file", type=Path)
@@ -395,7 +488,10 @@ def main(argv: list[str] | None = None) -> int:
parser.add_argument("--cd-run-jobs-http-status", type=int)
parser.add_argument("--cd-build-job-log-file", type=Path)
parser.add_argument("--cd-build-job-log-http-status", type=int)
parser.add_argument("--cd-tests-job-log-file", type=Path)
parser.add_argument("--cd-tests-job-log-http-status", type=int)
parser.add_argument("--skip-cd-build-job-log-read", action="store_true")
parser.add_argument("--skip-cd-tests-job-log-read", action="store_true")
parser.add_argument("--json", action="store_true")
args = parser.parse_args(argv)
@@ -469,6 +565,36 @@ def main(argv: list[str] | None = None) -> int:
cd_build_job_log_http_status = 0
cd_build_job_log_text = ""
if args.cd_tests_job_log_file:
cd_tests_job_log_http_status = args.cd_tests_job_log_http_status or 0
cd_tests_job_log_text = _read_text_file(args.cd_tests_job_log_file)
elif args.skip_cd_tests_job_log_read:
cd_tests_job_log_http_status = 0
cd_tests_job_log_text = ""
else:
latest_cd_run = next(
(
run
for run in parse_visible_runs(actions_html)
if run.get("workflow") == "cd.yaml"
),
{},
)
latest_cd_run_id = latest_cd_run.get("run_id", "")
if latest_cd_run_id:
tests_log_url = args.cd_tests_job_log_url_template.format(
run_id=latest_cd_run_id,
)
cd_tests_job_log_read = fetch_public_url(
tests_log_url,
args.timeout_seconds,
)
cd_tests_job_log_http_status = cd_tests_job_log_read.http_status
cd_tests_job_log_text = cd_tests_job_log_read.text
else:
cd_tests_job_log_http_status = 0
cd_tests_job_log_text = ""
payload = build_readback(
actions_html=actions_html,
actions_list_http_status=actions_list_http_status,
@@ -477,6 +603,8 @@ def main(argv: list[str] | None = None) -> int:
cd_jobs_payload=cd_jobs_payload,
latest_cd_build_log_http_status=cd_build_job_log_http_status,
latest_cd_build_log_text=cd_build_job_log_text,
latest_cd_tests_log_http_status=cd_tests_job_log_http_status,
latest_cd_tests_log_text=cd_tests_job_log_text,
)
if args.json:
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)

135
ops/runner/test_read_public_gitea_actions_queue.py
View File

@@ -122,6 +122,23 @@ def _harbor_blocked_log() -> str:
"""
def _host_pressure_waiting_log() -> str:
return """
2026-06-30T11:48:41.7864172Z ⏳ host web/build/smoke pressure detected (attempt 1/60); waiting 10s
2026-06-30T11:48:41.7918276Z host load5/core 0.935000 > 0.85
2026-06-30T11:52:35.1754675Z ⏳ host web/build/smoke pressure detected (attempt 24/60); waiting 10s
2026-06-30T11:52:35.1827409Z host load5/core 0.931667 > 0.85
"""
def _host_pressure_refused_log() -> str:
return """
2026-06-30T11:58:31.0000000Z ⏳ host web/build/smoke pressure detected (attempt 60/60); waiting 10s
2026-06-30T11:58:31.0000000Z host load5/core 0.920000 > 0.85
2026-06-30T11:58:41.0000000Z ❌ refusing to start AWOOI image build while host web/build/smoke pressure is still active
"""
def test_parse_visible_runs_extracts_no_matching_runner_label() -> None:
module = _load_module()
runs = module.parse_visible_runs(_actions_html())
@@ -188,6 +205,64 @@ def test_build_readback_classifies_harbor_public_route_blocker() -> None:
assert payload["rollups"]["current_main_cd_harbor_public_route_blocked"] is True
def test_build_readback_classifies_host_pressure_waiting() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_single_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={"jobs": [], "total_count": 0},
latest_cd_tests_log_http_status=200,
latest_cd_tests_log_text=_host_pressure_waiting_log(),
)
assert payload["status"] == "blocked_host_web_build_pressure"
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
"host_web_build_pressure_waiting"
)
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 24
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_limit"] == 60
assert (
payload["readback"]["latest_visible_cd_host_pressure_latest_load5_per_core"]
== "0.931667"
)
assert payload["readback"]["latest_visible_cd_host_pressure_load5_threshold"] == (
"0.85"
)
assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is True
assert payload["readback"]["latest_visible_cd_host_pressure_refused"] is False
assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is True
def test_host_pressure_refusal_takes_status_precedence_over_stale_jobs() -> None:
module = _load_module()
payload = module.build_readback(
actions_html=_actions_html_failed_cd_run(),
actions_list_http_status=401,
actions_list_payload={"message": "token is required"},
cd_jobs_http_status=200,
cd_jobs_payload={
"jobs": [
{
"run_id": 4047,
"head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923",
"conclusion": "success",
}
],
"total_count": 1,
},
latest_cd_tests_log_http_status=200,
latest_cd_tests_log_text=_host_pressure_refused_log(),
)
assert payload["status"] == "blocked_host_web_build_pressure"
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
"host_web_build_pressure_refused"
)
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 60
assert payload["readback"]["latest_visible_cd_host_pressure_refused"] is True
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
def test_build_readback_flags_stale_cd_jobs_api_payload() -> None:
module = _load_module()
payload = module.build_readback(
@@ -289,6 +364,7 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
list_path = tmp_path / "actions-list.json"
jobs_path = tmp_path / "jobs.json"
log_path = tmp_path / "build.log"
tests_log_path = tmp_path / "tests.log"
html_path.write_text(_actions_html(), encoding="utf-8")
list_path.write_text(
json.dumps(
@@ -301,6 +377,7 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
)
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
log_path.write_text("", encoding="utf-8")
tests_log_path.write_text("", encoding="utf-8")
result = subprocess.run(
[
@@ -320,6 +397,10 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
str(log_path),
"--cd-build-job-log-http-status",
"200",
"--cd-tests-job-log-file",
str(tests_log_path),
"--cd-tests-job-log-http-status",
"200",
"--json",
],
check=False,
@@ -342,10 +423,12 @@ def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) ->
list_path = tmp_path / "actions-list.json"
jobs_path = tmp_path / "jobs.json"
log_path = tmp_path / "build.log"
tests_log_path = tmp_path / "tests.log"
html_path.write_text(_actions_html_failed_cd_run(), encoding="utf-8")
list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8")
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
log_path.write_text(_harbor_blocked_log(), encoding="utf-8")
tests_log_path.write_text("", encoding="utf-8")
result = subprocess.run(
[
@@ -365,6 +448,10 @@ def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) ->
str(log_path),
"--cd-build-job-log-http-status",
"200",
"--cd-tests-job-log-file",
str(tests_log_path),
"--cd-tests-job-log-http-status",
"200",
"--json",
],
check=False,
@@ -377,3 +464,51 @@ def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) ->
assert payload["status"] == "blocked_harbor_public_route_unavailable"
assert payload["readback"]["latest_visible_cd_failure_status_code"] == "502"
assert "HARBOR_PASSWORD" not in result.stdout
def test_cli_json_classifies_host_pressure_from_fixture_log(tmp_path: Path) -> None:
html_path = tmp_path / "actions.html"
list_path = tmp_path / "actions-list.json"
jobs_path = tmp_path / "jobs.json"
build_log_path = tmp_path / "build.log"
tests_log_path = tmp_path / "tests.log"
html_path.write_text(_actions_html_single_cd_run(), encoding="utf-8")
list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8")
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
build_log_path.write_text("", encoding="utf-8")
tests_log_path.write_text(_host_pressure_waiting_log(), encoding="utf-8")
result = subprocess.run(
[
sys.executable,
str(SCRIPT),
"--actions-html-file",
str(html_path),
"--actions-list-json-file",
str(list_path),
"--actions-list-http-status",
"401",
"--cd-run-jobs-json-file",
str(jobs_path),
"--cd-run-jobs-http-status",
"200",
"--cd-build-job-log-file",
str(build_log_path),
"--cd-build-job-log-http-status",
"200",
"--cd-tests-job-log-file",
str(tests_log_path),
"--cd-tests-job-log-http-status",
"200",
"--json",
],
check=False,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert result.returncode == 0, result.stdout + result.stderr
payload = json.loads(result.stdout)
assert payload["status"] == "blocked_host_web_build_pressure"
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 24
assert "HARBOR_PASSWORD" not in result.stdout