515 lines
19 KiB
Python
515 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "ops/runner/read-public-gitea-actions-queue.py"
|
|
|
|
|
|
def _load_module():
|
|
spec = importlib.util.spec_from_file_location(
|
|
"read_public_gitea_actions_queue",
|
|
SCRIPT,
|
|
)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _actions_html() -> str:
|
|
return """
|
|
<span data-tooltip-content="No matching online runner with label: awoooi-non110-ubuntu">
|
|
<span><b>ai-technology-watch.yaml #3857</b>:</span>Scheduled</div>
|
|
<span data-tooltip-content="Canceled">
|
|
<span><b>ai-technology-watch.yaml #3856</b>:</span>Scheduled</div>
|
|
<span data-tooltip-content="Waiting">
|
|
<span><b>e2e-health.yaml #3855</b>:</span>Scheduled</div>
|
|
"""
|
|
|
|
|
|
def _actions_html_full_run_items() -> str:
|
|
return """
|
|
<div class="flex-list run-list">
|
|
<div class="flex-item tw-items-center">
|
|
<div class="flex-item-leading">
|
|
<span data-tooltip-content="Running"></span>
|
|
</div>
|
|
<div class="flex-item-main">
|
|
<a class="flex-item-title" title="chore(deploy): retry non110 cd after ci image seed" href="/wooo/awoooi/actions/runs/3863">
|
|
chore(deploy): retry non110 cd after ci image seed
|
|
</a>
|
|
<div class="flex-item-body">
|
|
<span><b>cd.yaml #3863</b>:</span>Commit
|
|
<a href="/wooo/awoooi/commit/a70c6756d9e76c33143676eef82bab7a49ac1839">a70c6756d9</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="flex-item tw-items-center">
|
|
<div class="flex-item-leading">
|
|
<span data-tooltip-content="No matching online runner with label: awoooi-non110-host"></span>
|
|
</div>
|
|
<div class="flex-item-main">
|
|
<a class="flex-item-title" title="fix(ci): restore non110 cd push trigger" href="/wooo/awoooi/actions/runs/3859">
|
|
fix(ci): restore non110 cd push trigger
|
|
</a>
|
|
<div class="flex-item-body">
|
|
<span><b>cd.yaml #3859</b>:</span>Commit
|
|
<a href="/wooo/awoooi/commit/f3634db18ec3e37381dd634f5f5b80d73f8a8e38">f3634db18e</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
"""
|
|
|
|
|
|
def _actions_html_single_cd_run() -> str:
|
|
return """
|
|
<div class="flex-list run-list">
|
|
<div class="flex-item tw-items-center">
|
|
<div class="flex-item-leading">
|
|
<span data-tooltip-content="Running"></span>
|
|
</div>
|
|
<div class="flex-item-main">
|
|
<a class="flex-item-title" title="chore(deploy): retry non110 cd after ci image seed" href="/wooo/awoooi/actions/runs/3863">
|
|
chore(deploy): retry non110 cd after ci image seed
|
|
</a>
|
|
<div class="flex-item-body">
|
|
<span><b>cd.yaml #3863</b>:</span>Commit
|
|
<a href="/wooo/awoooi/commit/a70c6756d9e76c33143676eef82bab7a49ac1839">a70c6756d9</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
"""
|
|
|
|
|
|
def _actions_html_failed_cd_run() -> str:
|
|
return """
|
|
<div class="flex-list run-list">
|
|
<div class="flex-item tw-items-center">
|
|
<div class="flex-item-leading">
|
|
<span data-tooltip-content="Failure"></span>
|
|
</div>
|
|
<div class="flex-item-main">
|
|
<a class="flex-item-title" title="fix(runner): flag stale gitea jobs readback" href="/wooo/awoooi/actions/runs/4043">
|
|
fix(runner): flag stale gitea jobs readback
|
|
</a>
|
|
<div class="flex-item-body">
|
|
<span><b>cd.yaml #4043</b>:</span>Commit
|
|
<a href="/wooo/awoooi/commit/c4fe100620a686b0e62c84113029ec81b040376c">c4fe10062</a>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
"""
|
|
|
|
|
|
def _harbor_blocked_log() -> str:
|
|
return """
|
|
2026-06-30T11:33:05.5822531Z harbor_login_attempt=1 registry_v2_status=000
|
|
2026-06-30T11:33:18.7703396Z harbor_login_attempt=2 registry_v2_status=502
|
|
2026-06-30T11:35:30.6670130Z harbor_login_attempt=12 registry_v2_status=502
|
|
2026-06-30T11:35:30.6670926Z BLOCKER harbor_registry_public_route_unavailable registry_v2_status=502
|
|
2026-06-30T11:35:30.6672047Z Failure - Main Login to Harbor
|
|
"""
|
|
|
|
|
|
def _host_pressure_waiting_log() -> str:
|
|
return """
|
|
2026-06-30T11:48:41.7864172Z ⏳ host web/build/smoke pressure detected (attempt 1/60); waiting 10s
|
|
2026-06-30T11:48:41.7918276Z host load5/core 0.935000 > 0.85
|
|
2026-06-30T11:52:35.1754675Z ⏳ host web/build/smoke pressure detected (attempt 24/60); waiting 10s
|
|
2026-06-30T11:52:35.1827409Z host load5/core 0.931667 > 0.85
|
|
"""
|
|
|
|
|
|
def _host_pressure_refused_log() -> str:
|
|
return """
|
|
2026-06-30T11:58:31.0000000Z ⏳ host web/build/smoke pressure detected (attempt 60/60); waiting 10s
|
|
2026-06-30T11:58:31.0000000Z host load5/core 0.920000 > 0.85
|
|
2026-06-30T11:58:41.0000000Z ❌ refusing to start AWOOI image build while host web/build/smoke pressure is still active
|
|
"""
|
|
|
|
|
|
def test_parse_visible_runs_extracts_no_matching_runner_label() -> None:
|
|
module = _load_module()
|
|
runs = module.parse_visible_runs(_actions_html())
|
|
assert runs[0]["run_id"] == "3857"
|
|
assert runs[0]["workflow"] == "ai-technology-watch.yaml"
|
|
assert runs[0]["kind"] == "Scheduled"
|
|
assert runs[0]["no_matching_runner_label"] == "awoooi-non110-ubuntu"
|
|
assert runs[1]["no_matching_runner_label"] == ""
|
|
|
|
|
|
def test_parse_visible_runs_keeps_full_gitea_rows_aligned() -> None:
|
|
module = _load_module()
|
|
runs = module.parse_visible_runs(_actions_html_full_run_items())
|
|
assert runs[0]["run_id"] == "3863"
|
|
assert runs[0]["workflow"] == "cd.yaml"
|
|
assert runs[0]["kind"] == "Commit"
|
|
assert runs[0]["status"] == "Running"
|
|
assert runs[0]["title"] == "chore(deploy): retry non110 cd after ci image seed"
|
|
assert runs[0]["commit_sha"] == "a70c6756d9e76c33143676eef82bab7a49ac1839"
|
|
assert runs[1]["run_id"] == "3859"
|
|
assert runs[1]["no_matching_runner_label"] == "awoooi-non110-host"
|
|
|
|
|
|
def test_build_readback_reports_latest_visible_cd_run() -> None:
|
|
module = _load_module()
|
|
payload = module.build_readback(
|
|
actions_html=_actions_html_single_cd_run(),
|
|
actions_list_http_status=401,
|
|
actions_list_payload={"message": "token is required"},
|
|
cd_jobs_http_status=200,
|
|
cd_jobs_payload={"jobs": [], "total_count": 0},
|
|
)
|
|
assert payload["readback"]["latest_visible_cd_run_id"] == "3863"
|
|
assert payload["readback"]["latest_visible_cd_run_status"] == "Running"
|
|
assert payload["readback"]["latest_visible_cd_run_kind"] == "Commit"
|
|
assert payload["readback"]["latest_visible_cd_run_commit_sha"] == (
|
|
"a70c6756d9e76c33143676eef82bab7a49ac1839"
|
|
)
|
|
assert payload["rollups"]["current_main_cd_run_visible"] is True
|
|
assert payload["rollups"]["current_main_cd_run_status"] == "Running"
|
|
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is False
|
|
|
|
|
|
def test_build_readback_classifies_harbor_public_route_blocker() -> None:
|
|
module = _load_module()
|
|
payload = module.build_readback(
|
|
actions_html=_actions_html_failed_cd_run(),
|
|
actions_list_http_status=401,
|
|
actions_list_payload={"message": "token is required"},
|
|
cd_jobs_http_status=200,
|
|
cd_jobs_payload={"jobs": [], "total_count": 0},
|
|
latest_cd_build_log_http_status=200,
|
|
latest_cd_build_log_text=_harbor_blocked_log(),
|
|
)
|
|
assert payload["status"] == "blocked_harbor_public_route_unavailable"
|
|
assert payload["readback"]["latest_visible_cd_run_id"] == "4043"
|
|
assert payload["readback"]["latest_visible_cd_run_status"] == "Failure"
|
|
assert payload["readback"]["latest_visible_cd_failure_classifier"] == (
|
|
"harbor_registry_public_route_unavailable"
|
|
)
|
|
assert payload["readback"]["latest_visible_cd_failure_status_code"] == "502"
|
|
assert payload["readback"]["latest_visible_cd_harbor_login_attempt_count"] == 12
|
|
assert payload["readback"]["latest_visible_cd_harbor_public_route_blocked"] is True
|
|
assert payload["rollups"]["current_main_cd_harbor_public_route_blocked"] is True
|
|
|
|
|
|
def test_build_readback_classifies_host_pressure_waiting() -> None:
|
|
module = _load_module()
|
|
payload = module.build_readback(
|
|
actions_html=_actions_html_single_cd_run(),
|
|
actions_list_http_status=401,
|
|
actions_list_payload={"message": "token is required"},
|
|
cd_jobs_http_status=200,
|
|
cd_jobs_payload={"jobs": [], "total_count": 0},
|
|
latest_cd_tests_log_http_status=200,
|
|
latest_cd_tests_log_text=_host_pressure_waiting_log(),
|
|
)
|
|
assert payload["status"] == "blocked_host_web_build_pressure"
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
|
|
"host_web_build_pressure_waiting"
|
|
)
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 24
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_limit"] == 60
|
|
assert (
|
|
payload["readback"]["latest_visible_cd_host_pressure_latest_load5_per_core"]
|
|
== "0.931667"
|
|
)
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_load5_threshold"] == (
|
|
"0.85"
|
|
)
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_waiting"] is True
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_refused"] is False
|
|
assert payload["rollups"]["current_main_cd_host_pressure_waiting"] is True
|
|
|
|
|
|
def test_host_pressure_refusal_takes_status_precedence_over_stale_jobs() -> None:
|
|
module = _load_module()
|
|
payload = module.build_readback(
|
|
actions_html=_actions_html_failed_cd_run(),
|
|
actions_list_http_status=401,
|
|
actions_list_payload={"message": "token is required"},
|
|
cd_jobs_http_status=200,
|
|
cd_jobs_payload={
|
|
"jobs": [
|
|
{
|
|
"run_id": 4047,
|
|
"head_sha": "df89bdf00b9413b4db61b7f2d9bbca57e9fc9923",
|
|
"conclusion": "success",
|
|
}
|
|
],
|
|
"total_count": 1,
|
|
},
|
|
latest_cd_tests_log_http_status=200,
|
|
latest_cd_tests_log_text=_host_pressure_refused_log(),
|
|
)
|
|
assert payload["status"] == "blocked_host_web_build_pressure"
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_classifier"] == (
|
|
"host_web_build_pressure_refused"
|
|
)
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 60
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_refused"] is True
|
|
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
|
|
|
|
|
|
def test_build_readback_flags_stale_cd_jobs_api_payload() -> None:
|
|
module = _load_module()
|
|
payload = module.build_readback(
|
|
actions_html=_actions_html_single_cd_run(),
|
|
actions_list_http_status=401,
|
|
actions_list_payload={"message": "token is required"},
|
|
cd_jobs_http_status=200,
|
|
cd_jobs_payload={
|
|
"jobs": [
|
|
{
|
|
"run_id": 3863,
|
|
"name": "tests",
|
|
"head_sha": "b17a28c29375afd207a90d05b4898daaa0f3f730",
|
|
"conclusion": "success",
|
|
}
|
|
],
|
|
"total_count": 1,
|
|
},
|
|
)
|
|
assert payload["status"] == "cd_jobs_stale_or_mismatched"
|
|
assert payload["readback"]["latest_visible_cd_run_commit_sha"] == (
|
|
"a70c6756d9e76c33143676eef82bab7a49ac1839"
|
|
)
|
|
assert payload["readback"]["cd_run_jobs_head_shas"] == [
|
|
"b17a28c29375afd207a90d05b4898daaa0f3f730"
|
|
]
|
|
assert payload["readback"]["cd_run_jobs_head_sha_matches_visible"] is False
|
|
assert payload["readback"]["cd_run_jobs_run_id_matches_visible"] is True
|
|
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
|
|
assert payload["rollups"]["cd_run_jobs_stale_or_mismatched"] is True
|
|
|
|
|
|
def test_harbor_blocker_takes_status_precedence_over_stale_jobs_payload() -> None:
|
|
module = _load_module()
|
|
payload = module.build_readback(
|
|
actions_html=_actions_html_failed_cd_run(),
|
|
actions_list_http_status=401,
|
|
actions_list_payload={"message": "token is required"},
|
|
cd_jobs_http_status=200,
|
|
cd_jobs_payload={
|
|
"jobs": [
|
|
{
|
|
"run_id": 4045,
|
|
"head_sha": "f4fb078100000000000000000000000000000000",
|
|
"conclusion": "success",
|
|
}
|
|
],
|
|
"total_count": 1,
|
|
},
|
|
latest_cd_build_log_http_status=200,
|
|
latest_cd_build_log_text=_harbor_blocked_log(),
|
|
)
|
|
assert payload["status"] == "blocked_harbor_public_route_unavailable"
|
|
assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
|
|
assert payload["readback"]["latest_visible_cd_failure_classifier"] == (
|
|
"harbor_registry_public_route_unavailable"
|
|
)
|
|
|
|
|
|
def test_derive_jobs_api_url_tracks_latest_visible_run_id() -> None:
|
|
module = _load_module()
|
|
|
|
assert module.DEFAULT_CD_RUN_JOBS_API_URL == ""
|
|
assert module.derive_jobs_api_url(
|
|
"https://gitea.wooo.work/api/v1/repos/wooo/awoooi/actions/runs?limit=10",
|
|
"4043",
|
|
) == "https://gitea.wooo.work/api/v1/repos/wooo/awoooi/actions/runs/4043/jobs"
|
|
|
|
|
|
def test_build_readback_sanitizes_actions_api_internal_url() -> None:
|
|
module = _load_module()
|
|
payload = module.build_readback(
|
|
actions_html=_actions_html(),
|
|
actions_list_http_status=401,
|
|
actions_list_payload={
|
|
"message": "token is required",
|
|
"url": "http://192.168.0.110:3001/api/swagger",
|
|
},
|
|
cd_jobs_http_status=200,
|
|
cd_jobs_payload={"jobs": [], "total_count": 0},
|
|
)
|
|
text = json.dumps(payload, sort_keys=True)
|
|
assert payload["schema_version"] == module.SCHEMA_VERSION
|
|
assert payload["status"] == "blocked_no_matching_online_runner"
|
|
assert payload["readback"]["actions_list_without_token_message"] == (
|
|
"token is required"
|
|
)
|
|
assert payload["readback"]["latest_visible_no_matching_runner_run_id"] == "3857"
|
|
assert payload["readback"]["latest_visible_no_matching_runner_label"] == (
|
|
"awoooi-non110-ubuntu"
|
|
)
|
|
assert payload["rollups"]["actions_list_requires_token"] is True
|
|
assert payload["operation_boundaries"]["github_api_used"] is False
|
|
assert "192.168.0." not in text
|
|
|
|
|
|
def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
|
|
html_path = tmp_path / "actions.html"
|
|
list_path = tmp_path / "actions-list.json"
|
|
jobs_path = tmp_path / "jobs.json"
|
|
log_path = tmp_path / "build.log"
|
|
tests_log_path = tmp_path / "tests.log"
|
|
html_path.write_text(_actions_html(), encoding="utf-8")
|
|
list_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"message": "token is required",
|
|
"url": "http://192.168.0.110:3001/api/swagger",
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
|
|
log_path.write_text("", encoding="utf-8")
|
|
tests_log_path.write_text("", encoding="utf-8")
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--actions-html-file",
|
|
str(html_path),
|
|
"--actions-list-json-file",
|
|
str(list_path),
|
|
"--actions-list-http-status",
|
|
"401",
|
|
"--cd-run-jobs-json-file",
|
|
str(jobs_path),
|
|
"--cd-run-jobs-http-status",
|
|
"200",
|
|
"--cd-build-job-log-file",
|
|
str(log_path),
|
|
"--cd-build-job-log-http-status",
|
|
"200",
|
|
"--cd-tests-job-log-file",
|
|
str(tests_log_path),
|
|
"--cd-tests-job-log-http-status",
|
|
"200",
|
|
"--json",
|
|
],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = json.loads(result.stdout)
|
|
assert payload["readback"]["actions_page_visible_run_count"] == 3
|
|
assert payload["readback"]["cd_run_jobs_total_count"] == 0
|
|
assert payload["readback"]["latest_visible_no_matching_runner_label"] == (
|
|
"awoooi-non110-ubuntu"
|
|
)
|
|
assert "192.168.0." not in result.stdout
|
|
|
|
|
|
def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) -> None:
|
|
html_path = tmp_path / "actions.html"
|
|
list_path = tmp_path / "actions-list.json"
|
|
jobs_path = tmp_path / "jobs.json"
|
|
log_path = tmp_path / "build.log"
|
|
tests_log_path = tmp_path / "tests.log"
|
|
html_path.write_text(_actions_html_failed_cd_run(), encoding="utf-8")
|
|
list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8")
|
|
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
|
|
log_path.write_text(_harbor_blocked_log(), encoding="utf-8")
|
|
tests_log_path.write_text("", encoding="utf-8")
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--actions-html-file",
|
|
str(html_path),
|
|
"--actions-list-json-file",
|
|
str(list_path),
|
|
"--actions-list-http-status",
|
|
"401",
|
|
"--cd-run-jobs-json-file",
|
|
str(jobs_path),
|
|
"--cd-run-jobs-http-status",
|
|
"200",
|
|
"--cd-build-job-log-file",
|
|
str(log_path),
|
|
"--cd-build-job-log-http-status",
|
|
"200",
|
|
"--cd-tests-job-log-file",
|
|
str(tests_log_path),
|
|
"--cd-tests-job-log-http-status",
|
|
"200",
|
|
"--json",
|
|
],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = json.loads(result.stdout)
|
|
assert payload["status"] == "blocked_harbor_public_route_unavailable"
|
|
assert payload["readback"]["latest_visible_cd_failure_status_code"] == "502"
|
|
assert "HARBOR_PASSWORD" not in result.stdout
|
|
|
|
|
|
def test_cli_json_classifies_host_pressure_from_fixture_log(tmp_path: Path) -> None:
|
|
html_path = tmp_path / "actions.html"
|
|
list_path = tmp_path / "actions-list.json"
|
|
jobs_path = tmp_path / "jobs.json"
|
|
build_log_path = tmp_path / "build.log"
|
|
tests_log_path = tmp_path / "tests.log"
|
|
html_path.write_text(_actions_html_single_cd_run(), encoding="utf-8")
|
|
list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8")
|
|
jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
|
|
build_log_path.write_text("", encoding="utf-8")
|
|
tests_log_path.write_text(_host_pressure_waiting_log(), encoding="utf-8")
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--actions-html-file",
|
|
str(html_path),
|
|
"--actions-list-json-file",
|
|
str(list_path),
|
|
"--actions-list-http-status",
|
|
"401",
|
|
"--cd-run-jobs-json-file",
|
|
str(jobs_path),
|
|
"--cd-run-jobs-http-status",
|
|
"200",
|
|
"--cd-build-job-log-file",
|
|
str(build_log_path),
|
|
"--cd-build-job-log-http-status",
|
|
"200",
|
|
"--cd-tests-job-log-file",
|
|
str(tests_log_path),
|
|
"--cd-tests-job-log-http-status",
|
|
"200",
|
|
"--json",
|
|
],
|
|
check=False,
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = json.loads(result.stdout)
|
|
assert payload["status"] == "blocked_host_web_build_pressure"
|
|
assert payload["readback"]["latest_visible_cd_host_pressure_attempt_count"] == 24
|
|
assert "HARBOR_PASSWORD" not in result.stdout
|