diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 84015f90a..ebb6359d0 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,19 @@ +## 2026-06-30 — 21:02 Harbor 110 repair jobs stale / generic label classifier + +**照主線修正的問題**： +- Live jobs API 對 `harbor-110-local-repair.yaml #4067` 回傳的 job name 是 `ai-code-review`、label=`ubuntu-latest`，不是 Harbor 110 repair workflow 的 `workflow-shape` / `harbor-110-local-repair`；先前不能把這種 jobs payload 當成 repair success。 +- `ops/runner/read-public-gitea-actions-queue.py` 新增 Harbor repair expected job name gate：只有 job names 符合 `workflow-shape` / `harbor-110-local-repair`，且 run id match，才允許 `harbor_110_repair_jobs_all_success=true`。 +- 新增 `harbor_110_repair_jobs_stale_or_mismatched`、`harbor_110_repair_jobs_unexpected_names`、`harbor_110_repair_jobs_stale_or_mismatched` status 與 `blocked_harbor_110_repair_jobs_stale_or_mismatched`，把 stale jobs API / generic `ubuntu-latest` runtime truth 直接升成 P0 classifier，不再被 public HTML `Waiting` 蓋掉。 + +**驗證**： +- `py_compile`、`ruff check`、`pytest ops/runner/test_read_public_gitea_actions_queue.py -q`（19 passed）、`git diff --check` 通過。 +- `guard-gitea-runner-pressure.py --root .` 通過：`auto_branch_events_on_110=0`、`generic_runner_labels=0`。 +- `check-gitea-step-env-secrets.js` 通過：沒有 Gitea run/with secrets 或 legacy Telegram route。 +- 21:06 live reader 回 `status=harbor_110_repair_jobs_stale_or_mismatched`、`latest_visible_cd_run_id=4068`、`latest_visible_harbor_110_repair_run_id=4067`、jobs=`ai-code-review`、label=`ubuntu-latest`、`harbor_110_repair_jobs_all_success=false`。 +- 21:06 registry readback：`https://registry.wooo.work/v2/` 與 `http://192.168.0.110:5000/v2/` 皆為 502；下一步不是宣稱 repair 成功，而是修正 Harbor repair jobs API / workflow runtime classifier 與 runner mapping。 + +**邊界**：只讀 public Gitea / production HTTP 與本機 source；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未 SSH / Docker / Nginx / K3s / DB / firewall runtime 寫入。 + ## 2026-06-30 — 20:52 Harbor 110 repair waiting status precedence **照主線修正的問題**： diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index 732156516..0145e9a1b 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -25,6 +25,10 @@ DEFAULT_CD_TESTS_JOB_LOG_URL_TEMPLATE = ( "https://gitea.wooo.work/wooo/awoooi/actions/runs/{run_id}/jobs/2/logs" ) SCHEMA_VERSION = "awoooi_public_gitea_actions_queue_readback_v1" +EXPECTED_HARBOR_110_REPAIR_JOB_NAMES = { + "workflow-shape", + "harbor-110-local-repair", +} _RUN_ROW_RE = re.compile( r'.*?' @@ -231,11 +235,15 @@ def build_readback( job_conclusion_counts[conclusion] = job_conclusion_counts.get(conclusion, 0) + 1 harbor_job_conclusion_counts: dict[str, int] = {} harbor_job_run_ids: set[str] = set() + harbor_job_names: set[str] = set() harbor_job_labels: set[str] = set() harbor_job_runner_names: set[str] = set() for job in harbor_jobs: if not isinstance(job, dict): continue + job_name = str(job.get("name") or "") + if job_name: + harbor_job_names.add(job_name) conclusion = str(job.get("conclusion") or job.get("status") or "unknown") harbor_job_conclusion_counts[conclusion] = ( harbor_job_conclusion_counts.get(conclusion, 0) + 1 @@ -277,9 +285,23 @@ def build_readback( bool(harbor_110_repair_run_id) and harbor_110_repair_run_id in harbor_job_run_ids ) + harbor_110_repair_jobs_unexpected_names = sorted( + harbor_job_names - EXPECTED_HARBOR_110_REPAIR_JOB_NAMES + ) + harbor_110_repair_jobs_match_expected_workflow = ( + bool(harbor_job_names) + and not harbor_110_repair_jobs_unexpected_names + and "harbor-110-local-repair" in harbor_job_names + ) + harbor_110_repair_jobs_stale_or_mismatched = ( + harbor_110_repair_jobs_http_status == 200 + and harbor_jobs_total_count > 0 + and not harbor_110_repair_jobs_match_expected_workflow + ) harbor_110_repair_jobs_all_success = ( harbor_110_repair_jobs_http_status == 200 and harbor_jobs_total_count > 0 + and harbor_110_repair_jobs_match_expected_workflow and harbor_job_conclusion_counts.get("success") == harbor_jobs_total_count and harbor_110_repair_jobs_run_id_matches_visible ) @@ -369,11 +391,21 @@ def build_readback( "harbor_110_repair_jobs_total_count": harbor_jobs_total_count, "harbor_110_repair_jobs_conclusion_counts": harbor_job_conclusion_counts, "harbor_110_repair_jobs_run_ids": sorted(harbor_job_run_ids), + "harbor_110_repair_jobs_names": sorted(harbor_job_names), + "harbor_110_repair_jobs_unexpected_names": ( + harbor_110_repair_jobs_unexpected_names + ), "harbor_110_repair_jobs_labels": sorted(harbor_job_labels), "harbor_110_repair_jobs_runner_names": sorted(harbor_job_runner_names), "harbor_110_repair_jobs_run_id_matches_visible": ( harbor_110_repair_jobs_run_id_matches_visible ), + "harbor_110_repair_jobs_match_expected_workflow": ( + harbor_110_repair_jobs_match_expected_workflow + ), + "harbor_110_repair_jobs_stale_or_mismatched": ( + harbor_110_repair_jobs_stale_or_mismatched + ), "harbor_110_repair_jobs_all_success": harbor_110_repair_jobs_all_success, "harbor_110_repair_visible_waiting_stale": ( harbor_110_repair_visible_waiting_stale @@ -416,12 +448,19 @@ def build_readback( build_log_classifier["harbor_public_route_blocked"] and harbor_110_repair_jobs_all_success ) + else "blocked_harbor_110_repair_jobs_stale_or_mismatched" + if ( + build_log_classifier["harbor_public_route_blocked"] + and harbor_110_repair_jobs_stale_or_mismatched + ) else "blocked_harbor_110_repair_workflow_waiting" if harbor_110_repair_waiting_after_cd_harbor_blocker else "blocked_harbor_public_route_unavailable" if build_log_classifier["harbor_public_route_blocked"] else "blocked_host_web_build_pressure" if tests_log_classifier["host_pressure_blocked_or_waiting"] + else "harbor_110_repair_jobs_stale_or_mismatched" + if harbor_110_repair_jobs_stale_or_mismatched else "harbor_110_repair_waiting_for_runner_or_queue" if harbor_110_repair_waiting else "harbor_110_repair_running" @@ -475,6 +514,13 @@ def build_readback( latest_harbor_110_repair_run.get("no_matching_runner_label", "") ), "harbor_110_repair_jobs_total_count": harbor_jobs_total_count, + "harbor_110_repair_jobs_names": sorted(harbor_job_names), + "harbor_110_repair_jobs_stale_or_mismatched": ( + harbor_110_repair_jobs_stale_or_mismatched + ), + "harbor_110_repair_jobs_unexpected_names": ( + harbor_110_repair_jobs_unexpected_names + ), "harbor_110_repair_jobs_all_success": ( harbor_110_repair_jobs_all_success ), @@ -649,6 +695,10 @@ def _human_summary(payload: dict[str, Any]) -> str: "HARBOR_110_REPAIR_JOBS_ALL_SUCCESS=" f"{int(readback['harbor_110_repair_jobs_all_success'])}" ), + ( + "HARBOR_110_REPAIR_JOBS_STALE_OR_MISMATCHED=" + f"{int(readback['harbor_110_repair_jobs_stale_or_mismatched'])}" + ), ( "HARBOR_110_REPAIR_VISIBLE_WAITING_STALE=" f"{int(readback['harbor_110_repair_visible_waiting_stale'])}" diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index a153b6a9e..b9103fa88 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -166,21 +166,21 @@ def _harbor_blocked_log() -> str: def _harbor_110_repair_success_jobs() -> dict: return { - "total_count": 3, + "total_count": 2, "jobs": [ { "id": 5821, - "name": "tests", + "name": "workflow-shape", "status": "completed", "conclusion": "success", - "labels": ["awoooi-host"], + "labels": ["awoooi-non110-host"], "runner_name": "wooo-runner", "run_id": 4060, "head_sha": "7c8bb3645bdf1fa5ac1aaa7041c237fce8c19c0e", }, { "id": 5822, - "name": "build-and-deploy", + "name": "harbor-110-local-repair", "status": "completed", "conclusion": "success", "labels": ["awoooi-host"], @@ -188,15 +188,23 @@ def _harbor_110_repair_success_jobs() -> dict: "run_id": 4060, "head_sha": "7c8bb3645bdf1fa5ac1aaa7041c237fce8c19c0e", }, + ], + } + + +def _harbor_110_repair_stale_code_review_jobs() -> dict: + return { + "total_count": 1, + "jobs": [ { - "id": 5823, - "name": "post-deploy-checks", + "id": 5834, + "name": "ai-code-review", "status": "completed", "conclusion": "success", - "labels": ["awoooi-host"], + "labels": ["ubuntu-latest"], "runner_name": "wooo-runner", - "run_id": 4060, - "head_sha": "7c8bb3645bdf1fa5ac1aaa7041c237fce8c19c0e", + "run_id": 4067, + "head_sha": "acaae99986aee2e1f5630984981ccb0f2b676bb8", }, ], } @@ -310,10 +318,15 @@ def test_build_readback_classifies_harbor_502_after_110_repair_jobs_success() -> "blocked_harbor_public_route_unavailable_after_harbor_110_repair_success" ) assert payload["readback"]["latest_visible_harbor_110_repair_run_id"] == "4060" - assert payload["readback"]["harbor_110_repair_jobs_total_count"] == 3 + assert payload["readback"]["harbor_110_repair_jobs_total_count"] == 2 assert payload["readback"]["harbor_110_repair_jobs_all_success"] is True assert payload["readback"]["harbor_110_repair_visible_waiting_stale"] is True - assert payload["readback"]["harbor_110_repair_jobs_labels"] == ["awoooi-host"] + assert payload["readback"]["harbor_110_repair_jobs_match_expected_workflow"] is True + assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is False + assert payload["readback"]["harbor_110_repair_jobs_labels"] == [ + "awoooi-host", + "awoooi-non110-host", + ] assert payload["readback"]["harbor_110_repair_jobs_runner_names"] == [ "wooo-runner" ] @@ -322,6 +335,47 @@ def test_build_readback_classifies_harbor_502_after_110_repair_jobs_success() -> assert payload["operation_boundaries"]["host_write_performed"] is False +def test_build_readback_rejects_stale_harbor_110_repair_jobs_payload() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_cd_running_harbor_repair_waiting(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + harbor_110_repair_jobs_http_status=200, + harbor_110_repair_jobs_payload=_harbor_110_repair_stale_code_review_jobs(), + latest_cd_build_log_http_status=200, + latest_cd_build_log_text=_harbor_blocked_log(), + ) + + assert payload["status"] == "blocked_harbor_110_repair_jobs_stale_or_mismatched" + assert payload["readback"]["harbor_110_repair_jobs_all_success"] is False + assert payload["readback"]["harbor_110_repair_jobs_stale_or_mismatched"] is True + assert payload["readback"]["harbor_110_repair_jobs_unexpected_names"] == [ + "ai-code-review" + ] + assert payload["readback"]["harbor_110_repair_jobs_labels"] == ["ubuntu-latest"] + assert payload["rollups"]["harbor_110_repair_jobs_stale_or_mismatched"] is True + + +def test_build_readback_prioritizes_harbor_repair_jobs_stale_status() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_cd_running_harbor_repair_waiting(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + harbor_110_repair_jobs_http_status=200, + harbor_110_repair_jobs_payload=_harbor_110_repair_stale_code_review_jobs(), + ) + + assert payload["status"] == "harbor_110_repair_jobs_stale_or_mismatched" + assert payload["readback"]["harbor_110_repair_jobs_all_success"] is False + assert payload["readback"]["latest_visible_harbor_110_repair_waiting"] is True + + def test_build_readback_classifies_harbor_public_route_blocker() -> None: module = _load_module() payload = module.build_readback(