fix(runner): classify harbor public route failures

docs/LOGBOOK.md

@@ -1,3 +1,16 @@
+## 2026-06-30 — 20:16 Public Gitea CD Harbor blocker classifier
+
+**照主線修正的問題**：
+- #4043 / #4044 顯示 tests 已可過，但 build-and-deploy 在 Harbor `/v2/` public route 502 / timeout 上 fail-closed；不能再只把 latest CD 顯示成泛化 Failure。
+- `ops/runner/read-public-gitea-actions-queue.py` 現在會在 public/read-only 邊界讀 latest visible CD run 的 build job log，分類 `harbor_registry_public_route_unavailable`、記錄 final `registry_v2_status` 與 Harbor login retry attempt count。
+- 這和前一段 dynamic jobs API URL guard 疊在一起：jobs API 舊資料不再被誤當成功，build log blocker 也能成為 closure verifier 的精準 evidence。
+
+**驗證**：
+- `test_read_public_gitea_actions_queue.py` 新增 fixture，覆蓋 Harbor `/v2/` `000/502`、第 12 次 retry、`BLOCKER harbor_registry_public_route_unavailable`、CLI fixture log readback。
+- focused verifier tests、`py_compile`、`git diff --check` 通過。
+
+**邊界**：只讀 public Gitea HTML / API / public job log；未 workflow_dispatch，未 SSH 寫主機，未重啟主機，未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall，未讀 secret / token / raw sessions / SQLite / `.env`，未使用 GitHub / `gh` / GitHub API。
+
 ## 2026-06-30 — 20:05 Harbor public route retry for P0 CD closure
 
 **照主線修正的問題**：

ops/runner/read-public-gitea-actions-queue.py

@@ -18,6 +18,9 @@ DEFAULT_ACTIONS_LIST_API_URL = (
     "https://gitea.wooo.work/api/v1/repos/wooo/awoooi/actions/runs?limit=10"
 )
 DEFAULT_CD_RUN_JOBS_API_URL = ""
+DEFAULT_CD_BUILD_JOB_LOG_URL_TEMPLATE = (
+    "https://gitea.wooo.work/wooo/awoooi/actions/runs/{run_id}/jobs/3/logs"
+)
 SCHEMA_VERSION = "awoooi_public_gitea_actions_queue_readback_v1"
 
 _RUN_ROW_RE = re.compile(
@@ -38,6 +41,13 @@ _RUN_NAME_RE = re.compile(r"^(?P<workflow>.+)\s+#(?P<run_id>\d+)$")
 _NO_MATCHING_LABEL_RE = re.compile(
     r"No matching online runner with label:\s*(?P<label>[A-Za-z0-9_.:-]+)"
 )
+_HARBOR_ATTEMPT_RE = re.compile(
+    r"harbor_login_attempt=(?P<attempt>\d+)\s+registry_v2_status=(?P<status>\d{3})"
+)
+_HARBOR_BLOCKER_RE = re.compile(
+    r"BLOCKER harbor_registry_public_route_unavailable "
+    r"registry_v2_status=(?P<status>\d{3})"
+)
 
 
 @dataclass(frozen=True)
@@ -139,6 +149,8 @@ def build_readback(
     actions_list_payload: Any,
     cd_jobs_http_status: int,
     cd_jobs_payload: Any,
+    latest_cd_build_log_http_status: int = 0,
+    latest_cd_build_log_text: str = "",
 ) -> dict[str, Any]:
     visible_runs = parse_visible_runs(actions_html)
     no_matching = next(
@@ -192,6 +204,7 @@ def build_readback(
             or not cd_jobs_run_id_matches_visible
         )
     )
+    build_log_classifier = classify_cd_build_log(latest_cd_build_log_text)
 
     readback = {
         "actions_page_visible_run_count": len(visible_runs),
@@ -219,6 +232,19 @@ def build_readback(
         "latest_visible_cd_run_kind": latest_cd_run.get("kind", ""),
         "latest_visible_cd_run_title": latest_cd_run.get("title", ""),
         "latest_visible_cd_run_commit_sha": latest_cd_run.get("commit_sha", ""),
+        "latest_visible_cd_build_log_http_status": latest_cd_build_log_http_status,
+        "latest_visible_cd_failure_classifier": build_log_classifier[
+            "failure_classifier"
+        ],
+        "latest_visible_cd_failure_status_code": build_log_classifier[
+            "failure_status_code"
+        ],
+        "latest_visible_cd_harbor_login_attempt_count": build_log_classifier[
+            "harbor_login_attempt_count"
+        ],
+        "latest_visible_cd_harbor_public_route_blocked": build_log_classifier[
+            "harbor_public_route_blocked"
+        ],
         "no_matching_online_runner_visible": bool(no_matching),
         "top_visible_runs": visible_runs[:10],
     }
@@ -227,6 +253,8 @@ def build_readback(
         "status": (
             "blocked_no_matching_online_runner"
             if no_matching
+            else "blocked_harbor_public_route_unavailable"
+            if build_log_classifier["harbor_public_route_blocked"]
             else "cd_jobs_stale_or_mismatched"
             if cd_jobs_stale_or_mismatched
             else "no_matching_runner_not_visible"
@@ -239,6 +267,12 @@ def build_readback(
             "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched,
             "current_main_cd_run_visible": bool(latest_cd_run),
             "current_main_cd_run_status": latest_cd_run.get("status", ""),
+            "current_main_cd_failure_classifier": build_log_classifier[
+                "failure_classifier"
+            ],
+            "current_main_cd_harbor_public_route_blocked": build_log_classifier[
+                "harbor_public_route_blocked"
+            ],
             "no_matching_online_runner_visible": bool(no_matching),
         },
         "operation_boundaries": {
@@ -255,6 +289,29 @@ def build_readback(
     }
 
 
+def classify_cd_build_log(text: str) -> dict[str, Any]:
+    attempt_statuses: list[str] = []
+    attempt_numbers: list[int] = []
+    for match in _HARBOR_ATTEMPT_RE.finditer(text):
+        attempt_numbers.append(_int(match.group("attempt")))
+        attempt_statuses.append(match.group("status"))
+
+    blocker_match = _HARBOR_BLOCKER_RE.search(text)
+    harbor_public_route_blocked = blocker_match is not None
+    failure_status_code = blocker_match.group("status") if blocker_match else ""
+    return {
+        "failure_classifier": (
+            "harbor_registry_public_route_unavailable"
+            if harbor_public_route_blocked
+            else ""
+        ),
+        "failure_status_code": failure_status_code,
+        "harbor_login_attempt_count": max(attempt_numbers) if attempt_numbers else 0,
+        "harbor_public_route_blocked": harbor_public_route_blocked,
+        "harbor_registry_v2_statuses": attempt_statuses[-12:],
+    }
+
+
 def load_json_text(text: str) -> Any:
     try:
         return json.loads(text)
@@ -306,6 +363,10 @@ def _human_summary(payload: dict[str, Any]) -> str:
         ),
         f"LATEST_VISIBLE_CD_RUN_ID={readback['latest_visible_cd_run_id']}",
         f"LATEST_VISIBLE_CD_RUN_STATUS={readback['latest_visible_cd_run_status']}",
+        (
+            "LATEST_VISIBLE_CD_FAILURE_CLASSIFIER="
+            f"{readback['latest_visible_cd_failure_classifier']}"
+        ),
         "WRITE_PERFORMED=false",
         "TOKEN_COLLECTED=false",
     ]
@@ -322,12 +383,19 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--actions-url", default=DEFAULT_ACTIONS_URL)
     parser.add_argument("--actions-list-api-url", default=DEFAULT_ACTIONS_LIST_API_URL)
     parser.add_argument("--cd-run-jobs-api-url", default=DEFAULT_CD_RUN_JOBS_API_URL)
+    parser.add_argument(
+        "--cd-build-job-log-url-template",
+        default=DEFAULT_CD_BUILD_JOB_LOG_URL_TEMPLATE,
+    )
     parser.add_argument("--timeout-seconds", type=float, default=10.0)
     parser.add_argument("--actions-html-file", type=Path)
     parser.add_argument("--actions-list-json-file", type=Path)
     parser.add_argument("--actions-list-http-status", type=int)
     parser.add_argument("--cd-run-jobs-json-file", type=Path)
     parser.add_argument("--cd-run-jobs-http-status", type=int)
+    parser.add_argument("--cd-build-job-log-file", type=Path)
+    parser.add_argument("--cd-build-job-log-http-status", type=int)
+    parser.add_argument("--skip-cd-build-job-log-read", action="store_true")
     parser.add_argument("--json", action="store_true")
     args = parser.parse_args(argv)
 
@@ -371,12 +439,44 @@ def main(argv: list[str] | None = None) -> int:
             cd_jobs_http_status = 0
             cd_jobs_payload = {"jobs": [], "total_count": 0}
 
+    if args.cd_build_job_log_file:
+        cd_build_job_log_http_status = args.cd_build_job_log_http_status or 0
+        cd_build_job_log_text = _read_text_file(args.cd_build_job_log_file)
+    elif args.skip_cd_build_job_log_read:
+        cd_build_job_log_http_status = 0
+        cd_build_job_log_text = ""
+    else:
+        latest_cd_run = next(
+            (
+                run
+                for run in parse_visible_runs(actions_html)
+                if run.get("workflow") == "cd.yaml"
+            ),
+            {},
+        )
+        latest_cd_run_id = latest_cd_run.get("run_id", "")
+        if latest_cd_run_id:
+            log_url = args.cd_build_job_log_url_template.format(
+                run_id=latest_cd_run_id,
+            )
+            cd_build_job_log_read = fetch_public_url(
+                log_url,
+                args.timeout_seconds,
+            )
+            cd_build_job_log_http_status = cd_build_job_log_read.http_status
+            cd_build_job_log_text = cd_build_job_log_read.text
+        else:
+            cd_build_job_log_http_status = 0
+            cd_build_job_log_text = ""
+
     payload = build_readback(
         actions_html=actions_html,
         actions_list_http_status=actions_list_http_status,
         actions_list_payload=actions_list_payload,
         cd_jobs_http_status=cd_jobs_http_status,
         cd_jobs_payload=cd_jobs_payload,
+        latest_cd_build_log_http_status=cd_build_job_log_http_status,
+        latest_cd_build_log_text=cd_build_job_log_text,
     )
     if args.json:
         json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, sort_keys=True)

ops/runner/test_read_public_gitea_actions_queue.py

@@ -91,6 +91,37 @@ def _actions_html_single_cd_run() -> str:
 """
 
 
+def _actions_html_failed_cd_run() -> str:
+    return """
+<div class="flex-list run-list">
+  <div class="flex-item tw-items-center">
+    <div class="flex-item-leading">
+      <span data-tooltip-content="Failure"></span>
+    </div>
+    <div class="flex-item-main">
+      <a class="flex-item-title" title="fix(runner): flag stale gitea jobs readback" href="/wooo/awoooi/actions/runs/4043">
+        fix(runner): flag stale gitea jobs readback
+      </a>
+      <div class="flex-item-body">
+        <span><b>cd.yaml #4043</b>:</span>Commit
+        <a href="/wooo/awoooi/commit/c4fe100620a686b0e62c84113029ec81b040376c">c4fe10062</a>
+      </div>
+    </div>
+  </div>
+</div>
+"""
+
+
+def _harbor_blocked_log() -> str:
+    return """
+2026-06-30T11:33:05.5822531Z harbor_login_attempt=1 registry_v2_status=000
+2026-06-30T11:33:18.7703396Z harbor_login_attempt=2 registry_v2_status=502
+2026-06-30T11:35:30.6670130Z harbor_login_attempt=12 registry_v2_status=502
+2026-06-30T11:35:30.6670926Z BLOCKER harbor_registry_public_route_unavailable registry_v2_status=502
+2026-06-30T11:35:30.6672047Z   Failure - Main Login to Harbor
+"""
+
+
 def test_parse_visible_runs_extracts_no_matching_runner_label() -> None:
     module = _load_module()
     runs = module.parse_visible_runs(_actions_html())
@@ -134,6 +165,29 @@ def test_build_readback_reports_latest_visible_cd_run() -> None:
     assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is False
 
 
+def test_build_readback_classifies_harbor_public_route_blocker() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_failed_cd_run(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={"jobs": [], "total_count": 0},
+        latest_cd_build_log_http_status=200,
+        latest_cd_build_log_text=_harbor_blocked_log(),
+    )
+    assert payload["status"] == "blocked_harbor_public_route_unavailable"
+    assert payload["readback"]["latest_visible_cd_run_id"] == "4043"
+    assert payload["readback"]["latest_visible_cd_run_status"] == "Failure"
+    assert payload["readback"]["latest_visible_cd_failure_classifier"] == (
+        "harbor_registry_public_route_unavailable"
+    )
+    assert payload["readback"]["latest_visible_cd_failure_status_code"] == "502"
+    assert payload["readback"]["latest_visible_cd_harbor_login_attempt_count"] == 12
+    assert payload["readback"]["latest_visible_cd_harbor_public_route_blocked"] is True
+    assert payload["rollups"]["current_main_cd_harbor_public_route_blocked"] is True
+
+
 def test_build_readback_flags_stale_cd_jobs_api_payload() -> None:
     module = _load_module()
     payload = module.build_readback(
@@ -166,6 +220,33 @@ def test_build_readback_flags_stale_cd_jobs_api_payload() -> None:
     assert payload["rollups"]["cd_run_jobs_stale_or_mismatched"] is True
 
 
+def test_harbor_blocker_takes_status_precedence_over_stale_jobs_payload() -> None:
+    module = _load_module()
+    payload = module.build_readback(
+        actions_html=_actions_html_failed_cd_run(),
+        actions_list_http_status=401,
+        actions_list_payload={"message": "token is required"},
+        cd_jobs_http_status=200,
+        cd_jobs_payload={
+            "jobs": [
+                {
+                    "run_id": 4045,
+                    "head_sha": "f4fb078100000000000000000000000000000000",
+                    "conclusion": "success",
+                }
+            ],
+            "total_count": 1,
+        },
+        latest_cd_build_log_http_status=200,
+        latest_cd_build_log_text=_harbor_blocked_log(),
+    )
+    assert payload["status"] == "blocked_harbor_public_route_unavailable"
+    assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True
+    assert payload["readback"]["latest_visible_cd_failure_classifier"] == (
+        "harbor_registry_public_route_unavailable"
+    )
+
+
 def test_derive_jobs_api_url_tracks_latest_visible_run_id() -> None:
     module = _load_module()
 
@@ -207,6 +288,7 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
     html_path = tmp_path / "actions.html"
     list_path = tmp_path / "actions-list.json"
     jobs_path = tmp_path / "jobs.json"
+    log_path = tmp_path / "build.log"
     html_path.write_text(_actions_html(), encoding="utf-8")
     list_path.write_text(
         json.dumps(
@@ -218,6 +300,7 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
         encoding="utf-8",
     )
     jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
+    log_path.write_text("", encoding="utf-8")
 
     result = subprocess.run(
         [
@@ -233,6 +316,10 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
             str(jobs_path),
             "--cd-run-jobs-http-status",
             "200",
+            "--cd-build-job-log-file",
+            str(log_path),
+            "--cd-build-job-log-http-status",
+            "200",
             "--json",
         ],
         check=False,
@@ -248,3 +335,45 @@ def test_cli_json_uses_fixture_files_without_network(tmp_path: Path) -> None:
         "awoooi-non110-ubuntu"
     )
     assert "192.168.0." not in result.stdout
+
+
+def test_cli_json_classifies_harbor_blocker_from_fixture_log(tmp_path: Path) -> None:
+    html_path = tmp_path / "actions.html"
+    list_path = tmp_path / "actions-list.json"
+    jobs_path = tmp_path / "jobs.json"
+    log_path = tmp_path / "build.log"
+    html_path.write_text(_actions_html_failed_cd_run(), encoding="utf-8")
+    list_path.write_text(json.dumps({"message": "token is required"}), encoding="utf-8")
+    jobs_path.write_text(json.dumps({"jobs": [], "total_count": 0}), encoding="utf-8")
+    log_path.write_text(_harbor_blocked_log(), encoding="utf-8")
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(SCRIPT),
+            "--actions-html-file",
+            str(html_path),
+            "--actions-list-json-file",
+            str(list_path),
+            "--actions-list-http-status",
+            "401",
+            "--cd-run-jobs-json-file",
+            str(jobs_path),
+            "--cd-run-jobs-http-status",
+            "200",
+            "--cd-build-job-log-file",
+            str(log_path),
+            "--cd-build-job-log-http-status",
+            "200",
+            "--json",
+        ],
+        check=False,
+        text=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    payload = json.loads(result.stdout)
+    assert payload["status"] == "blocked_harbor_public_route_unavailable"
+    assert payload["readback"]["latest_visible_cd_failure_status_code"] == "502"
+    assert "HARBOR_PASSWORD" not in result.stdout