diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index ab003e3cf..68433efbc 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -53465,3 +53465,28 @@ production browser smoke: **下一步**： - 轉回 P0 reboot SLO 主線：99 Windows / VMware autostart readback、111 reachability、backup full-dump coverage、service green 與 Wazuh degraded 依 scorecard blockers 順序收斂。 + +## 2026-07-02 — Gitea CD queue readback residual blocked-noise hardening + +**完成內容**： +- 修正 `ops/runner/read-public-gitea-actions-queue.py`：當 Gitea run-level UI 曾短暫顯示 `Blocked`，但 build log 已同時具備 `Job succeeded`、deploy marker pushed、`Production deploy readback matches this build` 與 build-deploy success notification 時，不再把 current main CD 誤判為 blocked。 +- 新增分層欄位：`latest_visible_cd_run_ui_blocked` 保留 UI 殘留訊號，`latest_visible_cd_run_blocked` 只代表仍缺成功部署證據的實際 blocker；另回傳 deploy marker SHA、source SHA、production deploy readback matched 與 notification receipt。 +- 新增單元測試固定今天的 false-blocked 模式，避免「成功部署但 Gitea UI residual blocked」再次干擾 reboot / CD readiness scorecard。 + +**live readback**： +- CD `#4467` 已讀回 `Success`；deploy marker 為 `14768cf6f chore(cd): deploy af3a289 [skip ci]`。 +- queue readback 讀回 `current_main_cd_run_blocked=false`、`latest_visible_cd_run_status=Success`、`latest_visible_cd_deploy_marker_sha_from_log=14768cf`、`latest_visible_cd_production_deploy_readback_matched_from_log=true`、`latest_visible_cd_build_deploy_success_notification_from_log=true`。 +- 歷史 `#4466` Harbor repair failure 保留為 `harbor_110_repair_blocked_raw=true`，但因 `harbor_110_repair_historical_after_latest_cd_success=true`，不再阻擋 current CD。 + +**本地驗證結果**： +- `python3.11 -m pytest ops/runner/test_read_public_gitea_actions_queue.py -q -p no:cacheprovider`：`40 passed`。 +- `python3.11 -m py_compile ops/runner/read-public-gitea-actions-queue.py`：通過。 +- `git diff --check`：通過。 + +**仍維持**： +- 沒有讀 secret / runner token / `.runner` 內容 / `.env` / raw sessions / SQLite / auth。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune。 + +**下一步**： +- commit / push 到 Gitea main 並讀回 CD；deploy 後回到 P0 reboot SLO scorecard blockers：99 VMware autostart readback、111 reachability、full Gitea dump/DB/issues/packages/LFS backup contract、service green 與 Wazuh degraded。 diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index 6a55c5c71..f825f074d 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -119,6 +119,17 @@ _HOST_PRESSURE_REFUSAL_RE = re.compile( r"refusing to start AWOOI image build while host web/build/smoke pressure is still active" ) _HOST_PRESSURE_INTERRUPTED_RE = re.compile(r"signal:\s*interrupt") +_CD_DEPLOY_MARKER_RE = re.compile( + r"Z\s+\[main\s+(?P[0-9a-f]{7,40})\]\s+" + r"chore\(cd\): deploy (?P[0-9a-f]{7,40}) \[skip ci\]" +) +_CD_PRODUCTION_DEPLOY_READBACK_MATCHED_RE = re.compile( + r"Z\s+(?:[^\w\"']+\s+)?Production deploy readback matches this build" +) +_CD_BUILD_DEPLOY_SUCCESS_NOTIFICATION_RE = re.compile( + r"Z\s+(?:[^\w\"']+\s+)?CI/CD build-deploy success notification mirrored" +) +_JOB_SUCCEEDED_RE = re.compile(r"Z\s+(?:[^\w\"']+\s+)?Job succeeded") @dataclass(frozen=True) @@ -404,7 +415,21 @@ def build_readback( ) latest_cd_status = latest_cd_run.get("status", "") latest_cd_success = latest_cd_status == "Success" - latest_cd_visible_blocked = latest_cd_status == "Blocked" + latest_cd_run_ui_blocked = latest_cd_status == "Blocked" + latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool( + latest_cd_run_ui_blocked + and build_log_classifier["job_succeeded"] + and build_log_classifier["deploy_marker_pushed"] + and build_log_classifier["production_deploy_readback_matched"] + and build_log_classifier["build_deploy_success_notification_sent"] + ) + latest_cd_visible_blocked = bool( + latest_cd_run_ui_blocked + and not latest_cd_deploy_readback_succeeded_despite_ui_blocked + ) + latest_cd_effective_success = bool( + latest_cd_success or latest_cd_deploy_readback_succeeded_despite_ui_blocked + ) latest_cd_waiting = latest_cd_status == "Waiting" host_pressure_waiting_from_stale_jobs = ( cd_jobs_stale_or_mismatched @@ -517,7 +542,7 @@ def build_readback( latest_cd_waiting and harbor_110_repair_running ) harbor_110_repair_historical_after_latest_cd_success = bool( - latest_cd_success + latest_cd_effective_success and latest_cd_run_id and harbor_110_repair_run_id and harbor_110_repair_run_id != latest_cd_run_id @@ -658,6 +683,10 @@ def build_readback( "latest_visible_cd_run_status": latest_cd_status, "latest_visible_cd_run_waiting": latest_cd_waiting, "latest_visible_cd_run_blocked": latest_cd_visible_blocked, + "latest_visible_cd_run_ui_blocked": latest_cd_run_ui_blocked, + "latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked": ( + latest_cd_deploy_readback_succeeded_despite_ui_blocked + ), "latest_visible_cd_run_kind": latest_cd_run.get("kind", ""), "latest_visible_cd_run_title": latest_cd_run.get("title", ""), "latest_visible_cd_run_commit_sha": latest_cd_run.get("commit_sha", ""), @@ -671,6 +700,24 @@ def build_readback( controlled_profile_no_matching_runner_labels ), "latest_visible_cd_build_log_http_status": latest_cd_build_log_http_status, + "latest_visible_cd_job_succeeded_from_log": build_log_classifier[ + "job_succeeded" + ], + "latest_visible_cd_deploy_marker_pushed_from_log": build_log_classifier[ + "deploy_marker_pushed" + ], + "latest_visible_cd_deploy_marker_sha_from_log": build_log_classifier[ + "deploy_marker_sha" + ], + "latest_visible_cd_deploy_marker_source_sha_from_log": ( + build_log_classifier["deploy_marker_source_sha"] + ), + "latest_visible_cd_production_deploy_readback_matched_from_log": ( + build_log_classifier["production_deploy_readback_matched"] + ), + "latest_visible_cd_build_deploy_success_notification_from_log": ( + build_log_classifier["build_deploy_success_notification_sent"] + ), "latest_visible_cd_failure_classifier": effective_cd_failure_classifier, "latest_visible_cd_failure_status_code": build_log_classifier[ "failure_status_code" @@ -983,7 +1030,7 @@ def build_readback( else "harbor_110_repair_jobs_stale_or_mismatched" if effective_harbor_110_repair_jobs_stale_or_mismatched else "cd_jobs_stale_or_mismatched" - if cd_jobs_stale_or_mismatched and not latest_cd_success + if cd_jobs_stale_or_mismatched and not latest_cd_effective_success else "no_matching_runner_not_visible" ), "readback": readback, @@ -1004,6 +1051,10 @@ def build_readback( current_cd_waiting_behind_harbor_110_repair_running ), "current_main_cd_run_blocked": latest_cd_visible_blocked, + "current_main_cd_run_ui_blocked": latest_cd_run_ui_blocked, + "current_main_cd_deploy_readback_succeeded_despite_ui_blocked": ( + latest_cd_deploy_readback_succeeded_despite_ui_blocked + ), "current_main_cd_no_matching_runner_label": ( latest_cd_no_matching_runner_label ), @@ -1502,7 +1553,23 @@ def classify_cd_build_log(text: str) -> dict[str, Any]: or bool(repair_skip_matches) or bool(repair_status_matches) ) + deploy_marker_matches = list(_CD_DEPLOY_MARKER_RE.finditer(text)) + latest_deploy_marker = deploy_marker_matches[-1] if deploy_marker_matches else None return { + "job_succeeded": _JOB_SUCCEEDED_RE.search(text) is not None, + "deploy_marker_pushed": latest_deploy_marker is not None, + "deploy_marker_sha": ( + latest_deploy_marker.group("marker_sha") if latest_deploy_marker else "" + ), + "deploy_marker_source_sha": ( + latest_deploy_marker.group("source_sha") if latest_deploy_marker else "" + ), + "production_deploy_readback_matched": ( + _CD_PRODUCTION_DEPLOY_READBACK_MATCHED_RE.search(text) is not None + ), + "build_deploy_success_notification_sent": ( + _CD_BUILD_DEPLOY_SUCCESS_NOTIFICATION_RE.search(text) is not None + ), "failure_classifier": ( "harbor_registry_public_route_unavailable" if harbor_public_route_blocked diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index a641cc292..067f4acd4 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -308,6 +308,16 @@ def _harbor_blocked_log() -> str: """ +def _cd_deploy_success_log() -> str: + return """ +2026-07-02T15:41:19.8265526Z [main 14768cf] chore(cd): deploy af3a289 [skip ci] +2026-07-02T15:41:22.3412803Z af3a289..14768cf main -> main +2026-07-02T15:42:01.6866643Z Production deploy readback matches this build and GitOps desired image tag (af3a2898c3) on attempt 1/36;matches_main=True +2026-07-02T15:42:11.9558448Z CI/CD build-deploy success notification mirrored through AWOOI API +2026-07-02T15:42:12.5523925Z Job succeeded +""" + + def _harbor_retrying_unavailable_log() -> str: return """ 2026-06-30T14:49:17.1327272Z harbor_login_attempt=1 registry_v2_status=502 @@ -1241,6 +1251,57 @@ def test_build_readback_prefers_visible_blocked_over_stale_jobs() -> None: assert payload["rollups"]["current_main_cd_run_blocked"] is True +def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_blocked_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + latest_cd_build_log_http_status=200, + latest_cd_build_log_text=_cd_deploy_success_log(), + ) + + assert payload["status"] == "no_matching_runner_not_visible" + assert payload["readback"]["latest_visible_cd_run_status"] == "Blocked" + assert payload["readback"]["latest_visible_cd_run_ui_blocked"] is True + assert payload["readback"]["latest_visible_cd_run_blocked"] is False + assert ( + payload["readback"][ + "latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked" + ] + is True + ) + assert payload["readback"]["latest_visible_cd_job_succeeded_from_log"] is True + assert payload["readback"]["latest_visible_cd_deploy_marker_pushed_from_log"] is True + assert payload["readback"]["latest_visible_cd_deploy_marker_sha_from_log"] == "14768cf" + assert ( + payload["readback"]["latest_visible_cd_deploy_marker_source_sha_from_log"] + == "af3a289" + ) + assert ( + payload["readback"][ + "latest_visible_cd_production_deploy_readback_matched_from_log" + ] + is True + ) + assert ( + payload["readback"][ + "latest_visible_cd_build_deploy_success_notification_from_log" + ] + is True + ) + assert payload["rollups"]["current_main_cd_run_ui_blocked"] is True + assert payload["rollups"]["current_main_cd_run_blocked"] is False + assert ( + payload["rollups"][ + "current_main_cd_deploy_readback_succeeded_despite_ui_blocked" + ] + is True + ) + + def test_build_readback_classifies_host_pressure_waiting() -> None: module = _load_module() payload = module.build_readback(