From 1b3b0fd572fde7bef3cd40a64ec992e9871abf91 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 23:56:17 +0800 Subject: [PATCH] fix(cd): treat deploy evidence as effective success --- docs/LOGBOOK.md | 4 +- ops/runner/read-public-gitea-actions-queue.py | 34 ++++++++--- .../test_read_public_gitea_actions_queue.py | 59 +++++++++++++++++++ 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 68433efbc..3bbcd52aa 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -53470,13 +53470,15 @@ production browser smoke: **完成內容**： - 修正 `ops/runner/read-public-gitea-actions-queue.py`：當 Gitea run-level UI 曾短暫顯示 `Blocked`，但 build log 已同時具備 `Job succeeded`、deploy marker pushed、`Production deploy readback matches this build` 與 build-deploy success notification 時，不再把 current main CD 誤判為 blocked。 +- 擴大 effective-success 判定：當 run-level UI 仍停在 `Running`，但 build log 已具備完整成功鏈，也視為 `latest_visible_cd_deploy_evidence_succeeded_from_log=true`，同時用 `latest_visible_cd_run_level_lag_after_deploy_success=true` 保留 UI lag 訊號。 - 新增分層欄位：`latest_visible_cd_run_ui_blocked` 保留 UI 殘留訊號，`latest_visible_cd_run_blocked` 只代表仍缺成功部署證據的實際 blocker；另回傳 deploy marker SHA、source SHA、production deploy readback matched 與 notification receipt。 -- 新增單元測試固定今天的 false-blocked 模式，避免「成功部署但 Gitea UI residual blocked」再次干擾 reboot / CD readiness scorecard。 +- 新增單元測試固定今天的 false-blocked / false-running 模式，避免「成功部署但 Gitea UI residual blocked/running」再次干擾 reboot / CD readiness scorecard。 **live readback**： - CD `#4467` 已讀回 `Success`；deploy marker 為 `14768cf6f chore(cd): deploy af3a289 [skip ci]`。 - queue readback 讀回 `current_main_cd_run_blocked=false`、`latest_visible_cd_run_status=Success`、`latest_visible_cd_deploy_marker_sha_from_log=14768cf`、`latest_visible_cd_production_deploy_readback_matched_from_log=true`、`latest_visible_cd_build_deploy_success_notification_from_log=true`。 - 歷史 `#4466` Harbor repair failure 保留為 `harbor_110_repair_blocked_raw=true`，但因 `harbor_110_repair_historical_after_latest_cd_success=true`，不再阻擋 current CD。 +- 後續 CD `#4468` 觀察到 run-level 仍停 `Running`，但 log 已有 deploy marker `93a1606`、production deploy readback matched、success notification 與 `Job succeeded`；已把此模式納入同一個 residual-noise hardening。 **本地驗證結果**： - `python3.11 -m pytest ops/runner/test_read_public_gitea_actions_queue.py -q -p no:cacheprovider`：`40 passed`。 diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index f825f074d..2f4694a54 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -415,22 +415,30 @@ def build_readback( ) latest_cd_status = latest_cd_run.get("status", "") latest_cd_success = latest_cd_status == "Success" - latest_cd_run_ui_blocked = latest_cd_status == "Blocked" - latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool( - latest_cd_run_ui_blocked - and build_log_classifier["job_succeeded"] + latest_cd_deploy_evidence_succeeded = bool( + build_log_classifier["job_succeeded"] and build_log_classifier["deploy_marker_pushed"] and build_log_classifier["production_deploy_readback_matched"] and build_log_classifier["build_deploy_success_notification_sent"] ) + latest_cd_run_ui_blocked = latest_cd_status == "Blocked" + latest_cd_deploy_readback_succeeded_despite_ui_blocked = bool( + latest_cd_run_ui_blocked and latest_cd_deploy_evidence_succeeded + ) + latest_cd_run_level_lag_after_deploy_success = bool( + latest_cd_status in {"Waiting", "Running", "Blocked"} + and latest_cd_deploy_evidence_succeeded + ) latest_cd_visible_blocked = bool( latest_cd_run_ui_blocked - and not latest_cd_deploy_readback_succeeded_despite_ui_blocked + and not latest_cd_deploy_evidence_succeeded ) latest_cd_effective_success = bool( - latest_cd_success or latest_cd_deploy_readback_succeeded_despite_ui_blocked + latest_cd_success or latest_cd_deploy_evidence_succeeded + ) + latest_cd_waiting = bool( + latest_cd_status == "Waiting" and not latest_cd_deploy_evidence_succeeded ) - latest_cd_waiting = latest_cd_status == "Waiting" host_pressure_waiting_from_stale_jobs = ( cd_jobs_stale_or_mismatched and latest_cd_status in {"Blocked", "Canceled", "Failure"} @@ -684,6 +692,12 @@ def build_readback( "latest_visible_cd_run_waiting": latest_cd_waiting, "latest_visible_cd_run_blocked": latest_cd_visible_blocked, "latest_visible_cd_run_ui_blocked": latest_cd_run_ui_blocked, + "latest_visible_cd_deploy_evidence_succeeded_from_log": ( + latest_cd_deploy_evidence_succeeded + ), + "latest_visible_cd_run_level_lag_after_deploy_success": ( + latest_cd_run_level_lag_after_deploy_success + ), "latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked": ( latest_cd_deploy_readback_succeeded_despite_ui_blocked ), @@ -1052,6 +1066,12 @@ def build_readback( ), "current_main_cd_run_blocked": latest_cd_visible_blocked, "current_main_cd_run_ui_blocked": latest_cd_run_ui_blocked, + "current_main_cd_deploy_evidence_succeeded_from_log": ( + latest_cd_deploy_evidence_succeeded + ), + "current_main_cd_run_level_lag_after_deploy_success": ( + latest_cd_run_level_lag_after_deploy_success + ), "current_main_cd_deploy_readback_succeeded_despite_ui_blocked": ( latest_cd_deploy_readback_succeeded_despite_ui_blocked ), diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index 067f4acd4..7bc6babea 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -1267,6 +1267,14 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() -> assert payload["readback"]["latest_visible_cd_run_status"] == "Blocked" assert payload["readback"]["latest_visible_cd_run_ui_blocked"] is True assert payload["readback"]["latest_visible_cd_run_blocked"] is False + assert ( + payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"] + is True + ) + assert ( + payload["readback"]["latest_visible_cd_run_level_lag_after_deploy_success"] + is True + ) assert ( payload["readback"][ "latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked" @@ -1294,6 +1302,14 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() -> ) assert payload["rollups"]["current_main_cd_run_ui_blocked"] is True assert payload["rollups"]["current_main_cd_run_blocked"] is False + assert ( + payload["rollups"]["current_main_cd_deploy_evidence_succeeded_from_log"] + is True + ) + assert ( + payload["rollups"]["current_main_cd_run_level_lag_after_deploy_success"] + is True + ) assert ( payload["rollups"][ "current_main_cd_deploy_readback_succeeded_despite_ui_blocked" @@ -1302,6 +1318,49 @@ def test_build_readback_downgrades_ui_blocked_after_deploy_readback_success() -> ) +def test_build_readback_downgrades_running_lag_after_deploy_success() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_single_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={"jobs": [], "total_count": 0}, + latest_cd_build_log_http_status=200, + latest_cd_build_log_text=_cd_deploy_success_log(), + ) + + assert payload["status"] == "no_matching_runner_not_visible" + assert payload["readback"]["latest_visible_cd_run_status"] == "Running" + assert payload["readback"]["latest_visible_cd_run_waiting"] is False + assert payload["readback"]["latest_visible_cd_run_blocked"] is False + assert ( + payload["readback"]["latest_visible_cd_deploy_evidence_succeeded_from_log"] + is True + ) + assert ( + payload["readback"]["latest_visible_cd_run_level_lag_after_deploy_success"] + is True + ) + assert ( + payload["readback"][ + "latest_visible_cd_deploy_readback_succeeded_despite_ui_blocked" + ] + is False + ) + assert payload["rollups"]["current_main_cd_run_status"] == "Running" + assert payload["rollups"]["current_main_cd_run_waiting"] is False + assert payload["rollups"]["current_main_cd_run_blocked"] is False + assert ( + payload["rollups"]["current_main_cd_deploy_evidence_succeeded_from_log"] + is True + ) + assert ( + payload["rollups"]["current_main_cd_run_level_lag_after_deploy_success"] + is True + ) + + def test_build_readback_classifies_host_pressure_waiting() -> None: module = _load_module() payload = module.build_readback(