From c4fe100620a686b0e62c84113029ec81b040376c Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Jun 2026 19:31:46 +0800 Subject: [PATCH] fix(runner): flag stale gitea jobs readback --- docs/LOGBOOK.md | 9 +++ ...oot-cold-start-backup-recovery-workplan.md | 2 +- ops/runner/read-public-gitea-actions-queue.py | 48 ++++++++++++++++ .../test_read_public_gitea_actions_queue.py | 56 ++++++++++++++++++- 4 files changed, 113 insertions(+), 2 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index b0842175b..97a8bddad 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -21,6 +21,15 @@ **邊界**：未 SSH 寫 110，未重啟主機，未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall，未 restore / prune / DB write，未讀 secret / token / raw sessions / SQLite / `.env`，未使用 GitHub / `gh` / GitHub API。 +## 2026-06-30 — current P0-4 Gitea Actions jobs readback stale guard + +**照主線修正的問題**： +- Public Gitea visible run 顯示 `cd.yaml #4041` 對 `78418babf` 為 `Failure`，但 `actions/runs/4041/jobs` 回來的 jobs `head_sha` 是舊 commit，不能拿 jobs conclusion 當作這次 CD 成功證據。 +- `ops/runner/read-public-gitea-actions-queue.py` 新增 `cd_run_jobs_head_shas`、`cd_run_jobs_head_sha_matches_visible`、`cd_run_jobs_run_id_matches_visible` 與 `cd_run_jobs_stale_or_mismatched`；live #4041 現在正確分類為 `status=cd_jobs_stale_or_mismatched`。 +- `test_read_public_gitea_actions_queue.py` 增加 stale jobs payload regression test；P0-4 後續只以 visible run / production deploy readback / deploy marker 三層一致作為成功，不再被 jobs API 舊資料誤導。 + +**邊界**：只讀 public Gitea HTML / API；job logs endpoint 回 `401 token is required`，未讀 token / secret / raw sessions / SQLite / `.env`，未 workflow_dispatch，未使用 GitHub / `gh` / GitHub API。 + ## 2026-06-30 — current P0 readback after verified Gitea bundle **目前不能宣稱完成的事實**： diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 4d6ccc4e3..59b0d8411 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -18,7 +18,7 @@ | P0-1 | BLOCKED | 全主機 cold-start / 10 分鐘自動恢復 SLO | 最新 `full-stack-cold-start-check.sh --monitor-read-only --no-color` 回 `PASS=67 WARN=4 BLOCKED=5`；110 registry external `/v2`、110 SSH read-only check、K3s registry pull refused、AWOOOI internal API probe、SigNoz TLS/public route 仍 blocked。 | 先修第一個 cold-start blocker，重跑同一 scorecard 到 `BLOCKED=0`；不可只用 route 200 宣稱恢復。 | | P0-2 | DONE_THIS_INCIDENT | 使用者可見 502：Tsenyang | `www.tsenyang.com` / `tsenyang.com` 由 502 恢復為 200；188 `tsenyang-website` container running；local `127.0.0.1:3000` 回 200。 | 下次同類 502 先查 release symlink / image / container；不先動 Nginx、DNS、DB、主機重啟。 | | P0-3 | BLOCKED | StockPlatform data freshness | public `/healthz`、`/api/healthz` 回 200；freshness / ingestion 回 `not_configured`、`postgres_not_ready`。 | 恢復 110 control path 後，read-only 查 `/home/wooo/stockplatform-v2` compose / DB schema / migration status；禁止 fake freshness、manual DB rows、restore/prune。 | -| P0-4 | BLOCKED | AWOOOI production 版本最新性 | source/Gitea `main=3b65f8762`；production runtime readback 仍為 `7890778b83`，`runtime_build_readback_status=runtime_build_diverges_from_committed_deploy_readback`。 | 補 deploy marker / runtime SHA / endpoint readback 一致；未一致前不可宣稱 AWOOOI 最新。 | +| P0-4 | BLOCKED | AWOOOI production 版本最新性 | source/Gitea `main=78418babf`；production runtime readback 仍為 `7890778b83`，`runtime_build_readback_status=runtime_build_diverges_from_committed_deploy_readback`。Public Gitea visible run `cd.yaml #4041` 是 Failure；jobs API 與 visible run `head_sha` 不一致，已標 `cd_jobs_stale_or_mismatched`。 | 補 deploy marker / runtime SHA / endpoint readback 一致；未一致前不可宣稱 AWOOOI 最新。 | | P0-5 | BLOCKED | 110 control path | `diagnose-110-ssh-publickey-auth.sh`：node-exporter / SSH banner 正常；`NODE_LOAD_CLASSIFIER=high_load`、`NODE_PROCS_BLOCKED=0`；`wooo` publickey `publickey_offer_timeout`，`root` publickey `permission_denied`，`git` / `ollama` `preauth_timeout`。 | 集中查 110 sshd publickey auth / authorized_keys / PAM / account lookup path，並把 load / runner pressure 視為同一 blocker 的共因；可在 110 console/local 跑 `repair-110-ssh-publickey-auth-local.sh --check` / `--apply` 修 metadata 權限。恢復 SSH read-only command path 後才能驗證 Stock DB、Gitea dump、110 backup completeness。 | | P0-6 | BLOCKED_BACKUP_COMPLETENESS | Gitea repo visibility 與完整備份 | Gitea version API 200；public repo search 只列 4 個 public repo；`stockplatform-v2` public page/API 404，但 internal `git ls-remote` 成功；188 `/home/ollama/backup/110/gitea` 起初為空。已建立 verified emergency bundle `/home/ollama/backup/110/gitea/git-bundles/20260630-190931`：4 個 public/internal repo bundle verify + checksum 成功，`AwoooGo`、`stockplatform-v2`、`vibework` 因 private auth fail-closed。 | 188 `gitea_repo_mirror_from_110` subtree metric / alert 已補；下一步仍是恢復 110 SSH command path 後跑正式 `gitea dump`、private repo 非互動備份、repo count 與 restore drill readback。 | | P0-7 | SOURCE_READY_RUNTIME_BLOCKED | 99 VMware / VM autostart | repo 已有 `windows99-vmware-autostart.ps1`，但 99 SSH/WinRM 尚未可用；VM host 111 仍不可達。 | 恢復 99 可控通道或由 console 套用腳本；完成後讀回 111/188/120/121/112 boot evidence。 | diff --git a/ops/runner/read-public-gitea-actions-queue.py b/ops/runner/read-public-gitea-actions-queue.py index 4013bde40..862917b41 100644 --- a/ops/runner/read-public-gitea-actions-queue.py +++ b/ops/runner/read-public-gitea-actions-queue.py @@ -155,6 +155,45 @@ def build_readback( actions_list = actions_list_payload if isinstance(actions_list_payload, dict) else {} actions_list_message = str(actions_list.get("message") or "") jobs_total_count = _int(cd_jobs.get("total_count")) + jobs = cd_jobs.get("jobs") if isinstance(cd_jobs.get("jobs"), list) else [] + latest_cd_run_id = latest_cd_run.get("run_id", "") + latest_cd_commit_sha = latest_cd_run.get("commit_sha", "") + job_head_shas = sorted( + { + str(job.get("head_sha") or "") + for job in jobs + if isinstance(job, dict) and job.get("head_sha") + } + ) + job_run_ids = sorted( + { + str(job.get("run_id") or "") + for job in jobs + if isinstance(job, dict) and job.get("run_id") is not None + } + ) + job_conclusion_counts: dict[str, int] = {} + for job in jobs: + if isinstance(job, dict): + conclusion = str(job.get("conclusion") or job.get("status") or "unknown") + job_conclusion_counts[conclusion] = job_conclusion_counts.get(conclusion, 0) + 1 + cd_jobs_head_sha_matches_visible = ( + bool(latest_cd_commit_sha) + and latest_cd_commit_sha in job_head_shas + ) + cd_jobs_run_id_matches_visible = ( + bool(latest_cd_run_id) + and latest_cd_run_id in job_run_ids + ) + cd_jobs_stale_or_mismatched = ( + cd_jobs_http_status == 200 + and jobs_total_count > 0 + and bool(latest_cd_run) + and ( + not cd_jobs_head_sha_matches_visible + or not cd_jobs_run_id_matches_visible + ) + ) readback = { "actions_page_visible_run_count": len(visible_runs), @@ -162,6 +201,12 @@ def build_readback( "actions_list_without_token_message": actions_list_message, "cd_run_jobs_http_status": cd_jobs_http_status, "cd_run_jobs_total_count": jobs_total_count, + "cd_run_jobs_head_shas": job_head_shas, + "cd_run_jobs_run_ids": job_run_ids, + "cd_run_jobs_conclusion_counts": job_conclusion_counts, + "cd_run_jobs_head_sha_matches_visible": cd_jobs_head_sha_matches_visible, + "cd_run_jobs_run_id_matches_visible": cd_jobs_run_id_matches_visible, + "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched, "latest_visible_no_matching_runner_run_id": no_matching.get("run_id", ""), "latest_visible_no_matching_runner_workflow": no_matching.get( "workflow", "" @@ -184,6 +229,8 @@ def build_readback( "status": ( "blocked_no_matching_online_runner" if no_matching + else "cd_jobs_stale_or_mismatched" + if cd_jobs_stale_or_mismatched else "no_matching_runner_not_visible" ), "readback": readback, @@ -191,6 +238,7 @@ def build_readback( "public_actions_readback_count": len(visible_runs), "actions_list_requires_token": actions_list_http_status == 401, "cd_run_jobs_total_count": jobs_total_count, + "cd_run_jobs_stale_or_mismatched": cd_jobs_stale_or_mismatched, "current_main_cd_run_visible": bool(latest_cd_run), "current_main_cd_run_status": latest_cd_run.get("status", ""), "no_matching_online_runner_visible": bool(no_matching), diff --git a/ops/runner/test_read_public_gitea_actions_queue.py b/ops/runner/test_read_public_gitea_actions_queue.py index a34ad8b83..8582bba81 100644 --- a/ops/runner/test_read_public_gitea_actions_queue.py +++ b/ops/runner/test_read_public_gitea_actions_queue.py @@ -70,6 +70,27 @@ def _actions_html_full_run_items() -> str: """ +def _actions_html_single_cd_run() -> str: + return """ +
+
+
+ +
+
+ + chore(deploy): retry non110 cd after ci image seed + +
+ cd.yaml #3863:Commit + a70c6756d9 +
+
+
+
+""" + + def test_parse_visible_runs_extracts_no_matching_runner_label() -> None: module = _load_module() runs = module.parse_visible_runs(_actions_html()) @@ -96,7 +117,7 @@ def test_parse_visible_runs_keeps_full_gitea_rows_aligned() -> None: def test_build_readback_reports_latest_visible_cd_run() -> None: module = _load_module() payload = module.build_readback( - actions_html=_actions_html_full_run_items(), + actions_html=_actions_html_single_cd_run(), actions_list_http_status=401, actions_list_payload={"message": "token is required"}, cd_jobs_http_status=200, @@ -110,6 +131,39 @@ def test_build_readback_reports_latest_visible_cd_run() -> None: ) assert payload["rollups"]["current_main_cd_run_visible"] is True assert payload["rollups"]["current_main_cd_run_status"] == "Running" + assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is False + + +def test_build_readback_flags_stale_cd_jobs_api_payload() -> None: + module = _load_module() + payload = module.build_readback( + actions_html=_actions_html_single_cd_run(), + actions_list_http_status=401, + actions_list_payload={"message": "token is required"}, + cd_jobs_http_status=200, + cd_jobs_payload={ + "jobs": [ + { + "run_id": 3863, + "name": "tests", + "head_sha": "b17a28c29375afd207a90d05b4898daaa0f3f730", + "conclusion": "success", + } + ], + "total_count": 1, + }, + ) + assert payload["status"] == "cd_jobs_stale_or_mismatched" + assert payload["readback"]["latest_visible_cd_run_commit_sha"] == ( + "a70c6756d9e76c33143676eef82bab7a49ac1839" + ) + assert payload["readback"]["cd_run_jobs_head_shas"] == [ + "b17a28c29375afd207a90d05b4898daaa0f3f730" + ] + assert payload["readback"]["cd_run_jobs_head_sha_matches_visible"] is False + assert payload["readback"]["cd_run_jobs_run_id_matches_visible"] is True + assert payload["readback"]["cd_run_jobs_stale_or_mismatched"] is True + assert payload["rollups"]["cd_run_jobs_stale_or_mismatched"] is True def test_build_readback_sanitizes_actions_api_internal_url() -> None: