From d6e5961296a5d6751074b2eda74433710cc2f224 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 1 Jul 2026 08:50:50 +0800 Subject: [PATCH] docs(recovery): record cpu circuit breaker readback [skip ci] --- docs/LOGBOOK.md | 22 +++++++++++++++ ...re-drain-readback-2026-07-01.snapshot.json | 28 ++++++++++++------- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 703036d50..40adcbb07 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -51189,3 +51189,25 @@ production browser smoke: **下一步**： - rebase / commit / push truth-chain source split；讀回 Gitea queue 與 188 DB active query。若 deploy 收斂，立即 reset `awoooi` role 的 temporary `max_parallel_workers_per_gather=0`；若 110 SSH 恢復，第一動作仍是套用新版 systemd exporter 或暫停舊 exporter，再回讀 systemd/logind、Gitea CPU 與 load5/core。 + +## 2026-07-01 — 08:50 P0 188 DB circuit breaker post-push readback + +**完成內容**： +- source split 已 commit / rebase / push 到 Gitea main：`abc512a7b fix(db): split inbound truth-chain hot lookup`。 +- Gitea queue 讀回新 CD `#4200`，commit `abc512a7b3911f44c48668d06965423f56471ab1`，狀態 `Waiting`；代表 120 production 尚未跑到新 source split，不能宣稱 deploy 完成。 +- 因 CD waiting，188 加第二層暫時 circuit breaker：`ALTER ROLE awoooi SET statement_timeout='1500ms'`，取消舊 OR 查詢並重建 120 app DB sessions。 +- 因舊 OR 查詢仍反覆出現，188 再加 planner nudge：`ALTER ROLE awoooi SET enable_seqscan=off`，配合已套用的 GIN / trigram indexes 讓舊查詢優先吃索引；這是 temporary runtime mitigation，不是永久架構。 +- 最新 188 post-readback：role config=`max_parallel_workers_per_gather=0,statement_timeout=1500ms,enable_seqscan=off`；`local_parallel_workers=0`；active 舊 OR query 約 `3` 條且 age < `250ms`；`k3s-postgres-recovery` 從約 `400.19%` 降到 `173.33%`；host load 約 `4.50 / 5.79 / 7.52`。 +- 110 最新 readback：`node_load1=11.11`、`node_load5=20.14`、`node_load15=21.94`、`awoooi_host_load5_per_core=1.9`、`gitea=3.4019` cores；短 SSH 仍 timeout，尚未能套用新版 `systemd-units-textfile-exporter.py` 或停掉舊 exporter / htop。 + +**rollback / 後續必做**： +- 120 部署 source split 並讀回舊 OR query 消失後，立即執行： + - `ALTER ROLE awoooi RESET max_parallel_workers_per_gather;` + - `ALTER ROLE awoooi RESET statement_timeout;` + - `ALTER ROLE awoooi RESET enable_seqscan;` +- 110 SSH/control path 恢復後第一個 runtime 動作仍是套用新版 systemd exporter 或暫停舊 exporter，再回讀 systemd/logind、Gitea CPU、`systemd_units.prom` 與 load5/core。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有讀 `.runner` 內容。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune。 diff --git a/docs/operations/host-cpu-pressure-drain-readback-2026-07-01.snapshot.json b/docs/operations/host-cpu-pressure-drain-readback-2026-07-01.snapshot.json index 9d6e22b9b..c0e6c7bbf 100644 --- a/docs/operations/host-cpu-pressure-drain-readback-2026-07-01.snapshot.json +++ b/docs/operations/host-cpu-pressure-drain-readback-2026-07-01.snapshot.json @@ -1,6 +1,6 @@ { "schema_version": "awoooi_host_cpu_pressure_drain_readback_v1", - "generated_at": "2026-07-01T08:44:00+08:00", + "generated_at": "2026-07-01T08:50:00+08:00", "status": "partial_188_bounded_source_fix_ready_110_control_path_blocked", "scope": { "hosts": ["188", "110"], @@ -39,10 +39,10 @@ { "host": "188", "target_selector": "awoooi PostgreSQL role sessions from 192.168.0.120 running old awooop_conversation_event OR lookup", - "action": "set max_parallel_workers_per_gather=0 for role, cancel active old OR queries, and terminate 192.168.0.120 app DB sessions so new sessions inherit the bound", + "action": "set max_parallel_workers_per_gather=0, statement_timeout=1500ms, and enable_seqscan=off for role; cancel active old OR queries and terminate 192.168.0.120 app DB sessions so new sessions inherit the bound", "reason": "the live 120 API was still repeatedly issuing the pre-fix OR query and spawning parallel workers after index apply", - "rollback": "ALTER ROLE awoooi RESET max_parallel_workers_per_gather; then let the app reconnect after source split deploys", - "post_apply_readback": "rolconfig shows max_parallel_workers_per_gather=0; local parallel workers for the old query dropped to 0" + "rollback": "ALTER ROLE awoooi RESET max_parallel_workers_per_gather; ALTER ROLE awoooi RESET statement_timeout; ALTER ROLE awoooi RESET enable_seqscan; then let the app reconnect after source split deploys", + "post_apply_readback": "rolconfig shows max_parallel_workers_per_gather=0, statement_timeout=1500ms, enable_seqscan=off; local parallel workers for the old query stayed 0 and k3s-postgres-recovery dropped to 173.33 percent" } ], "source_fixes": [ @@ -84,19 +84,26 @@ "docker_stats_textfile_k3s_postgres_recovery_cores": "0.406900" }, "post_bounded_db_mitigation_signals": { - "awoooi_role_config": "max_parallel_workers_per_gather=0", - "active_old_or_queries": "4", + "awoooi_role_config": "max_parallel_workers_per_gather=0, statement_timeout=1500ms, enable_seqscan=off", + "active_old_or_queries": "3", "local_parallel_workers_for_old_or_query": "0", - "k3s_postgres_recovery_docker_stats_percent": "400.19", + "k3s_postgres_recovery_docker_stats_percent": "173.33", + "host_load_average": "4.50, 5.79, 7.52", "remaining_root_cause": "120 live API still runs the pre-fix OR query until the source split is deployed" + }, + "deployment_readback": { + "commit": "abc512a7b3911f44c48668d06965423f56471ab1", + "gitea_cd_run": "4200", + "gitea_cd_status": "Waiting", + "source_split_live_on_120": false } }, "host_110": { "signals": { "load5_initial": "39.14", - "load5_latest": "29.34", - "load5_per_core_latest": "2.068333", - "node_procs_running_latest": "31", + "load5_latest": "20.14", + "load5_per_core_latest": "1.9", + "node_procs_running_latest": "39", "gitea_container_cpu_cores": "3.4019", "ssh_control_path": "timeout", "systemd_dbus_symptom": "systemd-logind pending replies exhausted; systemctl list/show timeout", @@ -105,6 +112,7 @@ "not_completed_live": [ "could not apply pkill/exporter drain on 110 because SSH command sessions timeout", "scp and stdin-over-ssh attempts to place the new exporter file timed out", + "latest short SSH readback still timed out with server not responding", "could not restart or reexec systemd/logind because that would require a control path that is currently unavailable" ] }