docs(recovery): record cpu circuit breaker readback [skip ci]
This commit is contained in:
@@ -51189,3 +51189,25 @@ production browser smoke:
|
||||
|
||||
**下一步**:
|
||||
- rebase / commit / push truth-chain source split;讀回 Gitea queue 與 188 DB active query。若 deploy 收斂,立即 reset `awoooi` role 的 temporary `max_parallel_workers_per_gather=0`;若 110 SSH 恢復,第一動作仍是套用新版 systemd exporter 或暫停舊 exporter,再回讀 systemd/logind、Gitea CPU 與 load5/core。
|
||||
|
||||
## 2026-07-01 — 08:50 P0 188 DB circuit breaker post-push readback
|
||||
|
||||
**完成內容**:
|
||||
- source split 已 commit / rebase / push 到 Gitea main:`abc512a7b fix(db): split inbound truth-chain hot lookup`。
|
||||
- Gitea queue 讀回新 CD `#4200`,commit `abc512a7b3911f44c48668d06965423f56471ab1`,狀態 `Waiting`;代表 120 production 尚未跑到新 source split,不能宣稱 deploy 完成。
|
||||
- 因 CD waiting,188 加第二層暫時 circuit breaker:`ALTER ROLE awoooi SET statement_timeout='1500ms'`,取消舊 OR 查詢並重建 120 app DB sessions。
|
||||
- 因舊 OR 查詢仍反覆出現,188 再加 planner nudge:`ALTER ROLE awoooi SET enable_seqscan=off`,配合已套用的 GIN / trigram indexes 讓舊查詢優先吃索引;這是 temporary runtime mitigation,不是永久架構。
|
||||
- 最新 188 post-readback:role config=`max_parallel_workers_per_gather=0,statement_timeout=1500ms,enable_seqscan=off`;`local_parallel_workers=0`;active 舊 OR query 約 `3` 條且 age < `250ms`;`k3s-postgres-recovery` 從約 `400.19%` 降到 `173.33%`;host load 約 `4.50 / 5.79 / 7.52`。
|
||||
- 110 最新 readback:`node_load1=11.11`、`node_load5=20.14`、`node_load15=21.94`、`awoooi_host_load5_per_core=1.9`、`gitea=3.4019` cores;短 SSH 仍 timeout,尚未能套用新版 `systemd-units-textfile-exporter.py` 或停掉舊 exporter / htop。
|
||||
|
||||
**rollback / 後續必做**:
|
||||
- 120 部署 source split 並讀回舊 OR query 消失後,立即執行:
|
||||
- `ALTER ROLE awoooi RESET max_parallel_workers_per_gather;`
|
||||
- `ALTER ROLE awoooi RESET statement_timeout;`
|
||||
- `ALTER ROLE awoooi RESET enable_seqscan;`
|
||||
- 110 SSH/control path 恢復後第一個 runtime 動作仍是套用新版 systemd exporter 或暫停舊 exporter,再回讀 systemd/logind、Gitea CPU、`systemd_units.prom` 與 load5/core。
|
||||
|
||||
**仍維持**:
|
||||
- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth;沒有讀 `.runner` 內容。
|
||||
- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。
|
||||
- 沒有重啟主機,沒有 Docker / Nginx / K3s / DB restart,沒有 workflow_dispatch,沒有 DROP / TRUNCATE / restore / prune。
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schema_version": "awoooi_host_cpu_pressure_drain_readback_v1",
|
||||
"generated_at": "2026-07-01T08:44:00+08:00",
|
||||
"generated_at": "2026-07-01T08:50:00+08:00",
|
||||
"status": "partial_188_bounded_source_fix_ready_110_control_path_blocked",
|
||||
"scope": {
|
||||
"hosts": ["188", "110"],
|
||||
@@ -39,10 +39,10 @@
|
||||
{
|
||||
"host": "188",
|
||||
"target_selector": "awoooi PostgreSQL role sessions from 192.168.0.120 running old awooop_conversation_event OR lookup",
|
||||
"action": "set max_parallel_workers_per_gather=0 for role, cancel active old OR queries, and terminate 192.168.0.120 app DB sessions so new sessions inherit the bound",
|
||||
"action": "set max_parallel_workers_per_gather=0, statement_timeout=1500ms, and enable_seqscan=off for role; cancel active old OR queries and terminate 192.168.0.120 app DB sessions so new sessions inherit the bound",
|
||||
"reason": "the live 120 API was still repeatedly issuing the pre-fix OR query and spawning parallel workers after index apply",
|
||||
"rollback": "ALTER ROLE awoooi RESET max_parallel_workers_per_gather; then let the app reconnect after source split deploys",
|
||||
"post_apply_readback": "rolconfig shows max_parallel_workers_per_gather=0; local parallel workers for the old query dropped to 0"
|
||||
"rollback": "ALTER ROLE awoooi RESET max_parallel_workers_per_gather; ALTER ROLE awoooi RESET statement_timeout; ALTER ROLE awoooi RESET enable_seqscan; then let the app reconnect after source split deploys",
|
||||
"post_apply_readback": "rolconfig shows max_parallel_workers_per_gather=0, statement_timeout=1500ms, enable_seqscan=off; local parallel workers for the old query stayed 0 and k3s-postgres-recovery dropped to 173.33 percent"
|
||||
}
|
||||
],
|
||||
"source_fixes": [
|
||||
@@ -84,19 +84,26 @@
|
||||
"docker_stats_textfile_k3s_postgres_recovery_cores": "0.406900"
|
||||
},
|
||||
"post_bounded_db_mitigation_signals": {
|
||||
"awoooi_role_config": "max_parallel_workers_per_gather=0",
|
||||
"active_old_or_queries": "4",
|
||||
"awoooi_role_config": "max_parallel_workers_per_gather=0, statement_timeout=1500ms, enable_seqscan=off",
|
||||
"active_old_or_queries": "3",
|
||||
"local_parallel_workers_for_old_or_query": "0",
|
||||
"k3s_postgres_recovery_docker_stats_percent": "400.19",
|
||||
"k3s_postgres_recovery_docker_stats_percent": "173.33",
|
||||
"host_load_average": "4.50, 5.79, 7.52",
|
||||
"remaining_root_cause": "120 live API still runs the pre-fix OR query until the source split is deployed"
|
||||
},
|
||||
"deployment_readback": {
|
||||
"commit": "abc512a7b3911f44c48668d06965423f56471ab1",
|
||||
"gitea_cd_run": "4200",
|
||||
"gitea_cd_status": "Waiting",
|
||||
"source_split_live_on_120": false
|
||||
}
|
||||
},
|
||||
"host_110": {
|
||||
"signals": {
|
||||
"load5_initial": "39.14",
|
||||
"load5_latest": "29.34",
|
||||
"load5_per_core_latest": "2.068333",
|
||||
"node_procs_running_latest": "31",
|
||||
"load5_latest": "20.14",
|
||||
"load5_per_core_latest": "1.9",
|
||||
"node_procs_running_latest": "39",
|
||||
"gitea_container_cpu_cores": "3.4019",
|
||||
"ssh_control_path": "timeout",
|
||||
"systemd_dbus_symptom": "systemd-logind pending replies exhausted; systemctl list/show timeout",
|
||||
@@ -105,6 +112,7 @@
|
||||
"not_completed_live": [
|
||||
"could not apply pkill/exporter drain on 110 because SSH command sessions timeout",
|
||||
"scp and stdin-over-ssh attempts to place the new exporter file timed out",
|
||||
"latest short SSH readback still timed out with server not responding",
|
||||
"could not restart or reexec systemd/logind because that would require a control path that is currently unavailable"
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user