From 0ca72df655242d679b2564cc5dc1428b7da6588b Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 00:40:48 +0800 Subject: [PATCH] docs(ops): record 110 gitea pressure closure [skip ci] --- docs/LOGBOOK.md | 20 ++++++++++++++++ ...recovery-readback-2026-07-01.snapshot.json | 23 ++++++++++++++++++- docs/runbooks/FULL-STACK-COLD-START-SOP.md | 10 +++++++- .../full-stack-cold-start-baseline.yml | 5 ++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 843775760..7cb6a4019 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -52012,6 +52012,26 @@ production browser smoke: **下一步**： - 同步新版 exporter / evidence script 到 110 live textfile path，跑一次 read-only exporter，讀回 active CI metrics；再 rebase / commit / push source。 +## 2026-07-02 — 00:40 P0 110 Gitea cap 收斂與 CPU alerts 清除 + +**完成內容**： +- 新版 `host-runaway-process-exporter.py` / `host-sustained-load-evidence.py` 已同步到 110 `/home/wooo/scripts/`，舊檔備份 suffix `before-hostedtoolcache-ci-classifier-20260702-003258`；手動跑 exporter 成功寫入 `/home/wooo/node_exporter_textfiles/host_runaway_process.prom`。 +- 同輪 Gitea 仍貼近 2 CPU cap，Prometheus 讀回 `gitea=2.2697 cores` 並觸發 `DockerContainerCpuSustainedHigh`；這不是 Stock DB 未限流，Stock DB 已在 cgroup 下。 +- 已 live 套用：`docker update --cpus=1.5 gitea`。此操作只改 Docker cgroup metadata，沒有重啟 Gitea container。 +- post-apply inspect：`running=true nanocpus=1500000000 memory=3221225472 restart=always`；Gitea API `/api/v1/version` 回 `1.25.5`。 +- `StockWoooWorkDown` 曾短暫 firing，但 public route 立即讀回 `/` 200、`/healthz` 200、`/api/healthz` 200、`/api/v1/system/freshness` 200 且 `status=ok`、`latest_trading_date=2026-07-01`；Prometheus `ALERTS{alertname="StockWoooWorkDown",alertstate="firing"}` 已清空，判定為 Stock web/admin/edge warmup transient。 +- Prometheus 收斂後：`load5/core=0.725833`、`gitea=0.9882 cores`、`stockplatform-v2-postgres-1=0.7296 cores`。 +- firing alerts 回到只剩 `HostRebootEventDetected` 與 `RebootAutoRecoverySLOMissed`；`Host110SustainedModeratePressure`、`HostLoadAverageSustainedHigh`、`DockerContainerCpuSustainedHigh` 已清除。 +- `ops/reboot-recovery/full-stack-cold-start-baseline.yml` 已補 Gitea `1.5 CPU / 3G / restart=always`，rollback 只允許暫回 2 CPU 並需 Prometheus readback。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune / process kill。 + +**下一步**： +- rebase 最新 Gitea main，commit / push classifier + baseline + receipt；再回到 Gitea queue / reboot SLO blocker 主線。 + ## 2026-07-02 — P0 AI loop stale control-path blocker production readback 修正 **完成內容**： diff --git a/docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json b/docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json index 073c367d6..3e7602a0b 100644 --- a/docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json +++ b/docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json @@ -108,6 +108,24 @@ "latest_trading_date": "2026-07-01" } } + }, + { + "host": "110", + "target": "gitea", + "action": "docker update --cpus=1.5 gitea", + "blast_radius": "container CPU quota update only; no container restart", + "readback": { + "running": true, + "nanocpus": 1500000000, + "memory_bytes": 3221225472, + "restart_policy": "always", + "public_version": "1.25.5", + "prometheus_after_convergence": { + "gitea_cpu_cores": 0.9882, + "stockplatform_v2_postgres_cpu_cores": 0.7296, + "host_110_load5_per_core": 0.725833 + } + } } ], "prometheus_readback": { @@ -129,7 +147,10 @@ "HostOutOfDiskSpace", "HostDiskUsageHigh", "RebootEventDetectorMissing", - "Host110SustainedModeratePressure" + "Host110SustainedModeratePressure", + "HostLoadAverageSustainedHigh", + "DockerContainerCpuSustainedHigh", + "StockWoooWorkDown" ], "filesystem_used_percent": { "110": 79.68293081715491, diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 7c8007dff..e22b9dc71 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -86,7 +86,7 @@ v1.82 bounded summary rule：`post-start-quick-check.sh` 與 `188-host-hygiene-m 2026-07-01 14:05 追加 controller / SLO stale-attribution guard：`host-sustained-load-controller.py` 與 `host-sustained-load-evidence.py` 必須把超過 `300s` 的 Docker stats 樣本標成 untrusted；`top_container_cpu` / `top_containers` 不得使用 stale `docker_container_cpu_cores`，舊值只能留在 `top_container_cpu_untrusted` / `top_containers_untrusted` 當證據。`reboot-auto-recovery-slo-scorecard.py` 若收到 `docker_stats.fresh=false` 或 `top_containers_fresh=false`，只能保留 `host_pressure_high_load` 與 `host_container_cpu_attribution_stale`，不得產生 `host_110_gitea_cpu_pressure`。此時下一步固定為恢復 Docker stats textfile exporter 或收集 sanitized host pressure，且仍不得重啟 Docker / Nginx / K3s / DB / firewall、不得恢復 generic runner、不得用 stale Gitea CPU 樣本取消或 drain 任何工作。 -2026-07-01 23:28 追加 110 主動告警與 controlled quota 收斂：`HostLoadAverageSustainedHigh` 的 `load5/core > 1.5 for 15m` 只能當 critical 門檻，不足以回答「110 CPU 又持續偏高為何沒告警」。Prometheus 必須同時有 `Host110SustainedModeratePressure`，在 `awoooi_host_load5_per_core{host="110"} > 0.75` 或 Gitea / StockPlatform 關鍵容器 `docker_container_cpu_cores > 2.0` 持續 1 分鐘時告警，auto-repair action 必須指向 110 已部署的 `/home/wooo/scripts/host-sustained-load-controller.py --load5-per-core-threshold 0.75`。Gitea runtime CPU quota 已以 `docker update --cpus=2 gitea` 收斂；rollback 是 `docker update --cpus=3 gitea`，不可重啟 Docker daemon 或 Gitea container。若 `BackupAggregateRunFailed` 只因 `exported_job="backup_all"` 舊 aggregate failed_count firing，而 component jobs / `backup-status.sh` 已 green，必須視為噪音並用 component failed count 判斷，不得讓它干擾 cold-start / Telegram 主線。 +2026-07-01 23:28 追加 110 主動告警與 controlled quota 收斂；2026-07-02 00:40 更新 Gitea cap：`HostLoadAverageSustainedHigh` 的 `load5/core > 1.5 for 15m` 只能當 critical 門檻，不足以回答「110 CPU 又持續偏高為何沒告警」。Prometheus 必須同時有 `Host110SustainedModeratePressure`，在 `awoooi_host_load5_per_core{host="110"} > 0.75` 或 Gitea / StockPlatform 關鍵容器 `docker_container_cpu_cores > 2.0` 持續 1 分鐘時告警，auto-repair action 必須指向 110 已部署的 `/home/wooo/scripts/host-sustained-load-controller.py --load5-per-core-threshold 0.75`。Gitea runtime CPU quota 已從 `docker update --cpus=2 gitea` 進一步收斂為 `docker update --cpus=1.5 gitea`；rollback 只允許暫回 `docker update --cpus=2 gitea` 並需 Prometheus readback，不可重啟 Docker daemon 或 Gitea container。若 `BackupAggregateRunFailed` 只因 `exported_job="backup_all"` 舊 aggregate failed_count firing，而 component jobs / `backup-status.sh` 已 green，必須視為噪音並用 component failed count 判斷，不得讓它干擾 cold-start / Telegram 主線。 2026-06-25 20:25 orphan Chrome cleanup / scorecard refresh supersedes the 20:11 CPU wording. 110 high CPU was traced to two `stockplatform-review-bulk-ux` Chrome process groups `2756503` and `2829627` with root Chrome process `PPID=1`, elapsed about 5h, no active parent smoke, and sustained GPU/renderer CPU. With user approval, only those two process groups received targeted `SIGTERM` at 20:24. Post-check showed no remaining PGID entries; `vmstat` showed CPU idle around `85-90%`, `si/so=0`, and no immediate swap thrash. No Docker/systemd/Nginx/firewall/K8s action, CI cancellation, manual data ingestion, manual DB write, Wazuh/SOC runtime change, or secret read was performed. The 20:25 full post-start wrapper then returned cold-start `PASS=89 WARN=0 BLOCKED=0`, but overall `POST_START_QUICK_CHECK PASS=37 WARN=2 BLOCKED=1`, `RESULT=BLOCKED`, because StockPlatform data freshness was still blocked at that time and DR remained incomplete. @@ -2494,6 +2494,14 @@ docker update --cpus=1.5 --memory=4g --memory-swap=6g --restart unless-stopped s | Product data freshness | `/api/v1/system/freshness` `status=ok`、`latest_trading_date=2026-07-01` | | Alert readback | `Host110SustainedModeratePressure` cleared；firing alerts only `HostRebootEventDetected` and `RebootAutoRecoverySLOMissed` | +後續同一輪 Gitea 仍曾貼近 2 CPU cap 並觸發 `DockerContainerCpuSustainedHigh`；已再套用： + +```bash +docker update --cpus=1.5 gitea +``` + +讀回：`nanocpus=1500000000 memory=3221225472 restart=always`、Gitea API `/api/v1/version` 回 `1.25.5`。Prometheus 收斂後 `load5/core=0.725833`、`gitea=0.9882 cores`、`stockplatform-v2-postgres-1=0.7296 cores`，CPU 類 firing alerts 清空。 + 宣告限制： - 可宣稱：110 sustained moderate CPU alert 已主動觸發、已定位、已用 Docker guardrail 降壓，StockPlatform route / health / freshness 未受損。 diff --git a/ops/reboot-recovery/full-stack-cold-start-baseline.yml b/ops/reboot-recovery/full-stack-cold-start-baseline.yml index a31dbbb36..e17f42d2f 100644 --- a/ops/reboot-recovery/full-stack-cold-start-baseline.yml +++ b/ops/reboot-recovery/full-stack-cold-start-baseline.yml @@ -297,6 +297,11 @@ resource_guardrails: cpu_quota: 200% memory_max: 2G watchdog: disabled + gitea: + cpus: 1.5 + memory: 3G + restart_policy: always + note: "2026-07-02 live recovery 將 gitea 從 2 CPU cap 收斂到 1.5 CPU；不得回退到無限額或 3 CPU，rollback 只允許暫回 2 CPU 並需 Prometheus readback。" stockplatform_v2_postgres: cpus: 1.5 memory: 4G