diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index fa111a96c..13b1dbb3f 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -536,7 +536,7 @@ jobs: ;; docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json) ;; - ops/runner/full-stack-cold-start-baseline.yml) + ops/reboot-recovery/full-stack-cold-start-baseline.yml) ;; ops/monitoring/alerts-unified.yml) ;; @@ -806,7 +806,7 @@ jobs: ../../scripts/security/gitea-authenticated-inventory-payload-validator.py python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts-unified.yml')); print('alerts-unified YAML OK')" python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts.yml')); print('alerts YAML OK')" - python3.11 -c "import yaml; yaml.safe_load(open('../../ops/runner/full-stack-cold-start-baseline.yml')); print('full-stack-cold-start-baseline YAML OK')" + python3.11 -c "import yaml; yaml.safe_load(open('../../ops/reboot-recovery/full-stack-cold-start-baseline.yml')); print('full-stack-cold-start-baseline YAML OK')" bash -n \ ../../ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh \ ../../ops/runner/check-awoooi-non110-runner-readiness.sh \ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 32bb25e02..9535817a8 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -51968,6 +51968,28 @@ production browser smoke: - rebase 到 Gitea main 最新 deploy marker，commit / push 本次 source 修法與 receipt。 - 讀回 Gitea CD / Prometheus；下一條主線是把 `RebootAutoRecoverySLOMissed` 的 6 個 blocker 轉成可自動判斷與可回滾 remediation，不消音告警。 +## 2026-07-02 — 00:24 P0 110 sustained moderate CPU alert 實際降壓 + +**完成內容**： +- Prometheus 主動觸發 `Host110SustainedModeratePressure`，證明 110 CPU 壓力已被監控抓到；這次沒有等使用者人工發現後才開始排查。 +- 讀回 `awoooi_host_load5_per_core{host="110"}=0.786667`，orphan browser group `0`，active Gitea Actions process count / CPU 均為 `0`，swap ratio 約 `0.01273`。 +- top container 顯示 `stockplatform-v2-postgres-1` 曾達約 `292.96% CPU`，且 inspect 為 `nanocpus=0 memory=0 memswap=0 restart=no`；這是 110 host pressure 的實際風險，不是噪音。 +- 已 live 套用：`docker update --cpus=1.5 --memory=4g --memory-swap=6g --restart unless-stopped stockplatform-v2-postgres-1`。此操作只改 Docker cgroup / restart policy metadata，沒有重啟 DB container，沒有 DB write。 +- post-apply inspect：`running=true health=healthy nanocpus=1500000000 memory=4294967296 memswap=6442450944 restart=unless-stopped`。 +- direct Docker stats after apply：`stockplatform-v2-postgres-1 cpu=149.38% mem=506.2MiB / 4GiB`。 +- Prometheus 收斂後：`stockplatform-v2-postgres-1=1.6508 cores`、`gitea=1.8097 cores`、`load5/core=0.673333`。 +- StockPlatform public route / health / data freshness 未受損：`/` 200、`/healthz` 200、`/api/healthz` 200、`/api/v1/system/freshness` 200 且 `status=ok`、`latest_trading_date=2026-07-01`。 +- `Host110SustainedModeratePressure` 已從 firing alerts 清除；目前 firing alerts 只剩本次 reboot lane 的 `HostRebootEventDetected` 與 `RebootAutoRecoverySLOMissed`。 +- `ops/reboot-recovery/full-stack-cold-start-baseline.yml` 已補 `stockplatform_v2_postgres` resource guardrail，避免下次重啟回退成無限額。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune / query kill / manual DB write。 + +**下一步**： +- 以 `[skip ci]` source commit 推回 Gitea main，避免只剩 live drift；然後回到 Gitea queue / 110 control-path / reboot SLO blocker 主線。 + ## 2026-07-02 — P0 AI loop stale control-path blocker production readback 修正 **完成內容**： diff --git a/docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json b/docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json index 89f7cadfd..a3ad4d734 100644 --- a/docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json +++ b/docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json @@ -81,6 +81,33 @@ "awoooi_reboot_auto_recovery_slo_blocker_count": 6, "awoooi_reboot_event_target_seconds_remaining": 0 } + }, + { + "host": "110", + "target": "stockplatform-v2-postgres-1", + "action": "docker update --cpus=1.5 --memory=4g --memory-swap=6g --restart unless-stopped stockplatform-v2-postgres-1", + "blast_radius": "container resource guardrail only; no container restart and no database write", + "readback": { + "running": true, + "health": "healthy", + "nanocpus": 1500000000, + "memory_bytes": 4294967296, + "memswap_bytes": 6442450944, + "restart_policy": "unless-stopped", + "direct_docker_stats_after": "cpu=149.38% mem=506.2MiB / 4GiB", + "prometheus_after_convergence": { + "stockplatform_v2_postgres_cpu_cores": 1.6508, + "gitea_cpu_cores": 1.8097, + "host_110_load5_per_core": 0.673333 + }, + "public_route_health": { + "root": 200, + "healthz": 200, + "api_healthz": 200, + "freshness_status": "ok", + "latest_trading_date": "2026-07-01" + } + } } ], "prometheus_readback": { @@ -101,7 +128,8 @@ "DockerContainerMemoryLimitPressure", "HostOutOfDiskSpace", "HostDiskUsageHigh", - "RebootEventDetectorMissing" + "RebootEventDetectorMissing", + "Host110SustainedModeratePressure" ], "filesystem_used_percent": { "110": 79.68293081715491, diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index a816aeb13..6fb7205e0 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -2472,3 +2472,30 @@ python3 scripts/ops/docker-disk-pressure-retention-cleanup.py --host-label 188 - - 可以宣稱：這次回讀的 runtime full-stack 已 GREEN，110 / 188 disk pressure、momo-db missing resource limit、Sentry relay memory pressure、RebootEventDetectorMissing 已解除。 - 不可宣稱：這次重啟有達成 10 分鐘 SLO。Prometheus 仍正確保留 `RebootAutoRecoverySLOMissed`，代表本次事件已 missed，必須做為後續自動化改善 evidence，而不是消音。 - 不可用 stale API snapshot 覆蓋 live truth；`https://aiops.wooo.work/api/v1/agents/reboot-auto-recovery-slo-scorecard` 若 `generated_at` 舊於最新 textfile / Prometheus readback，只能列為 stale artifact。 + +### 16.4 2026-07-02 110 StockPlatform DB pressure guardrail + +2026-07-02 00:20 後，`Host110SustainedModeratePressure` 正確觸發。這不是缺監控；Prometheus 已主動抓到 110 壓力。即時 evidence 顯示 orphan browser group `0`、active Gitea Actions process `0`、swap ratio 約 `0.0127`，壓力來源是 `gitea` 接近既有 2 CPU cap，以及 `stockplatform-v2-postgres-1` 沒有 Docker resource guardrail、曾瞬間到約 `292.96% CPU`。 + +已完成 live 修復： + +```bash +docker update --cpus=1.5 --memory=4g --memory-swap=6g --restart unless-stopped stockplatform-v2-postgres-1 +``` + +讀回證據： + +| Check | Evidence | +|-------|----------| +| Docker inspect | `running=true health=healthy nanocpus=1500000000 memory=4294967296 memswap=6442450944 restart=unless-stopped` | +| Direct Docker stats | `stockplatform-v2-postgres-1 cpu=149.38% mem=506.2MiB / 4GiB` after apply | +| Prometheus convergence | `stockplatform-v2-postgres-1=1.6508 cores`、`gitea=1.8097 cores`、`load5/core=0.673333` | +| Product route | `https://stock.wooo.work/`、`/healthz`、`/api/healthz` all `200` | +| Product data freshness | `/api/v1/system/freshness` `status=ok`、`latest_trading_date=2026-07-01` | +| Alert readback | `Host110SustainedModeratePressure` cleared；firing alerts only `HostRebootEventDetected` and `RebootAutoRecoverySLOMissed` | + +宣告限制： + +- 可宣稱：110 sustained moderate CPU alert 已主動觸發、已定位、已用 Docker guardrail 降壓，StockPlatform route / health / freshness 未受損。 +- 不可宣稱：110 永久不會再高負載；Gitea queue / hook backlog 仍需在後續 queue / control-path lane 持續觀察。 +- 不得用 DB restart、container restart、manual DB write、query kill 或 fake freshness 來處理 StockPlatform pressure。 diff --git a/ops/reboot-recovery/full-stack-cold-start-baseline.yml b/ops/reboot-recovery/full-stack-cold-start-baseline.yml index 013a55bc1..a31dbbb36 100644 --- a/ops/reboot-recovery/full-stack-cold-start-baseline.yml +++ b/ops/reboot-recovery/full-stack-cold-start-baseline.yml @@ -297,6 +297,12 @@ resource_guardrails: cpu_quota: 200% memory_max: 2G watchdog: disabled + stockplatform_v2_postgres: + cpus: 1.5 + memory: 4G + memswap: 6G + restart_policy: unless-stopped + note: "2026-07-02 live recovery 將 stockplatform-v2-postgres-1 補上 Docker resource guardrail；不得回退到 nanocpus=0 / memory=0 / restart=no 造成 110 host pressure。" "188": ollama_systemd: cpu_quota: 300% diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 7ae0bc64e..193587fb3 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -708,7 +708,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md)", "ops/monitoring/alerts-unified.yml)", "ops/monitoring/alerts.yml)", - "ops/runner/full-stack-cold-start-baseline.yml)", + "ops/reboot-recovery/full-stack-cold-start-baseline.yml)", "ops/runner/awoooi-cd-lane-drain.service)", "ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh)", "ops/runner/test_check_awoooi_110_controlled_cd_lane_readiness.py)", @@ -743,7 +743,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "../../scripts/backup/gitea-repo-bundle-backup.sh", "../../ops/monitoring/alerts-unified.yml", "../../ops/monitoring/alerts.yml", - "../../ops/runner/full-stack-cold-start-baseline.yml", + "../../ops/reboot-recovery/full-stack-cold-start-baseline.yml", "../../ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh", "../../scripts/reboot-recovery/deploy-to-110.sh", "../../scripts/reboot-recovery/recover-110-control-path-and-harbor-local.sh",