From 7249d8101727137b5d075ecbac382e50b3f50f6a Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 11:55:45 +0800 Subject: [PATCH] fix(ops): alert on 110 process-family pressure --- docs/LOGBOOK.md | 33 +++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 2 +- ops/monitoring/alerts-unified.yml | 10 +- scripts/ops/host-runaway-process-exporter.py | 119 +++++++++ scripts/ops/host-sustained-load-controller.py | 235 +++++++++++++++++- .../test_host_pressure_alert_contract.py | 7 +- .../test_host_runaway_process_exporter.py | 183 ++++++++++++++ 7 files changed, 578 insertions(+), 11 deletions(-) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 7a383b278..744f160fe 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -52434,6 +52434,39 @@ production browser smoke: **下一步**： - commit / push 後讀回新的 Gitea CD；目標是 controlled-runtime 直接跳過 B5，不再因 cold-start metadata/script 變更要求 Docker socket。 +## 2026-07-02 — P0 110 CPU 主動告警與 process-family controlled packet + +**完成內容**： +- `scripts/ops/host-runaway-process-exporter.py` 新增 sanitized process-family metrics：`awoooi_host_process_family_cpu_percent`、`process_count`、`oldest_age_seconds`、`top_info`；只輸出 `family` / `comm`，不輸出 raw command line、workspace path、URL 或 secret。 +- `scripts/ops/host-sustained-load-controller.py` 新增 live `ps` process-family 分類，並把中度壓力分流到 `control_plane_saturation`、`gitea_queue_or_hook_backlog`、`stockplatform_hot_query_or_api_pressure` check-mode packet；controller 仍只產生 controlled packet，不做 Docker / systemd / DB / Nginx restart。 +- `ops/monitoring/alerts-unified.yml` 將 `Host110SustainedModeratePressure` 從只看 `load5/core` / container CPU，擴充為同時看 `awoooi_host_process_family_cpu_percent{family=~"systemd_control_plane|ssh_control_plane|gitea_service|postgres"} > 50`；Gitea / StockPlatform container early triage 門檻從 `> 2.0 core` 降到 `> 1.0 core`。 +- 已 live 部署到 110： + - `/home/wooo/scripts/host-runaway-process-exporter.py` SHA `d85d27c81ea76a8f2f370ee85c92381bec4440eea4fd37efb2efb9f43dbd1a8a` + - `/home/wooo/scripts/host-sustained-load-controller.py` SHA `c731c6b5fe5e1683931cf949b3b18c8b290c9a5d5973fac826487ad8de05434a` + - `bash scripts/ops/deploy-alerts.sh` 完成，Prometheus 已載入 `159` 條規則。 + +**live readback 證據**： +- 110 textfile 已輸出 process-family metric：`systemd_control_plane=72.4`、`gitea_service=53.2`、`postgres=23.3`。 +- Prometheus rule readback：`Host110SustainedModeratePressure state=firing health=ok`，query 已包含 `docker_container_cpu_cores > 1` 與 `awoooi_host_process_family_cpu_percent > 50`。 +- Prometheus `ALERTS{alertname="Host110SustainedModeratePressure"}` 已有 `firing`：`container_name="gitea"`、`family="systemd_control_plane"`、`family="gitea_service"`；`stockplatform-v2-postgres-1` 在部署後短時間為 `pending`。 +- Alertmanager `/api/v2/alerts` 已有 active alert，`status.state=active`、`notification_type=TYPE-1`、`auto_repair=true`、`team=ops`。 +- live controller readback：`classification=blocked_control_plane_saturation_requires_playbook`、`next_action=run_control_plane_saturation_playbook_check_mode`、`control_plane_process_cpu_percent=72.4`、`gitea_process_cpu_percent=53.2`、`top_container_cpu.gitea=0.2507~1.5789`，且 `docker_stats.fresh=true`。 + +**本地驗證結果**： +- `python3.11 -m py_compile scripts/ops/host-runaway-process-exporter.py scripts/ops/host-sustained-load-controller.py`：通過。 +- `python3.11 -m pytest scripts/ops/tests/test_host_runaway_process_exporter.py scripts/ops/tests/test_host_pressure_alert_contract.py -q`：`27 passed`。 +- `python3.11 -c "import yaml; yaml.safe_load(open('ops/monitoring/alerts-unified.yml'))"`：通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`：通過，`auto_branch_events_on_110=0`、`generic_runner_labels=0`。 +- `git diff --check`：通過。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有讀 `.runner` 內容。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune。 + +**下一步**： +- commit / push 到 Gitea main 後讀回 CD；接續 `control_plane_saturation` check-mode packet，處理 110 D-state / control-plane 排隊根因，不從 restart 服務下手。 + ## 2026-07-02 — P0 110 controlled CD lane fail-closed enforcer staging 修正 **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index b9889a513..b59a9b46f 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -90,7 +90,7 @@ v1.82 bounded summary rule：`post-start-quick-check.sh` 與 `188-host-hygiene-m 2026-07-01 14:05 追加 controller / SLO stale-attribution guard：`host-sustained-load-controller.py` 與 `host-sustained-load-evidence.py` 必須把超過 `300s` 的 Docker stats 樣本標成 untrusted；`top_container_cpu` / `top_containers` 不得使用 stale `docker_container_cpu_cores`，舊值只能留在 `top_container_cpu_untrusted` / `top_containers_untrusted` 當證據。`reboot-auto-recovery-slo-scorecard.py` 若收到 `docker_stats.fresh=false` 或 `top_containers_fresh=false`，只能保留 `host_pressure_high_load` 與 `host_container_cpu_attribution_stale`，不得產生 `host_110_gitea_cpu_pressure`。此時下一步固定為恢復 Docker stats textfile exporter 或收集 sanitized host pressure，且仍不得重啟 Docker / Nginx / K3s / DB / firewall、不得恢復 generic runner、不得用 stale Gitea CPU 樣本取消或 drain 任何工作。 -2026-07-01 23:28 追加 110 主動告警與 controlled quota 收斂；2026-07-02 00:40 更新 Gitea cap：`HostLoadAverageSustainedHigh` 的 `load5/core > 1.5 for 15m` 只能當 critical 門檻，不足以回答「110 CPU 又持續偏高為何沒告警」。Prometheus 必須同時有 `Host110SustainedModeratePressure`，在 `awoooi_host_load5_per_core{host="110"} > 0.75` 或 Gitea / StockPlatform 關鍵容器 `docker_container_cpu_cores > 2.0` 持續 1 分鐘時告警，auto-repair action 必須指向 110 已部署的 `/home/wooo/scripts/host-sustained-load-controller.py --load5-per-core-threshold 0.75`。Gitea runtime CPU quota 已從 `docker update --cpus=2 gitea` 進一步收斂為 `docker update --cpus=1.5 gitea`；rollback 只允許暫回 `docker update --cpus=2 gitea` 並需 Prometheus readback，不可重啟 Docker daemon 或 Gitea container。若 `BackupAggregateRunFailed` 只因 `exported_job="backup_all"` 舊 aggregate failed_count firing，而 component jobs / `backup-status.sh` 已 green，必須視為噪音並用 component failed count 判斷，不得讓它干擾 cold-start / Telegram 主線。 +2026-07-01 23:28 追加 110 主動告警與 controlled quota 收斂；2026-07-02 00:40 更新 Gitea cap；2026-07-02 11:54 追加 process-family 主動告警：`HostLoadAverageSustainedHigh` 的 `load5/core > 1.5 for 15m` 只能當 critical 門檻，不足以回答「110 CPU 又持續偏高為何沒告警」。Prometheus 必須同時有 `Host110SustainedModeratePressure`，在 `awoooi_host_load5_per_core{host="110"} > 0.75`、Gitea / StockPlatform 關鍵容器 `docker_container_cpu_cores > 1.0`，或 `awoooi_host_process_family_cpu_percent{family=~"systemd_control_plane|ssh_control_plane|gitea_service|postgres"} > 50` 持續 1 分鐘時告警，auto-repair action 必須指向 110 已部署的 `/home/wooo/scripts/host-sustained-load-controller.py --load5-per-core-threshold 0.75 --hot-container-cpu-threshold 1.0 --process-family-cpu-threshold 50`。`host-runaway-process-exporter.py` 必須輸出 sanitized process-family metrics，只允許 family / comm labels，不得輸出 raw command line、workspace path、URL 或 secret。Gitea runtime CPU quota 已從 `docker update --cpus=2 gitea` 進一步收斂為 `docker update --cpus=1.5 gitea`；rollback 只允許暫回 `docker update --cpus=2 gitea` 並需 Prometheus readback，不可重啟 Docker daemon 或 Gitea container。若 `BackupAggregateRunFailed` 只因 `exported_job="backup_all"` 舊 aggregate failed_count firing，而 component jobs / `backup-status.sh` 已 green，必須視為噪音並用 component failed count 判斷，不得讓它干擾 cold-start / Telegram 主線。 2026-06-25 20:25 orphan Chrome cleanup / scorecard refresh supersedes the 20:11 CPU wording. 110 high CPU was traced to two `stockplatform-review-bulk-ux` Chrome process groups `2756503` and `2829627` with root Chrome process `PPID=1`, elapsed about 5h, no active parent smoke, and sustained GPU/renderer CPU. With user approval, only those two process groups received targeted `SIGTERM` at 20:24. Post-check showed no remaining PGID entries; `vmstat` showed CPU idle around `85-90%`, `si/so=0`, and no immediate swap thrash. No Docker/systemd/Nginx/firewall/K8s action, CI cancellation, manual data ingestion, manual DB write, Wazuh/SOC runtime change, or secret read was performed. The 20:25 full post-start wrapper then returned cold-start `PASS=89 WARN=0 BLOCKED=0`, but overall `POST_START_QUICK_CHECK PASS=37 WARN=2 BLOCKED=1`, `RESULT=BLOCKED`, because StockPlatform data freshness was still blocked at that time and DR remained incomplete. diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index c3f348a2c..d923052a9 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -308,7 +308,9 @@ groups: expr: | (awoooi_host_load5_per_core{host="110"} > 0.75) or - (docker_container_cpu_cores{host="110",container_name=~"gitea|stockplatform-v2-postgres-1|stockplatform-v2-api-1"} > 2.0) + (docker_container_cpu_cores{host="110",container_name=~"gitea|stockplatform-v2-postgres-1|stockplatform-v2-api-1"} > 1.0) + or + (awoooi_host_process_family_cpu_percent{host="110",family=~"systemd_control_plane|ssh_control_plane|gitea_service|postgres"} > 50) for: 1m labels: severity: warning @@ -323,9 +325,9 @@ groups: host_type: "bare_metal" annotations: summary: "110 sustained pressure needs triage" - description: "110 load5/core > 0.75 或 Gitea / StockPlatform 關鍵容器 CPU > 2.0 core 持續 1 分鐘；這是 critical 之前的主動偵測，避免等到 load5/core > 1.5 才反應。" - auto_repair_action: "ssh 192.168.0.110 '/home/wooo/scripts/host-sustained-load-controller.py --host 110 --load5-per-core-threshold 0.75 --container-cpu-threshold 2.0 --metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom --docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom --json'" - runbook: "controller 只產生 controlled packet，不讀 secret、不重啟服務。若分類為 gitea_queue_or_hook_backlog，先跑 host-sustained-load-evidence.py 取得脫敏 top family / container，再選 Gitea queue/hook backlog playbook；若是 orphan browser 才允許 gated SIGTERM；若是 StockPlatform postgres/API，轉 Stock hot-query/source freshness playbook。禁止 Docker / systemd / Nginx / DB restart、reboot、firewall。" + description: "110 load5/core > 0.75、Gitea / StockPlatform 關鍵容器 CPU > 1.0 core，或 control-plane / Gitea / Postgres process family CPU > 50% 持續 1 分鐘；這是 critical 之前的主動偵測，避免等到 load5/core > 1.5 或關鍵容器 > 2.0 core 才反應。" + auto_repair_action: "ssh 192.168.0.110 '/home/wooo/scripts/host-sustained-load-controller.py --host 110 --load5-per-core-threshold 0.75 --hot-container-cpu-threshold 1.0 --container-cpu-threshold 2.0 --process-family-cpu-threshold 50 --metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom --docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom --json'" + runbook: "controller 只產生 controlled packet，不讀 secret、不重啟服務。若分類為 gitea_queue_or_hook_backlog，先跑 host-sustained-load-evidence.py 取得脫敏 top family / container，再選 Gitea queue/hook backlog playbook；若是 orphan browser 才允許 gated SIGTERM；若是 StockPlatform postgres/API，轉 Stock hot-query/source freshness playbook；若是 control_plane_saturation，先處理 D-state 掃描/控制面排隊，不可直接 restart。禁止 Docker / systemd / Nginx / DB restart、reboot、firewall。" - alert: HostCiRunnerLoadSaturation expr: | diff --git a/scripts/ops/host-runaway-process-exporter.py b/scripts/ops/host-runaway-process-exporter.py index 381efcab7..598e1938d 100755 --- a/scripts/ops/host-runaway-process-exporter.py +++ b/scripts/ops/host-runaway-process-exporter.py @@ -87,6 +87,22 @@ GITEA_ACTION_PROCESS_RE = re.compile( r"\bnext build\b|\bpnpm install\b|\bnpm ci\b|\byarn install\b)" ) HOST_PRESSURE_GATE_RE = re.compile(r"wait-host-web-build-pressure\.sh|awoooi-wait-host-web-build-pressure\.sh") +KNOWN_PROCESS_FAMILIES = ( + "systemd_control_plane", + "ssh_control_plane", + "gitea_service", + "postgres", + "clickhouse", + "kafka", + "sentry", + "python_job", + "node_service", + "gitea_actions_runner", + "docker_build", + "web_build", + "headless_browser", + "unknown", +) def escape_label(value: str) -> str: @@ -261,6 +277,79 @@ def active_gitea_action_process_load(rows: list[ProcessRow]) -> ActiveCiLoad: ) +def classify_process_family(row: ProcessRow) -> str: + text = f"{row.comm} {row.args}".lower() + if ( + "act_runner" in text + or "gitea-actions-task" in text + or "/.cache/act/" in text + or "/opt/hostedtoolcache/" in text + or "pnpm install" in text + or "npm ci" in text + or "yarn install" in text + ): + return "gitea_actions_runner" + if "docker build" in text or "buildx" in text or "buildkit" in text: + return "docker_build" + if "next build" in text or "turbo build" in text or ("pnpm" in text and " build" in text): + return "web_build" + if "chrome" in text or "chromium" in text or "playwright" in text: + return "headless_browser" + if "gitea" in text: + return "gitea_service" + if "postgres" in text or "postmaster" in text: + return "postgres" + if "clickhouse" in text: + return "clickhouse" + if "kafka" in text: + return "kafka" + if "sentry" in text: + return "sentry" + if "systemctl" in text or "systemd" in text or "dbus" in text: + return "systemd_control_plane" + if "sshd" in text: + return "ssh_control_plane" + if "python" in text: + return "python_job" + if "node" in text: + return "node_service" + return "unknown" + + +def process_family_load(rows: list[ProcessRow]) -> dict[str, dict[str, object]]: + families: dict[str, dict[str, object]] = { + family: { + "cpu_percent": 0.0, + "process_count": 0, + "oldest_age_seconds": 0, + "sample_comm": "", + "_sample_cpu": -1.0, + } + for family in KNOWN_PROCESS_FAMILIES + } + for row in rows: + family = classify_process_family(row) + current = families.setdefault( + family, + { + "cpu_percent": 0.0, + "process_count": 0, + "oldest_age_seconds": 0, + "sample_comm": "", + "_sample_cpu": -1.0, + }, + ) + current["cpu_percent"] = float(current["cpu_percent"]) + row.pcpu + current["process_count"] = int(current["process_count"]) + 1 + current["oldest_age_seconds"] = max(int(current["oldest_age_seconds"]), row.etimes) + if row.pcpu > float(current["_sample_cpu"]): + current["sample_comm"] = row.comm[:48] + current["_sample_cpu"] = row.pcpu + for current in families.values(): + current.pop("_sample_cpu", None) + return families + + def load5_per_core() -> float: try: load5 = float(Path("/proc/loadavg").read_text(encoding="utf-8").split()[1]) @@ -296,6 +385,7 @@ def render_metrics( groups: list[ProcessGroup], active_action_containers: int, active_action_process_load: ActiveCiLoad, + process_family_summary: dict[str, dict[str, object]], min_age_seconds: int, min_cpu_percent: float, now: int, @@ -336,6 +426,14 @@ def render_metrics( "# TYPE awoooi_host_load5_per_core gauge", "# HELP awoooi_host_swap_used_ratio Host swap used ratio from /proc/meminfo.", "# TYPE awoooi_host_swap_used_ratio gauge", + "# HELP awoooi_host_process_family_cpu_percent Sanitized CPU percent grouped by process family.", + "# TYPE awoooi_host_process_family_cpu_percent gauge", + "# HELP awoooi_host_process_family_process_count Sanitized process count grouped by process family.", + "# TYPE awoooi_host_process_family_process_count gauge", + "# HELP awoooi_host_process_family_oldest_age_seconds Oldest process age grouped by process family.", + "# TYPE awoooi_host_process_family_oldest_age_seconds gauge", + "# HELP awoooi_host_process_family_top_info Sample command name for a sanitized process family.", + "# TYPE awoooi_host_process_family_top_info gauge", "# HELP awoooi_host_runaway_process_remediation_authorized Static guardrail: remediation is not authorized by this exporter.", "# TYPE awoooi_host_runaway_process_remediation_authorized gauge", f"awoooi_host_runaway_process_monitor_up{{{labels_host},mode=\"read_only\"}} 1", @@ -350,6 +448,26 @@ def render_metrics( f"awoooi_host_runaway_process_remediation_authorized{{{labels_host}}} 0", ] + for family in sorted(process_family_summary): + item = process_family_summary[family] + family_labels = f'{labels_host},family="{escape_label(family)}"' + lines.append( + f"awoooi_host_process_family_cpu_percent{{{family_labels}}} " + f"{float(item['cpu_percent']):.6f}" + ) + lines.append( + f"awoooi_host_process_family_process_count{{{family_labels}}} " + f"{int(item['process_count'])}" + ) + lines.append( + f"awoooi_host_process_family_oldest_age_seconds{{{family_labels}}} " + f"{int(item['oldest_age_seconds'])}" + ) + sample_comm = str(item.get("sample_comm") or "none") + lines.append( + f'awoooi_host_process_family_top_info{{{family_labels},comm="{escape_label(sample_comm)}"}} 1' + ) + for rule_id in rule_ids: rule_labels = ( f'{labels_host},rule="{escape_label(rule_id)}",' @@ -393,6 +511,7 @@ def collect(args: argparse.Namespace) -> str: groups=groups, active_action_containers=active_gitea_action_containers(args.docker_ps_file), active_action_process_load=active_gitea_action_process_load(rows), + process_family_summary=process_family_load(rows), min_age_seconds=args.min_age_seconds, min_cpu_percent=args.min_cpu_percent, now=int(time.time()), diff --git a/scripts/ops/host-sustained-load-controller.py b/scripts/ops/host-sustained-load-controller.py index 6c2e751d5..ecc448817 100755 --- a/scripts/ops/host-sustained-load-controller.py +++ b/scripts/ops/host-sustained-load-controller.py @@ -20,6 +20,7 @@ from __future__ import annotations import argparse import json import re +import subprocess import time from pathlib import Path from typing import Any @@ -50,7 +51,11 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--load5-per-core-threshold", type=float, default=1.5) parser.add_argument("--container-cpu-threshold", type=float, default=2.0) + parser.add_argument("--hot-container-cpu-threshold", type=float, default=1.0) + parser.add_argument("--process-family-cpu-threshold", type=float, default=50.0) parser.add_argument("--ci-stale-age-seconds", type=int, default=1800) + parser.add_argument("--ps-file", type=Path) + parser.add_argument("--top-n", type=int, default=8) parser.add_argument("--json", action="store_true", help="Print JSON only.") return parser.parse_args() @@ -211,14 +216,171 @@ def _top_container_cpu(samples: list[dict[str, Any]], *, host: str) -> dict[str, return sorted(candidates, key=lambda item: (-item["cpu_cores"], item["container_name"]))[0] +def _read_text(path: Path | None) -> str: + if path is None: + return "" + try: + return path.read_text(encoding="utf-8") + except FileNotFoundError: + return "" + + +def _collect_ps_text(ps_file: Path | None) -> str: + if ps_file is not None: + return _read_text(ps_file) + try: + result = subprocess.run( + ["ps", "-eo", "pid=,ppid=,pgid=,etimes=,pcpu=,pmem=,comm=,args="], + check=True, + capture_output=True, + text=True, + timeout=10, + ) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError): + return "" + return result.stdout + + +def _classify_process_family(comm: str, args: str) -> str: + text = f"{comm} {args}".lower() + if ( + "act_runner" in text + or "gitea-actions-task" in text + or "/.cache/act/" in text + or "/opt/hostedtoolcache/" in text + or "pnpm install" in text + or "npm ci" in text + or "yarn install" in text + ): + return "gitea_actions_runner" + if "docker build" in text or "buildx" in text or "buildkit" in text: + return "docker_build" + if "next build" in text or "turbo build" in text or ("pnpm" in text and " build" in text): + return "web_build" + if "chrome" in text or "chromium" in text or "playwright" in text: + return "headless_browser" + if "gitea" in text: + return "gitea_service" + if "postgres" in text or "postmaster" in text: + return "postgres" + if "clickhouse" in text: + return "clickhouse" + if "kafka" in text: + return "kafka" + if "sentry" in text: + return "sentry" + if "systemctl" in text or "systemd" in text or "dbus" in text: + return "systemd_control_plane" + if "sshd" in text: + return "ssh_control_plane" + if "python" in text: + return "python_job" + if "node" in text: + return "node_service" + return "unknown" + + +def _parse_ps_text(text: str) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line: + continue + parts = line.split(None, 7) + if len(parts) < 7: + continue + pid, ppid, pgid, etimes, pcpu, pmem, comm = parts[:7] + args = parts[7] if len(parts) > 7 else comm + try: + rows.append( + { + "pid": int(pid), + "ppid": int(ppid), + "pgid": int(pgid), + "etimes": int(float(etimes)), + "cpu_percent": float(pcpu), + "mem_percent": float(pmem), + "comm": Path(comm).name[:48], + "family": _classify_process_family(comm, args), + } + ) + except ValueError: + continue + return rows + + +def _summarize_processes(rows: list[dict[str, Any]], *, top_n: int) -> dict[str, Any]: + top_rows = sorted(rows, key=lambda item: (-item["cpu_percent"], item["comm"], item["pid"]))[:top_n] + families: dict[str, dict[str, Any]] = {} + for row in rows: + family = row["family"] + current = families.setdefault( + family, + { + "family": family, + "process_count": 0, + "cpu_percent": 0.0, + "max_age_seconds": 0, + "sample_comm": "", + }, + ) + current["process_count"] += 1 + current["cpu_percent"] += row["cpu_percent"] + current["max_age_seconds"] = max(current["max_age_seconds"], row["etimes"]) + if not current["sample_comm"] or row["cpu_percent"] > current.get("_sample_cpu", -1): + current["sample_comm"] = row["comm"] + current["_sample_cpu"] = row["cpu_percent"] + + family_rows = [] + for item in families.values(): + item.pop("_sample_cpu", None) + item["cpu_percent"] = round(float(item["cpu_percent"]), 3) + family_rows.append(item) + + return { + "top_processes": [ + { + "pid": row["pid"], + "ppid": row["ppid"], + "pgid": row["pgid"], + "cpu_percent": round(row["cpu_percent"], 3), + "mem_percent": round(row["mem_percent"], 3), + "age_seconds": row["etimes"], + "comm": row["comm"], + "family": row["family"], + } + for row in top_rows + ], + "families": sorted(family_rows, key=lambda item: (-item["cpu_percent"], item["family"]))[ + :top_n + ], + } + + +def _family_cpu(process_summary: dict[str, Any], *families: str) -> float: + return sum( + float(item.get("cpu_percent") or 0.0) + for item in process_summary.get("families", []) + if item.get("family") in families + ) + + +def _top_family(process_summary: dict[str, Any]) -> dict[str, Any] | None: + families = process_summary.get("families", []) + return families[0] if families else None + + def build_packet( *, host: str, samples: list[dict[str, Any]], docker_samples: list[dict[str, Any]], docker_stats_status: dict[str, Any], + process_summary: dict[str, Any], load5_per_core_threshold: float, container_cpu_threshold: float, + hot_container_cpu_threshold: float, + process_family_cpu_threshold: float, ci_stale_age_seconds: int, ) -> dict[str, Any]: monitor_up = int( @@ -272,6 +434,10 @@ def build_packet( top_container = raw_top_container if docker_stats_status.get("fresh") is True else None top_container_name = str((top_container or {}).get("container_name") or "").lower() top_container_cpu = float((top_container or {}).get("cpu_cores") or 0.0) + stock_process_cpu = _family_cpu(process_summary, "postgres") + gitea_process_cpu = _family_cpu(process_summary, "gitea_service") + control_plane_cpu = _family_cpu(process_summary, "systemd_control_plane", "ssh_control_plane") + top_family = _top_family(process_summary) classification = "observing_load_within_threshold" severity = "info" @@ -329,19 +495,58 @@ def build_packet( if controlled_apply_allowed else "keep_pressure_gate_fail_closed_until_ci_load_clears" ) - elif top_container_name == "gitea" and top_container_cpu >= container_cpu_threshold: + elif ( + ( + "stockplatform-v2-postgres-1" in top_container_name + or top_container_name == "stockplatform-v2-api-1" + ) + and top_container_cpu >= container_cpu_threshold + ) or stock_process_cpu >= process_family_cpu_threshold * 2: + classification = "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook" + severity = ( + "critical" + if load5_per_core > load5_per_core_threshold + or stock_process_cpu >= process_family_cpu_threshold * 2 + else "warning" + ) + dry_run_command = ( + "scripts/ops/host-sustained-load-evidence.py " + f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} " + f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json" + ) + next_action = "run_stockplatform_hot_query_or_api_pressure_playbook_check_mode" + elif ( + top_container_name == "gitea" and top_container_cpu >= container_cpu_threshold + ) or gitea_process_cpu >= process_family_cpu_threshold * 2: classification = "blocked_gitea_queue_or_hook_backlog_requires_playbook" - severity = "critical" if load5_per_core > load5_per_core_threshold else "warning" + severity = ( + "critical" + if load5_per_core > load5_per_core_threshold + or gitea_process_cpu >= process_family_cpu_threshold * 2 + else "warning" + ) dry_run_command = ( "scripts/ops/host-sustained-load-evidence.py " f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} " f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json" ) next_action = "run_gitea_queue_or_hook_backlog_playbook_check_mode" + elif control_plane_cpu >= process_family_cpu_threshold: + classification = "blocked_control_plane_saturation_requires_playbook" + severity = "critical" if load5_per_core > load5_per_core_threshold else "warning" + dry_run_command = ( + "scripts/ops/host-sustained-load-evidence.py " + f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} " + f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json" + ) + next_action = "run_control_plane_saturation_playbook_check_mode" elif ( - top_container_name in {"stockplatform-v2-postgres-1", "stockplatform-v2-api-1"} - and top_container_cpu >= container_cpu_threshold - ): + "stockplatform-v2-postgres-1" in top_container_name + and top_container_cpu >= hot_container_cpu_threshold + ) or ( + top_container_name == "stockplatform-v2-api-1" + and top_container_cpu >= hot_container_cpu_threshold + ) or stock_process_cpu >= process_family_cpu_threshold: classification = "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook" severity = "critical" if load5_per_core > load5_per_core_threshold else "warning" dry_run_command = ( @@ -350,6 +555,17 @@ def build_packet( f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json" ) next_action = "run_stockplatform_hot_query_or_api_pressure_playbook_check_mode" + elif ( + top_container_name == "gitea" and top_container_cpu >= hot_container_cpu_threshold + ) or gitea_process_cpu >= process_family_cpu_threshold: + classification = "blocked_gitea_queue_or_hook_backlog_requires_playbook" + severity = "critical" if load5_per_core > load5_per_core_threshold else "warning" + dry_run_command = ( + "scripts/ops/host-sustained-load-evidence.py " + f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} " + f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json" + ) + next_action = "run_gitea_queue_or_hook_backlog_playbook_check_mode" elif load5_per_core > load5_per_core_threshold and swap_used_ratio >= 0.85: classification = "blocked_memory_or_swap_pressure_requires_service_playbook" severity = "critical" @@ -378,6 +594,8 @@ def build_packet( "load5_per_core": round(load5_per_core, 6), "load5_per_core_threshold": load5_per_core_threshold, "container_cpu_threshold": container_cpu_threshold, + "hot_container_cpu_threshold": hot_container_cpu_threshold, + "process_family_cpu_threshold": process_family_cpu_threshold, "swap_used_ratio": round(swap_used_ratio, 6), "remediation_authorized": remediation_authorized, "active_ci_container_count": active_ci_containers, @@ -388,6 +606,10 @@ def build_packet( "top_container_cpu": top_container, "top_container_cpu_untrusted": raw_top_container, "docker_stats": docker_stats_status, + "top_process_family": top_family, + "stock_process_cpu_percent": round(stock_process_cpu, 3), + "gitea_process_cpu_percent": round(gitea_process_cpu, 3), + "control_plane_process_cpu_percent": round(control_plane_cpu, 3), }, "commands": { "dry_run": dry_run_command, @@ -440,8 +662,11 @@ def main() -> int: docker_stats_file=args.docker_stats_file, max_age_seconds=args.docker_stats_max_age_seconds, ), + process_summary=_summarize_processes(_parse_ps_text(_collect_ps_text(args.ps_file)), top_n=args.top_n), load5_per_core_threshold=args.load5_per_core_threshold, container_cpu_threshold=args.container_cpu_threshold, + hot_container_cpu_threshold=args.hot_container_cpu_threshold, + process_family_cpu_threshold=args.process_family_cpu_threshold, ci_stale_age_seconds=args.ci_stale_age_seconds, ) if args.json: diff --git a/scripts/ops/tests/test_host_pressure_alert_contract.py b/scripts/ops/tests/test_host_pressure_alert_contract.py index ad895d82d..f39ec7a9c 100644 --- a/scripts/ops/tests/test_host_pressure_alert_contract.py +++ b/scripts/ops/tests/test_host_pressure_alert_contract.py @@ -29,14 +29,19 @@ def test_110_moderate_pressure_alert_routes_to_live_controller() -> None: assert 'awoooi_host_load5_per_core{host="110"} > 0.75' in expr assert 'docker_container_cpu_cores{host="110"' in expr - assert "> 2.0" in expr + assert 'awoooi_host_process_family_cpu_percent{host="110"' in expr + assert "> 1.0" in expr + assert "> 50" in expr + assert "systemd_control_plane" in expr assert "gitea" in expr assert "stockplatform-v2-postgres-1" in expr assert rule["for"] == "1m" assert rule["labels"]["auto_repair"] == "true" assert "/home/wooo/scripts/host-sustained-load-controller.py" in action assert "--load5-per-core-threshold 0.75" in action + assert "--hot-container-cpu-threshold 1.0" in action assert "--container-cpu-threshold 2.0" in action + assert "--process-family-cpu-threshold 50" in action assert "不讀 secret" in annotations["runbook"] assert "禁止 Docker / systemd / Nginx / DB restart" in annotations["runbook"] diff --git a/scripts/ops/tests/test_host_runaway_process_exporter.py b/scripts/ops/tests/test_host_runaway_process_exporter.py index 9dfe1c7ad..2558c2fc3 100644 --- a/scripts/ops/tests/test_host_runaway_process_exporter.py +++ b/scripts/ops/tests/test_host_runaway_process_exporter.py @@ -88,6 +88,11 @@ def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) -> cpu_percent=188.5, oldest_age_seconds=240, ), + process_family_summary=exporter.process_family_load( + exporter.parse_ps_rows( + "200 1 200 200 7200 55.0 S gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini" + ) + ), min_age_seconds=1800, min_cpu_percent=50, now=123, @@ -103,6 +108,9 @@ def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) -> assert 'awoooi_host_gitea_actions_active_process_oldest_age_seconds{host="110"} 240' in metrics assert 'awoooi_host_swap_used_ratio{host="110"} 1.000000' in metrics assert 'awoooi_host_runaway_process_remediation_authorized{host="110"} 0' in metrics + assert 'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 55.000000' in metrics + assert 'awoooi_host_process_family_top_info{host="110",family="gitea_service",comm="gitea"} 1' in metrics + assert "/home/wooo/gitea/app.ini" not in metrics assert 'rule="stockplatform_headless_smoke"' in metrics @@ -604,6 +612,181 @@ def test_sustained_load_controller_routes_unknown_load_to_sanitized_evidence(tmp assert payload["operation_boundaries"]["process_signal_performed"] is False +def test_sustained_load_controller_routes_moderate_stock_container_pressure(tmp_path: Path) -> None: + metrics_file = tmp_path / "host.prom" + metrics_file.write_text( + "\n".join( + [ + 'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1', + 'awoooi_host_load5_per_core{host="110"} 0.62', + 'awoooi_host_swap_used_ratio{host="110"} 0.27', + 'awoooi_host_runaway_process_remediation_authorized{host="110"} 0', + 'awoooi_host_gitea_actions_active_container_count{host="110"} 0', + 'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0', + 'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 0', + ] + ), + encoding="utf-8", + ) + docker_file = tmp_path / "docker.prom" + docker_file.write_text( + "\n".join( + [ + 'docker_container_cpu_cores{host="110",container_name="stockplatform-v2-postgres-1"} 1.654', + 'docker_container_cpu_cores{host="110",container_name="gitea"} 1.092', + ] + ), + encoding="utf-8", + ) + ps_file = tmp_path / "ps.txt" + ps_file.write_text( + "\n".join( + [ + "100 1 100 73767 20.0 0.0 systemd /sbin/init", + "200 1 200 73608 30.0 1.1 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini", + "300 1 300 60 20.8 0.2 postgres postgres: stockplatform stockplatform SELECT", + ] + ), + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(CONTROLLER_PATH), + "--host", + "110", + "--load5-per-core-threshold", + "0.75", + "--container-cpu-threshold", + "2.0", + "--metrics-file", + str(metrics_file), + "--docker-stats-file", + str(docker_file), + "--ps-file", + str(ps_file), + "--json", + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 75 + payload = json.loads(result.stdout) + assert ( + payload["classification"] + == "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook" + ) + assert payload["severity"] == "warning" + assert payload["readback"]["hot_container_cpu_threshold"] == 1.0 + assert payload["readback"]["top_container_cpu"]["container_name"] == "stockplatform-v2-postgres-1" + assert payload["readback"]["top_process_family"]["family"] == "gitea_service" + assert payload["controlled_apply_allowed"] is False + assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"] + assert "/home/wooo/gitea/app.ini" not in result.stdout + + +def test_sustained_load_controller_routes_gitea_process_pressure_without_hot_container( + tmp_path: Path, +) -> None: + metrics_file = tmp_path / "host.prom" + metrics_file.write_text( + "\n".join( + [ + 'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1', + 'awoooi_host_load5_per_core{host="110"} 0.55', + 'awoooi_host_swap_used_ratio{host="110"} 0.1', + 'awoooi_host_runaway_process_remediation_authorized{host="110"} 0', + ] + ), + encoding="utf-8", + ) + docker_file = tmp_path / "docker.prom" + docker_file.write_text( + 'docker_container_cpu_cores{host="110",container_name="gitea"} 0.20\n', + encoding="utf-8", + ) + ps_file = tmp_path / "ps.txt" + ps_file.write_text( + "200 1 200 73608 55.5 1.1 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini\n", + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(CONTROLLER_PATH), + "--host", + "110", + "--metrics-file", + str(metrics_file), + "--docker-stats-file", + str(docker_file), + "--ps-file", + str(ps_file), + "--json", + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 75 + payload = json.loads(result.stdout) + assert payload["classification"] == "blocked_gitea_queue_or_hook_backlog_requires_playbook" + assert payload["readback"]["gitea_process_cpu_percent"] == 55.5 + assert payload["controlled_apply_allowed"] is False + assert "/home/wooo/gitea/app.ini" not in result.stdout + + +def test_sustained_load_controller_routes_control_plane_family_pressure(tmp_path: Path) -> None: + metrics_file = tmp_path / "host.prom" + metrics_file.write_text( + "\n".join( + [ + 'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1', + 'awoooi_host_load5_per_core{host="110"} 0.70', + 'awoooi_host_swap_used_ratio{host="110"} 0.1', + 'awoooi_host_runaway_process_remediation_authorized{host="110"} 0', + ] + ), + encoding="utf-8", + ) + ps_file = tmp_path / "ps.txt" + ps_file.write_text( + "\n".join( + [ + "100 1 100 73767 40.0 0.0 systemd /sbin/init", + "101 1 101 73767 15.0 0.0 dbus-daemon @dbus-daemon --system", + "200 1 200 73608 20.0 1.1 gitea /usr/local/bin/gitea web", + ] + ), + encoding="utf-8", + ) + + result = subprocess.run( + [ + sys.executable, + str(CONTROLLER_PATH), + "--host", + "110", + "--metrics-file", + str(metrics_file), + "--ps-file", + str(ps_file), + "--json", + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 75 + payload = json.loads(result.stdout) + assert payload["classification"] == "blocked_control_plane_saturation_requires_playbook" + assert payload["readback"]["control_plane_process_cpu_percent"] == 55.0 + assert payload["next_action"] == "run_control_plane_saturation_playbook_check_mode" + + def test_sustained_load_evidence_emits_sanitized_gitea_recommendation(tmp_path: Path) -> None: ps_file = tmp_path / "ps.txt" ps_file.write_text(