fix(ops): alert on 110 process-family pressure
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 2m23s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
Your Name
2026-07-02 11:55:45 +08:00
parent 1e29fbb7bd
commit 7249d81017
7 changed files with 578 additions and 11 deletions

View File

@@ -87,6 +87,22 @@ GITEA_ACTION_PROCESS_RE = re.compile(
r"\bnext build\b|\bpnpm install\b|\bnpm ci\b|\byarn install\b)"
)
HOST_PRESSURE_GATE_RE = re.compile(r"wait-host-web-build-pressure\.sh|awoooi-wait-host-web-build-pressure\.sh")
KNOWN_PROCESS_FAMILIES = (
"systemd_control_plane",
"ssh_control_plane",
"gitea_service",
"postgres",
"clickhouse",
"kafka",
"sentry",
"python_job",
"node_service",
"gitea_actions_runner",
"docker_build",
"web_build",
"headless_browser",
"unknown",
)
def escape_label(value: str) -> str:
@@ -261,6 +277,79 @@ def active_gitea_action_process_load(rows: list[ProcessRow]) -> ActiveCiLoad:
)
def classify_process_family(row: ProcessRow) -> str:
text = f"{row.comm} {row.args}".lower()
if (
"act_runner" in text
or "gitea-actions-task" in text
or "/.cache/act/" in text
or "/opt/hostedtoolcache/" in text
or "pnpm install" in text
or "npm ci" in text
or "yarn install" in text
):
return "gitea_actions_runner"
if "docker build" in text or "buildx" in text or "buildkit" in text:
return "docker_build"
if "next build" in text or "turbo build" in text or ("pnpm" in text and " build" in text):
return "web_build"
if "chrome" in text or "chromium" in text or "playwright" in text:
return "headless_browser"
if "gitea" in text:
return "gitea_service"
if "postgres" in text or "postmaster" in text:
return "postgres"
if "clickhouse" in text:
return "clickhouse"
if "kafka" in text:
return "kafka"
if "sentry" in text:
return "sentry"
if "systemctl" in text or "systemd" in text or "dbus" in text:
return "systemd_control_plane"
if "sshd" in text:
return "ssh_control_plane"
if "python" in text:
return "python_job"
if "node" in text:
return "node_service"
return "unknown"
def process_family_load(rows: list[ProcessRow]) -> dict[str, dict[str, object]]:
families: dict[str, dict[str, object]] = {
family: {
"cpu_percent": 0.0,
"process_count": 0,
"oldest_age_seconds": 0,
"sample_comm": "",
"_sample_cpu": -1.0,
}
for family in KNOWN_PROCESS_FAMILIES
}
for row in rows:
family = classify_process_family(row)
current = families.setdefault(
family,
{
"cpu_percent": 0.0,
"process_count": 0,
"oldest_age_seconds": 0,
"sample_comm": "",
"_sample_cpu": -1.0,
},
)
current["cpu_percent"] = float(current["cpu_percent"]) + row.pcpu
current["process_count"] = int(current["process_count"]) + 1
current["oldest_age_seconds"] = max(int(current["oldest_age_seconds"]), row.etimes)
if row.pcpu > float(current["_sample_cpu"]):
current["sample_comm"] = row.comm[:48]
current["_sample_cpu"] = row.pcpu
for current in families.values():
current.pop("_sample_cpu", None)
return families
def load5_per_core() -> float:
try:
load5 = float(Path("/proc/loadavg").read_text(encoding="utf-8").split()[1])
@@ -296,6 +385,7 @@ def render_metrics(
groups: list[ProcessGroup],
active_action_containers: int,
active_action_process_load: ActiveCiLoad,
process_family_summary: dict[str, dict[str, object]],
min_age_seconds: int,
min_cpu_percent: float,
now: int,
@@ -336,6 +426,14 @@ def render_metrics(
"# TYPE awoooi_host_load5_per_core gauge",
"# HELP awoooi_host_swap_used_ratio Host swap used ratio from /proc/meminfo.",
"# TYPE awoooi_host_swap_used_ratio gauge",
"# HELP awoooi_host_process_family_cpu_percent Sanitized CPU percent grouped by process family.",
"# TYPE awoooi_host_process_family_cpu_percent gauge",
"# HELP awoooi_host_process_family_process_count Sanitized process count grouped by process family.",
"# TYPE awoooi_host_process_family_process_count gauge",
"# HELP awoooi_host_process_family_oldest_age_seconds Oldest process age grouped by process family.",
"# TYPE awoooi_host_process_family_oldest_age_seconds gauge",
"# HELP awoooi_host_process_family_top_info Sample command name for a sanitized process family.",
"# TYPE awoooi_host_process_family_top_info gauge",
"# HELP awoooi_host_runaway_process_remediation_authorized Static guardrail: remediation is not authorized by this exporter.",
"# TYPE awoooi_host_runaway_process_remediation_authorized gauge",
f"awoooi_host_runaway_process_monitor_up{{{labels_host},mode=\"read_only\"}} 1",
@@ -350,6 +448,26 @@ def render_metrics(
f"awoooi_host_runaway_process_remediation_authorized{{{labels_host}}} 0",
]
for family in sorted(process_family_summary):
item = process_family_summary[family]
family_labels = f'{labels_host},family="{escape_label(family)}"'
lines.append(
f"awoooi_host_process_family_cpu_percent{{{family_labels}}} "
f"{float(item['cpu_percent']):.6f}"
)
lines.append(
f"awoooi_host_process_family_process_count{{{family_labels}}} "
f"{int(item['process_count'])}"
)
lines.append(
f"awoooi_host_process_family_oldest_age_seconds{{{family_labels}}} "
f"{int(item['oldest_age_seconds'])}"
)
sample_comm = str(item.get("sample_comm") or "none")
lines.append(
f'awoooi_host_process_family_top_info{{{family_labels},comm="{escape_label(sample_comm)}"}} 1'
)
for rule_id in rule_ids:
rule_labels = (
f'{labels_host},rule="{escape_label(rule_id)}",'
@@ -393,6 +511,7 @@ def collect(args: argparse.Namespace) -> str:
groups=groups,
active_action_containers=active_gitea_action_containers(args.docker_ps_file),
active_action_process_load=active_gitea_action_process_load(rows),
process_family_summary=process_family_load(rows),
min_age_seconds=args.min_age_seconds,
min_cpu_percent=args.min_cpu_percent,
now=int(time.time()),

View File

@@ -20,6 +20,7 @@ from __future__ import annotations
import argparse
import json
import re
import subprocess
import time
from pathlib import Path
from typing import Any
@@ -50,7 +51,11 @@ def parse_args() -> argparse.Namespace:
)
parser.add_argument("--load5-per-core-threshold", type=float, default=1.5)
parser.add_argument("--container-cpu-threshold", type=float, default=2.0)
parser.add_argument("--hot-container-cpu-threshold", type=float, default=1.0)
parser.add_argument("--process-family-cpu-threshold", type=float, default=50.0)
parser.add_argument("--ci-stale-age-seconds", type=int, default=1800)
parser.add_argument("--ps-file", type=Path)
parser.add_argument("--top-n", type=int, default=8)
parser.add_argument("--json", action="store_true", help="Print JSON only.")
return parser.parse_args()
@@ -211,14 +216,171 @@ def _top_container_cpu(samples: list[dict[str, Any]], *, host: str) -> dict[str,
return sorted(candidates, key=lambda item: (-item["cpu_cores"], item["container_name"]))[0]
def _read_text(path: Path | None) -> str:
if path is None:
return ""
try:
return path.read_text(encoding="utf-8")
except FileNotFoundError:
return ""
def _collect_ps_text(ps_file: Path | None) -> str:
if ps_file is not None:
return _read_text(ps_file)
try:
result = subprocess.run(
["ps", "-eo", "pid=,ppid=,pgid=,etimes=,pcpu=,pmem=,comm=,args="],
check=True,
capture_output=True,
text=True,
timeout=10,
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
return ""
return result.stdout
def _classify_process_family(comm: str, args: str) -> str:
text = f"{comm} {args}".lower()
if (
"act_runner" in text
or "gitea-actions-task" in text
or "/.cache/act/" in text
or "/opt/hostedtoolcache/" in text
or "pnpm install" in text
or "npm ci" in text
or "yarn install" in text
):
return "gitea_actions_runner"
if "docker build" in text or "buildx" in text or "buildkit" in text:
return "docker_build"
if "next build" in text or "turbo build" in text or ("pnpm" in text and " build" in text):
return "web_build"
if "chrome" in text or "chromium" in text or "playwright" in text:
return "headless_browser"
if "gitea" in text:
return "gitea_service"
if "postgres" in text or "postmaster" in text:
return "postgres"
if "clickhouse" in text:
return "clickhouse"
if "kafka" in text:
return "kafka"
if "sentry" in text:
return "sentry"
if "systemctl" in text or "systemd" in text or "dbus" in text:
return "systemd_control_plane"
if "sshd" in text:
return "ssh_control_plane"
if "python" in text:
return "python_job"
if "node" in text:
return "node_service"
return "unknown"
def _parse_ps_text(text: str) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for raw_line in text.splitlines():
line = raw_line.strip()
if not line:
continue
parts = line.split(None, 7)
if len(parts) < 7:
continue
pid, ppid, pgid, etimes, pcpu, pmem, comm = parts[:7]
args = parts[7] if len(parts) > 7 else comm
try:
rows.append(
{
"pid": int(pid),
"ppid": int(ppid),
"pgid": int(pgid),
"etimes": int(float(etimes)),
"cpu_percent": float(pcpu),
"mem_percent": float(pmem),
"comm": Path(comm).name[:48],
"family": _classify_process_family(comm, args),
}
)
except ValueError:
continue
return rows
def _summarize_processes(rows: list[dict[str, Any]], *, top_n: int) -> dict[str, Any]:
top_rows = sorted(rows, key=lambda item: (-item["cpu_percent"], item["comm"], item["pid"]))[:top_n]
families: dict[str, dict[str, Any]] = {}
for row in rows:
family = row["family"]
current = families.setdefault(
family,
{
"family": family,
"process_count": 0,
"cpu_percent": 0.0,
"max_age_seconds": 0,
"sample_comm": "",
},
)
current["process_count"] += 1
current["cpu_percent"] += row["cpu_percent"]
current["max_age_seconds"] = max(current["max_age_seconds"], row["etimes"])
if not current["sample_comm"] or row["cpu_percent"] > current.get("_sample_cpu", -1):
current["sample_comm"] = row["comm"]
current["_sample_cpu"] = row["cpu_percent"]
family_rows = []
for item in families.values():
item.pop("_sample_cpu", None)
item["cpu_percent"] = round(float(item["cpu_percent"]), 3)
family_rows.append(item)
return {
"top_processes": [
{
"pid": row["pid"],
"ppid": row["ppid"],
"pgid": row["pgid"],
"cpu_percent": round(row["cpu_percent"], 3),
"mem_percent": round(row["mem_percent"], 3),
"age_seconds": row["etimes"],
"comm": row["comm"],
"family": row["family"],
}
for row in top_rows
],
"families": sorted(family_rows, key=lambda item: (-item["cpu_percent"], item["family"]))[
:top_n
],
}
def _family_cpu(process_summary: dict[str, Any], *families: str) -> float:
return sum(
float(item.get("cpu_percent") or 0.0)
for item in process_summary.get("families", [])
if item.get("family") in families
)
def _top_family(process_summary: dict[str, Any]) -> dict[str, Any] | None:
families = process_summary.get("families", [])
return families[0] if families else None
def build_packet(
*,
host: str,
samples: list[dict[str, Any]],
docker_samples: list[dict[str, Any]],
docker_stats_status: dict[str, Any],
process_summary: dict[str, Any],
load5_per_core_threshold: float,
container_cpu_threshold: float,
hot_container_cpu_threshold: float,
process_family_cpu_threshold: float,
ci_stale_age_seconds: int,
) -> dict[str, Any]:
monitor_up = int(
@@ -272,6 +434,10 @@ def build_packet(
top_container = raw_top_container if docker_stats_status.get("fresh") is True else None
top_container_name = str((top_container or {}).get("container_name") or "").lower()
top_container_cpu = float((top_container or {}).get("cpu_cores") or 0.0)
stock_process_cpu = _family_cpu(process_summary, "postgres")
gitea_process_cpu = _family_cpu(process_summary, "gitea_service")
control_plane_cpu = _family_cpu(process_summary, "systemd_control_plane", "ssh_control_plane")
top_family = _top_family(process_summary)
classification = "observing_load_within_threshold"
severity = "info"
@@ -329,19 +495,58 @@ def build_packet(
if controlled_apply_allowed
else "keep_pressure_gate_fail_closed_until_ci_load_clears"
)
elif top_container_name == "gitea" and top_container_cpu >= container_cpu_threshold:
elif (
(
"stockplatform-v2-postgres-1" in top_container_name
or top_container_name == "stockplatform-v2-api-1"
)
and top_container_cpu >= container_cpu_threshold
) or stock_process_cpu >= process_family_cpu_threshold * 2:
classification = "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook"
severity = (
"critical"
if load5_per_core > load5_per_core_threshold
or stock_process_cpu >= process_family_cpu_threshold * 2
else "warning"
)
dry_run_command = (
"scripts/ops/host-sustained-load-evidence.py "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
)
next_action = "run_stockplatform_hot_query_or_api_pressure_playbook_check_mode"
elif (
top_container_name == "gitea" and top_container_cpu >= container_cpu_threshold
) or gitea_process_cpu >= process_family_cpu_threshold * 2:
classification = "blocked_gitea_queue_or_hook_backlog_requires_playbook"
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
severity = (
"critical"
if load5_per_core > load5_per_core_threshold
or gitea_process_cpu >= process_family_cpu_threshold * 2
else "warning"
)
dry_run_command = (
"scripts/ops/host-sustained-load-evidence.py "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
)
next_action = "run_gitea_queue_or_hook_backlog_playbook_check_mode"
elif control_plane_cpu >= process_family_cpu_threshold:
classification = "blocked_control_plane_saturation_requires_playbook"
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
dry_run_command = (
"scripts/ops/host-sustained-load-evidence.py "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
)
next_action = "run_control_plane_saturation_playbook_check_mode"
elif (
top_container_name in {"stockplatform-v2-postgres-1", "stockplatform-v2-api-1"}
and top_container_cpu >= container_cpu_threshold
):
"stockplatform-v2-postgres-1" in top_container_name
and top_container_cpu >= hot_container_cpu_threshold
) or (
top_container_name == "stockplatform-v2-api-1"
and top_container_cpu >= hot_container_cpu_threshold
) or stock_process_cpu >= process_family_cpu_threshold:
classification = "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook"
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
dry_run_command = (
@@ -350,6 +555,17 @@ def build_packet(
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
)
next_action = "run_stockplatform_hot_query_or_api_pressure_playbook_check_mode"
elif (
top_container_name == "gitea" and top_container_cpu >= hot_container_cpu_threshold
) or gitea_process_cpu >= process_family_cpu_threshold:
classification = "blocked_gitea_queue_or_hook_backlog_requires_playbook"
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
dry_run_command = (
"scripts/ops/host-sustained-load-evidence.py "
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
)
next_action = "run_gitea_queue_or_hook_backlog_playbook_check_mode"
elif load5_per_core > load5_per_core_threshold and swap_used_ratio >= 0.85:
classification = "blocked_memory_or_swap_pressure_requires_service_playbook"
severity = "critical"
@@ -378,6 +594,8 @@ def build_packet(
"load5_per_core": round(load5_per_core, 6),
"load5_per_core_threshold": load5_per_core_threshold,
"container_cpu_threshold": container_cpu_threshold,
"hot_container_cpu_threshold": hot_container_cpu_threshold,
"process_family_cpu_threshold": process_family_cpu_threshold,
"swap_used_ratio": round(swap_used_ratio, 6),
"remediation_authorized": remediation_authorized,
"active_ci_container_count": active_ci_containers,
@@ -388,6 +606,10 @@ def build_packet(
"top_container_cpu": top_container,
"top_container_cpu_untrusted": raw_top_container,
"docker_stats": docker_stats_status,
"top_process_family": top_family,
"stock_process_cpu_percent": round(stock_process_cpu, 3),
"gitea_process_cpu_percent": round(gitea_process_cpu, 3),
"control_plane_process_cpu_percent": round(control_plane_cpu, 3),
},
"commands": {
"dry_run": dry_run_command,
@@ -440,8 +662,11 @@ def main() -> int:
docker_stats_file=args.docker_stats_file,
max_age_seconds=args.docker_stats_max_age_seconds,
),
process_summary=_summarize_processes(_parse_ps_text(_collect_ps_text(args.ps_file)), top_n=args.top_n),
load5_per_core_threshold=args.load5_per_core_threshold,
container_cpu_threshold=args.container_cpu_threshold,
hot_container_cpu_threshold=args.hot_container_cpu_threshold,
process_family_cpu_threshold=args.process_family_cpu_threshold,
ci_stale_age_seconds=args.ci_stale_age_seconds,
)
if args.json:

View File

@@ -29,14 +29,19 @@ def test_110_moderate_pressure_alert_routes_to_live_controller() -> None:
assert 'awoooi_host_load5_per_core{host="110"} > 0.75' in expr
assert 'docker_container_cpu_cores{host="110"' in expr
assert "> 2.0" in expr
assert 'awoooi_host_process_family_cpu_percent{host="110"' in expr
assert "> 1.0" in expr
assert "> 50" in expr
assert "systemd_control_plane" in expr
assert "gitea" in expr
assert "stockplatform-v2-postgres-1" in expr
assert rule["for"] == "1m"
assert rule["labels"]["auto_repair"] == "true"
assert "/home/wooo/scripts/host-sustained-load-controller.py" in action
assert "--load5-per-core-threshold 0.75" in action
assert "--hot-container-cpu-threshold 1.0" in action
assert "--container-cpu-threshold 2.0" in action
assert "--process-family-cpu-threshold 50" in action
assert "不讀 secret" in annotations["runbook"]
assert "禁止 Docker / systemd / Nginx / DB restart" in annotations["runbook"]

View File

@@ -88,6 +88,11 @@ def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) ->
cpu_percent=188.5,
oldest_age_seconds=240,
),
process_family_summary=exporter.process_family_load(
exporter.parse_ps_rows(
"200 1 200 200 7200 55.0 S gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini"
)
),
min_age_seconds=1800,
min_cpu_percent=50,
now=123,
@@ -103,6 +108,9 @@ def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) ->
assert 'awoooi_host_gitea_actions_active_process_oldest_age_seconds{host="110"} 240' in metrics
assert 'awoooi_host_swap_used_ratio{host="110"} 1.000000' in metrics
assert 'awoooi_host_runaway_process_remediation_authorized{host="110"} 0' in metrics
assert 'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 55.000000' in metrics
assert 'awoooi_host_process_family_top_info{host="110",family="gitea_service",comm="gitea"} 1' in metrics
assert "/home/wooo/gitea/app.ini" not in metrics
assert 'rule="stockplatform_headless_smoke"' in metrics
@@ -604,6 +612,181 @@ def test_sustained_load_controller_routes_unknown_load_to_sanitized_evidence(tmp
assert payload["operation_boundaries"]["process_signal_performed"] is False
def test_sustained_load_controller_routes_moderate_stock_container_pressure(tmp_path: Path) -> None:
metrics_file = tmp_path / "host.prom"
metrics_file.write_text(
"\n".join(
[
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
'awoooi_host_load5_per_core{host="110"} 0.62',
'awoooi_host_swap_used_ratio{host="110"} 0.27',
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 0',
]
),
encoding="utf-8",
)
docker_file = tmp_path / "docker.prom"
docker_file.write_text(
"\n".join(
[
'docker_container_cpu_cores{host="110",container_name="stockplatform-v2-postgres-1"} 1.654',
'docker_container_cpu_cores{host="110",container_name="gitea"} 1.092',
]
),
encoding="utf-8",
)
ps_file = tmp_path / "ps.txt"
ps_file.write_text(
"\n".join(
[
"100 1 100 73767 20.0 0.0 systemd /sbin/init",
"200 1 200 73608 30.0 1.1 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini",
"300 1 300 60 20.8 0.2 postgres postgres: stockplatform stockplatform SELECT",
]
),
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
str(CONTROLLER_PATH),
"--host",
"110",
"--load5-per-core-threshold",
"0.75",
"--container-cpu-threshold",
"2.0",
"--metrics-file",
str(metrics_file),
"--docker-stats-file",
str(docker_file),
"--ps-file",
str(ps_file),
"--json",
],
capture_output=True,
text=True,
)
assert result.returncode == 75
payload = json.loads(result.stdout)
assert (
payload["classification"]
== "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook"
)
assert payload["severity"] == "warning"
assert payload["readback"]["hot_container_cpu_threshold"] == 1.0
assert payload["readback"]["top_container_cpu"]["container_name"] == "stockplatform-v2-postgres-1"
assert payload["readback"]["top_process_family"]["family"] == "gitea_service"
assert payload["controlled_apply_allowed"] is False
assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"]
assert "/home/wooo/gitea/app.ini" not in result.stdout
def test_sustained_load_controller_routes_gitea_process_pressure_without_hot_container(
tmp_path: Path,
) -> None:
metrics_file = tmp_path / "host.prom"
metrics_file.write_text(
"\n".join(
[
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
'awoooi_host_load5_per_core{host="110"} 0.55',
'awoooi_host_swap_used_ratio{host="110"} 0.1',
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
]
),
encoding="utf-8",
)
docker_file = tmp_path / "docker.prom"
docker_file.write_text(
'docker_container_cpu_cores{host="110",container_name="gitea"} 0.20\n',
encoding="utf-8",
)
ps_file = tmp_path / "ps.txt"
ps_file.write_text(
"200 1 200 73608 55.5 1.1 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini\n",
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
str(CONTROLLER_PATH),
"--host",
"110",
"--metrics-file",
str(metrics_file),
"--docker-stats-file",
str(docker_file),
"--ps-file",
str(ps_file),
"--json",
],
capture_output=True,
text=True,
)
assert result.returncode == 75
payload = json.loads(result.stdout)
assert payload["classification"] == "blocked_gitea_queue_or_hook_backlog_requires_playbook"
assert payload["readback"]["gitea_process_cpu_percent"] == 55.5
assert payload["controlled_apply_allowed"] is False
assert "/home/wooo/gitea/app.ini" not in result.stdout
def test_sustained_load_controller_routes_control_plane_family_pressure(tmp_path: Path) -> None:
metrics_file = tmp_path / "host.prom"
metrics_file.write_text(
"\n".join(
[
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
'awoooi_host_load5_per_core{host="110"} 0.70',
'awoooi_host_swap_used_ratio{host="110"} 0.1',
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
]
),
encoding="utf-8",
)
ps_file = tmp_path / "ps.txt"
ps_file.write_text(
"\n".join(
[
"100 1 100 73767 40.0 0.0 systemd /sbin/init",
"101 1 101 73767 15.0 0.0 dbus-daemon @dbus-daemon --system",
"200 1 200 73608 20.0 1.1 gitea /usr/local/bin/gitea web",
]
),
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
str(CONTROLLER_PATH),
"--host",
"110",
"--metrics-file",
str(metrics_file),
"--ps-file",
str(ps_file),
"--json",
],
capture_output=True,
text=True,
)
assert result.returncode == 75
payload = json.loads(result.stdout)
assert payload["classification"] == "blocked_control_plane_saturation_requires_playbook"
assert payload["readback"]["control_plane_process_cpu_percent"] == 55.0
assert payload["next_action"] == "run_control_plane_saturation_playbook_check_mode"
def test_sustained_load_evidence_emits_sanitized_gitea_recommendation(tmp_path: Path) -> None:
ps_file = tmp_path / "ps.txt"
ps_file.write_text(