fix(ops): alert on 110 process-family pressure
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 2m23s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 2m23s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
This commit is contained in:
@@ -87,6 +87,22 @@ GITEA_ACTION_PROCESS_RE = re.compile(
|
||||
r"\bnext build\b|\bpnpm install\b|\bnpm ci\b|\byarn install\b)"
|
||||
)
|
||||
HOST_PRESSURE_GATE_RE = re.compile(r"wait-host-web-build-pressure\.sh|awoooi-wait-host-web-build-pressure\.sh")
|
||||
KNOWN_PROCESS_FAMILIES = (
|
||||
"systemd_control_plane",
|
||||
"ssh_control_plane",
|
||||
"gitea_service",
|
||||
"postgres",
|
||||
"clickhouse",
|
||||
"kafka",
|
||||
"sentry",
|
||||
"python_job",
|
||||
"node_service",
|
||||
"gitea_actions_runner",
|
||||
"docker_build",
|
||||
"web_build",
|
||||
"headless_browser",
|
||||
"unknown",
|
||||
)
|
||||
|
||||
|
||||
def escape_label(value: str) -> str:
|
||||
@@ -261,6 +277,79 @@ def active_gitea_action_process_load(rows: list[ProcessRow]) -> ActiveCiLoad:
|
||||
)
|
||||
|
||||
|
||||
def classify_process_family(row: ProcessRow) -> str:
|
||||
text = f"{row.comm} {row.args}".lower()
|
||||
if (
|
||||
"act_runner" in text
|
||||
or "gitea-actions-task" in text
|
||||
or "/.cache/act/" in text
|
||||
or "/opt/hostedtoolcache/" in text
|
||||
or "pnpm install" in text
|
||||
or "npm ci" in text
|
||||
or "yarn install" in text
|
||||
):
|
||||
return "gitea_actions_runner"
|
||||
if "docker build" in text or "buildx" in text or "buildkit" in text:
|
||||
return "docker_build"
|
||||
if "next build" in text or "turbo build" in text or ("pnpm" in text and " build" in text):
|
||||
return "web_build"
|
||||
if "chrome" in text or "chromium" in text or "playwright" in text:
|
||||
return "headless_browser"
|
||||
if "gitea" in text:
|
||||
return "gitea_service"
|
||||
if "postgres" in text or "postmaster" in text:
|
||||
return "postgres"
|
||||
if "clickhouse" in text:
|
||||
return "clickhouse"
|
||||
if "kafka" in text:
|
||||
return "kafka"
|
||||
if "sentry" in text:
|
||||
return "sentry"
|
||||
if "systemctl" in text or "systemd" in text or "dbus" in text:
|
||||
return "systemd_control_plane"
|
||||
if "sshd" in text:
|
||||
return "ssh_control_plane"
|
||||
if "python" in text:
|
||||
return "python_job"
|
||||
if "node" in text:
|
||||
return "node_service"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def process_family_load(rows: list[ProcessRow]) -> dict[str, dict[str, object]]:
|
||||
families: dict[str, dict[str, object]] = {
|
||||
family: {
|
||||
"cpu_percent": 0.0,
|
||||
"process_count": 0,
|
||||
"oldest_age_seconds": 0,
|
||||
"sample_comm": "",
|
||||
"_sample_cpu": -1.0,
|
||||
}
|
||||
for family in KNOWN_PROCESS_FAMILIES
|
||||
}
|
||||
for row in rows:
|
||||
family = classify_process_family(row)
|
||||
current = families.setdefault(
|
||||
family,
|
||||
{
|
||||
"cpu_percent": 0.0,
|
||||
"process_count": 0,
|
||||
"oldest_age_seconds": 0,
|
||||
"sample_comm": "",
|
||||
"_sample_cpu": -1.0,
|
||||
},
|
||||
)
|
||||
current["cpu_percent"] = float(current["cpu_percent"]) + row.pcpu
|
||||
current["process_count"] = int(current["process_count"]) + 1
|
||||
current["oldest_age_seconds"] = max(int(current["oldest_age_seconds"]), row.etimes)
|
||||
if row.pcpu > float(current["_sample_cpu"]):
|
||||
current["sample_comm"] = row.comm[:48]
|
||||
current["_sample_cpu"] = row.pcpu
|
||||
for current in families.values():
|
||||
current.pop("_sample_cpu", None)
|
||||
return families
|
||||
|
||||
|
||||
def load5_per_core() -> float:
|
||||
try:
|
||||
load5 = float(Path("/proc/loadavg").read_text(encoding="utf-8").split()[1])
|
||||
@@ -296,6 +385,7 @@ def render_metrics(
|
||||
groups: list[ProcessGroup],
|
||||
active_action_containers: int,
|
||||
active_action_process_load: ActiveCiLoad,
|
||||
process_family_summary: dict[str, dict[str, object]],
|
||||
min_age_seconds: int,
|
||||
min_cpu_percent: float,
|
||||
now: int,
|
||||
@@ -336,6 +426,14 @@ def render_metrics(
|
||||
"# TYPE awoooi_host_load5_per_core gauge",
|
||||
"# HELP awoooi_host_swap_used_ratio Host swap used ratio from /proc/meminfo.",
|
||||
"# TYPE awoooi_host_swap_used_ratio gauge",
|
||||
"# HELP awoooi_host_process_family_cpu_percent Sanitized CPU percent grouped by process family.",
|
||||
"# TYPE awoooi_host_process_family_cpu_percent gauge",
|
||||
"# HELP awoooi_host_process_family_process_count Sanitized process count grouped by process family.",
|
||||
"# TYPE awoooi_host_process_family_process_count gauge",
|
||||
"# HELP awoooi_host_process_family_oldest_age_seconds Oldest process age grouped by process family.",
|
||||
"# TYPE awoooi_host_process_family_oldest_age_seconds gauge",
|
||||
"# HELP awoooi_host_process_family_top_info Sample command name for a sanitized process family.",
|
||||
"# TYPE awoooi_host_process_family_top_info gauge",
|
||||
"# HELP awoooi_host_runaway_process_remediation_authorized Static guardrail: remediation is not authorized by this exporter.",
|
||||
"# TYPE awoooi_host_runaway_process_remediation_authorized gauge",
|
||||
f"awoooi_host_runaway_process_monitor_up{{{labels_host},mode=\"read_only\"}} 1",
|
||||
@@ -350,6 +448,26 @@ def render_metrics(
|
||||
f"awoooi_host_runaway_process_remediation_authorized{{{labels_host}}} 0",
|
||||
]
|
||||
|
||||
for family in sorted(process_family_summary):
|
||||
item = process_family_summary[family]
|
||||
family_labels = f'{labels_host},family="{escape_label(family)}"'
|
||||
lines.append(
|
||||
f"awoooi_host_process_family_cpu_percent{{{family_labels}}} "
|
||||
f"{float(item['cpu_percent']):.6f}"
|
||||
)
|
||||
lines.append(
|
||||
f"awoooi_host_process_family_process_count{{{family_labels}}} "
|
||||
f"{int(item['process_count'])}"
|
||||
)
|
||||
lines.append(
|
||||
f"awoooi_host_process_family_oldest_age_seconds{{{family_labels}}} "
|
||||
f"{int(item['oldest_age_seconds'])}"
|
||||
)
|
||||
sample_comm = str(item.get("sample_comm") or "none")
|
||||
lines.append(
|
||||
f'awoooi_host_process_family_top_info{{{family_labels},comm="{escape_label(sample_comm)}"}} 1'
|
||||
)
|
||||
|
||||
for rule_id in rule_ids:
|
||||
rule_labels = (
|
||||
f'{labels_host},rule="{escape_label(rule_id)}",'
|
||||
@@ -393,6 +511,7 @@ def collect(args: argparse.Namespace) -> str:
|
||||
groups=groups,
|
||||
active_action_containers=active_gitea_action_containers(args.docker_ps_file),
|
||||
active_action_process_load=active_gitea_action_process_load(rows),
|
||||
process_family_summary=process_family_load(rows),
|
||||
min_age_seconds=args.min_age_seconds,
|
||||
min_cpu_percent=args.min_cpu_percent,
|
||||
now=int(time.time()),
|
||||
|
||||
@@ -20,6 +20,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -50,7 +51,11 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
parser.add_argument("--load5-per-core-threshold", type=float, default=1.5)
|
||||
parser.add_argument("--container-cpu-threshold", type=float, default=2.0)
|
||||
parser.add_argument("--hot-container-cpu-threshold", type=float, default=1.0)
|
||||
parser.add_argument("--process-family-cpu-threshold", type=float, default=50.0)
|
||||
parser.add_argument("--ci-stale-age-seconds", type=int, default=1800)
|
||||
parser.add_argument("--ps-file", type=Path)
|
||||
parser.add_argument("--top-n", type=int, default=8)
|
||||
parser.add_argument("--json", action="store_true", help="Print JSON only.")
|
||||
return parser.parse_args()
|
||||
|
||||
@@ -211,14 +216,171 @@ def _top_container_cpu(samples: list[dict[str, Any]], *, host: str) -> dict[str,
|
||||
return sorted(candidates, key=lambda item: (-item["cpu_cores"], item["container_name"]))[0]
|
||||
|
||||
|
||||
def _read_text(path: Path | None) -> str:
|
||||
if path is None:
|
||||
return ""
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except FileNotFoundError:
|
||||
return ""
|
||||
|
||||
|
||||
def _collect_ps_text(ps_file: Path | None) -> str:
|
||||
if ps_file is not None:
|
||||
return _read_text(ps_file)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["ps", "-eo", "pid=,ppid=,pgid=,etimes=,pcpu=,pmem=,comm=,args="],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return ""
|
||||
return result.stdout
|
||||
|
||||
|
||||
def _classify_process_family(comm: str, args: str) -> str:
|
||||
text = f"{comm} {args}".lower()
|
||||
if (
|
||||
"act_runner" in text
|
||||
or "gitea-actions-task" in text
|
||||
or "/.cache/act/" in text
|
||||
or "/opt/hostedtoolcache/" in text
|
||||
or "pnpm install" in text
|
||||
or "npm ci" in text
|
||||
or "yarn install" in text
|
||||
):
|
||||
return "gitea_actions_runner"
|
||||
if "docker build" in text or "buildx" in text or "buildkit" in text:
|
||||
return "docker_build"
|
||||
if "next build" in text or "turbo build" in text or ("pnpm" in text and " build" in text):
|
||||
return "web_build"
|
||||
if "chrome" in text or "chromium" in text or "playwright" in text:
|
||||
return "headless_browser"
|
||||
if "gitea" in text:
|
||||
return "gitea_service"
|
||||
if "postgres" in text or "postmaster" in text:
|
||||
return "postgres"
|
||||
if "clickhouse" in text:
|
||||
return "clickhouse"
|
||||
if "kafka" in text:
|
||||
return "kafka"
|
||||
if "sentry" in text:
|
||||
return "sentry"
|
||||
if "systemctl" in text or "systemd" in text or "dbus" in text:
|
||||
return "systemd_control_plane"
|
||||
if "sshd" in text:
|
||||
return "ssh_control_plane"
|
||||
if "python" in text:
|
||||
return "python_job"
|
||||
if "node" in text:
|
||||
return "node_service"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _parse_ps_text(text: str) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split(None, 7)
|
||||
if len(parts) < 7:
|
||||
continue
|
||||
pid, ppid, pgid, etimes, pcpu, pmem, comm = parts[:7]
|
||||
args = parts[7] if len(parts) > 7 else comm
|
||||
try:
|
||||
rows.append(
|
||||
{
|
||||
"pid": int(pid),
|
||||
"ppid": int(ppid),
|
||||
"pgid": int(pgid),
|
||||
"etimes": int(float(etimes)),
|
||||
"cpu_percent": float(pcpu),
|
||||
"mem_percent": float(pmem),
|
||||
"comm": Path(comm).name[:48],
|
||||
"family": _classify_process_family(comm, args),
|
||||
}
|
||||
)
|
||||
except ValueError:
|
||||
continue
|
||||
return rows
|
||||
|
||||
|
||||
def _summarize_processes(rows: list[dict[str, Any]], *, top_n: int) -> dict[str, Any]:
|
||||
top_rows = sorted(rows, key=lambda item: (-item["cpu_percent"], item["comm"], item["pid"]))[:top_n]
|
||||
families: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
family = row["family"]
|
||||
current = families.setdefault(
|
||||
family,
|
||||
{
|
||||
"family": family,
|
||||
"process_count": 0,
|
||||
"cpu_percent": 0.0,
|
||||
"max_age_seconds": 0,
|
||||
"sample_comm": "",
|
||||
},
|
||||
)
|
||||
current["process_count"] += 1
|
||||
current["cpu_percent"] += row["cpu_percent"]
|
||||
current["max_age_seconds"] = max(current["max_age_seconds"], row["etimes"])
|
||||
if not current["sample_comm"] or row["cpu_percent"] > current.get("_sample_cpu", -1):
|
||||
current["sample_comm"] = row["comm"]
|
||||
current["_sample_cpu"] = row["cpu_percent"]
|
||||
|
||||
family_rows = []
|
||||
for item in families.values():
|
||||
item.pop("_sample_cpu", None)
|
||||
item["cpu_percent"] = round(float(item["cpu_percent"]), 3)
|
||||
family_rows.append(item)
|
||||
|
||||
return {
|
||||
"top_processes": [
|
||||
{
|
||||
"pid": row["pid"],
|
||||
"ppid": row["ppid"],
|
||||
"pgid": row["pgid"],
|
||||
"cpu_percent": round(row["cpu_percent"], 3),
|
||||
"mem_percent": round(row["mem_percent"], 3),
|
||||
"age_seconds": row["etimes"],
|
||||
"comm": row["comm"],
|
||||
"family": row["family"],
|
||||
}
|
||||
for row in top_rows
|
||||
],
|
||||
"families": sorted(family_rows, key=lambda item: (-item["cpu_percent"], item["family"]))[
|
||||
:top_n
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _family_cpu(process_summary: dict[str, Any], *families: str) -> float:
|
||||
return sum(
|
||||
float(item.get("cpu_percent") or 0.0)
|
||||
for item in process_summary.get("families", [])
|
||||
if item.get("family") in families
|
||||
)
|
||||
|
||||
|
||||
def _top_family(process_summary: dict[str, Any]) -> dict[str, Any] | None:
|
||||
families = process_summary.get("families", [])
|
||||
return families[0] if families else None
|
||||
|
||||
|
||||
def build_packet(
|
||||
*,
|
||||
host: str,
|
||||
samples: list[dict[str, Any]],
|
||||
docker_samples: list[dict[str, Any]],
|
||||
docker_stats_status: dict[str, Any],
|
||||
process_summary: dict[str, Any],
|
||||
load5_per_core_threshold: float,
|
||||
container_cpu_threshold: float,
|
||||
hot_container_cpu_threshold: float,
|
||||
process_family_cpu_threshold: float,
|
||||
ci_stale_age_seconds: int,
|
||||
) -> dict[str, Any]:
|
||||
monitor_up = int(
|
||||
@@ -272,6 +434,10 @@ def build_packet(
|
||||
top_container = raw_top_container if docker_stats_status.get("fresh") is True else None
|
||||
top_container_name = str((top_container or {}).get("container_name") or "").lower()
|
||||
top_container_cpu = float((top_container or {}).get("cpu_cores") or 0.0)
|
||||
stock_process_cpu = _family_cpu(process_summary, "postgres")
|
||||
gitea_process_cpu = _family_cpu(process_summary, "gitea_service")
|
||||
control_plane_cpu = _family_cpu(process_summary, "systemd_control_plane", "ssh_control_plane")
|
||||
top_family = _top_family(process_summary)
|
||||
|
||||
classification = "observing_load_within_threshold"
|
||||
severity = "info"
|
||||
@@ -329,19 +495,58 @@ def build_packet(
|
||||
if controlled_apply_allowed
|
||||
else "keep_pressure_gate_fail_closed_until_ci_load_clears"
|
||||
)
|
||||
elif top_container_name == "gitea" and top_container_cpu >= container_cpu_threshold:
|
||||
elif (
|
||||
(
|
||||
"stockplatform-v2-postgres-1" in top_container_name
|
||||
or top_container_name == "stockplatform-v2-api-1"
|
||||
)
|
||||
and top_container_cpu >= container_cpu_threshold
|
||||
) or stock_process_cpu >= process_family_cpu_threshold * 2:
|
||||
classification = "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook"
|
||||
severity = (
|
||||
"critical"
|
||||
if load5_per_core > load5_per_core_threshold
|
||||
or stock_process_cpu >= process_family_cpu_threshold * 2
|
||||
else "warning"
|
||||
)
|
||||
dry_run_command = (
|
||||
"scripts/ops/host-sustained-load-evidence.py "
|
||||
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
|
||||
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
|
||||
)
|
||||
next_action = "run_stockplatform_hot_query_or_api_pressure_playbook_check_mode"
|
||||
elif (
|
||||
top_container_name == "gitea" and top_container_cpu >= container_cpu_threshold
|
||||
) or gitea_process_cpu >= process_family_cpu_threshold * 2:
|
||||
classification = "blocked_gitea_queue_or_hook_backlog_requires_playbook"
|
||||
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
|
||||
severity = (
|
||||
"critical"
|
||||
if load5_per_core > load5_per_core_threshold
|
||||
or gitea_process_cpu >= process_family_cpu_threshold * 2
|
||||
else "warning"
|
||||
)
|
||||
dry_run_command = (
|
||||
"scripts/ops/host-sustained-load-evidence.py "
|
||||
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
|
||||
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
|
||||
)
|
||||
next_action = "run_gitea_queue_or_hook_backlog_playbook_check_mode"
|
||||
elif control_plane_cpu >= process_family_cpu_threshold:
|
||||
classification = "blocked_control_plane_saturation_requires_playbook"
|
||||
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
|
||||
dry_run_command = (
|
||||
"scripts/ops/host-sustained-load-evidence.py "
|
||||
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
|
||||
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
|
||||
)
|
||||
next_action = "run_control_plane_saturation_playbook_check_mode"
|
||||
elif (
|
||||
top_container_name in {"stockplatform-v2-postgres-1", "stockplatform-v2-api-1"}
|
||||
and top_container_cpu >= container_cpu_threshold
|
||||
):
|
||||
"stockplatform-v2-postgres-1" in top_container_name
|
||||
and top_container_cpu >= hot_container_cpu_threshold
|
||||
) or (
|
||||
top_container_name == "stockplatform-v2-api-1"
|
||||
and top_container_cpu >= hot_container_cpu_threshold
|
||||
) or stock_process_cpu >= process_family_cpu_threshold:
|
||||
classification = "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook"
|
||||
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
|
||||
dry_run_command = (
|
||||
@@ -350,6 +555,17 @@ def build_packet(
|
||||
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
|
||||
)
|
||||
next_action = "run_stockplatform_hot_query_or_api_pressure_playbook_check_mode"
|
||||
elif (
|
||||
top_container_name == "gitea" and top_container_cpu >= hot_container_cpu_threshold
|
||||
) or gitea_process_cpu >= process_family_cpu_threshold:
|
||||
classification = "blocked_gitea_queue_or_hook_backlog_requires_playbook"
|
||||
severity = "critical" if load5_per_core > load5_per_core_threshold else "warning"
|
||||
dry_run_command = (
|
||||
"scripts/ops/host-sustained-load-evidence.py "
|
||||
f"--host {host} --metrics-file {DEFAULT_METRICS_FILE} "
|
||||
f"--docker-stats-file {DEFAULT_DOCKER_STATS_FILE} --json"
|
||||
)
|
||||
next_action = "run_gitea_queue_or_hook_backlog_playbook_check_mode"
|
||||
elif load5_per_core > load5_per_core_threshold and swap_used_ratio >= 0.85:
|
||||
classification = "blocked_memory_or_swap_pressure_requires_service_playbook"
|
||||
severity = "critical"
|
||||
@@ -378,6 +594,8 @@ def build_packet(
|
||||
"load5_per_core": round(load5_per_core, 6),
|
||||
"load5_per_core_threshold": load5_per_core_threshold,
|
||||
"container_cpu_threshold": container_cpu_threshold,
|
||||
"hot_container_cpu_threshold": hot_container_cpu_threshold,
|
||||
"process_family_cpu_threshold": process_family_cpu_threshold,
|
||||
"swap_used_ratio": round(swap_used_ratio, 6),
|
||||
"remediation_authorized": remediation_authorized,
|
||||
"active_ci_container_count": active_ci_containers,
|
||||
@@ -388,6 +606,10 @@ def build_packet(
|
||||
"top_container_cpu": top_container,
|
||||
"top_container_cpu_untrusted": raw_top_container,
|
||||
"docker_stats": docker_stats_status,
|
||||
"top_process_family": top_family,
|
||||
"stock_process_cpu_percent": round(stock_process_cpu, 3),
|
||||
"gitea_process_cpu_percent": round(gitea_process_cpu, 3),
|
||||
"control_plane_process_cpu_percent": round(control_plane_cpu, 3),
|
||||
},
|
||||
"commands": {
|
||||
"dry_run": dry_run_command,
|
||||
@@ -440,8 +662,11 @@ def main() -> int:
|
||||
docker_stats_file=args.docker_stats_file,
|
||||
max_age_seconds=args.docker_stats_max_age_seconds,
|
||||
),
|
||||
process_summary=_summarize_processes(_parse_ps_text(_collect_ps_text(args.ps_file)), top_n=args.top_n),
|
||||
load5_per_core_threshold=args.load5_per_core_threshold,
|
||||
container_cpu_threshold=args.container_cpu_threshold,
|
||||
hot_container_cpu_threshold=args.hot_container_cpu_threshold,
|
||||
process_family_cpu_threshold=args.process_family_cpu_threshold,
|
||||
ci_stale_age_seconds=args.ci_stale_age_seconds,
|
||||
)
|
||||
if args.json:
|
||||
|
||||
@@ -29,14 +29,19 @@ def test_110_moderate_pressure_alert_routes_to_live_controller() -> None:
|
||||
|
||||
assert 'awoooi_host_load5_per_core{host="110"} > 0.75' in expr
|
||||
assert 'docker_container_cpu_cores{host="110"' in expr
|
||||
assert "> 2.0" in expr
|
||||
assert 'awoooi_host_process_family_cpu_percent{host="110"' in expr
|
||||
assert "> 1.0" in expr
|
||||
assert "> 50" in expr
|
||||
assert "systemd_control_plane" in expr
|
||||
assert "gitea" in expr
|
||||
assert "stockplatform-v2-postgres-1" in expr
|
||||
assert rule["for"] == "1m"
|
||||
assert rule["labels"]["auto_repair"] == "true"
|
||||
assert "/home/wooo/scripts/host-sustained-load-controller.py" in action
|
||||
assert "--load5-per-core-threshold 0.75" in action
|
||||
assert "--hot-container-cpu-threshold 1.0" in action
|
||||
assert "--container-cpu-threshold 2.0" in action
|
||||
assert "--process-family-cpu-threshold 50" in action
|
||||
assert "不讀 secret" in annotations["runbook"]
|
||||
assert "禁止 Docker / systemd / Nginx / DB restart" in annotations["runbook"]
|
||||
|
||||
|
||||
@@ -88,6 +88,11 @@ def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) ->
|
||||
cpu_percent=188.5,
|
||||
oldest_age_seconds=240,
|
||||
),
|
||||
process_family_summary=exporter.process_family_load(
|
||||
exporter.parse_ps_rows(
|
||||
"200 1 200 200 7200 55.0 S gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini"
|
||||
)
|
||||
),
|
||||
min_age_seconds=1800,
|
||||
min_cpu_percent=50,
|
||||
now=123,
|
||||
@@ -103,6 +108,9 @@ def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) ->
|
||||
assert 'awoooi_host_gitea_actions_active_process_oldest_age_seconds{host="110"} 240' in metrics
|
||||
assert 'awoooi_host_swap_used_ratio{host="110"} 1.000000' in metrics
|
||||
assert 'awoooi_host_runaway_process_remediation_authorized{host="110"} 0' in metrics
|
||||
assert 'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 55.000000' in metrics
|
||||
assert 'awoooi_host_process_family_top_info{host="110",family="gitea_service",comm="gitea"} 1' in metrics
|
||||
assert "/home/wooo/gitea/app.ini" not in metrics
|
||||
assert 'rule="stockplatform_headless_smoke"' in metrics
|
||||
|
||||
|
||||
@@ -604,6 +612,181 @@ def test_sustained_load_controller_routes_unknown_load_to_sanitized_evidence(tmp
|
||||
assert payload["operation_boundaries"]["process_signal_performed"] is False
|
||||
|
||||
|
||||
def test_sustained_load_controller_routes_moderate_stock_container_pressure(tmp_path: Path) -> None:
|
||||
metrics_file = tmp_path / "host.prom"
|
||||
metrics_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
||||
'awoooi_host_load5_per_core{host="110"} 0.62',
|
||||
'awoooi_host_swap_used_ratio{host="110"} 0.27',
|
||||
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
||||
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
|
||||
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
|
||||
'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 0',
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
docker_file = tmp_path / "docker.prom"
|
||||
docker_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
'docker_container_cpu_cores{host="110",container_name="stockplatform-v2-postgres-1"} 1.654',
|
||||
'docker_container_cpu_cores{host="110",container_name="gitea"} 1.092',
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
ps_file = tmp_path / "ps.txt"
|
||||
ps_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"100 1 100 73767 20.0 0.0 systemd /sbin/init",
|
||||
"200 1 200 73608 30.0 1.1 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini",
|
||||
"300 1 300 60 20.8 0.2 postgres postgres: stockplatform stockplatform SELECT",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(CONTROLLER_PATH),
|
||||
"--host",
|
||||
"110",
|
||||
"--load5-per-core-threshold",
|
||||
"0.75",
|
||||
"--container-cpu-threshold",
|
||||
"2.0",
|
||||
"--metrics-file",
|
||||
str(metrics_file),
|
||||
"--docker-stats-file",
|
||||
str(docker_file),
|
||||
"--ps-file",
|
||||
str(ps_file),
|
||||
"--json",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 75
|
||||
payload = json.loads(result.stdout)
|
||||
assert (
|
||||
payload["classification"]
|
||||
== "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook"
|
||||
)
|
||||
assert payload["severity"] == "warning"
|
||||
assert payload["readback"]["hot_container_cpu_threshold"] == 1.0
|
||||
assert payload["readback"]["top_container_cpu"]["container_name"] == "stockplatform-v2-postgres-1"
|
||||
assert payload["readback"]["top_process_family"]["family"] == "gitea_service"
|
||||
assert payload["controlled_apply_allowed"] is False
|
||||
assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"]
|
||||
assert "/home/wooo/gitea/app.ini" not in result.stdout
|
||||
|
||||
|
||||
def test_sustained_load_controller_routes_gitea_process_pressure_without_hot_container(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
metrics_file = tmp_path / "host.prom"
|
||||
metrics_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
||||
'awoooi_host_load5_per_core{host="110"} 0.55',
|
||||
'awoooi_host_swap_used_ratio{host="110"} 0.1',
|
||||
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
docker_file = tmp_path / "docker.prom"
|
||||
docker_file.write_text(
|
||||
'docker_container_cpu_cores{host="110",container_name="gitea"} 0.20\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
ps_file = tmp_path / "ps.txt"
|
||||
ps_file.write_text(
|
||||
"200 1 200 73608 55.5 1.1 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(CONTROLLER_PATH),
|
||||
"--host",
|
||||
"110",
|
||||
"--metrics-file",
|
||||
str(metrics_file),
|
||||
"--docker-stats-file",
|
||||
str(docker_file),
|
||||
"--ps-file",
|
||||
str(ps_file),
|
||||
"--json",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 75
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["classification"] == "blocked_gitea_queue_or_hook_backlog_requires_playbook"
|
||||
assert payload["readback"]["gitea_process_cpu_percent"] == 55.5
|
||||
assert payload["controlled_apply_allowed"] is False
|
||||
assert "/home/wooo/gitea/app.ini" not in result.stdout
|
||||
|
||||
|
||||
def test_sustained_load_controller_routes_control_plane_family_pressure(tmp_path: Path) -> None:
|
||||
metrics_file = tmp_path / "host.prom"
|
||||
metrics_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
|
||||
'awoooi_host_load5_per_core{host="110"} 0.70',
|
||||
'awoooi_host_swap_used_ratio{host="110"} 0.1',
|
||||
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
ps_file = tmp_path / "ps.txt"
|
||||
ps_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"100 1 100 73767 40.0 0.0 systemd /sbin/init",
|
||||
"101 1 101 73767 15.0 0.0 dbus-daemon @dbus-daemon --system",
|
||||
"200 1 200 73608 20.0 1.1 gitea /usr/local/bin/gitea web",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(CONTROLLER_PATH),
|
||||
"--host",
|
||||
"110",
|
||||
"--metrics-file",
|
||||
str(metrics_file),
|
||||
"--ps-file",
|
||||
str(ps_file),
|
||||
"--json",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 75
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["classification"] == "blocked_control_plane_saturation_requires_playbook"
|
||||
assert payload["readback"]["control_plane_process_cpu_percent"] == 55.0
|
||||
assert payload["next_action"] == "run_control_plane_saturation_playbook_check_mode"
|
||||
|
||||
|
||||
def test_sustained_load_evidence_emits_sanitized_gitea_recommendation(tmp_path: Path) -> None:
|
||||
ps_file = tmp_path / "ps.txt"
|
||||
ps_file.write_text(
|
||||
|
||||
Reference in New Issue
Block a user