wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(ops): alert on 110 process-family pressure
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Failing after 2m23s
Details
CD Pipeline / build-and-deploy (push) Has been skipped
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details

Browse Source
This commit is contained in:
Your Name
2026-07-02 11:55:45 +08:00
parent 1e29fbb7bd
commit 7249d81017
7 changed files with 578 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scripts/ops/tests/test_host_pressure_alert_contract.py
View File

@@ -29,14 +29,19 @@ def test_110_moderate_pressure_alert_routes_to_live_controller() -> None:
assert 'awoooi_host_load5_per_core{host="110"} > 0.75' in expr
assert 'docker_container_cpu_cores{host="110"' in expr
assert "> 2.0" in expr
assert 'awoooi_host_process_family_cpu_percent{host="110"' in expr
assert "> 1.0" in expr
assert "> 50" in expr
assert "systemd_control_plane" in expr
assert "gitea" in expr
assert "stockplatform-v2-postgres-1" in expr
assert rule["for"] == "1m"
assert rule["labels"]["auto_repair"] == "true"
assert "/home/wooo/scripts/host-sustained-load-controller.py" in action
assert "--load5-per-core-threshold 0.75" in action
assert "--hot-container-cpu-threshold 1.0" in action
assert "--container-cpu-threshold 2.0" in action
assert "--process-family-cpu-threshold 50" in action
assert "不讀 secret" in annotations["runbook"]
assert "禁止 Docker / systemd / Nginx / DB restart" in annotations["runbook"]

183
scripts/ops/tests/test_host_runaway_process_exporter.py
View File

@@ -88,6 +88,11 @@ def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) ->
cpu_percent=188.5,
oldest_age_seconds=240,
),
process_family_summary=exporter.process_family_load(
exporter.parse_ps_rows(
"200 1 200 200 7200 55.0 S gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini"
)
),
min_age_seconds=1800,
min_cpu_percent=50,
now=123,
@@ -103,6 +108,9 @@ def test_renders_ci_load_and_swap_without_authorizing_repair(tmp_path: Path) ->
assert 'awoooi_host_gitea_actions_active_process_oldest_age_seconds{host="110"} 240' in metrics
assert 'awoooi_host_swap_used_ratio{host="110"} 1.000000' in metrics
assert 'awoooi_host_runaway_process_remediation_authorized{host="110"} 0' in metrics
assert 'awoooi_host_process_family_cpu_percent{host="110",family="gitea_service"} 55.000000' in metrics
assert 'awoooi_host_process_family_top_info{host="110",family="gitea_service",comm="gitea"} 1' in metrics
assert "/home/wooo/gitea/app.ini" not in metrics
assert 'rule="stockplatform_headless_smoke"' in metrics
@@ -604,6 +612,181 @@ def test_sustained_load_controller_routes_unknown_load_to_sanitized_evidence(tmp
assert payload["operation_boundaries"]["process_signal_performed"] is False
def test_sustained_load_controller_routes_moderate_stock_container_pressure(tmp_path: Path) -> None:
metrics_file = tmp_path / "host.prom"
metrics_file.write_text(
"\n".join(
[
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
'awoooi_host_load5_per_core{host="110"} 0.62',
'awoooi_host_swap_used_ratio{host="110"} 0.27',
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 0',
]
),
encoding="utf-8",
)
docker_file = tmp_path / "docker.prom"
docker_file.write_text(
"\n".join(
[
'docker_container_cpu_cores{host="110",container_name="stockplatform-v2-postgres-1"} 1.654',
'docker_container_cpu_cores{host="110",container_name="gitea"} 1.092',
]
),
encoding="utf-8",
)
ps_file = tmp_path / "ps.txt"
ps_file.write_text(
"\n".join(
[
"100 1 100 73767 20.0 0.0 systemd /sbin/init",
"200 1 200 73608 30.0 1.1 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini",
"300 1 300 60 20.8 0.2 postgres postgres: stockplatform stockplatform SELECT",
]
),
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
str(CONTROLLER_PATH),
"--host",
"110",
"--load5-per-core-threshold",
"0.75",
"--container-cpu-threshold",
"2.0",
"--metrics-file",
str(metrics_file),
"--docker-stats-file",
str(docker_file),
"--ps-file",
str(ps_file),
"--json",
],
capture_output=True,
text=True,
)
assert result.returncode == 75
payload = json.loads(result.stdout)
assert (
payload["classification"]
== "blocked_stockplatform_hot_query_or_api_pressure_requires_playbook"
)
assert payload["severity"] == "warning"
assert payload["readback"]["hot_container_cpu_threshold"] == 1.0
assert payload["readback"]["top_container_cpu"]["container_name"] == "stockplatform-v2-postgres-1"
assert payload["readback"]["top_process_family"]["family"] == "gitea_service"
assert payload["controlled_apply_allowed"] is False
assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"]
assert "/home/wooo/gitea/app.ini" not in result.stdout
def test_sustained_load_controller_routes_gitea_process_pressure_without_hot_container(
tmp_path: Path,
) -> None:
metrics_file = tmp_path / "host.prom"
metrics_file.write_text(
"\n".join(
[
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
'awoooi_host_load5_per_core{host="110"} 0.55',
'awoooi_host_swap_used_ratio{host="110"} 0.1',
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
]
),
encoding="utf-8",
)
docker_file = tmp_path / "docker.prom"
docker_file.write_text(
'docker_container_cpu_cores{host="110",container_name="gitea"} 0.20\n',
encoding="utf-8",
)
ps_file = tmp_path / "ps.txt"
ps_file.write_text(
"200 1 200 73608 55.5 1.1 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini\n",
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
str(CONTROLLER_PATH),
"--host",
"110",
"--metrics-file",
str(metrics_file),
"--docker-stats-file",
str(docker_file),
"--ps-file",
str(ps_file),
"--json",
],
capture_output=True,
text=True,
)
assert result.returncode == 75
payload = json.loads(result.stdout)
assert payload["classification"] == "blocked_gitea_queue_or_hook_backlog_requires_playbook"
assert payload["readback"]["gitea_process_cpu_percent"] == 55.5
assert payload["controlled_apply_allowed"] is False
assert "/home/wooo/gitea/app.ini" not in result.stdout
def test_sustained_load_controller_routes_control_plane_family_pressure(tmp_path: Path) -> None:
metrics_file = tmp_path / "host.prom"
metrics_file.write_text(
"\n".join(
[
'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
'awoooi_host_load5_per_core{host="110"} 0.70',
'awoooi_host_swap_used_ratio{host="110"} 0.1',
'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
]
),
encoding="utf-8",
)
ps_file = tmp_path / "ps.txt"
ps_file.write_text(
"\n".join(
[
"100 1 100 73767 40.0 0.0 systemd /sbin/init",
"101 1 101 73767 15.0 0.0 dbus-daemon @dbus-daemon --system",
"200 1 200 73608 20.0 1.1 gitea /usr/local/bin/gitea web",
]
),
encoding="utf-8",
)
result = subprocess.run(
[
sys.executable,
str(CONTROLLER_PATH),
"--host",
"110",
"--metrics-file",
str(metrics_file),
"--ps-file",
str(ps_file),
"--json",
],
capture_output=True,
text=True,
)
assert result.returncode == 75
payload = json.loads(result.stdout)
assert payload["classification"] == "blocked_control_plane_saturation_requires_playbook"
assert payload["readback"]["control_plane_process_cpu_percent"] == 55.0
assert payload["next_action"] == "run_control_plane_saturation_playbook_check_mode"
def test_sustained_load_evidence_emits_sanitized_gitea_recommendation(tmp_path: Path) -> None:
ps_file = tmp_path / "ps.txt"
ps_file.write_text(