feat(agent): automate sustained host load response

scripts/ops/tests/test_host_runaway_process_exporter.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import importlib.util
+import json
 import subprocess
 import sys
 from pathlib import Path
@@ -9,6 +10,8 @@ from pathlib import Path
 SCRIPT_ROOT = Path(__file__).resolve().parents[1]
 EXPORTER_PATH = SCRIPT_ROOT / "host-runaway-process-exporter.py"
 REMEDIATION_PATH = SCRIPT_ROOT / "host-runaway-process-remediation.py"
+CONTROLLER_PATH = SCRIPT_ROOT / "host-sustained-load-controller.py"
+EVIDENCE_PATH = SCRIPT_ROOT / "host-sustained-load-evidence.py"
 
 
 def load_exporter():
@@ -167,7 +170,7 @@ def test_ignores_the_host_pressure_gate_process_group() -> None:
 def test_remediation_defaults_to_dry_run(tmp_path: Path) -> None:
     ps_file = tmp_path / "ps.txt"
     ps_file.write_text(
-        "100 1 100 100 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa\n",
+        "999999 1 999999 999999 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa\n",
         encoding="utf-8",
     )
 
@@ -193,7 +196,7 @@ def test_remediation_defaults_to_dry_run(tmp_path: Path) -> None:
 def test_remediation_refuses_apply_without_gates(tmp_path: Path) -> None:
     ps_file = tmp_path / "ps.txt"
     ps_file.write_text(
-        "100 1 100 100 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa\n",
+        "999999 1 999999 999999 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa\n",
         encoding="utf-8",
     )
 
@@ -213,3 +216,329 @@ def test_remediation_refuses_apply_without_gates(tmp_path: Path) -> None:
 
     assert result.returncode != 0
     assert "Refusing apply" in result.stderr
+    assert "--controlled-apply-id" in result.stderr
+    assert "--confirm-apply" in result.stderr
+    assert "--post-apply-verifier" in result.stderr
+
+
+def test_remediation_accepts_controlled_apply_gate_without_owner_gate(tmp_path: Path) -> None:
+    ps_file = tmp_path / "ps.txt"
+    ps_file.write_text(
+        "100 1 1 100 7200 65.0 S chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa\n",
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(REMEDIATION_PATH),
+            "--ps-file",
+            str(ps_file),
+            "--apply",
+            "--confirm-apply",
+            "--rule",
+            "stockplatform_headless_smoke",
+            "--controlled-apply-id",
+            "CAP-20260701-HOSTLOAD",
+            "--evidence-ref",
+            "HostLoadAverageSustainedHigh:110",
+            "--post-apply-verifier",
+            "scripts/ops/host-sustained-load-controller.py --host 110 --json",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    assert '"mode": "apply_sigterm"' in result.stdout
+    assert '"runtime_gate": 1' in result.stdout
+    assert '"controlled_apply_id": "CAP-20260701-HOSTLOAD"' in result.stdout
+    assert '"owner_approval_id": ""' in result.stdout
+    assert '"blocked_reason": "unsafe_pgid"' in result.stdout
+    assert '"missing_process_group_count": 0' in result.stdout
+    assert '"signal_error_count": 0' in result.stdout
+    assert '"signaled_process_group_count": 0' in result.stdout
+
+
+def test_sustained_load_controller_routes_orphan_browser_to_controlled_remediation(tmp_path: Path) -> None:
+    metrics_file = tmp_path / "host.prom"
+    metrics_file.write_text(
+        "\n".join(
+            [
+                'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
+                'awoooi_host_load5_per_core{host="110"} 2.2',
+                'awoooi_host_swap_used_ratio{host="110"} 0.1',
+                'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
+                'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
+                'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
+                'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 1',
+                'awoooi_host_runaway_browser_orphan_cpu_percent{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 155.5',
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(CONTROLLER_PATH),
+            "--host",
+            "110",
+            "--metrics-file",
+            str(metrics_file),
+            "--json",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    payload = json.loads(result.stdout)
+    assert payload["classification"] == "controlled_orphan_browser_remediation_ready"
+    assert payload["controlled_apply_allowed"] is True
+    assert "host-runaway-process-remediation.py --rule stockplatform_headless_smoke" in payload["commands"]["dry_run"]
+    assert "--controlled-apply-id" in payload["commands"]["controlled_apply"]
+    assert payload["operation_boundaries"]["process_signal_performed"] is False
+
+
+def test_sustained_load_controller_keeps_ci_saturation_on_runner_path(tmp_path: Path) -> None:
+    metrics_file = tmp_path / "host.prom"
+    metrics_file.write_text(
+        "\n".join(
+            [
+                'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
+                'awoooi_host_load5_per_core{host="110"} 2.0',
+                'awoooi_host_swap_used_ratio{host="110"} 0.1',
+                'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
+                'awoooi_host_gitea_actions_active_container_count{host="110"} 2',
+                'awoooi_host_gitea_actions_active_process_group_count{host="110"} 1',
+                'awoooi_host_gitea_actions_active_process_cpu_percent{host="110"} 180.0',
+                'awoooi_host_gitea_actions_active_process_oldest_age_seconds{host="110"} 1900',
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(CONTROLLER_PATH),
+            "--host",
+            "110",
+            "--metrics-file",
+            str(metrics_file),
+            "--json",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    payload = json.loads(result.stdout)
+    assert payload["classification"] == "controlled_ci_runner_saturation_guarded"
+    assert payload["controlled_apply_allowed"] is True
+    assert "fail_closed" in payload["commands"]["controlled_apply"]
+    assert "process_kill" not in payload["commands"]["controlled_apply"]
+
+
+def test_sustained_load_controller_blocks_monitor_authority_violation(tmp_path: Path) -> None:
+    metrics_file = tmp_path / "host.prom"
+    metrics_file.write_text(
+        "\n".join(
+            [
+                'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
+                'awoooi_host_load5_per_core{host="110"} 2.0',
+                'awoooi_host_runaway_process_remediation_authorized{host="110"} 1',
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(CONTROLLER_PATH),
+            "--host",
+            "110",
+            "--metrics-file",
+            str(metrics_file),
+            "--json",
+        ],
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 75
+    payload = json.loads(result.stdout)
+    assert payload["classification"] == "blocked_monitor_authority_violation"
+    assert payload["controlled_apply_allowed"] is False
+
+
+def test_sustained_load_controller_routes_unknown_load_to_sanitized_evidence(tmp_path: Path) -> None:
+    metrics_file = tmp_path / "host.prom"
+    metrics_file.write_text(
+        "\n".join(
+            [
+                'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
+                'awoooi_host_load5_per_core{host="110"} 2.0',
+                'awoooi_host_swap_used_ratio{host="110"} 0.1',
+                'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
+                'awoooi_host_gitea_actions_active_container_count{host="110"} 0',
+                'awoooi_host_gitea_actions_active_process_group_count{host="110"} 0',
+                'awoooi_host_runaway_browser_orphan_group_count{host="110",rule="stockplatform_headless_smoke",min_age_seconds="1800",min_cpu_percent="50"} 0',
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(CONTROLLER_PATH),
+            "--host",
+            "110",
+            "--metrics-file",
+            str(metrics_file),
+            "--json",
+        ],
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 75
+    payload = json.loads(result.stdout)
+    assert payload["classification"] == "blocked_unknown_sustained_load_requires_source_specific_playbook"
+    assert payload["controlled_apply_allowed"] is False
+    assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"]
+    assert payload["operation_boundaries"]["process_signal_performed"] is False
+
+
+def test_sustained_load_evidence_emits_sanitized_gitea_recommendation(tmp_path: Path) -> None:
+    ps_file = tmp_path / "ps.txt"
+    ps_file.write_text(
+        "\n".join(
+            [
+                "100 1 100 7200 280.0 1.0 gitea /usr/local/bin/gitea web --config /home/wooo/gitea/app.ini",
+                "200 1 200 180 15.0 0.5 systemd systemctl show gitea-act-runner-host.service",
+            ]
+        ),
+        encoding="utf-8",
+    )
+    docker_file = tmp_path / "docker.prom"
+    docker_file.write_text(
+        'docker_container_cpu_cores{host="110",container_name="gitea"} 3.4\n',
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(EVIDENCE_PATH),
+            "--host",
+            "110",
+            "--ps-file",
+            str(ps_file),
+            "--docker-stats-file",
+            str(docker_file),
+            "--json",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    payload = json.loads(result.stdout)
+    assert payload["schema_version"] == "host_sustained_load_sanitized_evidence_v1"
+    assert payload["recommendation"] == "gitea_queue_or_hook_backlog_playbook"
+    assert payload["redaction"]["raw_command_lines_emitted"] is False
+    assert payload["operation_boundaries"]["host_write_performed"] is False
+    assert "/home/wooo" not in result.stdout
+
+
+def test_sustained_load_controller_routes_unknown_load_to_sanitized_evidence(tmp_path: Path) -> None:
+    metrics_file = tmp_path / "host.prom"
+    metrics_file.write_text(
+        "\n".join(
+            [
+                'awoooi_host_runaway_process_monitor_up{host="110",mode="read_only"} 1',
+                'awoooi_host_load5_per_core{host="110"} 2.4',
+                'awoooi_host_swap_used_ratio{host="110"} 0.1',
+                'awoooi_host_runaway_process_remediation_authorized{host="110"} 0',
+            ]
+        ),
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(CONTROLLER_PATH),
+            "--host",
+            "110",
+            "--metrics-file",
+            str(metrics_file),
+            "--json",
+        ],
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 75
+    payload = json.loads(result.stdout)
+    assert (
+        payload["classification"]
+        == "blocked_unknown_sustained_load_requires_source_specific_playbook"
+    )
+    assert payload["controlled_apply_allowed"] is False
+    assert "host-sustained-load-evidence.py" in payload["commands"]["dry_run"]
+    assert payload["operation_boundaries"]["host_write_performed"] is False
+
+
+def test_sustained_load_evidence_sanitizes_process_details(tmp_path: Path) -> None:
+    ps_file = tmp_path / "ps.txt"
+    ps_file.write_text(
+        "\n".join(
+            [
+                "101 1 101 7200 65.0 2.5 chrome /opt/chrome/chrome --headless --user-data-dir=/tmp/stockplatform-review-bulk-ux-aa --url=https://example.invalid/token",
+                "102 1 102 3600 20.0 1.0 node node /srv/private/app/server.js --api-key=SECRET",
+            ]
+        ),
+        encoding="utf-8",
+    )
+    docker_stats_file = tmp_path / "docker.prom"
+    docker_stats_file.write_text(
+        'docker_container_cpu_cores{host="110",container_name="gitea"} 3.2\n',
+        encoding="utf-8",
+    )
+
+    result = subprocess.run(
+        [
+            sys.executable,
+            str(EVIDENCE_PATH),
+            "--host",
+            "110",
+            "--ps-file",
+            str(ps_file),
+            "--docker-stats-file",
+            str(docker_stats_file),
+            "--json",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    payload = json.loads(result.stdout)
+    assert payload["schema_version"] == "host_sustained_load_sanitized_evidence_v1"
+    assert payload["recommendation"] == "gitea_queue_or_hook_backlog_playbook"
+    assert payload["redaction"]["raw_command_lines_emitted"] is False
+    assert payload["redaction"]["workspace_paths_emitted"] is False
+    assert payload["redaction"]["urls_emitted"] is False
+    assert payload["operation_boundaries"]["host_write_performed"] is False
+    assert "https://example.invalid/token" not in result.stdout
+    assert "/tmp/stockplatform-review-bulk-ux-aa" not in result.stdout
+    assert "SECRET" not in result.stdout
+    assert {item["family"] for item in payload["top_process_families"]} >= {
+        "headless_browser",
+        "node_service",
+    }