fix(agent): route host pressure through bounded diagnosis
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s
This commit is contained in:
@@ -56,6 +56,26 @@ _CATALOG: tuple[dict[str, Any], ...] = (
|
||||
"risk_level": "low",
|
||||
"no_write_only": True,
|
||||
},
|
||||
{
|
||||
"catalog_id": "ansible:110-host-pressure-readonly",
|
||||
"playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
|
||||
"check_mode_playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
|
||||
"inventory_hosts": ["host_110"],
|
||||
"domains": ["host_pressure", "gitea", "runner", "postgres", "systemd_control_plane"],
|
||||
"keywords": [
|
||||
"host110sustainedmoderatepressure",
|
||||
"host-pressure-controller",
|
||||
"sustained pressure",
|
||||
"host_resource",
|
||||
"gitea_service",
|
||||
"systemd_control_plane",
|
||||
],
|
||||
"supports_check_mode": True,
|
||||
"auto_apply_enabled": False,
|
||||
"approval_required": False,
|
||||
"risk_level": "low",
|
||||
"no_write_only": True,
|
||||
},
|
||||
{
|
||||
"catalog_id": "ansible:110-devops",
|
||||
"playbook_path": "infra/ansible/playbooks/110-devops.yml",
|
||||
@@ -448,6 +468,7 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
|
||||
"auto_apply_enabled",
|
||||
"approval_required",
|
||||
"risk_level",
|
||||
"no_write_only",
|
||||
}
|
||||
}
|
||||
if matched:
|
||||
|
||||
@@ -58,6 +58,32 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = {
|
||||
"systemctl is-active --quiet wazuh-manager.service",
|
||||
),
|
||||
),
|
||||
"ansible:110-host-pressure-readonly": (
|
||||
AssetPostcondition(
|
||||
"host_110_pressure_controller_asset",
|
||||
"service",
|
||||
"host_110",
|
||||
"test -x /home/wooo/scripts/host-sustained-load-controller.py",
|
||||
),
|
||||
AssetPostcondition(
|
||||
"host_110_gitea_pressure_probe_asset",
|
||||
"service",
|
||||
"host_110",
|
||||
"test -x /home/wooo/scripts/gitea-queue-hook-backlog-playbook.py",
|
||||
),
|
||||
AssetPostcondition(
|
||||
"host_110_pressure_metrics_freshness",
|
||||
"metric",
|
||||
"host_110",
|
||||
"test -s /home/wooo/node_exporter_textfiles/host_runaway_process.prom",
|
||||
),
|
||||
AssetPostcondition(
|
||||
"host_110_docker_metrics_freshness",
|
||||
"metric",
|
||||
"host_110",
|
||||
"test -s /home/wooo/node_exporter_textfiles/docker_stats.prom",
|
||||
),
|
||||
),
|
||||
"ansible:110-devops": (
|
||||
AssetPostcondition(
|
||||
"host_110_docker_runtime",
|
||||
|
||||
@@ -1318,13 +1318,35 @@ class RepairCandidateService:
|
||||
])
|
||||
elif target_kind == "host_service":
|
||||
service_ref = target or "<service>"
|
||||
command_template = f"systemctl restart {service_ref}"
|
||||
rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120"
|
||||
route = "host_service_route_after_owner_review"
|
||||
verifier_plan.extend([
|
||||
f"systemctl is-active {service_ref}",
|
||||
f"journalctl -u {service_ref} -n 120 --no-pager",
|
||||
])
|
||||
if (
|
||||
alertname.strip().lower() == "host110sustainedmoderatepressure"
|
||||
or "host-pressure-controller" in service_ref.lower()
|
||||
):
|
||||
command_template = (
|
||||
"/home/wooo/scripts/host-sustained-load-controller.py "
|
||||
"--host 110 --load5-per-core-threshold 0.75 "
|
||||
"--hot-container-cpu-threshold 1.0 "
|
||||
"--container-cpu-threshold 2.0 "
|
||||
"--process-family-cpu-threshold 50 "
|
||||
"--metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom "
|
||||
"--docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom "
|
||||
"--json"
|
||||
)
|
||||
rollback_template = "no host mutation; retain current services and rerun bounded verifier"
|
||||
route = "host_pressure_readonly_after_owner_review"
|
||||
verifier_plan.extend([
|
||||
"rerun host-sustained-load-controller.py after 15m cooldown or active CD terminal",
|
||||
"require alert fingerprint to resolve or decrease without service restart",
|
||||
"never treat host-pressure-controller as a systemd service",
|
||||
])
|
||||
else:
|
||||
command_template = f"systemctl restart {service_ref}"
|
||||
rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120"
|
||||
route = "host_service_route_after_owner_review"
|
||||
verifier_plan.extend([
|
||||
f"systemctl is-active {service_ref}",
|
||||
f"journalctl -u {service_ref} -n 120 --no-pager",
|
||||
])
|
||||
elif target_kind == "database":
|
||||
db_ref = target or "postgres"
|
||||
command_template = (
|
||||
|
||||
84
apps/api/tests/test_host_pressure_ansible_catalog.py
Normal file
84
apps/api/tests/test_host_pressure_ansible_catalog.py
Normal file
@@ -0,0 +1,84 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from src.services.awooop_ansible_audit_service import (
|
||||
_catalog_hints,
|
||||
get_ansible_catalog_item,
|
||||
)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-host-pressure-readonly.yml"
|
||||
|
||||
|
||||
def test_host_pressure_alert_prefers_dedicated_no_write_catalog() -> None:
|
||||
hints = _catalog_hints(
|
||||
{
|
||||
"alertname": "Host110SustainedModeratePressure",
|
||||
"alert_category": "host_resource",
|
||||
"affected_services": ["host-pressure-controller", "gitea_service"],
|
||||
"signals": [
|
||||
{
|
||||
"alert_name": "Host110SustainedModeratePressure",
|
||||
"labels": {
|
||||
"component": "host-pressure-controller",
|
||||
"host": "110",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
None,
|
||||
)
|
||||
|
||||
candidates = hints["candidates"]
|
||||
assert candidates[0]["catalog_id"] == "ansible:110-host-pressure-readonly"
|
||||
generic = next(row for row in candidates if row["catalog_id"] == "ansible:110-devops")
|
||||
assert candidates[0]["match_score"] > generic["match_score"]
|
||||
assert candidates[0]["auto_apply_enabled"] is False
|
||||
assert candidates[0]["no_write_only"] is True
|
||||
|
||||
|
||||
def test_generic_110_fallback_remains_below_exact_host_pressure_route() -> None:
|
||||
hints = _catalog_hints(
|
||||
{
|
||||
"alertname": "Host110SustainedModeratePressure",
|
||||
"affected_services": ["host-pressure-controller"],
|
||||
},
|
||||
None,
|
||||
)
|
||||
|
||||
candidates = hints["candidates"]
|
||||
dedicated = next(
|
||||
row
|
||||
for row in candidates
|
||||
if row["catalog_id"] == "ansible:110-host-pressure-readonly"
|
||||
)
|
||||
generic = next(
|
||||
row for row in candidates if row["catalog_id"] == "ansible:110-devops"
|
||||
)
|
||||
assert dedicated["match_score"] > generic["match_score"]
|
||||
assert generic["matched_keywords"] == ["110"]
|
||||
|
||||
|
||||
def test_host_pressure_playbook_is_bounded_and_read_only() -> None:
|
||||
payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
|
||||
text = PLAYBOOK.read_text(encoding="utf-8")
|
||||
catalog = get_ansible_catalog_item("ansible:110-host-pressure-readonly")
|
||||
|
||||
assert payload[0]["hosts"] == "host_110"
|
||||
assert payload[0]["become"] is False
|
||||
assert catalog is not None
|
||||
assert catalog["auto_apply_enabled"] is False
|
||||
assert catalog["no_write_only"] is True
|
||||
assert "host-sustained-load-controller.py" in text
|
||||
assert "gitea-queue-hook-backlog-playbook.py" in text
|
||||
for forbidden in (
|
||||
"systemctl restart",
|
||||
"docker restart",
|
||||
"kill -9",
|
||||
"kubectl delete",
|
||||
"reboot",
|
||||
):
|
||||
assert forbidden not in text.lower()
|
||||
@@ -531,6 +531,34 @@ def test_promotion_summary_marks_controlled_runtime_when_apply_gate_passes() ->
|
||||
)
|
||||
|
||||
|
||||
def test_host_pressure_controller_is_never_drafted_as_systemd_restart() -> None:
|
||||
service = RepairCandidateService()
|
||||
|
||||
template = service._build_playbook_draft_template(
|
||||
coverage_gap={
|
||||
"target_kind": "host_service",
|
||||
"coverage_key": "host110sustainedmoderatepressure:host-pressure-controller",
|
||||
"blocking_stage": "service_playbook_coverage",
|
||||
"required_mcp_evidence_refs": ["metrics_or_logs_window"],
|
||||
},
|
||||
blockers=["playbook_observe_only"],
|
||||
lane="promote_diagnostic_to_repair_playbook",
|
||||
alertname="Host110SustainedModeratePressure",
|
||||
target_resource="host-pressure-controller",
|
||||
namespace="",
|
||||
evidence=None,
|
||||
playbook=None,
|
||||
)
|
||||
|
||||
assert template["suggested_route"] == "host_pressure_readonly_after_owner_review"
|
||||
assert template["repair_command_template"].startswith(
|
||||
"/home/wooo/scripts/host-sustained-load-controller.py"
|
||||
)
|
||||
assert template["rollback_command_template"].startswith("no host mutation")
|
||||
assert "systemctl restart" not in template["repair_command_template"]
|
||||
assert "systemctl is-active" not in " ".join(template["verifier_plan_template"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_postgres_slow_query_gap_prefills_database_owner_review_not_restart() -> None:
|
||||
incident = _incident()
|
||||
|
||||
Reference in New Issue
Block a user