fix(agent): route host pressure through bounded diagnosis
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s

This commit is contained in:
ogt
2026-07-11 10:53:37 +08:00
parent b44190edb8
commit db9b33cfa7
8 changed files with 532 additions and 36 deletions

View File

@@ -56,6 +56,26 @@ _CATALOG: tuple[dict[str, Any], ...] = (
"risk_level": "low",
"no_write_only": True,
},
{
"catalog_id": "ansible:110-host-pressure-readonly",
"playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
"inventory_hosts": ["host_110"],
"domains": ["host_pressure", "gitea", "runner", "postgres", "systemd_control_plane"],
"keywords": [
"host110sustainedmoderatepressure",
"host-pressure-controller",
"sustained pressure",
"host_resource",
"gitea_service",
"systemd_control_plane",
],
"supports_check_mode": True,
"auto_apply_enabled": False,
"approval_required": False,
"risk_level": "low",
"no_write_only": True,
},
{
"catalog_id": "ansible:110-devops",
"playbook_path": "infra/ansible/playbooks/110-devops.yml",
@@ -448,6 +468,7 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
"auto_apply_enabled",
"approval_required",
"risk_level",
"no_write_only",
}
}
if matched:

View File

@@ -58,6 +58,32 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = {
"systemctl is-active --quiet wazuh-manager.service",
),
),
"ansible:110-host-pressure-readonly": (
AssetPostcondition(
"host_110_pressure_controller_asset",
"service",
"host_110",
"test -x /home/wooo/scripts/host-sustained-load-controller.py",
),
AssetPostcondition(
"host_110_gitea_pressure_probe_asset",
"service",
"host_110",
"test -x /home/wooo/scripts/gitea-queue-hook-backlog-playbook.py",
),
AssetPostcondition(
"host_110_pressure_metrics_freshness",
"metric",
"host_110",
"test -s /home/wooo/node_exporter_textfiles/host_runaway_process.prom",
),
AssetPostcondition(
"host_110_docker_metrics_freshness",
"metric",
"host_110",
"test -s /home/wooo/node_exporter_textfiles/docker_stats.prom",
),
),
"ansible:110-devops": (
AssetPostcondition(
"host_110_docker_runtime",

View File

@@ -1318,13 +1318,35 @@ class RepairCandidateService:
])
elif target_kind == "host_service":
service_ref = target or "<service>"
command_template = f"systemctl restart {service_ref}"
rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120"
route = "host_service_route_after_owner_review"
verifier_plan.extend([
f"systemctl is-active {service_ref}",
f"journalctl -u {service_ref} -n 120 --no-pager",
])
if (
alertname.strip().lower() == "host110sustainedmoderatepressure"
or "host-pressure-controller" in service_ref.lower()
):
command_template = (
"/home/wooo/scripts/host-sustained-load-controller.py "
"--host 110 --load5-per-core-threshold 0.75 "
"--hot-container-cpu-threshold 1.0 "
"--container-cpu-threshold 2.0 "
"--process-family-cpu-threshold 50 "
"--metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom "
"--docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom "
"--json"
)
rollback_template = "no host mutation; retain current services and rerun bounded verifier"
route = "host_pressure_readonly_after_owner_review"
verifier_plan.extend([
"rerun host-sustained-load-controller.py after 15m cooldown or active CD terminal",
"require alert fingerprint to resolve or decrease without service restart",
"never treat host-pressure-controller as a systemd service",
])
else:
command_template = f"systemctl restart {service_ref}"
rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120"
route = "host_service_route_after_owner_review"
verifier_plan.extend([
f"systemctl is-active {service_ref}",
f"journalctl -u {service_ref} -n 120 --no-pager",
])
elif target_kind == "database":
db_ref = target or "postgres"
command_template = (

View File

@@ -0,0 +1,84 @@
from __future__ import annotations
from pathlib import Path
import yaml
from src.services.awooop_ansible_audit_service import (
_catalog_hints,
get_ansible_catalog_item,
)
ROOT = Path(__file__).resolve().parents[3]
PLAYBOOK = ROOT / "infra" / "ansible" / "playbooks" / "110-host-pressure-readonly.yml"
def test_host_pressure_alert_prefers_dedicated_no_write_catalog() -> None:
hints = _catalog_hints(
{
"alertname": "Host110SustainedModeratePressure",
"alert_category": "host_resource",
"affected_services": ["host-pressure-controller", "gitea_service"],
"signals": [
{
"alert_name": "Host110SustainedModeratePressure",
"labels": {
"component": "host-pressure-controller",
"host": "110",
},
}
],
},
None,
)
candidates = hints["candidates"]
assert candidates[0]["catalog_id"] == "ansible:110-host-pressure-readonly"
generic = next(row for row in candidates if row["catalog_id"] == "ansible:110-devops")
assert candidates[0]["match_score"] > generic["match_score"]
assert candidates[0]["auto_apply_enabled"] is False
assert candidates[0]["no_write_only"] is True
def test_generic_110_fallback_remains_below_exact_host_pressure_route() -> None:
hints = _catalog_hints(
{
"alertname": "Host110SustainedModeratePressure",
"affected_services": ["host-pressure-controller"],
},
None,
)
candidates = hints["candidates"]
dedicated = next(
row
for row in candidates
if row["catalog_id"] == "ansible:110-host-pressure-readonly"
)
generic = next(
row for row in candidates if row["catalog_id"] == "ansible:110-devops"
)
assert dedicated["match_score"] > generic["match_score"]
assert generic["matched_keywords"] == ["110"]
def test_host_pressure_playbook_is_bounded_and_read_only() -> None:
payload = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
text = PLAYBOOK.read_text(encoding="utf-8")
catalog = get_ansible_catalog_item("ansible:110-host-pressure-readonly")
assert payload[0]["hosts"] == "host_110"
assert payload[0]["become"] is False
assert catalog is not None
assert catalog["auto_apply_enabled"] is False
assert catalog["no_write_only"] is True
assert "host-sustained-load-controller.py" in text
assert "gitea-queue-hook-backlog-playbook.py" in text
for forbidden in (
"systemctl restart",
"docker restart",
"kill -9",
"kubectl delete",
"reboot",
):
assert forbidden not in text.lower()

View File

@@ -531,6 +531,34 @@ def test_promotion_summary_marks_controlled_runtime_when_apply_gate_passes() ->
)
def test_host_pressure_controller_is_never_drafted_as_systemd_restart() -> None:
service = RepairCandidateService()
template = service._build_playbook_draft_template(
coverage_gap={
"target_kind": "host_service",
"coverage_key": "host110sustainedmoderatepressure:host-pressure-controller",
"blocking_stage": "service_playbook_coverage",
"required_mcp_evidence_refs": ["metrics_or_logs_window"],
},
blockers=["playbook_observe_only"],
lane="promote_diagnostic_to_repair_playbook",
alertname="Host110SustainedModeratePressure",
target_resource="host-pressure-controller",
namespace="",
evidence=None,
playbook=None,
)
assert template["suggested_route"] == "host_pressure_readonly_after_owner_review"
assert template["repair_command_template"].startswith(
"/home/wooo/scripts/host-sustained-load-controller.py"
)
assert template["rollback_command_template"].startswith("no host mutation")
assert "systemctl restart" not in template["repair_command_template"]
assert "systemctl is-active" not in " ".join(template["verifier_plan_template"])
@pytest.mark.asyncio
async def test_postgres_slow_query_gap_prefills_database_owner_review_not_restart() -> None:
incident = _incident()