fix(agent): route host pressure through bounded diagnosis
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s

This commit is contained in:
ogt
2026-07-11 10:53:37 +08:00
parent b44190edb8
commit db9b33cfa7
8 changed files with 532 additions and 36 deletions

View File

@@ -56,6 +56,26 @@ _CATALOG: tuple[dict[str, Any], ...] = (
"risk_level": "low",
"no_write_only": True,
},
{
"catalog_id": "ansible:110-host-pressure-readonly",
"playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
"check_mode_playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
"inventory_hosts": ["host_110"],
"domains": ["host_pressure", "gitea", "runner", "postgres", "systemd_control_plane"],
"keywords": [
"host110sustainedmoderatepressure",
"host-pressure-controller",
"sustained pressure",
"host_resource",
"gitea_service",
"systemd_control_plane",
],
"supports_check_mode": True,
"auto_apply_enabled": False,
"approval_required": False,
"risk_level": "low",
"no_write_only": True,
},
{
"catalog_id": "ansible:110-devops",
"playbook_path": "infra/ansible/playbooks/110-devops.yml",
@@ -448,6 +468,7 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
"auto_apply_enabled",
"approval_required",
"risk_level",
"no_write_only",
}
}
if matched:

View File

@@ -58,6 +58,32 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = {
"systemctl is-active --quiet wazuh-manager.service",
),
),
"ansible:110-host-pressure-readonly": (
AssetPostcondition(
"host_110_pressure_controller_asset",
"service",
"host_110",
"test -x /home/wooo/scripts/host-sustained-load-controller.py",
),
AssetPostcondition(
"host_110_gitea_pressure_probe_asset",
"service",
"host_110",
"test -x /home/wooo/scripts/gitea-queue-hook-backlog-playbook.py",
),
AssetPostcondition(
"host_110_pressure_metrics_freshness",
"metric",
"host_110",
"test -s /home/wooo/node_exporter_textfiles/host_runaway_process.prom",
),
AssetPostcondition(
"host_110_docker_metrics_freshness",
"metric",
"host_110",
"test -s /home/wooo/node_exporter_textfiles/docker_stats.prom",
),
),
"ansible:110-devops": (
AssetPostcondition(
"host_110_docker_runtime",

View File

@@ -1318,13 +1318,35 @@ class RepairCandidateService:
])
elif target_kind == "host_service":
service_ref = target or "<service>"
command_template = f"systemctl restart {service_ref}"
rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120"
route = "host_service_route_after_owner_review"
verifier_plan.extend([
f"systemctl is-active {service_ref}",
f"journalctl -u {service_ref} -n 120 --no-pager",
])
if (
alertname.strip().lower() == "host110sustainedmoderatepressure"
or "host-pressure-controller" in service_ref.lower()
):
command_template = (
"/home/wooo/scripts/host-sustained-load-controller.py "
"--host 110 --load5-per-core-threshold 0.75 "
"--hot-container-cpu-threshold 1.0 "
"--container-cpu-threshold 2.0 "
"--process-family-cpu-threshold 50 "
"--metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom "
"--docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom "
"--json"
)
rollback_template = "no host mutation; retain current services and rerun bounded verifier"
route = "host_pressure_readonly_after_owner_review"
verifier_plan.extend([
"rerun host-sustained-load-controller.py after 15m cooldown or active CD terminal",
"require alert fingerprint to resolve or decrease without service restart",
"never treat host-pressure-controller as a systemd service",
])
else:
command_template = f"systemctl restart {service_ref}"
rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120"
route = "host_service_route_after_owner_review"
verifier_plan.extend([
f"systemctl is-active {service_ref}",
f"journalctl -u {service_ref} -n 120 --no-pager",
])
elif target_kind == "database":
db_ref = target or "postgres"
command_template = (