fix(agent): route host pressure through bounded diagnosis
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m29s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m53s
This commit is contained in:
@@ -56,6 +56,26 @@ _CATALOG: tuple[dict[str, Any], ...] = (
|
||||
"risk_level": "low",
|
||||
"no_write_only": True,
|
||||
},
|
||||
{
|
||||
"catalog_id": "ansible:110-host-pressure-readonly",
|
||||
"playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
|
||||
"check_mode_playbook_path": "infra/ansible/playbooks/110-host-pressure-readonly.yml",
|
||||
"inventory_hosts": ["host_110"],
|
||||
"domains": ["host_pressure", "gitea", "runner", "postgres", "systemd_control_plane"],
|
||||
"keywords": [
|
||||
"host110sustainedmoderatepressure",
|
||||
"host-pressure-controller",
|
||||
"sustained pressure",
|
||||
"host_resource",
|
||||
"gitea_service",
|
||||
"systemd_control_plane",
|
||||
],
|
||||
"supports_check_mode": True,
|
||||
"auto_apply_enabled": False,
|
||||
"approval_required": False,
|
||||
"risk_level": "low",
|
||||
"no_write_only": True,
|
||||
},
|
||||
{
|
||||
"catalog_id": "ansible:110-devops",
|
||||
"playbook_path": "infra/ansible/playbooks/110-devops.yml",
|
||||
@@ -448,6 +468,7 @@ def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None
|
||||
"auto_apply_enabled",
|
||||
"approval_required",
|
||||
"risk_level",
|
||||
"no_write_only",
|
||||
}
|
||||
}
|
||||
if matched:
|
||||
|
||||
@@ -58,6 +58,32 @@ _POSTCONDITION_REGISTRY: dict[str, tuple[AssetPostcondition, ...]] = {
|
||||
"systemctl is-active --quiet wazuh-manager.service",
|
||||
),
|
||||
),
|
||||
"ansible:110-host-pressure-readonly": (
|
||||
AssetPostcondition(
|
||||
"host_110_pressure_controller_asset",
|
||||
"service",
|
||||
"host_110",
|
||||
"test -x /home/wooo/scripts/host-sustained-load-controller.py",
|
||||
),
|
||||
AssetPostcondition(
|
||||
"host_110_gitea_pressure_probe_asset",
|
||||
"service",
|
||||
"host_110",
|
||||
"test -x /home/wooo/scripts/gitea-queue-hook-backlog-playbook.py",
|
||||
),
|
||||
AssetPostcondition(
|
||||
"host_110_pressure_metrics_freshness",
|
||||
"metric",
|
||||
"host_110",
|
||||
"test -s /home/wooo/node_exporter_textfiles/host_runaway_process.prom",
|
||||
),
|
||||
AssetPostcondition(
|
||||
"host_110_docker_metrics_freshness",
|
||||
"metric",
|
||||
"host_110",
|
||||
"test -s /home/wooo/node_exporter_textfiles/docker_stats.prom",
|
||||
),
|
||||
),
|
||||
"ansible:110-devops": (
|
||||
AssetPostcondition(
|
||||
"host_110_docker_runtime",
|
||||
|
||||
@@ -1318,13 +1318,35 @@ class RepairCandidateService:
|
||||
])
|
||||
elif target_kind == "host_service":
|
||||
service_ref = target or "<service>"
|
||||
command_template = f"systemctl restart {service_ref}"
|
||||
rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120"
|
||||
route = "host_service_route_after_owner_review"
|
||||
verifier_plan.extend([
|
||||
f"systemctl is-active {service_ref}",
|
||||
f"journalctl -u {service_ref} -n 120 --no-pager",
|
||||
])
|
||||
if (
|
||||
alertname.strip().lower() == "host110sustainedmoderatepressure"
|
||||
or "host-pressure-controller" in service_ref.lower()
|
||||
):
|
||||
command_template = (
|
||||
"/home/wooo/scripts/host-sustained-load-controller.py "
|
||||
"--host 110 --load5-per-core-threshold 0.75 "
|
||||
"--hot-container-cpu-threshold 1.0 "
|
||||
"--container-cpu-threshold 2.0 "
|
||||
"--process-family-cpu-threshold 50 "
|
||||
"--metrics-file /home/wooo/node_exporter_textfiles/host_runaway_process.prom "
|
||||
"--docker-stats-file /home/wooo/node_exporter_textfiles/docker_stats.prom "
|
||||
"--json"
|
||||
)
|
||||
rollback_template = "no host mutation; retain current services and rerun bounded verifier"
|
||||
route = "host_pressure_readonly_after_owner_review"
|
||||
verifier_plan.extend([
|
||||
"rerun host-sustained-load-controller.py after 15m cooldown or active CD terminal",
|
||||
"require alert fingerprint to resolve or decrease without service restart",
|
||||
"never treat host-pressure-controller as a systemd service",
|
||||
])
|
||||
else:
|
||||
command_template = f"systemctl restart {service_ref}"
|
||||
rollback_template = f"systemctl status {service_ref}; journalctl -u {service_ref} -n 120"
|
||||
route = "host_service_route_after_owner_review"
|
||||
verifier_plan.extend([
|
||||
f"systemctl is-active {service_ref}",
|
||||
f"journalctl -u {service_ref} -n 120 --no-pager",
|
||||
])
|
||||
elif target_kind == "database":
|
||||
db_ref = target or "postgres"
|
||||
command_template = (
|
||||
|
||||
Reference in New Issue
Block a user