feat(aiops): add typed CPU P99 recovery closure
This commit is contained in:
@@ -67,6 +67,10 @@ _NODE_EXPORTER_ALERTS = {
|
||||
"nodeexporterscrapedown",
|
||||
"nodeexporterunhealthy",
|
||||
}
|
||||
_HOST_CPU_PRESSURE_ALERTS = {
|
||||
"hosthighcpuload",
|
||||
"hostloadaveragesustainedhigh",
|
||||
}
|
||||
_ALERTMANAGER_DELIVERY_ALERTS = {"alertchainbrokenalertmanager"}
|
||||
_PROVIDER_FRESHNESS_ALERTS = {
|
||||
"providerfreshnesssignal",
|
||||
@@ -704,6 +708,18 @@ def resolve_typed_alert_target(
|
||||
host110_pressure_alert = compact_alert.startswith(
|
||||
"host110sustainedmoderatepressure"
|
||||
)
|
||||
host_cpu_pressure_alert = (
|
||||
compact_alert in _HOST_CPU_PRESSURE_ALERTS or host110_pressure_alert
|
||||
)
|
||||
pressure_target_identity = (
|
||||
registry.resolve_identity(target_resource)
|
||||
if host_cpu_pressure_alert and target_resource
|
||||
else {}
|
||||
)
|
||||
pressure_target_is_exact_service = bool(
|
||||
pressure_target_identity.get("resolution_status") == "resolved"
|
||||
and pressure_target_identity.get("asset_domain") != "unknown"
|
||||
)
|
||||
allowed_catalog_ids: list[str] = []
|
||||
if compact_alert in _DISK_ALERTS and host_scope is not None:
|
||||
if host_scope[0] in {"host_110", "host_188"}:
|
||||
@@ -714,9 +730,10 @@ def resolve_typed_alert_target(
|
||||
if host_scope[0] == "host_110":
|
||||
allowed_catalog_ids = ["ansible:110-devops"]
|
||||
elif (
|
||||
host110_pressure_alert
|
||||
host_cpu_pressure_alert
|
||||
and host_scope is not None
|
||||
and host_scope[0] == "host_110"
|
||||
and not pressure_target_is_exact_service
|
||||
):
|
||||
allowed_catalog_ids = ["ansible:110-host-pressure-readonly"]
|
||||
elif (
|
||||
@@ -729,7 +746,7 @@ def resolve_typed_alert_target(
|
||||
if host_scope is not None and (
|
||||
compact_alert in _DISK_ALERTS
|
||||
or compact_alert in _NODE_EXPORTER_ALERTS
|
||||
or host110_pressure_alert
|
||||
or (host_cpu_pressure_alert and not pressure_target_is_exact_service)
|
||||
or "wazuh" in text
|
||||
):
|
||||
return _typed_route(
|
||||
@@ -744,6 +761,7 @@ def resolve_typed_alert_target(
|
||||
risk_class="medium",
|
||||
allowed_catalog_ids=allowed_catalog_ids,
|
||||
allowed_inventory_hosts=[host_scope[0]],
|
||||
controlled_apply_allowed=(False if host_cpu_pressure_alert else None),
|
||||
)
|
||||
|
||||
identity = registry.resolve_identity(target_resource)
|
||||
|
||||
1423
apps/api/src/services/cpu_p99_controlled_recovery.py
Normal file
1423
apps/api/src/services/cpu_p99_controlled_recovery.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -111,6 +111,20 @@ _EXTERNAL_READBACKS: tuple[dict[str, Any], ...] = (
|
||||
"verifier": "agent99_alertmanager_pull_relay_verifier",
|
||||
"runtime_status": "receipt_pending",
|
||||
},
|
||||
{
|
||||
"readback_id": "cpu_resource_postcondition",
|
||||
"verifier": "cpu_resource_independent_verifier",
|
||||
"required_source": "prometheus_independent_query",
|
||||
"source_ref": "apps/api/src/services/cpu_p99_controlled_recovery.py",
|
||||
"runtime_status": "receipt_pending",
|
||||
},
|
||||
{
|
||||
"readback_id": "api_p99_postcondition",
|
||||
"verifier": "signoz_api_latency_independent_verifier",
|
||||
"required_source": "signoz_independent_query",
|
||||
"source_ref": "apps/api/src/services/cpu_p99_controlled_recovery.py",
|
||||
"runtime_status": "receipt_pending",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user