feat(aiops): add typed CPU P99 recovery closure

This commit is contained in:
Your Name
2026-07-22 16:00:05 +08:00
parent 3c9ec23400
commit 0aee245761
10 changed files with 2036 additions and 10 deletions

View File

@@ -67,6 +67,10 @@ _NODE_EXPORTER_ALERTS = {
"nodeexporterscrapedown",
"nodeexporterunhealthy",
}
_HOST_CPU_PRESSURE_ALERTS = {
"hosthighcpuload",
"hostloadaveragesustainedhigh",
}
_ALERTMANAGER_DELIVERY_ALERTS = {"alertchainbrokenalertmanager"}
_PROVIDER_FRESHNESS_ALERTS = {
"providerfreshnesssignal",
@@ -704,6 +708,18 @@ def resolve_typed_alert_target(
host110_pressure_alert = compact_alert.startswith(
"host110sustainedmoderatepressure"
)
host_cpu_pressure_alert = (
compact_alert in _HOST_CPU_PRESSURE_ALERTS or host110_pressure_alert
)
pressure_target_identity = (
registry.resolve_identity(target_resource)
if host_cpu_pressure_alert and target_resource
else {}
)
pressure_target_is_exact_service = bool(
pressure_target_identity.get("resolution_status") == "resolved"
and pressure_target_identity.get("asset_domain") != "unknown"
)
allowed_catalog_ids: list[str] = []
if compact_alert in _DISK_ALERTS and host_scope is not None:
if host_scope[0] in {"host_110", "host_188"}:
@@ -714,9 +730,10 @@ def resolve_typed_alert_target(
if host_scope[0] == "host_110":
allowed_catalog_ids = ["ansible:110-devops"]
elif (
host110_pressure_alert
host_cpu_pressure_alert
and host_scope is not None
and host_scope[0] == "host_110"
and not pressure_target_is_exact_service
):
allowed_catalog_ids = ["ansible:110-host-pressure-readonly"]
elif (
@@ -729,7 +746,7 @@ def resolve_typed_alert_target(
if host_scope is not None and (
compact_alert in _DISK_ALERTS
or compact_alert in _NODE_EXPORTER_ALERTS
or host110_pressure_alert
or (host_cpu_pressure_alert and not pressure_target_is_exact_service)
or "wazuh" in text
):
return _typed_route(
@@ -744,6 +761,7 @@ def resolve_typed_alert_target(
risk_class="medium",
allowed_catalog_ids=allowed_catalog_ids,
allowed_inventory_hosts=[host_scope[0]],
controlled_apply_allowed=(False if host_cpu_pressure_alert else None),
)
identity = registry.resolve_identity(target_resource)

File diff suppressed because it is too large Load Diff

View File

@@ -111,6 +111,20 @@ _EXTERNAL_READBACKS: tuple[dict[str, Any], ...] = (
"verifier": "agent99_alertmanager_pull_relay_verifier",
"runtime_status": "receipt_pending",
},
{
"readback_id": "cpu_resource_postcondition",
"verifier": "cpu_resource_independent_verifier",
"required_source": "prometheus_independent_query",
"source_ref": "apps/api/src/services/cpu_p99_controlled_recovery.py",
"runtime_status": "receipt_pending",
},
{
"readback_id": "api_p99_postcondition",
"verifier": "signoz_api_latency_independent_verifier",
"required_source": "signoz_independent_query",
"source_ref": "apps/api/src/services/cpu_p99_controlled_recovery.py",
"runtime_status": "receipt_pending",
},
)