fix(alerts): guard approval actions and wire playbook learning
This commit is contained in:
Your Name
151
apps/api/src/services/alert_approval_guard.py
Normal file
151
apps/api/src/services/alert_approval_guard.py
Normal file
@@ -0,0 +1,151 @@
|
||||
"""Alert approval guardrails for AI-generated remediation actions.
|
||||
|
||||
This service runs before an Alertmanager-derived action becomes an
|
||||
ApprovalRecord. It prevents a known failure mode: an LLM invents a kubectl
|
||||
target that does not belong to the current alert domain, then the approval
|
||||
pipeline faithfully executes or displays that bad command.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import structlog
|
||||
|
||||
from src.services.action_parser import ActionKind, parse_kubectl_action
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
_ALLOWED_K8S_NAMESPACES = frozenset({"awoooi-prod", "observability", "signoz", "langfuse"})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApprovalActionGuardResult:
|
||||
"""Guarded action payload returned to approval creation."""
|
||||
|
||||
action: str
|
||||
blocked: bool = False
|
||||
reason: str | None = None
|
||||
metadata: dict[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
async def guard_alert_approval_action(
|
||||
*,
|
||||
action: str,
|
||||
alert_namespace: str | None,
|
||||
alertname: str,
|
||||
alert_category: str,
|
||||
) -> ApprovalActionGuardResult:
|
||||
"""Validate an AI/rule action before it is persisted as an approval.
|
||||
|
||||
Non-kubectl actions are intentionally left to their domain-specific gates.
|
||||
Kubectl actions must satisfy the structured parser and must not jump to an
|
||||
unrelated namespace such as ``default`` or ``production`` when the alert
|
||||
came from AWOOOI's production namespace.
|
||||
"""
|
||||
|
||||
raw_action = (action or "").strip()
|
||||
if not raw_action.lower().startswith("kubectl"):
|
||||
return ApprovalActionGuardResult(action=action)
|
||||
|
||||
parsed = parse_kubectl_action(raw_action)
|
||||
if not parsed.ok:
|
||||
return _blocked(raw_action, f"invalid_kubectl:{parsed.reason}", alertname)
|
||||
|
||||
requested_namespace = parsed.namespace
|
||||
expected_namespace = (alert_namespace or "awoooi-prod").strip() or "awoooi-prod"
|
||||
if requested_namespace and requested_namespace not in _ALLOWED_K8S_NAMESPACES:
|
||||
return _blocked(
|
||||
raw_action,
|
||||
f"namespace_not_allowed:{requested_namespace}",
|
||||
alertname,
|
||||
expected_namespace=expected_namespace,
|
||||
)
|
||||
|
||||
if (
|
||||
requested_namespace
|
||||
and expected_namespace in _ALLOWED_K8S_NAMESPACES
|
||||
and requested_namespace != expected_namespace
|
||||
and requested_namespace != "observability"
|
||||
):
|
||||
return _blocked(
|
||||
raw_action,
|
||||
f"namespace_mismatch:{requested_namespace}!={expected_namespace}",
|
||||
alertname,
|
||||
expected_namespace=expected_namespace,
|
||||
)
|
||||
|
||||
# Read-only commands are safe enough to display once the namespace is sane.
|
||||
# Mutating commands still need resource existence checks to avoid executing
|
||||
# hallucinated deployments like "flywheelexecutionratemissing".
|
||||
if parsed.kind == ActionKind.READONLY and parsed.verb in {"get", "version"}:
|
||||
return ApprovalActionGuardResult(action=action)
|
||||
|
||||
if parsed.resource_name and parsed.resource_type in {
|
||||
"deployment",
|
||||
"statefulset",
|
||||
"daemonset",
|
||||
"pod",
|
||||
"service",
|
||||
}:
|
||||
try:
|
||||
from src.services.resource_resolver import get_resource_resolver
|
||||
|
||||
resolver = get_resource_resolver()
|
||||
resolved = await resolver.resolve(
|
||||
raw_resource=parsed.resource_name,
|
||||
namespace=requested_namespace or expected_namespace,
|
||||
resource_kind=parsed.resource_type,
|
||||
)
|
||||
if not resolved.success:
|
||||
return _blocked(
|
||||
raw_action,
|
||||
f"k8s_resource_not_found:{parsed.resource_type}/{parsed.resource_name}",
|
||||
alertname,
|
||||
expected_namespace=expected_namespace,
|
||||
candidates=resolved.candidates,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"approval_action_resource_guard_unavailable",
|
||||
alertname=alertname,
|
||||
alert_category=alert_category,
|
||||
action=raw_action[:160],
|
||||
error=str(exc),
|
||||
)
|
||||
return ApprovalActionGuardResult(
|
||||
action=action,
|
||||
metadata={"action_guard_warning": "resource_guard_unavailable"},
|
||||
)
|
||||
|
||||
return ApprovalActionGuardResult(action=action)
|
||||
|
||||
|
||||
def _blocked(
|
||||
raw_action: str,
|
||||
reason: str,
|
||||
alertname: str,
|
||||
*,
|
||||
expected_namespace: str | None = None,
|
||||
candidates: list[str] | None = None,
|
||||
) -> ApprovalActionGuardResult:
|
||||
logger.warning(
|
||||
"approval_action_blocked_before_persist",
|
||||
alertname=alertname,
|
||||
reason=reason,
|
||||
action=raw_action[:160],
|
||||
expected_namespace=expected_namespace,
|
||||
candidates=candidates or [],
|
||||
)
|
||||
return ApprovalActionGuardResult(
|
||||
action=f"NO_ACTION - INVALID_TARGET: {reason}; original={raw_action[:180]}",
|
||||
blocked=True,
|
||||
reason=reason,
|
||||
metadata={
|
||||
"action_guard": "blocked_before_persist",
|
||||
"blocked_action": raw_action[:300],
|
||||
"blocked_reason": reason,
|
||||
"expected_namespace": expected_namespace,
|
||||
"candidates": candidates or [],
|
||||
},
|
||||
)
|
||||
189
apps/api/src/services/playbook_match_resolver.py
Normal file
189
apps/api/src/services/playbook_match_resolver.py
Normal file
@@ -0,0 +1,189 @@
|
||||
"""Resolve alert/rule context to a real Playbook ID.
|
||||
|
||||
The learning loop updates EWMA trust only when ``approval_records`` carries the
|
||||
actual ``playbooks.playbook_id``. YAML rule IDs such as ``host_resource_alert``
|
||||
are not Playbook IDs, so this resolver bridges rule/alert context to the
|
||||
canonical DB identity before an ApprovalRecord is created.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
import structlog
|
||||
import yaml
|
||||
from sqlalchemy import text as sa_text
|
||||
|
||||
from src.db.base import get_db_context
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlaybookMatch:
|
||||
playbook_id: str
|
||||
source: str
|
||||
|
||||
|
||||
async def resolve_playbook_id_for_alert(
|
||||
*,
|
||||
rule_id: str | None = None,
|
||||
alertname: str | None = None,
|
||||
affected_services: list[str] | None = None,
|
||||
severity: str | None = None,
|
||||
) -> str | None:
|
||||
"""Return a real ``playbooks.playbook_id`` for alert context if available."""
|
||||
|
||||
match = await _resolve_exact_yaml_rule(rule_id=rule_id, alertname=alertname)
|
||||
if match:
|
||||
return match.playbook_id
|
||||
|
||||
match = await _resolve_by_recommendation(
|
||||
alertname=alertname,
|
||||
affected_services=affected_services or [],
|
||||
severity=severity,
|
||||
)
|
||||
return match.playbook_id if match else None
|
||||
|
||||
|
||||
async def _resolve_exact_yaml_rule(
|
||||
*,
|
||||
rule_id: str | None,
|
||||
alertname: str | None,
|
||||
) -> PlaybookMatch | None:
|
||||
"""Use deterministic DB fields before falling back to fuzzy recommendations."""
|
||||
|
||||
rule_id = (rule_id or "").strip()
|
||||
alertname = (alertname or "").strip()
|
||||
if not rule_id and not alertname:
|
||||
return None
|
||||
|
||||
alertname_candidates = [alertname]
|
||||
alertname_candidates.extend(_alertnames_for_rule_id(rule_id))
|
||||
alertname_candidates = list(dict.fromkeys(name for name in alertname_candidates if name))
|
||||
|
||||
try:
|
||||
async with get_db_context() as db:
|
||||
if rule_id:
|
||||
row = (
|
||||
await db.execute(
|
||||
sa_text(
|
||||
"""
|
||||
SELECT playbook_id
|
||||
FROM playbooks
|
||||
WHERE source = 'yaml_rule'
|
||||
AND status = 'approved'
|
||||
AND (
|
||||
name = ('AutoMigrated: ' || :rule_id)
|
||||
OR notes ILIKE ('%rule.id=' || :rule_id || '%')
|
||||
)
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
),
|
||||
{"rule_id": rule_id},
|
||||
)
|
||||
).first()
|
||||
if row:
|
||||
return PlaybookMatch(playbook_id=str(row[0]), source="exact_yaml_rule")
|
||||
|
||||
for candidate in alertname_candidates:
|
||||
row = (
|
||||
await db.execute(
|
||||
sa_text(
|
||||
"""
|
||||
SELECT playbook_id
|
||||
FROM playbooks
|
||||
WHERE source = 'yaml_rule'
|
||||
AND status = 'approved'
|
||||
AND (symptom_pattern::jsonb->'alert_names') ? :alertname
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1
|
||||
"""
|
||||
),
|
||||
{"alertname": candidate},
|
||||
)
|
||||
).first()
|
||||
if row:
|
||||
return PlaybookMatch(playbook_id=str(row[0]), source="exact_alertname")
|
||||
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"playbook_exact_match_failed",
|
||||
rule_id=rule_id,
|
||||
alertname=alertname,
|
||||
error=str(exc),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _rule_alertname_index() -> dict[str, tuple[str, ...]]:
|
||||
rules_path = Path(__file__).resolve().parents[2] / "alert_rules.yaml"
|
||||
try:
|
||||
data = yaml.safe_load(rules_path.read_text(encoding="utf-8")) or {}
|
||||
except Exception as exc:
|
||||
logger.debug("playbook_rule_index_load_failed", path=str(rules_path), error=str(exc))
|
||||
return {}
|
||||
|
||||
index: dict[str, tuple[str, ...]] = {}
|
||||
for rule in data.get("rules", []):
|
||||
if not isinstance(rule, dict):
|
||||
continue
|
||||
rule_id = str(rule.get("id") or "").strip()
|
||||
alertnames = rule.get("match", {}).get("alertname", [])
|
||||
if rule_id and isinstance(alertnames, list):
|
||||
index[rule_id] = tuple(str(name) for name in alertnames if name)
|
||||
return index
|
||||
|
||||
|
||||
def _alertnames_for_rule_id(rule_id: str) -> tuple[str, ...]:
|
||||
if not rule_id:
|
||||
return ()
|
||||
return _rule_alertname_index().get(rule_id, ())
|
||||
|
||||
|
||||
async def _resolve_by_recommendation(
|
||||
*,
|
||||
alertname: str | None,
|
||||
affected_services: list[str],
|
||||
severity: str | None,
|
||||
) -> PlaybookMatch | None:
|
||||
alertname = (alertname or "").strip()
|
||||
if not alertname and not affected_services:
|
||||
return None
|
||||
|
||||
try:
|
||||
from src.models.playbook import SymptomPattern
|
||||
from src.services.playbook_service import get_playbook_service
|
||||
|
||||
symptoms = SymptomPattern(
|
||||
alert_names=[alertname] if alertname else [],
|
||||
affected_services=affected_services,
|
||||
severity_range=[severity or "P2"],
|
||||
)
|
||||
recommendations = await get_playbook_service().get_recommendations(
|
||||
symptoms=symptoms,
|
||||
top_k=1,
|
||||
use_rag=False,
|
||||
)
|
||||
if not recommendations:
|
||||
return None
|
||||
best = recommendations[0]
|
||||
if best.similarity_score < 0.5:
|
||||
return None
|
||||
return PlaybookMatch(
|
||||
playbook_id=best.playbook.playbook_id,
|
||||
source="symptom_recommendation",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"playbook_recommendation_match_skipped",
|
||||
alertname=alertname,
|
||||
affected_services=affected_services,
|
||||
error=str(exc),
|
||||
)
|
||||
return None
|
||||
Reference in New Issue
Block a user