wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity

fix(agent99): dispatch durable alerts before LLM

Browse Source
This commit is contained in:
ogt
2026-07-15 01:55:48 +08:00
parent e3acc321a7
commit fa61d4e07a
2 changed files with 215 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
apps/api/src/api/v1/webhooks.py
View File

@@ -2135,7 +2135,6 @@ async def _process_new_alert_background(
"""
try:
service = get_approval_service()
openclaw = get_openclaw()
traced_alert_labels = {
**(alert_labels or {}),
"fingerprint": fingerprint,
@@ -2147,8 +2146,19 @@ async def _process_new_alert_background(
source_severity=severity,
)
agent99_durable_route = resolve_agent99_durable_route(
alertname=alertname,
severity=severity,
target_resource=target_resource,
namespace=namespace,
message=message,
labels=traced_alert_labels,
annotations=alert_context.get("annotations", {}),
)
rule_response = match_rule(alert_context)
should_bypass_llm = _should_use_alertmanager_rule_first(rule_response, alert_category)
should_bypass_llm = bool(agent99_durable_route) or (
_should_use_alertmanager_rule_first(rule_response, alert_category)
)
if should_bypass_llm:
logger.info(
@@ -2157,7 +2167,12 @@ async def _process_new_alert_background(
alertname=alertname,
rule_id=rule_response.get("rule_id", ""),
alert_category=alert_category,
reason="host/backup YAML 權威規則命中，跳過 LLM 避免產生錯誤 K8s 動作",
agent99_route=(agent99_durable_route or {}).get("route_id"),
reason=(
"agent99_durable_route_precedes_llm"
if agent99_durable_route
else "host_or_backup_authoritative_rule_precedes_llm"
),
)
risk_mapping = {
"low": RiskLevel.LOW,
@@ -2169,6 +2184,20 @@ async def _process_new_alert_background(
str(rule_response.get("risk_level", "low")).lower(),
RiskLevel.LOW,
)
if agent99_durable_route:
rule_risk = (
RiskLevel.MEDIUM
if agent99_durable_route.get("suggested_mode")
in {
"Recover",
"StartVMs",
"HarborRepair",
"AwoooRepair",
"Perf",
"LoadShed",
}
else RiskLevel.LOW
)
blast = rule_response.get("blast_radius", {}) or {}
impact_mapping = {
"NONE": DataImpact.NONE,
@@ -2180,6 +2209,12 @@ async def _process_new_alert_background(
str(blast.get("data_impact", "NONE")).upper(),
DataImpact.NONE,
)
if agent99_durable_route:
data_impact = (
DataImpact.WRITE
if rule_risk == RiskLevel.MEDIUM
else DataImpact.READ_ONLY
)
rule_kubectl = str(rule_response.get("kubectl_command", "")).strip()
rule_description = str(rule_response.get("description", message))
rule_action = (
@@ -2187,6 +2222,16 @@ async def _process_new_alert_background(
if rule_kubectl else
f"NO_ACTION - {rule_description[:120]}"
)
if agent99_durable_route:
rule_kubectl = ""
rule_description = (
"告警已命中 Agent99 durable route；先建立同一 incident/run ledger，"
"再由 99 主機執行受控處置與獨立 verifier。"
)
rule_action = (
"AGENT99_CONTROLLED_ROUTE "
f"{agent99_durable_route['route_id']}"
)
_matched_playbook_id_cs2 = await resolve_playbook_id_for_alert(
rule_id=str(rule_response.get("rule_id", "")),
alertname=alertname,
@@ -2214,6 +2259,34 @@ async def _process_new_alert_background(
"playbook_id": _matched_playbook_id_cs2,
**_guarded_action_cs2.metadata,
}
if agent99_durable_route:
_approval_metadata_cs2.update({
"source": "agent99_durable_route",
"agent99_route_id": agent99_durable_route["route_id"],
"agent99_kind": agent99_durable_route["kind"],
"agent99_mode": agent99_durable_route["suggested_mode"],
"llm_blocking_dependency": False,
})
rule_dry_run_checks = [
DryRunCheck(
name="規則命中",
passed=True,
message=str(rule_response.get("rule_id", "unknown")),
),
DryRunCheck(
name="來源",
passed=True,
message="alertmanager",
),
]
if agent99_durable_route:
rule_dry_run_checks.append(
DryRunCheck(
name="Agent99 durable route",
passed=True,
message=agent99_durable_route["route_id"],
)
)
approval_create = ApprovalRequestCreate(
action=rule_action,
description=f"[Rule: {rule_response.get('rule_id', 'unknown')}] {rule_description}",
@@ -2229,19 +2302,12 @@ async def _process_new_alert_background(
),
data_impact=data_impact,
),
dry_run_checks=[
DryRunCheck(
name="規則命中",
passed=True,
message=str(rule_response.get("rule_id", "unknown")),
),
DryRunCheck(
name="來源",
passed=True,
message="alertmanager",
),
],
requested_by="OpenClaw (rule-engine)",
dry_run_checks=rule_dry_run_checks,
requested_by=(
"Agent99 durable router"
if agent99_durable_route
else "OpenClaw (rule-engine)"
),
metadata=_approval_metadata_cs2,
matched_playbook_id=_matched_playbook_id_cs2,
)
@@ -2329,8 +2395,9 @@ async def _process_new_alert_background(
)
_is_heartbeat = is_heartbeat_alertname(alertname)
_controlled_handoff: dict[str, Any] | None = None
if _controlled_ai_policy_allows(rule_risk) and not _is_heartbeat:
await _try_auto_repair_background(
_controlled_handoff = await _try_auto_repair_background(
incident_id=incident_id,
approval_id=str(approval.id),
alert_type=alert_type,
@@ -2383,9 +2450,15 @@ async def _process_new_alert_background(
estimated_downtime=str(blast.get("estimated_downtime", "N/A")),
hit_count=1,
primary_responsibility=str(
rule_response.get("primary_responsibility", "INFRA")
"AGENT99_CONTROLLED_EXECUTOR"
if agent99_durable_route
else rule_response.get("primary_responsibility", "INFRA")
),
confidence=float(rule_response.get("confidence", 0.0) or 0.0),
automation_state=str(
(_controlled_handoff or {}).get("status")
or "agent99_dispatch_not_requested"
),
namespace=namespace,
incident_id=incident_id,
notification_type=notification_type,
@@ -2397,6 +2470,7 @@ async def _process_new_alert_background(
record_alert_chain_success("alertmanager")
return
openclaw = get_openclaw()
analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url, ai_tokens, ai_cost = await _analyze_alertmanager_with_timeout(
openclaw,
alert_context,

123
apps/api/tests/test_alertmanager_rule_bypass.py
View File

@@ -2,16 +2,20 @@ import ast
import asyncio
import inspect
from datetime import datetime
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from src.api.v1 import webhooks as webhooks_module
from src.api.v1.webhooks import (
_analyze_alertmanager_with_timeout,
_process_new_alert_background,
_should_bypass_alertmanager_llm,
_should_use_alertmanager_rule_first,
)
from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES
from src.services.alert_approval_guard import ApprovalActionGuardResult
from src.services.alertmanager_llm_guard import (
ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS,
alertmanager_llm_inflight_key,
@@ -175,6 +179,125 @@ async def test_alertmanager_analysis_error_returns_fallback():
assert result == (None, "fallback_error", "", None, "", 0, 0.0)
@pytest.mark.asyncio
async def test_agent99_durable_route_dispatches_before_llm(monkeypatch):
class ApprovalService:
created_request = None
async def create_approval_with_fingerprint(self, *, request, fingerprint):
self.created_request = request
assert fingerprint == "fp-agent99-durable-route"
return SimpleNamespace(id="approval-agent99-fast-path", incident_id=None)
async def update_incident_id(self, approval_id, incident_id):
assert str(approval_id) == "approval-agent99-fast-path"
assert incident_id == "INC-AGENT99-FAST-PATH"
service = ApprovalService()
openclaw_factory = Mock(
side_effect=AssertionError("durable Agent99 route must not create OpenClaw")
)
handoff = AsyncMock(
return_value={
"status": "agent99_dispatch_accepted_verifier_pending",
"queued": True,
}
)
telegram = AsyncMock()
monkeypatch.setattr(webhooks_module, "get_approval_service", lambda: service)
monkeypatch.setattr(webhooks_module, "get_openclaw", openclaw_factory)
monkeypatch.setattr(
webhooks_module,
"get_trusted_alert_canonical_route_context",
lambda *_args, **_kwargs: {},
)
monkeypatch.setattr(
webhooks_module,
"match_rule",
lambda _context: {
"rule_id": "generic_fallback",
"risk_level": "critical",
"blast_radius": {},
"kubectl_command": "",
"description": "generic fallback must not block Agent99",
"confidence": 0.0,
},
)
monkeypatch.setattr(
webhooks_module,
"resolve_playbook_id_for_alert",
AsyncMock(return_value=None),
)
monkeypatch.setattr(
webhooks_module,
"guard_alert_approval_action",
AsyncMock(
side_effect=lambda **kwargs: ApprovalActionGuardResult(
action=kwargs["action"]
)
),
)
monkeypatch.setattr(
webhooks_module,
"get_auto_approve_policy",
lambda: SimpleNamespace(
evaluate=lambda _proposal: SimpleNamespace(
should_auto_approve=False,
reason=SimpleNamespace(value="bounded_durable_route"),
)
),
)
monkeypatch.setattr(
webhooks_module,
"create_incident_for_approval",
AsyncMock(return_value="INC-AGENT99-FAST-PATH"),
)
monkeypatch.setattr(
webhooks_module,
"record_alertmanager_event",
AsyncMock(),
)
monkeypatch.setattr(webhooks_module, "_try_auto_repair_background", handoff)
monkeypatch.setattr(webhooks_module, "_push_to_telegram_background", telegram)
monkeypatch.setattr(webhooks_module, "record_alert_chain_success", Mock())
await _process_new_alert_background(
alert_context={
"alertname": "RebootAutoRecoveryActiveBlocker",
"annotations": {"summary": "cold-start gate blocked"},
"source_url": "http://prometheus.invalid/graph",
},
alert_id="alert-agent99-fast-path",
fingerprint="fp-agent99-durable-route",
target_resource="reboot-auto-recovery-slo",
namespace="default",
alert_type="custom",
message="cold-start-gate service recovery verifier",
alertname="RebootAutoRecoveryActiveBlocker",
severity="critical",
alert_labels={"service": "cold-start-gate"},
notification_type="TYPE-3",
alert_category="general",
can_auto_repair=True,
)
openclaw_factory.assert_not_called()
handoff.assert_awaited_once()
assert handoff.await_args.kwargs["source_alertname"] == (
"RebootAutoRecoveryActiveBlocker"
)
assert service.created_request.risk_level.value == "medium"
assert service.created_request.action == (
"AGENT99_CONTROLLED_ROUTE agent99:host_recovery:Recover"
)
assert service.created_request.metadata["source"] == "agent99_durable_route"
telegram.assert_awaited_once()
assert telegram.await_args.kwargs["automation_state"] == (
"agent99_dispatch_accepted_verifier_pending"
)
def test_resolved_guard_stamp_without_timestamp_is_clean():
assert _format_resolved_guard_stamp(None) == "✅ 此事件已解決"