wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(agent99): dispatch durable alerts before LLM

Browse Source
This commit is contained in:
ogt
2026-07-15 01:55:48 +08:00
parent e3acc321a7
commit fa61d4e07a
2 changed files with 215 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
apps/api/src/api/v1/webhooks.py
View File

@@ -2135,7 +2135,6 @@ async def _process_new_alert_background(
"""
try:
service = get_approval_service()
openclaw = get_openclaw()
traced_alert_labels = {
**(alert_labels or {}),
"fingerprint": fingerprint,
@@ -2147,8 +2146,19 @@ async def _process_new_alert_background(
source_severity=severity,
)
agent99_durable_route = resolve_agent99_durable_route(
alertname=alertname,
severity=severity,
target_resource=target_resource,
namespace=namespace,
message=message,
labels=traced_alert_labels,
annotations=alert_context.get("annotations", {}),
)
rule_response = match_rule(alert_context)
should_bypass_llm = _should_use_alertmanager_rule_first(rule_response, alert_category)
should_bypass_llm = bool(agent99_durable_route) or (
_should_use_alertmanager_rule_first(rule_response, alert_category)
)
if should_bypass_llm:
logger.info(
@@ -2157,7 +2167,12 @@ async def _process_new_alert_background(
alertname=alertname,
rule_id=rule_response.get("rule_id", ""),
alert_category=alert_category,
reason="host/backup YAML 權威規則命中，跳過 LLM 避免產生錯誤 K8s 動作",
agent99_route=(agent99_durable_route or {}).get("route_id"),
reason=(
"agent99_durable_route_precedes_llm"
if agent99_durable_route
else "host_or_backup_authoritative_rule_precedes_llm"
),
)
risk_mapping = {
"low": RiskLevel.LOW,
@@ -2169,6 +2184,20 @@ async def _process_new_alert_background(
str(rule_response.get("risk_level", "low")).lower(),
RiskLevel.LOW,
)
if agent99_durable_route:
rule_risk = (
RiskLevel.MEDIUM
if agent99_durable_route.get("suggested_mode")
in {
"Recover",
"StartVMs",
"HarborRepair",
"AwoooRepair",
"Perf",
"LoadShed",
}
else RiskLevel.LOW
)
blast = rule_response.get("blast_radius", {}) or {}
impact_mapping = {
"NONE": DataImpact.NONE,
@@ -2180,6 +2209,12 @@ async def _process_new_alert_background(
str(blast.get("data_impact", "NONE")).upper(),
DataImpact.NONE,
)
if agent99_durable_route:
data_impact = (
DataImpact.WRITE
if rule_risk == RiskLevel.MEDIUM
else DataImpact.READ_ONLY
)
rule_kubectl = str(rule_response.get("kubectl_command", "")).strip()
rule_description = str(rule_response.get("description", message))
rule_action = (
@@ -2187,6 +2222,16 @@ async def _process_new_alert_background(
if rule_kubectl else
f"NO_ACTION - {rule_description[:120]}"
)
if agent99_durable_route:
rule_kubectl = ""
rule_description = (
"告警已命中 Agent99 durable route；先建立同一 incident/run ledger，"
"再由 99 主機執行受控處置與獨立 verifier。"
)
rule_action = (
"AGENT99_CONTROLLED_ROUTE "
f"{agent99_durable_route['route_id']}"
)
_matched_playbook_id_cs2 = await resolve_playbook_id_for_alert(
rule_id=str(rule_response.get("rule_id", "")),
alertname=alertname,
@@ -2214,6 +2259,34 @@ async def _process_new_alert_background(
"playbook_id": _matched_playbook_id_cs2,
**_guarded_action_cs2.metadata,
}
if agent99_durable_route:
_approval_metadata_cs2.update({
"source": "agent99_durable_route",
"agent99_route_id": agent99_durable_route["route_id"],
"agent99_kind": agent99_durable_route["kind"],
"agent99_mode": agent99_durable_route["suggested_mode"],
"llm_blocking_dependency": False,
})
rule_dry_run_checks = [
DryRunCheck(
name="規則命中",
passed=True,
message=str(rule_response.get("rule_id", "unknown")),
),
DryRunCheck(
name="來源",
passed=True,
message="alertmanager",
),
]
if agent99_durable_route:
rule_dry_run_checks.append(
DryRunCheck(
name="Agent99 durable route",
passed=True,
message=agent99_durable_route["route_id"],
)
)
approval_create = ApprovalRequestCreate(
action=rule_action,
description=f"[Rule: {rule_response.get('rule_id', 'unknown')}] {rule_description}",
@@ -2229,19 +2302,12 @@ async def _process_new_alert_background(
),
data_impact=data_impact,
),
dry_run_checks=[
DryRunCheck(
name="規則命中",
passed=True,
message=str(rule_response.get("rule_id", "unknown")),
),
DryRunCheck(
name="來源",
passed=True,
message="alertmanager",
),
],
requested_by="OpenClaw (rule-engine)",
dry_run_checks=rule_dry_run_checks,
requested_by=(
"Agent99 durable router"
if agent99_durable_route
else "OpenClaw (rule-engine)"
),
metadata=_approval_metadata_cs2,
matched_playbook_id=_matched_playbook_id_cs2,
)
@@ -2329,8 +2395,9 @@ async def _process_new_alert_background(
)
_is_heartbeat = is_heartbeat_alertname(alertname)
_controlled_handoff: dict[str, Any] | None = None
if _controlled_ai_policy_allows(rule_risk) and not _is_heartbeat:
await _try_auto_repair_background(
_controlled_handoff = await _try_auto_repair_background(
incident_id=incident_id,
approval_id=str(approval.id),
alert_type=alert_type,
@@ -2383,9 +2450,15 @@ async def _process_new_alert_background(
estimated_downtime=str(blast.get("estimated_downtime", "N/A")),
hit_count=1,
primary_responsibility=str(
rule_response.get("primary_responsibility", "INFRA")
"AGENT99_CONTROLLED_EXECUTOR"
if agent99_durable_route
else rule_response.get("primary_responsibility", "INFRA")
),
confidence=float(rule_response.get("confidence", 0.0) or 0.0),
automation_state=str(
(_controlled_handoff or {}).get("status")
or "agent99_dispatch_not_requested"
),
namespace=namespace,
incident_id=incident_id,
notification_type=notification_type,
@@ -2397,6 +2470,7 @@ async def _process_new_alert_background(
record_alert_chain_success("alertmanager")
return
openclaw = get_openclaw()
analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url, ai_tokens, ai_cost = await _analyze_alertmanager_with_timeout(
openclaw,
alert_context,