diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 3089895fe..6f4c66155 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -2135,7 +2135,6 @@ async def _process_new_alert_background( """ try: service = get_approval_service() - openclaw = get_openclaw() traced_alert_labels = { **(alert_labels or {}), "fingerprint": fingerprint, @@ -2147,8 +2146,19 @@ async def _process_new_alert_background( source_severity=severity, ) + agent99_durable_route = resolve_agent99_durable_route( + alertname=alertname, + severity=severity, + target_resource=target_resource, + namespace=namespace, + message=message, + labels=traced_alert_labels, + annotations=alert_context.get("annotations", {}), + ) rule_response = match_rule(alert_context) - should_bypass_llm = _should_use_alertmanager_rule_first(rule_response, alert_category) + should_bypass_llm = bool(agent99_durable_route) or ( + _should_use_alertmanager_rule_first(rule_response, alert_category) + ) if should_bypass_llm: logger.info( @@ -2157,7 +2167,12 @@ async def _process_new_alert_background( alertname=alertname, rule_id=rule_response.get("rule_id", ""), alert_category=alert_category, - reason="host/backup YAML 權威規則命中，跳過 LLM 避免產生錯誤 K8s 動作", + agent99_route=(agent99_durable_route or {}).get("route_id"), + reason=( + "agent99_durable_route_precedes_llm" + if agent99_durable_route + else "host_or_backup_authoritative_rule_precedes_llm" + ), ) risk_mapping = { "low": RiskLevel.LOW, @@ -2169,6 +2184,20 @@ async def _process_new_alert_background( str(rule_response.get("risk_level", "low")).lower(), RiskLevel.LOW, ) + if agent99_durable_route: + rule_risk = ( + RiskLevel.MEDIUM + if agent99_durable_route.get("suggested_mode") + in { + "Recover", + "StartVMs", + "HarborRepair", + "AwoooRepair", + "Perf", + "LoadShed", + } + else RiskLevel.LOW + ) blast = rule_response.get("blast_radius", {}) or {} impact_mapping = { "NONE": DataImpact.NONE, @@ -2180,6 +2209,12 @@ async def _process_new_alert_background( str(blast.get("data_impact", "NONE")).upper(), DataImpact.NONE, ) + if agent99_durable_route: + data_impact = ( + DataImpact.WRITE + if rule_risk == RiskLevel.MEDIUM + else DataImpact.READ_ONLY + ) rule_kubectl = str(rule_response.get("kubectl_command", "")).strip() rule_description = str(rule_response.get("description", message)) rule_action = ( @@ -2187,6 +2222,16 @@ async def _process_new_alert_background( if rule_kubectl else f"NO_ACTION - {rule_description[:120]}" ) + if agent99_durable_route: + rule_kubectl = "" + rule_description = ( + "告警已命中 Agent99 durable route；先建立同一 incident/run ledger，" + "再由 99 主機執行受控處置與獨立 verifier。" + ) + rule_action = ( + "AGENT99_CONTROLLED_ROUTE " + f"{agent99_durable_route['route_id']}" + ) _matched_playbook_id_cs2 = await resolve_playbook_id_for_alert( rule_id=str(rule_response.get("rule_id", "")), alertname=alertname, @@ -2214,6 +2259,34 @@ async def _process_new_alert_background( "playbook_id": _matched_playbook_id_cs2, **_guarded_action_cs2.metadata, } + if agent99_durable_route: + _approval_metadata_cs2.update({ + "source": "agent99_durable_route", + "agent99_route_id": agent99_durable_route["route_id"], + "agent99_kind": agent99_durable_route["kind"], + "agent99_mode": agent99_durable_route["suggested_mode"], + "llm_blocking_dependency": False, + }) + rule_dry_run_checks = [ + DryRunCheck( + name="規則命中", + passed=True, + message=str(rule_response.get("rule_id", "unknown")), + ), + DryRunCheck( + name="來源", + passed=True, + message="alertmanager", + ), + ] + if agent99_durable_route: + rule_dry_run_checks.append( + DryRunCheck( + name="Agent99 durable route", + passed=True, + message=agent99_durable_route["route_id"], + ) + ) approval_create = ApprovalRequestCreate( action=rule_action, description=f"[Rule: {rule_response.get('rule_id', 'unknown')}] {rule_description}", @@ -2229,19 +2302,12 @@ async def _process_new_alert_background( ), data_impact=data_impact, ), - dry_run_checks=[ - DryRunCheck( - name="規則命中", - passed=True, - message=str(rule_response.get("rule_id", "unknown")), - ), - DryRunCheck( - name="來源", - passed=True, - message="alertmanager", - ), - ], - requested_by="OpenClaw (rule-engine)", + dry_run_checks=rule_dry_run_checks, + requested_by=( + "Agent99 durable router" + if agent99_durable_route + else "OpenClaw (rule-engine)" + ), metadata=_approval_metadata_cs2, matched_playbook_id=_matched_playbook_id_cs2, ) @@ -2329,8 +2395,9 @@ async def _process_new_alert_background( ) _is_heartbeat = is_heartbeat_alertname(alertname) + _controlled_handoff: dict[str, Any] | None = None if _controlled_ai_policy_allows(rule_risk) and not _is_heartbeat: - await _try_auto_repair_background( + _controlled_handoff = await _try_auto_repair_background( incident_id=incident_id, approval_id=str(approval.id), alert_type=alert_type, @@ -2383,9 +2450,15 @@ async def _process_new_alert_background( estimated_downtime=str(blast.get("estimated_downtime", "N/A")), hit_count=1, primary_responsibility=str( - rule_response.get("primary_responsibility", "INFRA") + "AGENT99_CONTROLLED_EXECUTOR" + if agent99_durable_route + else rule_response.get("primary_responsibility", "INFRA") ), confidence=float(rule_response.get("confidence", 0.0) or 0.0), + automation_state=str( + (_controlled_handoff or {}).get("status") + or "agent99_dispatch_not_requested" + ), namespace=namespace, incident_id=incident_id, notification_type=notification_type, @@ -2397,6 +2470,7 @@ async def _process_new_alert_background( record_alert_chain_success("alertmanager") return + openclaw = get_openclaw() analysis_result, ai_provider, raw_response, signoz_metrics, signoz_trace_url, ai_tokens, ai_cost = await _analyze_alertmanager_with_timeout( openclaw, alert_context, diff --git a/apps/api/tests/test_alertmanager_rule_bypass.py b/apps/api/tests/test_alertmanager_rule_bypass.py index cc053e082..64f09135b 100644 --- a/apps/api/tests/test_alertmanager_rule_bypass.py +++ b/apps/api/tests/test_alertmanager_rule_bypass.py @@ -2,16 +2,20 @@ import ast import asyncio import inspect from datetime import datetime +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock import pytest from src.api.v1 import webhooks as webhooks_module from src.api.v1.webhooks import ( _analyze_alertmanager_with_timeout, + _process_new_alert_background, _should_bypass_alertmanager_llm, _should_use_alertmanager_rule_first, ) from src.repositories.alert_operation_log_repository import ALERT_EVENT_TYPES +from src.services.alert_approval_guard import ApprovalActionGuardResult from src.services.alertmanager_llm_guard import ( ALERTMANAGER_LLM_INFLIGHT_LOCK_TTL_SECONDS, alertmanager_llm_inflight_key, @@ -175,6 +179,125 @@ async def test_alertmanager_analysis_error_returns_fallback(): assert result == (None, "fallback_error", "", None, "", 0, 0.0) +@pytest.mark.asyncio +async def test_agent99_durable_route_dispatches_before_llm(monkeypatch): + class ApprovalService: + created_request = None + + async def create_approval_with_fingerprint(self, *, request, fingerprint): + self.created_request = request + assert fingerprint == "fp-agent99-durable-route" + return SimpleNamespace(id="approval-agent99-fast-path", incident_id=None) + + async def update_incident_id(self, approval_id, incident_id): + assert str(approval_id) == "approval-agent99-fast-path" + assert incident_id == "INC-AGENT99-FAST-PATH" + + service = ApprovalService() + openclaw_factory = Mock( + side_effect=AssertionError("durable Agent99 route must not create OpenClaw") + ) + handoff = AsyncMock( + return_value={ + "status": "agent99_dispatch_accepted_verifier_pending", + "queued": True, + } + ) + telegram = AsyncMock() + + monkeypatch.setattr(webhooks_module, "get_approval_service", lambda: service) + monkeypatch.setattr(webhooks_module, "get_openclaw", openclaw_factory) + monkeypatch.setattr( + webhooks_module, + "get_trusted_alert_canonical_route_context", + lambda *_args, **_kwargs: {}, + ) + monkeypatch.setattr( + webhooks_module, + "match_rule", + lambda _context: { + "rule_id": "generic_fallback", + "risk_level": "critical", + "blast_radius": {}, + "kubectl_command": "", + "description": "generic fallback must not block Agent99", + "confidence": 0.0, + }, + ) + monkeypatch.setattr( + webhooks_module, + "resolve_playbook_id_for_alert", + AsyncMock(return_value=None), + ) + monkeypatch.setattr( + webhooks_module, + "guard_alert_approval_action", + AsyncMock( + side_effect=lambda **kwargs: ApprovalActionGuardResult( + action=kwargs["action"] + ) + ), + ) + monkeypatch.setattr( + webhooks_module, + "get_auto_approve_policy", + lambda: SimpleNamespace( + evaluate=lambda _proposal: SimpleNamespace( + should_auto_approve=False, + reason=SimpleNamespace(value="bounded_durable_route"), + ) + ), + ) + monkeypatch.setattr( + webhooks_module, + "create_incident_for_approval", + AsyncMock(return_value="INC-AGENT99-FAST-PATH"), + ) + monkeypatch.setattr( + webhooks_module, + "record_alertmanager_event", + AsyncMock(), + ) + monkeypatch.setattr(webhooks_module, "_try_auto_repair_background", handoff) + monkeypatch.setattr(webhooks_module, "_push_to_telegram_background", telegram) + monkeypatch.setattr(webhooks_module, "record_alert_chain_success", Mock()) + + await _process_new_alert_background( + alert_context={ + "alertname": "RebootAutoRecoveryActiveBlocker", + "annotations": {"summary": "cold-start gate blocked"}, + "source_url": "http://prometheus.invalid/graph", + }, + alert_id="alert-agent99-fast-path", + fingerprint="fp-agent99-durable-route", + target_resource="reboot-auto-recovery-slo", + namespace="default", + alert_type="custom", + message="cold-start-gate service recovery verifier", + alertname="RebootAutoRecoveryActiveBlocker", + severity="critical", + alert_labels={"service": "cold-start-gate"}, + notification_type="TYPE-3", + alert_category="general", + can_auto_repair=True, + ) + + openclaw_factory.assert_not_called() + handoff.assert_awaited_once() + assert handoff.await_args.kwargs["source_alertname"] == ( + "RebootAutoRecoveryActiveBlocker" + ) + assert service.created_request.risk_level.value == "medium" + assert service.created_request.action == ( + "AGENT99_CONTROLLED_ROUTE agent99:host_recovery:Recover" + ) + assert service.created_request.metadata["source"] == "agent99_durable_route" + telegram.assert_awaited_once() + assert telegram.await_args.kwargs["automation_state"] == ( + "agent99_dispatch_accepted_verifier_pending" + ) + + def test_resolved_guard_stamp_without_timestamp_is_clean(): assert _format_resolved_guard_stamp(None) == "✅ 此事件已解決"