From 2d15ac7e7158b0b557b9583365c6e5ee1dae8b66 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 17 Jul 2026 11:29:30 +0800 Subject: [PATCH] fix(sre): honor typed executor before Agent99 mutation --- apps/api/src/api/v1/webhooks.py | 86 ++++++++++++++++--- .../test_controlled_alert_target_router.py | 77 +++++++++++++++++ 2 files changed, 151 insertions(+), 12 deletions(-) diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index c0f4de027..ff492ed91 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -74,6 +74,7 @@ from src.services.channel_hub import ( ) from src.services.controlled_alert_target_router import ( build_controlled_recovery_handoff, + resolve_typed_alert_target, ) from src.services.converged_alert_recurrence_notifier import ( notify_converged_alert_recurrence, @@ -118,6 +119,52 @@ from src.utils.timezone import now_taipei router = APIRouter(prefix="/webhooks", tags=["Webhooks"]) logger = get_logger("awoooi.webhooks") +_AGENT99_MUTATING_MODES = frozenset( + {"Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed"} +) + + +def _resolve_typed_agent99_durable_route( + *, + alertname: str, + severity: str, + target_resource: str, + namespace: str, + message: str, + labels: dict[str, Any] | None, + annotations: dict[str, Any] | None, + alert_category: str, +) -> tuple[dict[str, Any], dict[str, str] | None]: + """Let canonical typed routing veto cross-domain Agent99 mutation.""" + + typed_target_route = resolve_typed_alert_target( + alertname=alertname, + target_resource=target_resource, + namespace=namespace, + labels=labels or {}, + alert_category=alert_category, + ) + durable_route = resolve_agent99_durable_route( + alertname=alertname, + severity=severity, + target_resource=target_resource, + namespace=namespace, + message=message, + labels=labels or {}, + annotations=annotations or {}, + ) + if durable_route is None: + return typed_target_route, None + + mutating = durable_route.get("suggested_mode") in _AGENT99_MUTATING_MODES + agent99_dispatch_allowed = bool( + (typed_target_route.get("agent99_bridge") or {}).get("dispatch_allowed") + is True + ) + if mutating and not agent99_dispatch_allowed: + return typed_target_route, None + return typed_target_route, durable_route + class Agent99OutcomeWebhookPayload(BaseModel): """Public-safe Agent99 outcome; learning refs are compatibility-only.""" @@ -463,15 +510,26 @@ async def _try_auto_repair_background( namespace=namespace, incident_id=incident_id, ) - durable_agent99_route = resolve_agent99_durable_route( + typed_target_route, durable_agent99_route = _resolve_typed_agent99_durable_route( alertname=source_alertname or alert_type, severity=source_severity or risk_level, target_resource=target_resource, namespace=namespace, message=source_message or target_resource, - labels=source_labels or {}, - annotations=source_annotations or {}, + labels=source_labels, + annotations=source_annotations, + alert_category=alert_category, ) + if typed_target_route.get("executor") != "Agent99": + logger.info( + "typed_executor_selected_before_agent99_mutation", + incident_id=incident_id, + target_kind=typed_target_route.get("target_kind"), + canonical_asset_id=typed_target_route.get("canonical_asset_id"), + executor=typed_target_route.get("executor"), + allowed_catalog_ids=typed_target_route.get("allowed_catalog_ids"), + cross_domain_fallback_allowed=False, + ) if handoff is None and durable_agent99_route is not None: # Domain-specific Agent99 routes (including read-only BackupCheck) are # durable single-owner lanes too; they must never fall through to the @@ -2246,7 +2304,7 @@ async def _process_new_alert_background( source_severity=severity, ) - agent99_durable_route = resolve_agent99_durable_route( + typed_target_route, agent99_durable_route = _resolve_typed_agent99_durable_route( alertname=alertname, severity=severity, target_resource=target_resource, @@ -2254,7 +2312,18 @@ async def _process_new_alert_background( message=message, labels=traced_alert_labels, annotations=alert_context.get("annotations", {}), + alert_category=alert_category, ) + if typed_target_route.get("executor") != "Agent99": + logger.info( + "typed_executor_selected_before_agent99_mutation", + alert_id=alert_id, + target_kind=typed_target_route.get("target_kind"), + canonical_asset_id=typed_target_route.get("canonical_asset_id"), + executor=typed_target_route.get("executor"), + allowed_catalog_ids=typed_target_route.get("allowed_catalog_ids"), + cross_domain_fallback_allowed=False, + ) rule_response = match_rule(alert_context) should_bypass_llm = bool(agent99_durable_route) or ( _should_use_alertmanager_rule_first(rule_response, alert_category) @@ -2288,14 +2357,7 @@ async def _process_new_alert_background( rule_risk = ( RiskLevel.MEDIUM if agent99_durable_route.get("suggested_mode") - in { - "Recover", - "StartVMs", - "HarborRepair", - "AwoooRepair", - "Perf", - "LoadShed", - } + in _AGENT99_MUTATING_MODES else RiskLevel.LOW ) blast = rule_response.get("blast_radius", {}) or {} diff --git a/apps/api/tests/test_controlled_alert_target_router.py b/apps/api/tests/test_controlled_alert_target_router.py index 11fa81e60..ca995018a 100644 --- a/apps/api/tests/test_controlled_alert_target_router.py +++ b/apps/api/tests/test_controlled_alert_target_router.py @@ -249,3 +249,80 @@ async def test_webhook_router_never_queues_generic_ansible_for_cold_start( "run_independent_cold_start_verifier_then_km_playbook_writeback" ) assert receipt["context"]["runtime_closure_verified"] is False + + +@pytest.mark.asyncio +async def test_webhook_router_queues_host188_callback_ansible_before_agent99( + monkeypatch: pytest.MonkeyPatch, +) -> None: + append = AsyncMock() + incident = SimpleNamespace( + incident_id="INC-H188-CALLBACK", + project_id="awoooi", + ) + queue = AsyncMock( + return_value={ + "schema_version": "ai_decision_ansible_candidate_handoff_v1", + "status": "ansible_candidate_queued", + "queued": True, + "side_effect_performed": False, + "single_writer_executor": "awoooi-ansible-executor-broker", + "automation_run_id": "run-h188-callback", + "active_blockers": [], + } + ) + bridge = AsyncMock( + side_effect=AssertionError( + "Docker Compose mutation must use Host188 Ansible, not Agent99" + ) + ) + + monkeypatch.setattr( + "src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate", + queue, + ) + monkeypatch.setattr( + "src.repositories.alert_operation_log_repository.get_alert_operation_log_repository", + lambda: SimpleNamespace(append=append), + ) + monkeypatch.setattr( + webhooks, + "get_incident_service", + lambda: SimpleNamespace( + get_from_working_memory=AsyncMock(return_value=incident) + ), + ) + monkeypatch.setattr(webhooks, "bridge_alertmanager_to_agent99", bridge) + + handoff = await webhooks._try_auto_repair_background( + incident_id="INC-H188-CALLBACK", + approval_id="00000000-0000-0000-0000-000000000188", + alert_type="custom", + target_resource="openclaw", + namespace="default", + risk_level="medium", + source_alert_id="alert-h188-callback", + source_alertname="TelegramCallbackForwarderDrift", + source_severity="warning", + source_message="callback ingress receipt missing", + source_labels={ + "alertname": "TelegramCallbackForwarderDrift", + "component": "openclaw", + "host": "188", + "instance": "192.168.0.188", + }, + source_fingerprint="callback-fingerprint", + alert_category="ai_agent", + notification_type="TYPE-3", + ) + + bridge.assert_not_awaited() + queue.assert_awaited_once() + proposal = queue.await_args.kwargs["proposal_data"] + assert proposal["source"] == "alert_webhook_controlled_router" + assert proposal["source_occurrence_id"] == "alert-h188-callback" + assert handoff["queued"] is True + assert handoff["single_writer_executor"] == "awoooi-ansible-executor-broker" + assert handoff["execution_priority"] == 20 + append.assert_awaited_once() + assert append.await_args.kwargs["success"] is True