fix(sre): honor typed executor before Agent99 mutation

This commit is contained in:
ogt
2026-07-17 11:29:30 +08:00
parent dd7053b01f
commit 2d15ac7e71
2 changed files with 151 additions and 12 deletions

View File

@@ -74,6 +74,7 @@ from src.services.channel_hub import (
)
from src.services.controlled_alert_target_router import (
build_controlled_recovery_handoff,
resolve_typed_alert_target,
)
from src.services.converged_alert_recurrence_notifier import (
notify_converged_alert_recurrence,
@@ -118,6 +119,52 @@ from src.utils.timezone import now_taipei
router = APIRouter(prefix="/webhooks", tags=["Webhooks"])
logger = get_logger("awoooi.webhooks")
_AGENT99_MUTATING_MODES = frozenset(
{"Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed"}
)
def _resolve_typed_agent99_durable_route(
*,
alertname: str,
severity: str,
target_resource: str,
namespace: str,
message: str,
labels: dict[str, Any] | None,
annotations: dict[str, Any] | None,
alert_category: str,
) -> tuple[dict[str, Any], dict[str, str] | None]:
"""Let canonical typed routing veto cross-domain Agent99 mutation."""
typed_target_route = resolve_typed_alert_target(
alertname=alertname,
target_resource=target_resource,
namespace=namespace,
labels=labels or {},
alert_category=alert_category,
)
durable_route = resolve_agent99_durable_route(
alertname=alertname,
severity=severity,
target_resource=target_resource,
namespace=namespace,
message=message,
labels=labels or {},
annotations=annotations or {},
)
if durable_route is None:
return typed_target_route, None
mutating = durable_route.get("suggested_mode") in _AGENT99_MUTATING_MODES
agent99_dispatch_allowed = bool(
(typed_target_route.get("agent99_bridge") or {}).get("dispatch_allowed")
is True
)
if mutating and not agent99_dispatch_allowed:
return typed_target_route, None
return typed_target_route, durable_route
class Agent99OutcomeWebhookPayload(BaseModel):
"""Public-safe Agent99 outcome; learning refs are compatibility-only."""
@@ -463,15 +510,26 @@ async def _try_auto_repair_background(
namespace=namespace,
incident_id=incident_id,
)
durable_agent99_route = resolve_agent99_durable_route(
typed_target_route, durable_agent99_route = _resolve_typed_agent99_durable_route(
alertname=source_alertname or alert_type,
severity=source_severity or risk_level,
target_resource=target_resource,
namespace=namespace,
message=source_message or target_resource,
labels=source_labels or {},
annotations=source_annotations or {},
labels=source_labels,
annotations=source_annotations,
alert_category=alert_category,
)
if typed_target_route.get("executor") != "Agent99":
logger.info(
"typed_executor_selected_before_agent99_mutation",
incident_id=incident_id,
target_kind=typed_target_route.get("target_kind"),
canonical_asset_id=typed_target_route.get("canonical_asset_id"),
executor=typed_target_route.get("executor"),
allowed_catalog_ids=typed_target_route.get("allowed_catalog_ids"),
cross_domain_fallback_allowed=False,
)
if handoff is None and durable_agent99_route is not None:
# Domain-specific Agent99 routes (including read-only BackupCheck) are
# durable single-owner lanes too; they must never fall through to the
@@ -2246,7 +2304,7 @@ async def _process_new_alert_background(
source_severity=severity,
)
agent99_durable_route = resolve_agent99_durable_route(
typed_target_route, agent99_durable_route = _resolve_typed_agent99_durable_route(
alertname=alertname,
severity=severity,
target_resource=target_resource,
@@ -2254,7 +2312,18 @@ async def _process_new_alert_background(
message=message,
labels=traced_alert_labels,
annotations=alert_context.get("annotations", {}),
alert_category=alert_category,
)
if typed_target_route.get("executor") != "Agent99":
logger.info(
"typed_executor_selected_before_agent99_mutation",
alert_id=alert_id,
target_kind=typed_target_route.get("target_kind"),
canonical_asset_id=typed_target_route.get("canonical_asset_id"),
executor=typed_target_route.get("executor"),
allowed_catalog_ids=typed_target_route.get("allowed_catalog_ids"),
cross_domain_fallback_allowed=False,
)
rule_response = match_rule(alert_context)
should_bypass_llm = bool(agent99_durable_route) or (
_should_use_alertmanager_rule_first(rule_response, alert_category)
@@ -2288,14 +2357,7 @@ async def _process_new_alert_background(
rule_risk = (
RiskLevel.MEDIUM
if agent99_durable_route.get("suggested_mode")
in {
"Recover",
"StartVMs",
"HarborRepair",
"AwoooRepair",
"Perf",
"LoadShed",
}
in _AGENT99_MUTATING_MODES
else RiskLevel.LOW
)
blast = rule_response.get("blast_radius", {}) or {}

View File

@@ -249,3 +249,80 @@ async def test_webhook_router_never_queues_generic_ansible_for_cold_start(
"run_independent_cold_start_verifier_then_km_playbook_writeback"
)
assert receipt["context"]["runtime_closure_verified"] is False
@pytest.mark.asyncio
async def test_webhook_router_queues_host188_callback_ansible_before_agent99(
monkeypatch: pytest.MonkeyPatch,
) -> None:
append = AsyncMock()
incident = SimpleNamespace(
incident_id="INC-H188-CALLBACK",
project_id="awoooi",
)
queue = AsyncMock(
return_value={
"schema_version": "ai_decision_ansible_candidate_handoff_v1",
"status": "ansible_candidate_queued",
"queued": True,
"side_effect_performed": False,
"single_writer_executor": "awoooi-ansible-executor-broker",
"automation_run_id": "run-h188-callback",
"active_blockers": [],
}
)
bridge = AsyncMock(
side_effect=AssertionError(
"Docker Compose mutation must use Host188 Ansible, not Agent99"
)
)
monkeypatch.setattr(
"src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate",
queue,
)
monkeypatch.setattr(
"src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
lambda: SimpleNamespace(append=append),
)
monkeypatch.setattr(
webhooks,
"get_incident_service",
lambda: SimpleNamespace(
get_from_working_memory=AsyncMock(return_value=incident)
),
)
monkeypatch.setattr(webhooks, "bridge_alertmanager_to_agent99", bridge)
handoff = await webhooks._try_auto_repair_background(
incident_id="INC-H188-CALLBACK",
approval_id="00000000-0000-0000-0000-000000000188",
alert_type="custom",
target_resource="openclaw",
namespace="default",
risk_level="medium",
source_alert_id="alert-h188-callback",
source_alertname="TelegramCallbackForwarderDrift",
source_severity="warning",
source_message="callback ingress receipt missing",
source_labels={
"alertname": "TelegramCallbackForwarderDrift",
"component": "openclaw",
"host": "188",
"instance": "192.168.0.188",
},
source_fingerprint="callback-fingerprint",
alert_category="ai_agent",
notification_type="TYPE-3",
)
bridge.assert_not_awaited()
queue.assert_awaited_once()
proposal = queue.await_args.kwargs["proposal_data"]
assert proposal["source"] == "alert_webhook_controlled_router"
assert proposal["source_occurrence_id"] == "alert-h188-callback"
assert handoff["queued"] is True
assert handoff["single_writer_executor"] == "awoooi-ansible-executor-broker"
assert handoff["execution_priority"] == 20
append.assert_awaited_once()
assert append.await_args.kwargs["success"] is True