fix(sre): honor typed executor before Agent99 mutation
This commit is contained in:
@@ -74,6 +74,7 @@ from src.services.channel_hub import (
|
||||
)
|
||||
from src.services.controlled_alert_target_router import (
|
||||
build_controlled_recovery_handoff,
|
||||
resolve_typed_alert_target,
|
||||
)
|
||||
from src.services.converged_alert_recurrence_notifier import (
|
||||
notify_converged_alert_recurrence,
|
||||
@@ -118,6 +119,52 @@ from src.utils.timezone import now_taipei
|
||||
router = APIRouter(prefix="/webhooks", tags=["Webhooks"])
|
||||
logger = get_logger("awoooi.webhooks")
|
||||
|
||||
_AGENT99_MUTATING_MODES = frozenset(
|
||||
{"Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed"}
|
||||
)
|
||||
|
||||
|
||||
def _resolve_typed_agent99_durable_route(
|
||||
*,
|
||||
alertname: str,
|
||||
severity: str,
|
||||
target_resource: str,
|
||||
namespace: str,
|
||||
message: str,
|
||||
labels: dict[str, Any] | None,
|
||||
annotations: dict[str, Any] | None,
|
||||
alert_category: str,
|
||||
) -> tuple[dict[str, Any], dict[str, str] | None]:
|
||||
"""Let canonical typed routing veto cross-domain Agent99 mutation."""
|
||||
|
||||
typed_target_route = resolve_typed_alert_target(
|
||||
alertname=alertname,
|
||||
target_resource=target_resource,
|
||||
namespace=namespace,
|
||||
labels=labels or {},
|
||||
alert_category=alert_category,
|
||||
)
|
||||
durable_route = resolve_agent99_durable_route(
|
||||
alertname=alertname,
|
||||
severity=severity,
|
||||
target_resource=target_resource,
|
||||
namespace=namespace,
|
||||
message=message,
|
||||
labels=labels or {},
|
||||
annotations=annotations or {},
|
||||
)
|
||||
if durable_route is None:
|
||||
return typed_target_route, None
|
||||
|
||||
mutating = durable_route.get("suggested_mode") in _AGENT99_MUTATING_MODES
|
||||
agent99_dispatch_allowed = bool(
|
||||
(typed_target_route.get("agent99_bridge") or {}).get("dispatch_allowed")
|
||||
is True
|
||||
)
|
||||
if mutating and not agent99_dispatch_allowed:
|
||||
return typed_target_route, None
|
||||
return typed_target_route, durable_route
|
||||
|
||||
|
||||
class Agent99OutcomeWebhookPayload(BaseModel):
|
||||
"""Public-safe Agent99 outcome; learning refs are compatibility-only."""
|
||||
@@ -463,15 +510,26 @@ async def _try_auto_repair_background(
|
||||
namespace=namespace,
|
||||
incident_id=incident_id,
|
||||
)
|
||||
durable_agent99_route = resolve_agent99_durable_route(
|
||||
typed_target_route, durable_agent99_route = _resolve_typed_agent99_durable_route(
|
||||
alertname=source_alertname or alert_type,
|
||||
severity=source_severity or risk_level,
|
||||
target_resource=target_resource,
|
||||
namespace=namespace,
|
||||
message=source_message or target_resource,
|
||||
labels=source_labels or {},
|
||||
annotations=source_annotations or {},
|
||||
labels=source_labels,
|
||||
annotations=source_annotations,
|
||||
alert_category=alert_category,
|
||||
)
|
||||
if typed_target_route.get("executor") != "Agent99":
|
||||
logger.info(
|
||||
"typed_executor_selected_before_agent99_mutation",
|
||||
incident_id=incident_id,
|
||||
target_kind=typed_target_route.get("target_kind"),
|
||||
canonical_asset_id=typed_target_route.get("canonical_asset_id"),
|
||||
executor=typed_target_route.get("executor"),
|
||||
allowed_catalog_ids=typed_target_route.get("allowed_catalog_ids"),
|
||||
cross_domain_fallback_allowed=False,
|
||||
)
|
||||
if handoff is None and durable_agent99_route is not None:
|
||||
# Domain-specific Agent99 routes (including read-only BackupCheck) are
|
||||
# durable single-owner lanes too; they must never fall through to the
|
||||
@@ -2246,7 +2304,7 @@ async def _process_new_alert_background(
|
||||
source_severity=severity,
|
||||
)
|
||||
|
||||
agent99_durable_route = resolve_agent99_durable_route(
|
||||
typed_target_route, agent99_durable_route = _resolve_typed_agent99_durable_route(
|
||||
alertname=alertname,
|
||||
severity=severity,
|
||||
target_resource=target_resource,
|
||||
@@ -2254,7 +2312,18 @@ async def _process_new_alert_background(
|
||||
message=message,
|
||||
labels=traced_alert_labels,
|
||||
annotations=alert_context.get("annotations", {}),
|
||||
alert_category=alert_category,
|
||||
)
|
||||
if typed_target_route.get("executor") != "Agent99":
|
||||
logger.info(
|
||||
"typed_executor_selected_before_agent99_mutation",
|
||||
alert_id=alert_id,
|
||||
target_kind=typed_target_route.get("target_kind"),
|
||||
canonical_asset_id=typed_target_route.get("canonical_asset_id"),
|
||||
executor=typed_target_route.get("executor"),
|
||||
allowed_catalog_ids=typed_target_route.get("allowed_catalog_ids"),
|
||||
cross_domain_fallback_allowed=False,
|
||||
)
|
||||
rule_response = match_rule(alert_context)
|
||||
should_bypass_llm = bool(agent99_durable_route) or (
|
||||
_should_use_alertmanager_rule_first(rule_response, alert_category)
|
||||
@@ -2288,14 +2357,7 @@ async def _process_new_alert_background(
|
||||
rule_risk = (
|
||||
RiskLevel.MEDIUM
|
||||
if agent99_durable_route.get("suggested_mode")
|
||||
in {
|
||||
"Recover",
|
||||
"StartVMs",
|
||||
"HarborRepair",
|
||||
"AwoooRepair",
|
||||
"Perf",
|
||||
"LoadShed",
|
||||
}
|
||||
in _AGENT99_MUTATING_MODES
|
||||
else RiskLevel.LOW
|
||||
)
|
||||
blast = rule_response.get("blast_radius", {}) or {}
|
||||
|
||||
@@ -249,3 +249,80 @@ async def test_webhook_router_never_queues_generic_ansible_for_cold_start(
|
||||
"run_independent_cold_start_verifier_then_km_playbook_writeback"
|
||||
)
|
||||
assert receipt["context"]["runtime_closure_verified"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_webhook_router_queues_host188_callback_ansible_before_agent99(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
append = AsyncMock()
|
||||
incident = SimpleNamespace(
|
||||
incident_id="INC-H188-CALLBACK",
|
||||
project_id="awoooi",
|
||||
)
|
||||
queue = AsyncMock(
|
||||
return_value={
|
||||
"schema_version": "ai_decision_ansible_candidate_handoff_v1",
|
||||
"status": "ansible_candidate_queued",
|
||||
"queued": True,
|
||||
"side_effect_performed": False,
|
||||
"single_writer_executor": "awoooi-ansible-executor-broker",
|
||||
"automation_run_id": "run-h188-callback",
|
||||
"active_blockers": [],
|
||||
}
|
||||
)
|
||||
bridge = AsyncMock(
|
||||
side_effect=AssertionError(
|
||||
"Docker Compose mutation must use Host188 Ansible, not Agent99"
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate",
|
||||
queue,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
|
||||
lambda: SimpleNamespace(append=append),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
webhooks,
|
||||
"get_incident_service",
|
||||
lambda: SimpleNamespace(
|
||||
get_from_working_memory=AsyncMock(return_value=incident)
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(webhooks, "bridge_alertmanager_to_agent99", bridge)
|
||||
|
||||
handoff = await webhooks._try_auto_repair_background(
|
||||
incident_id="INC-H188-CALLBACK",
|
||||
approval_id="00000000-0000-0000-0000-000000000188",
|
||||
alert_type="custom",
|
||||
target_resource="openclaw",
|
||||
namespace="default",
|
||||
risk_level="medium",
|
||||
source_alert_id="alert-h188-callback",
|
||||
source_alertname="TelegramCallbackForwarderDrift",
|
||||
source_severity="warning",
|
||||
source_message="callback ingress receipt missing",
|
||||
source_labels={
|
||||
"alertname": "TelegramCallbackForwarderDrift",
|
||||
"component": "openclaw",
|
||||
"host": "188",
|
||||
"instance": "192.168.0.188",
|
||||
},
|
||||
source_fingerprint="callback-fingerprint",
|
||||
alert_category="ai_agent",
|
||||
notification_type="TYPE-3",
|
||||
)
|
||||
|
||||
bridge.assert_not_awaited()
|
||||
queue.assert_awaited_once()
|
||||
proposal = queue.await_args.kwargs["proposal_data"]
|
||||
assert proposal["source"] == "alert_webhook_controlled_router"
|
||||
assert proposal["source_occurrence_id"] == "alert-h188-callback"
|
||||
assert handoff["queued"] is True
|
||||
assert handoff["single_writer_executor"] == "awoooi-ansible-executor-broker"
|
||||
assert handoff["execution_priority"] == 20
|
||||
append.assert_awaited_once()
|
||||
assert append.await_args.kwargs["success"] is True
|
||||
|
||||
Reference in New Issue
Block a user