541a32a9cb
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
332 lines
11 KiB
Python
332 lines
11 KiB
Python
"""Production AI route contract tests.
|
|
|
|
These tests intentionally exercise the router without any network, Redis, or
|
|
paid-provider call. Provider health observations must never reorder the
|
|
executable chain or authorize a direct Gemini jump.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from src.services import ai_router as ai_router_module
|
|
from src.services.ai_providers.interfaces import AIResult
|
|
from src.services.ai_router import (
|
|
AIProviderEnum,
|
|
AIProviderRegistry,
|
|
AIRouter,
|
|
AIRouterExecutor,
|
|
reset_ai_router,
|
|
)
|
|
from src.services.complexity_scorer import ComplexityScore
|
|
from src.services.intent_classifier import IntentResult, IntentType
|
|
|
|
PRODUCTION_ENUM_ORDER = [
|
|
AIProviderEnum.OLLAMA_GCP_A,
|
|
AIProviderEnum.OLLAMA_GCP_B,
|
|
AIProviderEnum.OLLAMA_LOCAL,
|
|
AIProviderEnum.CLAUDE,
|
|
AIProviderEnum.GEMINI,
|
|
]
|
|
PRODUCTION_NAME_ORDER = [provider.value for provider in PRODUCTION_ENUM_ORDER]
|
|
|
|
|
|
def _synthetic_cloud_context(prompt: str, **overrides: Any) -> dict[str, Any]:
|
|
return {
|
|
"intent_hint": "diagnose",
|
|
"synthetic_validation": True,
|
|
"data_classification": "sanitized",
|
|
"synthetic_prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(),
|
|
"infrastructure_remediation_write_allowed": False,
|
|
"run_id": "AIA-SRE-013-test",
|
|
**overrides,
|
|
}
|
|
|
|
|
|
class _NoCacheRedis:
|
|
def __init__(self, *, gemini_disabled: bool = False) -> None:
|
|
self.gemini_disabled = gemini_disabled
|
|
self.set_calls: list[tuple[str, str, int | None]] = []
|
|
|
|
async def get(self, key: str) -> bytes | None:
|
|
if key == "ai:control:disabled:gemini":
|
|
return b"1" if self.gemini_disabled else b"0"
|
|
if key == "ai:control:disabled:claude":
|
|
return b"0"
|
|
return None
|
|
|
|
async def set(self, key: str, value: str, ex: int | None = None) -> None:
|
|
self.set_calls.append((key, value, ex))
|
|
|
|
async def eval(self, *_args: object) -> list[object]:
|
|
if self.gemini_disabled:
|
|
return [0, "paired_paid_state_not_enabled", ""]
|
|
run_id = str(_args[-1] or "")
|
|
if not run_id:
|
|
return [0, "run_owned_canary_not_admitted", ""]
|
|
return [
|
|
1,
|
|
"run_owned_canary_admitted",
|
|
f"{run_id}.attempt-fixture",
|
|
]
|
|
|
|
|
|
class _Provider:
|
|
is_enabled = True
|
|
capabilities = {"rca", "chat", "tool_calling", "code_review"}
|
|
|
|
def __init__(self, name: str, *, success: bool) -> None:
|
|
self.name = name
|
|
self.success = success
|
|
self.calls = 0
|
|
self.privacy_level = "cloud" if name in {"claude", "gemini"} else "local"
|
|
|
|
async def analyze(
|
|
self,
|
|
prompt: str,
|
|
context: dict[str, Any] | None = None,
|
|
) -> AIResult:
|
|
self.calls += 1
|
|
return AIResult(
|
|
raw_response=f"{self.name}-response" if self.success else "",
|
|
success=self.success,
|
|
provider=self.name,
|
|
error=None if self.success else "forced_failure",
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_router_singletons(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("ENABLE_GEMINI", "true")
|
|
monkeypatch.setenv("ENABLE_CLAUDE", "true")
|
|
yield
|
|
reset_ai_router()
|
|
|
|
|
|
def _router_with_failover_primary(primary: str) -> AIRouter:
|
|
router = AIRouter()
|
|
router._failover_manager = MagicMock()
|
|
router._failover_manager.select_provider = AsyncMock(
|
|
return_value=SimpleNamespace(
|
|
primary=SimpleNamespace(provider_name=primary),
|
|
fallback_chain=[],
|
|
)
|
|
)
|
|
return router
|
|
|
|
|
|
def _registry(*providers: _Provider) -> AIProviderRegistry:
|
|
registry = AIProviderRegistry()
|
|
for provider in providers:
|
|
registry.register(provider)
|
|
return registry
|
|
|
|
|
|
def test_claude_is_executable_but_legacy_providers_remain_shadow_metadata() -> None:
|
|
registry = _registry(
|
|
_Provider("claude", success=True),
|
|
_Provider("nemotron", success=True),
|
|
_Provider("openclaw_nemo", success=True),
|
|
)
|
|
|
|
assert set(registry.names()) == {"claude", "nemotron", "openclaw_nemo"}
|
|
assert registry.all_enabled() == [registry.get("claude")]
|
|
assert registry.get("claude") is not None
|
|
assert registry.get("nemotron") is None
|
|
assert registry.get("openclaw_nemo") is None
|
|
|
|
|
|
def test_every_risk_and_complexity_starts_on_logical_ollama() -> None:
|
|
router = _router_with_failover_primary("gemini")
|
|
|
|
for intent in IntentType:
|
|
for score in range(1, 6):
|
|
provider, _model, _reason = router._select_provider_and_model(
|
|
intent,
|
|
IntentResult(
|
|
intent=intent,
|
|
confidence=1.0,
|
|
method="test",
|
|
),
|
|
ComplexityScore(score=score),
|
|
)
|
|
assert provider == AIProviderEnum.OLLAMA
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_health_selected_gemini_cannot_become_router_primary() -> None:
|
|
router = _router_with_failover_primary("gemini")
|
|
|
|
decision = await router.route(
|
|
"diagnose cascading failure",
|
|
context=_synthetic_cloud_context("diagnose alert"),
|
|
)
|
|
|
|
assert decision.selected_provider == AIProviderEnum.OLLAMA_GCP_A
|
|
assert [provider for provider, _model in decision.fallback_chain] == (
|
|
PRODUCTION_ENUM_ORDER[1:]
|
|
)
|
|
assert decision.to_dict()["production_route_contract"] == PRODUCTION_NAME_ORDER
|
|
|
|
|
|
def test_sync_and_tool_calling_publish_the_same_exact_route() -> None:
|
|
router = _router_with_failover_primary("gemini")
|
|
|
|
sync_decision = router.route_sync(
|
|
"review code",
|
|
context={"intent_hint": "code_review"},
|
|
)
|
|
tool_provider, _tool_model, tool_fallback = router.route_tool_calling()
|
|
|
|
assert sync_decision.selected_provider == AIProviderEnum.OLLAMA_GCP_A
|
|
assert [provider for provider, _model in sync_decision.fallback_chain] == (
|
|
PRODUCTION_ENUM_ORDER[1:]
|
|
)
|
|
assert tool_provider == AIProviderEnum.OLLAMA_GCP_A
|
|
assert [provider for provider, _model in tool_fallback] == (
|
|
PRODUCTION_ENUM_ORDER[1:]
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_executor_rejects_direct_cloud_or_legacy_only_route(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
legacy = _Provider("openclaw_nemo", success=True)
|
|
gemini = _Provider("gemini", success=True)
|
|
executor = AIRouterExecutor(_registry(legacy, gemini))
|
|
monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False)
|
|
|
|
result = await executor.execute(
|
|
prompt="bypass attempt",
|
|
provider_order=["openclaw_nemo", "gemini", "claude"],
|
|
context=_synthetic_cloud_context("diagnose alert"),
|
|
)
|
|
|
|
assert result.success is False
|
|
assert result.error == "production_route_missing_ollama_lane"
|
|
assert legacy.calls == 0
|
|
assert gemini.calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_executor_runs_exact_five_hops_in_order(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
providers = [
|
|
_Provider("ollama_gcp_a", success=False),
|
|
_Provider("ollama_gcp_b", success=False),
|
|
_Provider("ollama_local", success=False),
|
|
_Provider("claude", success=False),
|
|
_Provider("gemini", success=True),
|
|
]
|
|
redis = _NoCacheRedis()
|
|
monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False)
|
|
monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis)
|
|
|
|
result = await AIRouterExecutor(_registry(*providers)).execute(
|
|
prompt="diagnose alert",
|
|
provider_order=["ollama", "openclaw_nemo", "gemini", "claude"],
|
|
context=_synthetic_cloud_context("diagnose alert"),
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.provider == "gemini"
|
|
assert [provider.calls for provider in providers] == [1, 1, 1, 1, 1]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_disabled_gemini_is_never_readded_or_called(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
providers = [
|
|
_Provider("ollama_gcp_a", success=False),
|
|
_Provider("ollama_gcp_b", success=False),
|
|
_Provider("ollama_local", success=False),
|
|
_Provider("claude", success=False),
|
|
_Provider("gemini", success=True),
|
|
]
|
|
redis = _NoCacheRedis(gemini_disabled=True)
|
|
monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False)
|
|
monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis)
|
|
|
|
result = await AIRouterExecutor(_registry(*providers)).execute(
|
|
prompt="diagnose alert",
|
|
provider_order=PRODUCTION_NAME_ORDER,
|
|
context=_synthetic_cloud_context("diagnose alert"),
|
|
)
|
|
|
|
assert result.success is False
|
|
assert "claude: disabled_by_runtime_control" in str(result.error)
|
|
assert "gemini: disabled_by_runtime_control" in str(result.error)
|
|
assert [provider.calls for provider in providers] == [1, 1, 1, 0, 0]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_code_review_local_gate_removes_paid_fallback(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
providers = [
|
|
_Provider("ollama_gcp_a", success=False),
|
|
_Provider("ollama_gcp_b", success=False),
|
|
_Provider("ollama_local", success=True),
|
|
_Provider("claude", success=True),
|
|
_Provider("gemini", success=True),
|
|
]
|
|
redis = _NoCacheRedis()
|
|
monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False)
|
|
monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis)
|
|
|
|
result = await AIRouterExecutor(_registry(*providers)).execute(
|
|
prompt="review private code",
|
|
provider_order=PRODUCTION_NAME_ORDER,
|
|
context={"intent_hint": "code_review"},
|
|
require_local=True,
|
|
)
|
|
|
|
assert result.success is True
|
|
assert result.provider == "ollama_local"
|
|
assert providers[-1].calls == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"privacy_context",
|
|
[
|
|
{},
|
|
{"data_classification": "private"},
|
|
{"data_classification": "sanitized", "contains_secret": True},
|
|
{"data_classification": "sanitized", "secret_value_exposed": True},
|
|
{"data_classification": "sanitized", "raw_log": "unredacted"},
|
|
],
|
|
)
|
|
async def test_router_blocks_paid_cloud_before_cache_for_unclassified_or_sensitive_context(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
privacy_context: dict[str, object],
|
|
) -> None:
|
|
providers = [
|
|
_Provider("ollama_gcp_a", success=False),
|
|
_Provider("ollama_gcp_b", success=False),
|
|
_Provider("ollama_local", success=False),
|
|
_Provider("claude", success=True),
|
|
_Provider("gemini", success=True),
|
|
]
|
|
redis = _NoCacheRedis()
|
|
monkeypatch.setattr(ai_router_module._settings, "MOCK_MODE", False)
|
|
monkeypatch.setattr("src.core.redis_client.get_redis", lambda: redis)
|
|
|
|
result = await AIRouterExecutor(_registry(*providers)).execute(
|
|
prompt="opaque context",
|
|
provider_order=PRODUCTION_NAME_ORDER,
|
|
context={"intent_hint": "diagnose", **privacy_context},
|
|
)
|
|
|
|
assert result.success is False
|
|
assert [provider.calls for provider in providers] == [0, 0, 1, 0, 0]
|
|
assert "cloud_sanitization_receipt_missing" in str(result.error)
|