509 lines
18 KiB
Python
509 lines
18 KiB
Python
from __future__ import annotations
|
||
|
||
import hashlib
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock
|
||
|
||
import pytest
|
||
|
||
from src.services import ai_control as ai_control_module
|
||
from src.services import ai_rate_limiter as limiter_module
|
||
from src.services import chat_manager as chat_module
|
||
from src.services import ollama_failover_manager as failover_module
|
||
from src.services.ai_provider_policy import PRODUCTION_PROVIDER_ORDER
|
||
from src.services.ai_providers.interfaces import AIResult
|
||
from src.services.chat_manager import ChatManager
|
||
|
||
|
||
def _settings() -> SimpleNamespace:
|
||
return SimpleNamespace(
|
||
OPENCLAW_DEFAULT_MODEL="qwen3:14b",
|
||
OLLAMA_HEALTH_CHECK_MODEL="gemma3:4b",
|
||
CLAUDE_API_KEY="configured-test-value",
|
||
CLAUDE_EXECUTION_MODE="canary",
|
||
CLAUDE_CANARY_PERCENT=100,
|
||
CLAUDE_MODEL="claude-sonnet-5",
|
||
GEMINI_API_KEY="configured-test-value",
|
||
GEMINI_EXECUTION_MODE="canary",
|
||
GEMINI_CANARY_PERCENT=100,
|
||
GEMINI_MODEL="gemini-2.5-flash-lite",
|
||
)
|
||
|
||
|
||
class _Executor:
|
||
def __init__(self, result: AIResult) -> None:
|
||
self.result = result
|
||
self.calls: list[dict[str, Any]] = []
|
||
|
||
async def execute(
|
||
self,
|
||
prompt: str,
|
||
*,
|
||
provider_order: list[str],
|
||
context: dict[str, Any],
|
||
cache_ttl: int,
|
||
require_local: bool,
|
||
) -> AIResult:
|
||
self.calls.append(
|
||
{
|
||
"prompt": prompt,
|
||
"provider_order": list(provider_order),
|
||
"context": dict(context),
|
||
"cache_ttl": cache_ttl,
|
||
"require_local": require_local,
|
||
}
|
||
)
|
||
return self.result
|
||
|
||
|
||
class _Limiter:
|
||
async def get_usage_stats(self, provider: str) -> dict[str, Any]:
|
||
return {
|
||
"provider": provider,
|
||
"cost_exceeded": False,
|
||
"pricing_policy": {"supported": True},
|
||
}
|
||
|
||
async def get_provider_authentication_status(
|
||
self,
|
||
provider: str,
|
||
) -> dict[str, Any]:
|
||
authenticated = provider == "claude"
|
||
return {
|
||
"provider": provider,
|
||
"configured": True,
|
||
"authenticated": authenticated,
|
||
"authentication_verified": authenticated,
|
||
"verification_status": (
|
||
"authenticated_verified_by_generation"
|
||
if authenticated
|
||
else "authentication_rejected"
|
||
),
|
||
"verification_source": "durable_generation_receipt",
|
||
}
|
||
|
||
|
||
class _FailoverManager:
|
||
async def select_provider(
|
||
self,
|
||
*,
|
||
task_type: str,
|
||
emit_side_effects: bool,
|
||
) -> SimpleNamespace:
|
||
assert task_type == "chat_status"
|
||
assert emit_side_effects is False
|
||
return SimpleNamespace(
|
||
primary=SimpleNamespace(provider_name="ollama_gcp_a"),
|
||
observed_fallback=SimpleNamespace(provider_name="ollama_gcp_b"),
|
||
health_gcp_a=SimpleNamespace(status=SimpleNamespace(value="offline")),
|
||
health_gcp_b=SimpleNamespace(status=SimpleNamespace(value="healthy")),
|
||
health_local=SimpleNamespace(status=SimpleNamespace(value="offline")),
|
||
)
|
||
|
||
|
||
def _bounded_canary_context() -> dict[str, Any]:
|
||
return {
|
||
"trace_id": "trace-AIA-SRE-013-chat",
|
||
"run_id": "AIA-SRE-013-chat-canary-1",
|
||
"work_item_id": "AIA-SRE-013",
|
||
"synthetic_validation": True,
|
||
"data_classification": "sanitized",
|
||
"infrastructure_remediation_write_allowed": False,
|
||
"paid_canary_max_cost_usd": "0.25",
|
||
"paid_canary_max_output_tokens": 512,
|
||
}
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _configure(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
monkeypatch.setattr(chat_module, "get_settings", _settings)
|
||
monkeypatch.setenv("ENABLE_CLAUDE", "true")
|
||
monkeypatch.setenv("ENABLE_GEMINI", "true")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_sanitized_group_chat_uses_three_free_hops_and_never_executes_repair(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
executor = _Executor(
|
||
AIResult(
|
||
raw_response='{"message":"老闆，這是可驗證的本地判讀。"}',
|
||
success=True,
|
||
provider="ollama",
|
||
tokens=24,
|
||
)
|
||
)
|
||
monkeypatch.setattr(chat_module, "get_ai_executor", lambda: executor)
|
||
|
||
result = await ChatManager().route_chat(
|
||
role="openclaw",
|
||
system_prompt=("系統摘要 host=192.168.0.111 peer=10.8.0.2 node=172.16.1.9"),
|
||
user_message="請分析目前服務狀況",
|
||
)
|
||
|
||
assert result.success is True
|
||
assert result.provider == "ollama_gcp_a"
|
||
assert result.model == "gemma3:4b"
|
||
assert result.policy_order == PRODUCTION_PROVIDER_ORDER
|
||
assert result.admitted_provider_order == (
|
||
"ollama_gcp_a",
|
||
"ollama_gcp_b",
|
||
"ollama_local",
|
||
)
|
||
assert result.cost_receipt_id is None
|
||
assert result.fallback_reason == "canonical_primary_succeeded"
|
||
assert result.infrastructure_remediation_write_allowed is False
|
||
assert result.cross_domain_execution_allowed is False
|
||
|
||
call = executor.calls[0]
|
||
assert call["provider_order"] == [
|
||
"ollama_gcp_a",
|
||
"ollama_gcp_b",
|
||
"ollama_local",
|
||
]
|
||
assert call["require_local"] is False
|
||
assert call["context"]["executor_invocation_allowed"] is False
|
||
assert call["context"]["agent99_dispatch_allowed"] is False
|
||
assert call["context"]["cross_domain_execution_allowed"] is False
|
||
assert call["context"]["provider_timeout_seconds"] == 35
|
||
assert "synthetic_validation" not in call["context"]
|
||
assert "synthetic_prompt_sha256" not in call["context"]
|
||
assert call["context"]["cloud_transport_auto_preflight"] is True
|
||
receipt = call["context"]["sre_chat_sanitization_receipt"]
|
||
assert receipt["schema_version"] == "sre_chat_sanitization_receipt_v1"
|
||
assert receipt["status"] == "verified"
|
||
assert receipt["source_label"] == "awoooi_sre_group_chat"
|
||
assert (
|
||
receipt["prompt_sha256"]
|
||
== hashlib.sha256(call["prompt"].encode("utf-8")).hexdigest()
|
||
)
|
||
assert receipt["trace_id"] == call["context"]["trace_id"]
|
||
assert receipt["run_id"] == call["context"]["run_id"]
|
||
assert receipt["work_item_id"] == call["context"]["work_item_id"]
|
||
assert receipt["raw_input_forwarded"] is False
|
||
assert receipt["raw_log_payload_forwarded"] is False
|
||
assert receipt["secret_value_exposed"] is False
|
||
assert receipt["private_network_value_exposed"] is False
|
||
assert receipt["provider_timeout_seconds"] == 35
|
||
assert "192.168.0.111" not in call["prompt"]
|
||
assert "10.8.0.2" not in call["prompt"]
|
||
assert "172.16.1.9" not in call["prompt"]
|
||
assert call["prompt"].count("[PRIVATE_IP_REDACTED]") == 3
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_bounded_run_owned_canary_uses_exact_five_hop_order_and_receipt(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
executor = _Executor(
|
||
AIResult(
|
||
raw_response='{"reasoning":"Claude 已完成受控 critic 回應。"}',
|
||
success=True,
|
||
provider="claude",
|
||
tokens=45,
|
||
cost_usd=0.004,
|
||
audit_metadata={"generation_receipt_id": "gen-claude-chat-1"},
|
||
)
|
||
)
|
||
monkeypatch.setattr(chat_module, "get_ai_executor", lambda: executor)
|
||
manager = ChatManager()
|
||
monkeypatch.setattr(
|
||
manager,
|
||
"_paid_provider_candidates",
|
||
AsyncMock(
|
||
return_value=(
|
||
["claude", "gemini"],
|
||
{
|
||
"status": "admitted",
|
||
"run_owned_lease_verified": True,
|
||
"cost_policy_preflight_verified": True,
|
||
},
|
||
)
|
||
),
|
||
)
|
||
|
||
result = await manager.route_chat(
|
||
role="openclaw",
|
||
system_prompt="不含敏感資料的固定 canary",
|
||
user_message="評估固定的合成 SRE 狀態",
|
||
route_context=_bounded_canary_context(),
|
||
)
|
||
|
||
assert result.success is True
|
||
assert result.provider == "claude"
|
||
assert result.model == "claude-sonnet-5"
|
||
assert result.policy_order == PRODUCTION_PROVIDER_ORDER
|
||
assert result.admitted_provider_order == PRODUCTION_PROVIDER_ORDER
|
||
assert result.fallback_reason == "canonical_position_4_after_prior_hops_no_success"
|
||
assert result.cost_receipt_id == "gen-claude-chat-1"
|
||
assert result.cost_usd == 0.004
|
||
|
||
call = executor.calls[0]
|
||
assert call["provider_order"] == list(PRODUCTION_PROVIDER_ORDER)
|
||
assert call["require_local"] is False
|
||
assert call["context"]["synthetic_validation"] is True
|
||
assert (
|
||
call["context"]["synthetic_prompt_sha256"]
|
||
== hashlib.sha256(call["prompt"].encode("utf-8")).hexdigest()
|
||
)
|
||
assert call["context"]["paid_canary_max_cost_usd"] == "0.25"
|
||
assert "provider_timeout_seconds" not in call["context"]
|
||
assert call["context"]["infrastructure_remediation_write_allowed"] is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_paid_candidate_requires_canary_lease_bucket_and_cost_policy(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
monkeypatch.setattr(
|
||
ai_control_module,
|
||
"get_paid_provider_canary_control_readback",
|
||
AsyncMock(return_value={"leased_enabled": True}),
|
||
)
|
||
monkeypatch.setattr(
|
||
ai_control_module,
|
||
"is_provider_disabled",
|
||
AsyncMock(return_value=False),
|
||
)
|
||
monkeypatch.setattr(limiter_module, "get_ai_rate_limiter", lambda: _Limiter())
|
||
|
||
candidates, readback = await ChatManager()._paid_provider_candidates(
|
||
_bounded_canary_context()
|
||
)
|
||
|
||
assert candidates == ["claude", "gemini"]
|
||
assert readback["status"] == "admitted"
|
||
assert readback["run_owned_lease_verified"] is True
|
||
assert readback["cost_policy_preflight_verified"] is True
|
||
assert readback["provider_states"] == {
|
||
"claude": "admitted_atomic_reservation_pending",
|
||
"gemini": "admitted_atomic_reservation_pending",
|
||
}
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_paid_candidate_fails_closed_without_bounded_run_context() -> None:
|
||
candidates, readback = await ChatManager()._paid_provider_candidates(None)
|
||
|
||
assert candidates == []
|
||
assert readback["status"] == "bounded_canary_context_missing"
|
||
assert readback["run_owned_lease_verified"] is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_paid_success_without_valid_cost_receipt_is_not_rendered(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
executor = _Executor(
|
||
AIResult(
|
||
raw_response="must not be rendered",
|
||
success=True,
|
||
provider="gemini",
|
||
tokens=10,
|
||
cost_usd=0.003,
|
||
audit_metadata={},
|
||
)
|
||
)
|
||
monkeypatch.setattr(chat_module, "get_ai_executor", lambda: executor)
|
||
manager = ChatManager()
|
||
monkeypatch.setattr(
|
||
manager,
|
||
"_paid_provider_candidates",
|
||
AsyncMock(return_value=(["claude", "gemini"], {"status": "admitted"})),
|
||
)
|
||
|
||
result = await manager.route_chat(
|
||
role="openclaw",
|
||
system_prompt="固定 canary",
|
||
user_message="合成狀態",
|
||
route_context=_bounded_canary_context(),
|
||
)
|
||
|
||
assert result.success is False
|
||
assert result.provider == "gemini"
|
||
assert result.fallback_reason == "paid_generation_cost_receipt_invalid"
|
||
assert result.cost_receipt_id is None
|
||
assert "must not be rendered" not in result.text
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_provider_status_question_returns_structured_runtime_readback(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
readback = {
|
||
"schema_version": "sre_chat_provider_runtime_readback_v1",
|
||
"policy_order": list(PRODUCTION_PROVIDER_ORDER),
|
||
"ollama_health": {
|
||
"ollama_gcp_a": "offline",
|
||
"ollama_gcp_b": "healthy",
|
||
"ollama_local": "not_checked",
|
||
},
|
||
"paid": {
|
||
"claude": {
|
||
"configured": True,
|
||
"authenticated": True,
|
||
"authentication_verified": True,
|
||
"verification_status": "authenticated_verified_by_generation",
|
||
"execution_mode": "canary",
|
||
"canary_percent": 5,
|
||
"route_lease_state": "inactive_disabled_after_canary",
|
||
"runtime_state": (
|
||
"configured_authenticated_canary_waiting_for_run_owned_lease"
|
||
),
|
||
},
|
||
"gemini": {
|
||
"configured": True,
|
||
"authenticated": False,
|
||
"authentication_verified": False,
|
||
"verification_status": "authentication_rejected",
|
||
"execution_mode": "canary",
|
||
"canary_percent": 5,
|
||
"route_lease_state": "inactive_disabled_after_canary",
|
||
"runtime_state": "configured_canary_authentication_rejected",
|
||
},
|
||
},
|
||
"paid_control": {
|
||
"readback_status": "verified",
|
||
"route_lease_state": "inactive_disabled_after_canary",
|
||
},
|
||
"provider_route_switch_performed": False,
|
||
"infrastructure_remediation_write_allowed": False,
|
||
"cross_domain_execution_allowed": False,
|
||
}
|
||
manager = ChatManager()
|
||
monkeypatch.setattr(
|
||
manager,
|
||
"get_provider_runtime_readback",
|
||
AsyncMock(return_value=readback),
|
||
)
|
||
|
||
result = await manager.route_chat(
|
||
role="nemoclaw",
|
||
system_prompt="system",
|
||
user_message="Gemini API 現在是硬鎖定狀態嗎？",
|
||
)
|
||
|
||
assert result.provider == "runtime_readback"
|
||
assert result.model == "deterministic_policy"
|
||
assert result.fallback_reason == "provider_status_request_no_generation"
|
||
assert result.runtime_readback == readback
|
||
assert "Claude：configured=true｜authenticated=true" in result.text
|
||
assert "verification=authenticated_verified_by_generation" in result.text
|
||
assert "Gemini：configured=true｜authenticated=false" in result.text
|
||
assert "verification=authentication_rejected" in result.text
|
||
assert "canary=canary 5%" in result.text
|
||
assert "route_lease=inactive_disabled_after_canary" in result.text
|
||
assert "硬鎖" not in result.text
|
||
assert "locked" not in result.text.casefold()
|
||
assert "無法正常回應" not in result.text
|
||
assert result.to_dict()["cross_domain_execution_allowed"] is False
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_provider_runtime_readback_separates_auth_canary_and_route_lease(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
monkeypatch.setattr(
|
||
failover_module,
|
||
"get_ollama_failover_manager",
|
||
lambda: _FailoverManager(),
|
||
)
|
||
monkeypatch.setattr(limiter_module, "get_ai_rate_limiter", lambda: _Limiter())
|
||
monkeypatch.setattr(
|
||
ai_control_module,
|
||
"get_paid_provider_canary_control_readback",
|
||
AsyncMock(
|
||
return_value={
|
||
"claude_disabled": True,
|
||
"gemini_disabled": True,
|
||
"lock_present": False,
|
||
"lock_ttl_seconds": -2,
|
||
"disabled_verified": True,
|
||
"persistent_enabled": False,
|
||
"legacy_persistent_state_detected": False,
|
||
}
|
||
),
|
||
)
|
||
manager = ChatManager()
|
||
monkeypatch.setattr(
|
||
manager,
|
||
"_paid_provider_candidates",
|
||
AsyncMock(
|
||
return_value=(
|
||
[],
|
||
{
|
||
"status": "bounded_canary_context_missing",
|
||
"run_owned_lease_verified": False,
|
||
"provider_states": {},
|
||
},
|
||
)
|
||
),
|
||
)
|
||
|
||
readback = await manager.get_provider_runtime_readback()
|
||
|
||
assert readback["paid_control"] == {
|
||
"readback_status": "verified",
|
||
"route_lease_state": "inactive_disabled_after_canary",
|
||
"lock_present": False,
|
||
"lock_ttl_seconds": -2,
|
||
"claude_disabled": True,
|
||
"gemini_disabled": True,
|
||
"disabled_verified": True,
|
||
"persistent_enabled": False,
|
||
"legacy_persistent_state_detected": False,
|
||
}
|
||
claude = readback["paid"]["claude"]
|
||
assert claude["configured"] is True
|
||
assert claude["authenticated"] is True
|
||
assert claude["authentication_verified"] is True
|
||
assert claude["verification_status"] == ("authenticated_verified_by_generation")
|
||
assert claude["execution_mode"] == "canary"
|
||
assert claude["canary_percent"] == 100
|
||
assert claude["route_lease_state"] == "inactive_disabled_after_canary"
|
||
assert claude["runtime_state"] == (
|
||
"configured_authenticated_canary_waiting_for_run_owned_lease"
|
||
)
|
||
gemini = readback["paid"]["gemini"]
|
||
assert gemini["configured"] is True
|
||
assert gemini["authenticated"] is False
|
||
assert gemini["authentication_verified"] is False
|
||
assert gemini["verification_status"] == "authentication_rejected"
|
||
assert gemini["runtime_state"] == "configured_canary_authentication_rejected"
|
||
assert "configured-test-value" not in repr(readback)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_openclaw_compatibility_render_includes_route_evidence(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
executor = _Executor(
|
||
AIResult(
|
||
raw_response="老闆，服務狀態已完成判讀。",
|
||
success=True,
|
||
provider="ollama_local",
|
||
tokens=9,
|
||
)
|
||
)
|
||
monkeypatch.setattr(chat_module, "get_ai_executor", lambda: executor)
|
||
|
||
rendered = await ChatManager()._call_openclaw("system", "請看服務")
|
||
|
||
assert rendered is not None
|
||
assert "provider=ollama_local" in rendered
|
||
assert "model=qwen3:14b" in rendered
|
||
assert "cost_receipt=none" in rendered
|
||
|
||
|
||
def test_chat_manager_reuses_router_and_has_no_direct_provider_http_path() -> None:
|
||
source_path = Path(chat_module.__file__).resolve()
|
||
source = source_path.read_text(encoding="utf-8")
|
||
|
||
assert "get_ai_executor" in source
|
||
assert "httpx.AsyncClient" not in source
|
||
assert "generativelanguage.googleapis.com" not in source
|
||
assert "api.anthropic.com" not in source
|
||
assert "/api/chat" not in source
|