Files
awoooi/apps/api/tests/test_paid_provider_canary_gate5_run.py
Your Name 18b5ba25fa
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 51s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m45s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 33s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
fix(platform): bind AI runs to canonical correlation
2026-07-22 16:23:21 +08:00

646 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import os
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
from uuid import RFC_4122, UUID
import pytest
from fastapi import HTTPException
from pydantic import ValidationError
os.environ.setdefault(
"DATABASE_URL",
"postgresql+asyncpg://test:test@localhost/test",
)
from src.api.v1.platform import operator_runs # noqa: E402
from src.api.v1.platform import runs as public_runs # noqa: E402
from src.core.awooop_operator_auth import AwoooPOperatorPrincipal # noqa: E402
from src.db.awooop_models import AwoooPRunState # noqa: E402
from src.services import platform_operator_service, platform_runtime # noqa: E402
from src.services.paid_provider_canary_authorization import ( # noqa: E402
PAID_CANARY_AGENT_ID,
PAID_CANARY_PROJECT_ID,
PAID_CANARY_TRIGGER_REF,
PAID_CANARY_TRIGGER_TYPE,
paid_provider_canary_authorization_input_sha256,
paid_provider_canary_run_selected,
)
from src.services.paid_provider_canary_gate5_run import ( # noqa: E402
PAID_CANARY_AUTHORIZATION_SCOPE,
PaidProviderCanaryGate5RunRecord,
create_paid_provider_canary_gate5_run,
paid_provider_canary_approval_contract_errors,
)
from src.services.runtime_correlation import ( # noqa: E402
canonical_traceparent,
is_canonical_traceparent,
)
_SELECTED_RUN_ID = UUID("267a3d26-3c29-470f-8fe3-388a2f408f39")
_UNSELECTED_RUN_ID = UUID("00000000-0000-4000-8000-000000000001")
class _Repository:
def __init__(
self,
*,
duplicate: bool = False,
state: str = "waiting_approval",
mismatched_duplicate_trace: bool = False,
unselected_duplicate_run: bool = False,
) -> None:
self.duplicate = duplicate
self.state = state
self.mismatched_duplicate_trace = mismatched_duplicate_trace
self.unselected_duplicate_run = unselected_duplicate_run
self.calls: list[dict[str, object]] = []
async def create_or_get(self, **kwargs: object) -> tuple[
PaidProviderCanaryGate5RunRecord,
bool,
]:
self.calls.append(kwargs)
run_id = kwargs["run_id"]
timeout_at = kwargs["timeout_at"]
input_sha256 = kwargs["input_sha256"]
trace_id = kwargs["trace_id"]
assert isinstance(run_id, UUID)
assert isinstance(timeout_at, datetime)
assert isinstance(input_sha256, str)
assert isinstance(trace_id, str)
assert is_canonical_traceparent(trace_id, run_id=run_id) is True
if self.duplicate:
run_id = (
_UNSELECTED_RUN_ID
if self.unselected_duplicate_run
else _SELECTED_RUN_ID
)
if not self.mismatched_duplicate_trace:
trace_id = canonical_traceparent(run_id)
return (
PaidProviderCanaryGate5RunRecord(
run_id=run_id,
project_id=PAID_CANARY_PROJECT_ID,
agent_id=PAID_CANARY_AGENT_ID,
state=self.state,
trigger_type=PAID_CANARY_TRIGGER_TYPE,
trigger_ref=PAID_CANARY_TRIGGER_REF,
is_shadow=False,
input_sha256=input_sha256,
timeout_at=timeout_at,
trace_id=trace_id,
),
self.duplicate,
)
@pytest.mark.asyncio
async def test_operator_service_creates_exact_bounded_non_shadow_gate5_run() -> None:
now = datetime(2026, 7, 16, 9, 0, tzinfo=UTC)
repository = _Repository()
audit_calls: list[dict[str, object]] = []
async def audit_writer(**kwargs: object) -> None:
audit_calls.append(kwargs)
response = await create_paid_provider_canary_gate5_run(
operator_id="telegram:123456",
repository=repository,
audit_writer=audit_writer,
now=now,
)
assert response["project_id"] == "awoooi"
assert response["work_item_id"] == "AIA-SRE-013"
assert response["agent_id"] == "awoooi-paid-provider-canary"
assert response["state"] == "waiting_approval"
assert response["is_shadow"] is False
assert response["trigger_type"] == "api"
assert response["trigger_ref"] == "paid-provider-canary:AIA-SRE-013"
assert response["input_sha256"] == paid_provider_canary_authorization_input_sha256()
assert response["authorization_scope"] == PAID_CANARY_AUTHORIZATION_SCOPE
assert response["provider_order"] == [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local",
"claude",
"gemini",
]
assert response["paid_provider_canary_percent"] == 5
assert response["canary_bucket"] < 5
assert response["authorization_run_is_execution_run"] is True
assert response["max_output_tokens_per_paid_provider"] == 512
assert response["max_total_paid_cost_usd"] == "0.25"
assert response["timeout_at"] == now + timedelta(minutes=15)
assert response["approval_ttl_seconds"] == 900
assert response["provider_call_performed"] is False
assert response["route_change_performed"] is False
assert response["infrastructure_write_performed"] is False
assert response["raw_prompt_persistence_allowed"] is False
assert response["raw_response_persistence_allowed"] is False
assert response["next_action"] == "operator_gate5_decision_required"
assert UUID(response["authorization_ref"]) == UUID(response["run_id"])
assert "telegram:123456" not in str(response)
assert len(repository.calls) == 1
call = repository.calls[0]
assert call["now"] == now
assert call["timeout_at"] == now + timedelta(minutes=15)
assert call["input_sha256"] == paid_provider_canary_authorization_input_sha256()
assert len(audit_calls) == 1
assert audit_calls[0]["resource_id"] == response["run_id"]
assert audit_calls[0]["run_id"] == response["run_id"]
details = audit_calls[0]["details"]
assert isinstance(details, dict)
assert details["provider_call_performed"] is False
assert details["route_change_performed"] is False
@pytest.mark.asyncio
async def test_duplicate_active_authorization_is_reused_without_scope_expansion() -> None:
repository = _Repository(duplicate=True, state="running")
async def audit_writer(**_: object) -> None:
return None
response = await create_paid_provider_canary_gate5_run(
operator_id="ops@example.com",
repository=repository,
audit_writer=audit_writer,
now=datetime(2026, 7, 16, 9, 0, tzinfo=UTC),
)
assert response["is_duplicate"] is True
assert response["state"] == "running"
assert response["run_id"] == str(_SELECTED_RUN_ID)
assert response["max_total_paid_cost_usd"] == "0.25"
assert response["infrastructure_write_performed"] is False
assert response["next_action"] == (
"verify_authorization_then_run_bounded_canary"
)
@pytest.mark.asyncio
async def test_duplicate_active_authorization_rejects_unbound_trace() -> None:
repository = _Repository(
duplicate=True,
state="running",
mismatched_duplicate_trace=True,
)
with pytest.raises(
RuntimeError,
match="paid_provider_canary_gate5_run_trace_unbound",
):
await create_paid_provider_canary_gate5_run(
operator_id="ops@example.com",
repository=repository,
now=datetime(2026, 7, 16, 9, 0, tzinfo=UTC),
)
@pytest.mark.asyncio
async def test_injected_repository_rejects_unselected_execution_run() -> None:
assert paid_provider_canary_run_selected(_UNSELECTED_RUN_ID) is False
repository = _Repository(
duplicate=True,
state="running",
unselected_duplicate_run=True,
)
with pytest.raises(
RuntimeError,
match="paid_provider_canary_gate5_run_not_selected",
):
await create_paid_provider_canary_gate5_run(
operator_id="ops@example.com",
repository=repository,
now=datetime(2026, 7, 16, 9, 0, tzinfo=UTC),
)
def _run(
*,
now: datetime,
agent_id: str = PAID_CANARY_AGENT_ID,
trigger_ref: str = PAID_CANARY_TRIGGER_REF,
) -> AwoooPRunState:
return AwoooPRunState(
run_id=_SELECTED_RUN_ID,
project_id=PAID_CANARY_PROJECT_ID,
agent_id=agent_id,
state="waiting_approval",
trace_id=canonical_traceparent(_SELECTED_RUN_ID),
trigger_type=PAID_CANARY_TRIGGER_TYPE,
trigger_ref=trigger_ref,
is_shadow=False,
input_sha256=paid_provider_canary_authorization_input_sha256(),
timeout_at=now + timedelta(minutes=15),
)
def test_paid_canary_approval_contract_is_exact_and_expiry_bound() -> None:
now = datetime(2026, 7, 16, 9, 0, tzinfo=UTC)
exact = _run(now=now)
expired = _run(now=now)
expired.timeout_at = now - timedelta(seconds=1)
shadow = _run(now=now)
shadow.is_shadow = True
drifted_hash = _run(now=now)
drifted_hash.input_sha256 = "0" * 64
assert paid_provider_canary_approval_contract_errors(exact, now=now) == []
assert paid_provider_canary_approval_contract_errors(expired, now=now) == [
"timeout_not_expired"
]
assert paid_provider_canary_approval_contract_errors(shadow, now=now) == [
"non_shadow"
]
assert paid_provider_canary_approval_contract_errors(
drifted_hash,
now=now,
) == ["input_sha256_exact"]
def test_unrelated_generic_run_does_not_enter_paid_canary_contract() -> None:
now = datetime(2026, 7, 16, 9, 0, tzinfo=UTC)
generic = _run(
now=now,
agent_id="generic-agent",
trigger_ref="generic:api:request",
)
assert paid_provider_canary_approval_contract_errors(generic, now=now) is None
@pytest.mark.asyncio
async def test_generic_approval_service_rejects_expired_paid_canary_before_transition(
monkeypatch: pytest.MonkeyPatch,
) -> None:
now = datetime.now(UTC)
expired = _run(now=now)
expired.timeout_at = now - timedelta(seconds=1)
class _Result:
def scalar_one_or_none(self) -> AwoooPRunState:
return expired
class _Db:
async def execute(self, *_: object, **__: object) -> _Result:
return _Result()
@asynccontextmanager
async def fake_db_context(project_id: str):
assert project_id == "awoooi"
yield _Db()
monkeypatch.setattr(
platform_operator_service,
"get_db_context",
fake_db_context,
)
with pytest.raises(HTTPException) as exc:
await platform_operator_service.decide_approval(
run_id=str(expired.run_id),
project_id="awoooi",
decision="approve",
approver_id="telegram:123456",
reason="bounded canary",
)
assert exc.value.status_code == 409
assert "timeout_not_expired" in str(exc.value.detail)
@pytest.mark.asyncio
async def test_paid_canary_audit_failure_terminalizes_active_gate(
monkeypatch: pytest.MonkeyPatch,
) -> None:
run = _run(now=datetime.now(UTC))
class _Result:
def scalar_one_or_none(self) -> AwoooPRunState:
return run
class _Db:
async def execute(self, *_: object, **__: object) -> _Result:
return _Result()
@asynccontextmanager
async def fake_db_context(project_id: str):
assert project_id == "awoooi"
yield _Db()
transition_calls: list[dict[str, object]] = []
audit_calls: list[dict[str, object]] = []
async def fake_record_approval(**_: object) -> int:
return 1
async def fake_transition(
run_id: UUID,
project_id: str,
to_state: str,
**kwargs: object,
) -> None:
transition_calls.append(
{
"run_id": run_id,
"project_id": project_id,
"to_state": to_state,
**kwargs,
}
)
async def fake_record_step(**_: object) -> None:
return None
async def failing_write_audit(**kwargs: object) -> None:
audit_calls.append(kwargs)
raise platform_operator_service.AuditDurabilityError(
"audit_durable_write_failed"
)
monkeypatch.setattr(platform_operator_service, "get_db_context", fake_db_context)
monkeypatch.setattr(
platform_operator_service,
"issue_approval_token",
lambda **_: "e30.eyJqdGkiOiJqdGktdGVzdCJ9.sig",
)
monkeypatch.setattr(
platform_operator_service,
"record_approval",
fake_record_approval,
)
monkeypatch.setattr(platform_operator_service, "transition", fake_transition)
monkeypatch.setattr(
platform_operator_service,
"_record_approval_decision_step",
fake_record_step,
)
monkeypatch.setattr(
platform_operator_service,
"write_audit",
failing_write_audit,
)
with pytest.raises(HTTPException) as raised:
await platform_operator_service.decide_approval(
run_id=str(run.run_id),
project_id="awoooi",
decision="approve",
approver_id="operator:test",
reason="bounded canary",
)
assert raised.value.status_code == 503
assert raised.value.detail == "critical_approval_durable_audit_unavailable"
assert [call["to_state"] for call in transition_calls] == ["running", "failed"]
assert transition_calls[1]["error_code"] == "E-PAID-CANARY-AUDIT"
assert len(audit_calls) == 1
assert audit_calls[0]["require_durable"] is True
assert audit_calls[0]["run_id"] == str(run.run_id)
assert audit_calls[0]["trace_id"] == run.trace_id
assert "paid-provider-canary:approval:v1" in str(
audit_calls[0]["idempotency_key"]
)
@pytest.mark.asyncio
async def test_operator_route_uses_authenticated_principal_and_fixed_body(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: dict[str, object] = {}
async def fake_create(**kwargs: object) -> dict[str, object]:
captured.update(kwargs)
return {
"schema_version": "paid_provider_canary_gate5_run_v1",
"authorization_ref": "018f2d04-4c37-7a18-b764-df0df0cbe111",
"run_id": "018f2d04-4c37-7a18-b764-df0df0cbe111",
"project_id": "awoooi",
"work_item_id": "AIA-SRE-013",
"agent_id": "awoooi-paid-provider-canary",
"state": "waiting_approval",
"is_shadow": False,
"is_duplicate": False,
"trigger_type": "api",
"trigger_ref": "paid-provider-canary:AIA-SRE-013",
"input_sha256": "1" * 64,
"authorization_scope": PAID_CANARY_AUTHORIZATION_SCOPE,
"provider_order": [
"ollama_gcp_a",
"ollama_gcp_b",
"ollama_local",
"claude",
"gemini",
],
"paid_provider_canary_percent": 5,
"max_output_tokens_per_paid_provider": 512,
"max_total_paid_cost_usd": "0.25",
"timeout_at": datetime(2026, 7, 16, 9, 15, tzinfo=UTC),
"approval_ttl_seconds": 900,
"provider_call_performed": False,
"route_change_performed": False,
"infrastructure_write_performed": False,
"raw_prompt_persistence_allowed": False,
"raw_response_persistence_allowed": False,
"next_action": "operator_gate5_decision_required",
"approval_path": (
"/api/v1/platform/approvals/"
"018f2d04-4c37-7a18-b764-df0df0cbe111/decide"
),
}
monkeypatch.setattr(
operator_runs,
"create_paid_provider_canary_gate5_run_svc",
fake_create,
)
body = operator_runs.CreatePaidProviderCanaryAuthorizationRequest(
work_item_id="AIA-SRE-013",
authorization_scope=PAID_CANARY_AUTHORIZATION_SCOPE,
)
principal = AwoooPOperatorPrincipal(
operator_id="telegram:123456",
auth_method="operator_api_key",
)
response = await operator_runs.create_paid_provider_canary_authorization(
body,
principal,
)
validated = (
operator_runs.CreatePaidProviderCanaryAuthorizationResponse.model_validate(
response
)
)
assert response["state"] == "waiting_approval"
assert validated.provider_order[-2:] == ("claude", "gemini")
assert captured == {"operator_id": "telegram:123456"}
with pytest.raises(ValidationError):
operator_runs.CreatePaidProviderCanaryAuthorizationRequest.model_validate(
{
"work_item_id": "OTHER",
"authorization_scope": PAID_CANARY_AUTHORIZATION_SCOPE,
}
)
def test_paid_canary_route_declares_operator_auth_dependency() -> None:
route = next(
route
for route in operator_runs.router.routes
if getattr(route, "path", "")
== "/approvals/paid-provider-canary/authorize"
)
assert any(
dependency.call is operator_runs.verify_awooop_operator
for dependency in route.dependant.dependencies
)
@pytest.mark.asyncio
async def test_generic_runtime_service_persists_shadow_true(
monkeypatch: pytest.MonkeyPatch,
) -> None:
captured: list[AwoooPRunState] = []
class _Db:
def add(self, run: AwoooPRunState) -> None:
captured.append(run)
@asynccontextmanager
async def fake_db_context(project_id: str):
assert project_id == "awoooi"
yield _Db()
monkeypatch.setattr(platform_runtime, "get_db_context", fake_db_context)
_, is_duplicate = await platform_runtime.create_run(
project_id="awoooi",
agent_id="generic-agent",
trigger_type="api",
input_payload={"safe": True},
)
assert is_duplicate is False
assert len(captured) == 1
assert captured[0].is_shadow is True
assert captured[0].state == "pending"
assert captured[0].run_id.version == 7
assert captured[0].run_id.variant == RFC_4122
assert is_canonical_traceparent(
str(captured[0].trace_id),
run_id=captured[0].run_id,
) is True
@pytest.mark.asyncio
async def test_generic_public_run_creation_remains_shadow_only(
monkeypatch: pytest.MonkeyPatch,
) -> None:
run_id = UUID("018f2d04-4c37-7a18-b764-df0df0cbe222")
async def fake_create_run(**_: object) -> tuple[UUID, bool]:
return run_id, False
async def fake_read_run_status(
received_run_id: UUID,
project_id: str,
) -> dict[str, object]:
assert received_run_id == run_id
assert project_id == "awoooi"
return {
"trace_id": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
"work_item_id": f"platform-run:awoooi:{run_id}",
"correlation_status": "canonical",
}
async def fake_write_audit(**kwargs: object) -> None:
details = kwargs["details"]
assert isinstance(details, dict)
assert details["is_shadow"] is True
assert details["trace_id"] == (
"00-0123456789abcdef0123456789abcdef-0123456789abcdef-01"
)
assert details["work_item_id"] == f"platform-run:awoooi:{run_id}"
assert details["correlation_status"] == "canonical"
monkeypatch.setattr(public_runs, "create_run", fake_create_run)
monkeypatch.setattr(
public_runs,
"read_platform_run_status",
fake_read_run_status,
)
monkeypatch.setattr(public_runs, "write_audit", fake_write_audit)
response = await public_runs.create_platform_run(
public_runs.CreateRunRequest(
project_id="awoooi",
agent_id="generic-agent",
trigger_type="api",
)
)
assert response.is_shadow is True
assert response.trace_id.startswith("00-")
assert response.work_item_id == f"platform-run:awoooi:{run_id}"
assert response.correlation_status == "canonical"
assert response.message == "Run 已接受shadow mode"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"identity_readback",
[
None,
{
"trace_id": "legacy-trace",
"work_item_id": (
"platform-run:awoooi:019f88d5-f6c9-7a18-b764-df0df0cbe222"
),
"correlation_status": "legacy_or_invalid_trace",
},
],
)
async def test_generic_public_run_creation_fails_closed_without_canonical_identity_readback(
monkeypatch: pytest.MonkeyPatch,
identity_readback: dict[str, str] | None,
) -> None:
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
async def fake_create_run(**_: object) -> tuple[UUID, bool]:
return run_id, False
async def fake_read_run_status(
_run_id: UUID,
_project_id: str,
) -> dict[str, str] | None:
return identity_readback
monkeypatch.setattr(public_runs, "create_run", fake_create_run)
monkeypatch.setattr(
public_runs,
"read_platform_run_status",
fake_read_run_status,
)
with pytest.raises(HTTPException) as exc_info:
await public_runs.create_platform_run(
public_runs.CreateRunRequest(
project_id="awoooi",
agent_id="generic-agent",
trigger_type="api",
)
)
assert exc_info.value.status_code == 500
assert "correlation readback missing" in str(exc_info.value.detail)