from __future__ import annotations # ruff: noqa: E402 import asyncio import json import os from contextlib import asynccontextmanager from datetime import UTC, datetime from types import SimpleNamespace from uuid import UUID import pytest os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") from src.services.ai_providers.interfaces import AIResult from src.services.paid_provider_canary_validation import ( _ollama_lane_timeout_seconds, run_paid_provider_canary_validation, select_claude_canary_run_id, ) _AUTHORIZATION_REF = "267a3d26-3c29-470f-8fe3-388a2f408f39" _AUTHORIZATION_TRACE_ID = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01" def test_ollama_canary_uses_bounded_production_diagnose_timeout( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace(OLLAMA_DIAGNOSE_TIMEOUT_SECONDS=300), ) assert _ollama_lane_timeout_seconds() == 300.0 monkeypatch.setattr( module, "settings", SimpleNamespace(OLLAMA_DIAGNOSE_TIMEOUT_SECONDS=900), ) assert _ollama_lane_timeout_seconds() == 600.0 class _Provider: def __init__( self, name: str, *, action: str = "NO_ACTION", success: bool = True, include_receipt: bool = True, close_fails: bool = False, error: str = "fixture_provider_unavailable", cancel: bool = False, ) -> None: self.name = name self.closed = False self.action = action self.success = success self.include_receipt = include_receipt self.close_fails = close_fails self.error = error self.cancel = cancel self.calls = 0 async def analyze(self, prompt: str, context: dict) -> AIResult: self.calls += 1 assert "sanitized" in prompt assert context["data_classification"] == "sanitized" assert context["work_item_id"] == "AIA-SRE-013" assert context["task_type"] == "diagnose" assert context["ollama_model"] == "qwen3:14b" if self.cancel: raise asyncio.CancelledError if not self.success: return AIResult( raw_response="", success=False, provider=self.name, error=self.error, ) return AIResult( raw_response=json.dumps( { "suggested_action": self.action, "target_resource": "awoooi-api", "namespace": "awoooi-prod", "kubectl_command": "", "risk_level": "low", "blast_radius": { "affected_pods": 0, "estimated_downtime": "0s", "related_services": [], "data_impact": "NONE", }, "reasoning": "Synthetic workload is healthy.", "confidence": 0.95, } ), success=True, provider=self.name, tokens=120, cost_usd=0.001, latency_ms=25, audit_metadata=( {"generation_receipt_id": f"gen-{self.name}"} if self.include_receipt else {} ), ) async def close(self) -> None: self.closed = True if self.close_fails: raise RuntimeError("synthetic close failure") class _Repository: def __init__(self, *, omit_start: bool = False) -> None: self.rows: list[dict] = [] self.omit_start = omit_start async def append(self, event_type: str, **kwargs): self.rows.append({"event_type": event_type, **kwargs}) if event_type == "EXECUTION_STARTED" and self.omit_start: return None return SimpleNamespace(id=f"row-{len(self.rows)}") def _configure_runtime_control(monkeypatch) -> dict[str, object]: from src.services import paid_provider_canary_validation as module state: dict[str, object] = { "claude": True, "gemini": True, "lock_owner": None, "active_owner": None, "active_completion_record_id": None, "lease_ttl": -2, "execution_claim_owner": None, "gate5_terminal_transitions": [], } def _readback( owner: str | None = None, completion_record_id: str | None = None, ) -> dict: lock_matches = bool(owner and state["lock_owner"] == owner) active_matches = bool(owner and state["active_owner"] == owner) completion_present = bool(state["active_completion_record_id"]) claude_disabled = bool(state["claude"]) gemini_disabled = bool(state["gemini"]) return { "claude_disabled": claude_disabled, "gemini_disabled": gemini_disabled, "lock_present": state["lock_owner"] is not None, "lock_owner_matches": lock_matches, "lock_ttl_seconds": state["lease_ttl"], "active_receipt_present": state["active_owner"] is not None, "active_receipt_owner_matches": active_matches, "active_completion_receipt_present": completion_present, "active_completion_receipt_matches": bool( completion_record_id and state["active_completion_record_id"] == completion_record_id ), "leased_enabled": bool( lock_matches and state["active_owner"] is None and not claude_disabled and not gemini_disabled and int(state["lease_ttl"]) > 0 ), "persistent_promoted": bool( lock_matches and active_matches and completion_present and not claude_disabled and not gemini_disabled and state["lease_ttl"] == -1 ), "persistent_enabled": bool( state["lock_owner"] is None and active_matches and completion_present and ( completion_record_id is None or state["active_completion_record_id"] == completion_record_id ) and not claude_disabled and not gemini_disabled and state["lease_ttl"] == -1 ), "disabled_verified": bool( claude_disabled and gemini_disabled and state["lock_owner"] is None and state["active_owner"] is None and state["active_completion_record_id"] is None ), } async def _control_readback( owner: str | None = None, completion_record_id: str | None = None, ) -> dict: return _readback(owner, completion_record_id) async def _acquire(owner: str) -> bool: if not _readback()["disabled_verified"]: return False state.update( claude=False, gemini=False, lock_owner=owner, active_owner=None, active_completion_record_id=None, lease_ttl=300, ) return True async def _rollback(owner: str) -> bool: if state["lock_owner"] not in {None, owner}: return False if state["active_owner"] not in {None, owner}: return False state.update( claude=True, gemini=True, lock_owner=None, active_owner=None, active_completion_record_id=None, lease_ttl=-2, ) return True async def _acquire_execution_claim(run_id: str, owner: str) -> bool: assert run_id == _AUTHORIZATION_REF if state["execution_claim_owner"] is not None: return False state["execution_claim_owner"] = owner return True async def _release_execution_claim(run_id: str, owner: str) -> bool: assert run_id == _AUTHORIZATION_REF if state["execution_claim_owner"] != owner: return False state["execution_claim_owner"] = None return True async def _receipt_readback( *, provider: str, receipt_id: str, trace_id: str, run_id: str, ) -> dict: assert trace_id == _AUTHORIZATION_TRACE_ID assert run_id == _AUTHORIZATION_REF from src.services.ai_rate_limiter import AIRateLimiter budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013") budget_id = f"paid-run-{budget_hash[:32]}" return { "provider": provider, "receipt_id": receipt_id, "found": True, "correlation_verified": True, "finalized_accounted": True, "reservation_released": True, "success": True, "status": "succeeded", "accounted_tokens": 120, "accounted_cost_micro_usd": 1000, "accounted_cost_usd": 0.001, "run_budget_required": True, "run_budget_id": budget_id, "run_budget_max_micro_usd": 250_000, } async def _run_budget_readback(*, run_id: str) -> dict: from src.services.ai_rate_limiter import AIRateLimiter budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013") return { "schema_version": "ai_paid_run_budget_receipt_v1", "run_budget_id": f"paid-run-{budget_hash[:32]}", "found": True, "status": "finalized", "max_cost_micro_usd": 250_000, "reserved_cost_micro_usd": 0, "accounted_cost_micro_usd": 2000, "accounted_cost_usd": 0.002, "reservation_count": 2, "finalized_count": 2, "correlation_verified": True, "cap_verified": True, "terminal_verified": True, } async def _terminalize_gate5( run_id: str, *, expected_trace_id: str, succeeded: bool, error_code: str | None = None, ) -> dict: assert run_id == _AUTHORIZATION_REF assert expected_trace_id == _AUTHORIZATION_TRACE_ID transitions = state["gate5_terminal_transitions"] assert isinstance(transitions, list) transitions.append( { "run_id": run_id, "succeeded": succeeded, "error_code": error_code, "paid_routes_disabled": bool(state["claude"] and state["gemini"]), "activation_lock_absent": state["lock_owner"] is None, } ) return { "schema_version": "paid_provider_canary_gate5_terminal_readback_v1", "run_id": run_id, "project_id": "awoooi", "trace_id": expected_trace_id, "state": "completed" if succeeded else "failed", "completed_at": "2026-07-16T00:10:00", "error_code": None if succeeded else error_code, "checks": { "run_id_exact": True, "project_exact": True, "state_exact": True, "completed_at_present": True, "trace_id_exact": True, "error_code_exact": True, }, "durable_readback_verified": True, "same_run_trace_preserved": True, } async def _authorization_readback(authorization_ref: str) -> dict: assert authorization_ref == _AUTHORIZATION_REF return { "schema_version": "paid_provider_canary_authorization_receipt_v1", "status": "authorized", "authorized": True, "error_code": None, "authorization_ref": authorization_ref, "run_id": authorization_ref, "trace_id": _AUTHORIZATION_TRACE_ID, "project_id": "awoooi", "work_item_id": "AIA-SRE-013", "checked_at": "2026-07-16T00:00:00+00:00", "expires_at": "2026-07-16T00:15:00+00:00", "valid_for_seconds": 900, "contract": { "agent_id": "awoooi-paid-provider-canary", "state": "running", "trigger_type": "api", "trigger_ref": "paid-provider-canary:AIA-SRE-013", "input_sha256": "fixture-contract-sha256", "operator_approval_step": "operator_console.approve", "operator_approval_audit": "run.approval.approve", }, "prior_canary_start_count": 0, "reuse_check": "clear_at_read_time", "read_only_verification": True, "authorization_consumed_by_verifier": False, "raw_input_persisted_or_returned": False, "operator_identity_returned": False, "secret_value_read_or_returned": False, } async def _transport_receipt_issuer(**kwargs) -> dict: provider = kwargs["provider"] return { "schema_version": "cloud_transport_preflight_receipt_v1", "receipt_id": ( "cloud-transport:" + ("a" if provider.endswith("_a") else "b") * 32 ), "source": "cloud_transport_preflight.network_probe", "status": "degraded_transport_reachable", "transport_boundary": "public_http_sanitized_candidate_only", "transport_security_status": "degraded_public_http", "provider": provider, "endpoint_sha256": "a" * 64, "trace_id": kwargs["trace_id"], "run_id": kwargs["run_id"], "work_item_id": kwargs["work_item_id"], "check_completed": True, "http_status": 200, "ollama_contract_verified": True, "observed_at": "2026-07-16T00:00:00+00:00", "expires_at": "2026-07-16T00:01:30+00:00", "ttl_seconds": 90, "durable_write_ack": True, "verifier_status": "verified", "verified_by": "cloud_transport_preflight.redis_readback", "raw_response_persisted": False, "endpoint_value_persisted": False, "secret_value_persisted_or_returned": False, } monkeypatch.setattr( module, "_require_paid_canary_authorization", _authorization_readback, ) monkeypatch.setattr( module, "issue_cloud_transport_receipt", _transport_receipt_issuer, ) monkeypatch.setattr(module, "_paid_canary_control_readback", _control_readback) monkeypatch.setattr(module, "_acquire_paid_canary_lease", _acquire) monkeypatch.setattr( module, "_acquire_paid_canary_execution_claim", _acquire_execution_claim, ) monkeypatch.setattr( module, "_release_paid_canary_execution_claim", _release_execution_claim, ) monkeypatch.setattr(module, "_rollback_paid_canary", _rollback) monkeypatch.setattr(module, "_generation_receipt_readback", _receipt_readback) monkeypatch.setattr(module, "_run_budget_readback", _run_budget_readback) monkeypatch.setattr( module, "_terminalize_paid_canary_gate5_run", _terminalize_gate5, ) return state def test_canary_run_selection_is_deterministic_and_inside_percent() -> None: first = select_claude_canary_run_id("source-sha", 5) second = select_claude_canary_run_id("source-sha", 5) assert first == second assert first[1] < 5 @pytest.mark.asyncio async def test_gate5_terminal_readback_requires_exact_durable_state_and_trace( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module completed_at = datetime(2026, 7, 16, 0, 10, tzinfo=UTC) class _Result: @staticmethod def one_or_none(): return ( UUID(_AUTHORIZATION_REF), "awoooi", "completed", completed_at, _AUTHORIZATION_TRACE_ID, None, ) class _Db: @staticmethod async def execute(_statement): return _Result() @asynccontextmanager async def _db_context(project_id: str): assert project_id == "awoooi" yield _Db() monkeypatch.setattr(module, "get_db_context", _db_context) receipt = await module._read_paid_canary_gate5_terminal( UUID(_AUTHORIZATION_REF), expected_state="completed", expected_trace_id=_AUTHORIZATION_TRACE_ID, expected_error_code=None, ) assert receipt["durable_readback_verified"] is True assert receipt["state"] == "completed" assert receipt["completed_at"] == completed_at.isoformat() assert receipt["trace_id"] == _AUTHORIZATION_TRACE_ID assert all(receipt["checks"].values()) with pytest.raises( RuntimeError, match="paid_provider_canary_gate5_terminal_readback_failed:trace_id_exact", ): await module._read_paid_canary_gate5_terminal( UUID(_AUTHORIZATION_REF), expected_state="completed", expected_trace_id=( "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01" ), expected_error_code=None, ) @pytest.mark.asyncio async def test_gate5_terminal_transition_is_followed_by_exact_readback( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module from src.services import run_state_machine events: list[tuple] = [] async def _transition( run_id: UUID, project_id: str, to_state: str, *, error_code: str | None = None, ) -> None: events.append(("transition", run_id, project_id, to_state, error_code)) async def _readback( run_id: UUID, *, expected_state: str, expected_trace_id: str, expected_error_code: str | None, ) -> dict: events.append( ( "readback", run_id, expected_state, expected_trace_id, expected_error_code, ) ) return {"durable_readback_verified": True, "state": expected_state} monkeypatch.setattr(run_state_machine, "transition", _transition) monkeypatch.setattr(module, "_read_paid_canary_gate5_terminal", _readback) receipt = await module._terminalize_paid_canary_gate5_run( _AUTHORIZATION_REF, expected_trace_id=_AUTHORIZATION_TRACE_ID, succeeded=False, error_code="E-PAID-CANARY-VERIFY", ) assert receipt == {"durable_readback_verified": True, "state": "failed"} assert events == [ ( "transition", UUID(_AUTHORIZATION_REF), "awoooi", "failed", "E-PAID-CANARY-VERIFY", ), ( "readback", UUID(_AUTHORIZATION_REF), "failed", _AUTHORIZATION_TRACE_ID, "E-PAID-CANARY-VERIFY", ), ] @pytest.mark.asyncio async def test_optional_terminal_aol_projection_cannot_reopen_terminal_run() -> None: from src.services import paid_provider_canary_validation as module class _FailingRepository: @staticmethod async def append(*_args, **_kwargs): raise RuntimeError("synthetic_aol_unavailable") recorded, record_id = await module._append_paid_canary_gate5_terminal_receipt( _FailingRepository(), operator_id="test-operator", terminal_readback={ "run_id": _AUTHORIZATION_REF, "trace_id": _AUTHORIZATION_TRACE_ID, "state": "completed", "durable_readback_verified": True, }, ) assert recorded is False assert record_id is None @pytest.mark.asyncio async def test_durable_authorization_blocks_before_receipt_or_provider_call( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) async def _blocked(_authorization_ref: str) -> dict: raise RuntimeError("paid_canary_authorization_expired") monkeypatch.setattr(module, "_require_paid_canary_authorization", _blocked) providers = [ _Provider("ollama", include_receipt=False), _Provider("ollama_gcp_b", include_receipt=False), _Provider("ollama_local", include_receipt=False), _Provider("claude"), _Provider("gemini"), ] repository = _Repository() with pytest.raises(RuntimeError, match="paid_canary_authorization_expired"): await run_paid_provider_canary_validation( run_ref="source-sha-auth-blocked", operator_id="test-operator", authorization_ref="00000000-0000-4000-8000-000000000001", ollama_gcp_a_provider=providers[0], ollama_gcp_b_provider=providers[1], ollama_local_provider=providers[2], claude_provider=providers[3], gemini_provider=providers[4], operation_repository=repository, ) assert repository.rows == [] assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0] @pytest.mark.asyncio async def test_atomic_execution_claim_blocks_concurrent_authorization_reuse( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) async def _claim_held(_run_id: str, _owner: str) -> bool: return False monkeypatch.setattr( module, "_acquire_paid_canary_execution_claim", _claim_held, ) providers = [_Provider(name) for name in ("a", "b", "c", "d", "e")] repository = _Repository() with pytest.raises( RuntimeError, match="paid_canary_authorization_execution_claim_held", ): await run_paid_provider_canary_validation( run_ref="source-sha-concurrent", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=providers[0], ollama_gcp_b_provider=providers[1], ollama_local_provider=providers[2], claude_provider=providers[3], gemini_provider=providers[4], operation_repository=repository, ) assert repository.rows == [] assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0] assert state["gate5_terminal_transitions"] == [] @pytest.mark.asyncio async def test_canary_discards_content_and_writes_bounded_receipt(monkeypatch) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) gcp_a = _Provider("ollama", include_receipt=False) gcp_b = _Provider("ollama_gcp_b", include_receipt=False) local = _Provider("ollama_local", include_receipt=False) claude = _Provider("claude") gemini = _Provider("gemini") repository = _Repository() receipt = await run_paid_provider_canary_validation( run_ref="source-sha", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=gcp_a, ollama_gcp_b_provider=gcp_b, ollama_local_provider=local, claude_provider=claude, gemini_provider=gemini, operation_repository=repository, ) assert receipt["independent_verifier"]["passed"] is True assert receipt["independent_verifier"]["paid_activation_verifier_passed"] is True assert receipt["independent_verifier"]["five_lane_verifier_passed"] is True assert receipt["independent_verifier"]["paid_provider_comparison"] == ( "initial_canary_tie_no_overall_winner" ) assert ( receipt["policy_decision"]["durable_authorization_receipt"]["authorized"] is True ) assert receipt["bounded_execution"]["provider_order"] == [ "ollama_gcp_a", "ollama_gcp_b", "ollama_local", "claude", "gemini", ] assert receipt["source_receipt"]["gcp_transport"] == { "status": "degraded_public_http", "sanitized_only": True, "receipt_ids": [ "cloud-transport:" + "a" * 32, "cloud-transport:" + "b" * 32, ], "preflight": [ { "provider": "ollama_gcp_a", "status": "verified", "receipt_id": "cloud-transport:" + "a" * 32, }, { "provider": "ollama_gcp_b", "status": "verified", "receipt_id": "cloud-transport:" + "b" * 32, }, ], "same_run_bound": True, "durable_readback_verified": True, "encrypted_transport_claimed": False, } assert [ row["contract_score_percent"] for row in receipt["bounded_execution"]["results"] ] == [100, 100, 100, 100, 100] assert all( "raw_response" not in row for row in receipt["bounded_execution"]["results"] ) assert receipt["durable_operation_writeback"] == { "start_recorded": True, "lease_change_recorded": True, "lease_change_record_id": "row-2", "comparison_ready_recorded": True, "comparison_ready_record_id": "row-3", "rollback_change_recorded": True, "rollback_change_record_id": "row-4", "activation_failure_recorded": False, "activation_failure_record_id": None, "completion_recorded": True, "completion_record_id": "row-5", } assert [row["event_type"] for row in repository.rows] == [ "EXECUTION_STARTED", "CHANGE_APPLIED", "PRE_FLIGHT_PASSED", "CHANGE_APPLIED", "EXECUTION_COMPLETED", "CHANGE_APPLIED", ] assert claude.closed is True assert gemini.closed is True assert state["claude"] is True assert state["gemini"] is True assert state["lock_owner"] is None assert state["active_owner"] is None assert state["execution_claim_owner"] is None assert receipt["policy_decision"]["authorization_execution_claim"] == { "acquired": True, "released_after_durable_start": True, "same_run_bound": True, } assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == ( "validated_comparison_rollback_verified" ) assert receipt["policy_decision"]["persistent_promotion_authorized"] is False assert receipt["gate5_run_terminal"] == { "schema_version": "paid_provider_canary_gate5_terminal_readback_v1", "run_id": _AUTHORIZATION_REF, "project_id": "awoooi", "trace_id": _AUTHORIZATION_TRACE_ID, "state": "completed", "completed_at": "2026-07-16T00:10:00", "error_code": None, "checks": { "run_id_exact": True, "project_exact": True, "state_exact": True, "completed_at_present": True, "trace_id_exact": True, "error_code_exact": True, }, "durable_readback_verified": True, "same_run_trace_preserved": True, "rollback_verified_before_terminal": True, "disabled_readback_verified_before_terminal": True, "immutable_aol_terminal_receipt_recorded": True, "immutable_aol_terminal_receipt_record_id": "row-6", } assert state["gate5_terminal_transitions"] == [ { "run_id": _AUTHORIZATION_REF, "succeeded": True, "error_code": None, "paid_routes_disabled": True, "activation_lock_absent": True, } ] @pytest.mark.asyncio async def test_canary_verifier_rejects_a_mutating_false_positive(monkeypatch) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) receipt = await run_paid_provider_canary_validation( run_ref="source-sha-failed", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude", action="RESTART_DEPLOYMENT"), gemini_provider=_Provider("gemini"), operation_repository=_Repository(), ) assert receipt["independent_verifier"]["passed"] is False assert ( receipt["bounded_execution"]["results"][3]["contract_checks"][ "expected_no_action" ] is False ) assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True assert state["claude"] is True assert state["gemini"] is True assert state["lock_owner"] is None assert state["active_owner"] is None assert state["execution_claim_owner"] is None assert receipt["gate5_run_terminal"]["state"] == "failed" assert receipt["gate5_run_terminal"]["error_code"] == ("E-PAID-CANARY-VERIFY") assert state["gate5_terminal_transitions"] == [ { "run_id": _AUTHORIZATION_REF, "succeeded": False, "error_code": "E-PAID-CANARY-VERIFY", "paid_routes_disabled": True, "activation_lock_absent": True, } ] @pytest.mark.asyncio async def test_ollama_baseline_failure_forces_paired_paid_canary_rollback( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) receipt = await run_paid_provider_canary_validation( run_ref="source-sha-partial-baseline", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider("ollama", success=False), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude"), gemini_provider=_Provider("gemini"), operation_repository=_Repository(), ) assert receipt["independent_verifier"]["passed"] is False assert receipt["independent_verifier"]["paid_activation_verifier_passed"] is False assert ( receipt["independent_verifier"]["paid_provider_contract_verifier_passed"] is True ) assert receipt["independent_verifier"]["five_lane_verifier_passed"] is False assert receipt["independent_verifier"]["five_lane_comparison_ready"] is False assert receipt["independent_verifier"]["ollama_baseline_available_count"] == 2 assert receipt["bounded_execution"]["results"][0]["provider"] == ("ollama_gcp_a") assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == ( "failed_rollback_verified" ) assert state["claude"] is True assert state["gemini"] is True assert state["active_owner"] is None @pytest.mark.asyncio async def test_missing_paid_generation_receipt_forces_atomic_rollback( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) receipt = await run_paid_provider_canary_validation( run_ref="source-sha-missing-receipt", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude", include_receipt=False), gemini_provider=_Provider("gemini"), operation_repository=_Repository(), ) assert receipt["independent_verifier"]["passed"] is False assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == ( "failed_rollback_verified" ) assert state["claude"] is True assert state["gemini"] is True assert state["lock_owner"] is None assert state["active_owner"] is None assert state["gate5_terminal_transitions"] == [ { "run_id": _AUTHORIZATION_REF, "succeeded": False, "error_code": "E-PAID-CANARY-VERIFY", "paid_routes_disabled": True, "activation_lock_absent": True, } ] @pytest.mark.asyncio async def test_no_paid_or_baseline_call_without_durable_start_receipt( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) providers = [_Provider(name) for name in ("a", "b", "c", "d", "e")] with pytest.raises( RuntimeError, match="paid_provider_canary_start_receipt_not_recorded", ): await run_paid_provider_canary_validation( run_ref="source-sha-no-start", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=providers[0], ollama_gcp_b_provider=providers[1], ollama_local_provider=providers[2], claude_provider=providers[3], gemini_provider=providers[4], operation_repository=_Repository(omit_start=True), ) assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0] assert state["claude"] is True assert state["gemini"] is True assert state["lock_owner"] is None assert state["active_owner"] is None assert state["gate5_terminal_transitions"] == [ { "run_id": _AUTHORIZATION_REF, "succeeded": False, "error_code": "E-PAID-CANARY-EXECUTION", "paid_routes_disabled": True, "activation_lock_absent": True, } ] @pytest.mark.asyncio async def test_failed_exact_receipt_cannot_promote_and_rolls_back(monkeypatch) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) async def _failed_receipt(**kwargs) -> dict: return { "provider": kwargs["provider"], "receipt_id": kwargs["receipt_id"], "found": True, "correlation_verified": True, "finalized_accounted": True, "reservation_released": True, "success": False, "status": "failed_estimate_charged", "accounted_tokens": 120, "accounted_cost_usd": 0.001, } monkeypatch.setattr(module, "_generation_receipt_readback", _failed_receipt) receipt = await run_paid_provider_canary_validation( run_ref="source-sha-failed-exact-receipt", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude"), gemini_provider=_Provider("gemini"), operation_repository=_Repository(), ) assert receipt["independent_verifier"]["passed"] is False assert ( receipt["independent_verifier"]["generation_receipt_readback_passed"] is False ) assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True assert state["claude"] is True assert state["gemini"] is True @pytest.mark.asyncio async def test_aggregate_run_budget_must_be_finalized_under_exact_cap( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) async def _pending_budget(*, run_id: str) -> dict: from src.services.ai_rate_limiter import AIRateLimiter budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013") return { "schema_version": "ai_paid_run_budget_receipt_v1", "run_budget_id": f"paid-run-{budget_hash[:32]}", "found": True, "status": "pending_finalize", "max_cost_micro_usd": 250_000, "reserved_cost_micro_usd": 1000, "accounted_cost_micro_usd": 1000, "reservation_count": 2, "finalized_count": 1, "correlation_verified": True, "cap_verified": True, "terminal_verified": False, } monkeypatch.setattr(module, "_run_budget_readback", _pending_budget) receipt = await run_paid_provider_canary_validation( run_ref="source-sha-pending-run-budget", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude"), gemini_provider=_Provider("gemini"), operation_repository=_Repository(), ) assert receipt["independent_verifier"]["passed"] is False assert receipt["independent_verifier"]["run_budget_readback_passed"] is False assert ( receipt["independent_verifier"]["generation_receipt_readback_passed"] is False ) assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True assert state["claude"] is True assert state["gemini"] is True @pytest.mark.asyncio async def test_canary_rejects_generation_and_aggregate_accounting_mismatch( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) async def _mismatched_budget(*, run_id: str) -> dict: from src.services.ai_rate_limiter import AIRateLimiter budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013") return { "schema_version": "ai_paid_run_budget_receipt_v1", "run_budget_id": f"paid-run-{budget_hash[:32]}", "found": True, "status": "finalized", "max_cost_micro_usd": 250_000, "reserved_cost_micro_usd": 0, "accounted_cost_micro_usd": 1999, "reservation_count": 2, "finalized_count": 2, "correlation_verified": True, "cap_verified": True, "terminal_verified": True, } monkeypatch.setattr(module, "_run_budget_readback", _mismatched_budget) repository = _Repository() receipt = await run_paid_provider_canary_validation( run_ref="source-sha-accounting-mismatch", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude"), gemini_provider=_Provider("gemini"), operation_repository=repository, ) assert receipt["independent_verifier"]["run_budget_readback_passed"] is False assert receipt["independent_verifier"]["passed"] is False assert "PRE_FLIGHT_PASSED" not in {row["event_type"] for row in repository.rows} assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True assert state["claude"] is True assert state["gemini"] is True @pytest.mark.asyncio async def test_cancelled_paid_call_runs_verified_compensation(monkeypatch) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) repository = _Repository() with pytest.raises(asyncio.CancelledError): await run_paid_provider_canary_validation( run_ref="source-sha-cancelled", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude", cancel=True), gemini_provider=_Provider("gemini"), operation_repository=repository, ) assert state["claude"] is True assert state["gemini"] is True assert state["lock_owner"] is None assert state["active_owner"] is None assert repository.rows[-2]["event_type"] == "EXECUTION_COMPLETED" assert repository.rows[-2]["success"] is False assert repository.rows[-2]["context"]["rollback_verified"] is True assert repository.rows[-1]["event_type"] == "CHANGE_APPLIED" assert repository.rows[-1]["action_detail"] == ( "paid_provider_canary_gate5_terminal_readback" ) assert repository.rows[-1]["context"]["terminal_readback"]["state"] == ("failed") @pytest.mark.asyncio async def test_ambiguous_lease_failure_is_compensated_and_verified(monkeypatch) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) state = _configure_runtime_control(monkeypatch) async def _ambiguous_acquire(owner: str) -> bool: state.update( claude=False, gemini=False, lock_owner=owner, active_owner=None, lease_ttl=300, ) return False monkeypatch.setattr(module, "_acquire_paid_canary_lease", _ambiguous_acquire) with pytest.raises( RuntimeError, match="paid_provider_canary_activation_lease_failed", ): await run_paid_provider_canary_validation( run_ref="source-sha-ambiguous-lease", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider("ollama", include_receipt=False), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude"), gemini_provider=_Provider("gemini"), operation_repository=_Repository(), ) assert state["claude"] is True assert state["gemini"] is True assert state["lock_owner"] is None @pytest.mark.asyncio async def test_provider_error_text_is_never_written_to_safe_receipt( monkeypatch, ) -> None: from src.services import paid_provider_canary_validation as module monkeypatch.setattr( module, "settings", SimpleNamespace( CLAUDE_EXECUTION_MODE="canary", CLAUDE_CANARY_PERCENT=5, GEMINI_EXECUTION_MODE="canary", GEMINI_CANARY_PERCENT=5, ALERT_OLLAMA_MODEL="qwen3:14b", ), ) _configure_runtime_control(monkeypatch) unsafe_error = "http://192.168.0.111:11434/?key=secret-like-value" receipt = await run_paid_provider_canary_validation( run_ref="source-sha-sanitized-error", operator_id="test-operator", authorization_ref=_AUTHORIZATION_REF, ollama_gcp_a_provider=_Provider( "ollama", success=False, include_receipt=False, error=unsafe_error, ), ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False), ollama_local_provider=_Provider("ollama_local", include_receipt=False), claude_provider=_Provider("claude"), gemini_provider=_Provider("gemini"), operation_repository=_Repository(), ) serialized = json.dumps(receipt, sort_keys=True) assert unsafe_error not in serialized assert "192.168.0.111" not in serialized assert "secret-like-value" not in serialized assert receipt["bounded_execution"]["results"][0]["error_code"] == ( "provider_error_redacted" )