541a32a9cb
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
1077 lines
38 KiB
Python
1077 lines
38 KiB
Python
from __future__ import annotations
|
|
|
|
# ruff: noqa: E402
|
|
import asyncio
|
|
import json
|
|
import os
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test")
|
|
|
|
from src.services.ai_providers.interfaces import AIResult
|
|
from src.services.paid_provider_canary_validation import (
|
|
run_paid_provider_canary_validation,
|
|
select_claude_canary_run_id,
|
|
)
|
|
|
|
_AUTHORIZATION_REF = "267a3d26-3c29-470f-8fe3-388a2f408f39"
|
|
_AUTHORIZATION_TRACE_ID = (
|
|
"00-0123456789abcdef0123456789abcdef-0123456789abcdef-01"
|
|
)
|
|
|
|
|
|
class _Provider:
|
|
def __init__(
|
|
self,
|
|
name: str,
|
|
*,
|
|
action: str = "NO_ACTION",
|
|
success: bool = True,
|
|
include_receipt: bool = True,
|
|
close_fails: bool = False,
|
|
error: str = "fixture_provider_unavailable",
|
|
cancel: bool = False,
|
|
) -> None:
|
|
self.name = name
|
|
self.closed = False
|
|
self.action = action
|
|
self.success = success
|
|
self.include_receipt = include_receipt
|
|
self.close_fails = close_fails
|
|
self.error = error
|
|
self.cancel = cancel
|
|
self.calls = 0
|
|
|
|
async def analyze(self, prompt: str, context: dict) -> AIResult:
|
|
self.calls += 1
|
|
assert "sanitized" in prompt
|
|
assert context["data_classification"] == "sanitized"
|
|
assert context["work_item_id"] == "AIA-SRE-013"
|
|
assert context["task_type"] == "diagnose"
|
|
assert context["ollama_model"] == "qwen3:14b"
|
|
if self.cancel:
|
|
raise asyncio.CancelledError
|
|
if not self.success:
|
|
return AIResult(
|
|
raw_response="",
|
|
success=False,
|
|
provider=self.name,
|
|
error=self.error,
|
|
)
|
|
return AIResult(
|
|
raw_response=json.dumps(
|
|
{
|
|
"suggested_action": self.action,
|
|
"target_resource": "awoooi-api",
|
|
"namespace": "awoooi-prod",
|
|
"kubectl_command": "",
|
|
"risk_level": "low",
|
|
"blast_radius": {
|
|
"affected_pods": 0,
|
|
"estimated_downtime": "0s",
|
|
"related_services": [],
|
|
"data_impact": "NONE",
|
|
},
|
|
"reasoning": "Synthetic workload is healthy.",
|
|
"confidence": 0.95,
|
|
}
|
|
),
|
|
success=True,
|
|
provider=self.name,
|
|
tokens=120,
|
|
cost_usd=0.001,
|
|
latency_ms=25,
|
|
audit_metadata=(
|
|
{"generation_receipt_id": f"gen-{self.name}"}
|
|
if self.include_receipt
|
|
else {}
|
|
),
|
|
)
|
|
|
|
async def close(self) -> None:
|
|
self.closed = True
|
|
if self.close_fails:
|
|
raise RuntimeError("synthetic close failure")
|
|
|
|
|
|
class _Repository:
|
|
def __init__(self, *, omit_start: bool = False) -> None:
|
|
self.rows: list[dict] = []
|
|
self.omit_start = omit_start
|
|
|
|
async def append(self, event_type: str, **kwargs):
|
|
self.rows.append({"event_type": event_type, **kwargs})
|
|
if event_type == "EXECUTION_STARTED" and self.omit_start:
|
|
return None
|
|
return SimpleNamespace(id=f"row-{len(self.rows)}")
|
|
|
|
|
|
def _configure_runtime_control(monkeypatch) -> dict[str, object]:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
state: dict[str, object] = {
|
|
"claude": True,
|
|
"gemini": True,
|
|
"lock_owner": None,
|
|
"active_owner": None,
|
|
"active_completion_record_id": None,
|
|
"lease_ttl": -2,
|
|
"execution_claim_owner": None,
|
|
}
|
|
|
|
def _readback(
|
|
owner: str | None = None,
|
|
completion_record_id: str | None = None,
|
|
) -> dict:
|
|
lock_matches = bool(owner and state["lock_owner"] == owner)
|
|
active_matches = bool(owner and state["active_owner"] == owner)
|
|
completion_present = bool(state["active_completion_record_id"])
|
|
claude_disabled = bool(state["claude"])
|
|
gemini_disabled = bool(state["gemini"])
|
|
return {
|
|
"claude_disabled": claude_disabled,
|
|
"gemini_disabled": gemini_disabled,
|
|
"lock_present": state["lock_owner"] is not None,
|
|
"lock_owner_matches": lock_matches,
|
|
"lock_ttl_seconds": state["lease_ttl"],
|
|
"active_receipt_present": state["active_owner"] is not None,
|
|
"active_receipt_owner_matches": active_matches,
|
|
"active_completion_receipt_present": completion_present,
|
|
"active_completion_receipt_matches": bool(
|
|
completion_record_id
|
|
and state["active_completion_record_id"] == completion_record_id
|
|
),
|
|
"leased_enabled": bool(
|
|
lock_matches
|
|
and state["active_owner"] is None
|
|
and not claude_disabled
|
|
and not gemini_disabled
|
|
and int(state["lease_ttl"]) > 0
|
|
),
|
|
"persistent_promoted": bool(
|
|
lock_matches
|
|
and active_matches
|
|
and completion_present
|
|
and not claude_disabled
|
|
and not gemini_disabled
|
|
and state["lease_ttl"] == -1
|
|
),
|
|
"persistent_enabled": bool(
|
|
state["lock_owner"] is None
|
|
and active_matches
|
|
and completion_present
|
|
and (
|
|
completion_record_id is None
|
|
or state["active_completion_record_id"] == completion_record_id
|
|
)
|
|
and not claude_disabled
|
|
and not gemini_disabled
|
|
and state["lease_ttl"] == -1
|
|
),
|
|
"disabled_verified": bool(
|
|
claude_disabled
|
|
and gemini_disabled
|
|
and state["lock_owner"] is None
|
|
and state["active_owner"] is None
|
|
and state["active_completion_record_id"] is None
|
|
),
|
|
}
|
|
|
|
async def _control_readback(
|
|
owner: str | None = None,
|
|
completion_record_id: str | None = None,
|
|
) -> dict:
|
|
return _readback(owner, completion_record_id)
|
|
|
|
async def _acquire(owner: str) -> bool:
|
|
if not _readback()["disabled_verified"]:
|
|
return False
|
|
state.update(
|
|
claude=False,
|
|
gemini=False,
|
|
lock_owner=owner,
|
|
active_owner=None,
|
|
active_completion_record_id=None,
|
|
lease_ttl=300,
|
|
)
|
|
return True
|
|
|
|
async def _rollback(owner: str) -> bool:
|
|
if state["lock_owner"] not in {None, owner}:
|
|
return False
|
|
if state["active_owner"] not in {None, owner}:
|
|
return False
|
|
state.update(
|
|
claude=True,
|
|
gemini=True,
|
|
lock_owner=None,
|
|
active_owner=None,
|
|
active_completion_record_id=None,
|
|
lease_ttl=-2,
|
|
)
|
|
return True
|
|
|
|
async def _acquire_execution_claim(run_id: str, owner: str) -> bool:
|
|
assert run_id == _AUTHORIZATION_REF
|
|
if state["execution_claim_owner"] is not None:
|
|
return False
|
|
state["execution_claim_owner"] = owner
|
|
return True
|
|
|
|
async def _release_execution_claim(run_id: str, owner: str) -> bool:
|
|
assert run_id == _AUTHORIZATION_REF
|
|
if state["execution_claim_owner"] != owner:
|
|
return False
|
|
state["execution_claim_owner"] = None
|
|
return True
|
|
|
|
async def _receipt_readback(
|
|
*,
|
|
provider: str,
|
|
receipt_id: str,
|
|
trace_id: str,
|
|
run_id: str,
|
|
) -> dict:
|
|
assert trace_id == _AUTHORIZATION_TRACE_ID
|
|
assert run_id == _AUTHORIZATION_REF
|
|
from src.services.ai_rate_limiter import AIRateLimiter
|
|
|
|
budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013")
|
|
budget_id = f"paid-run-{budget_hash[:32]}"
|
|
return {
|
|
"provider": provider,
|
|
"receipt_id": receipt_id,
|
|
"found": True,
|
|
"correlation_verified": True,
|
|
"finalized_accounted": True,
|
|
"reservation_released": True,
|
|
"success": True,
|
|
"status": "succeeded",
|
|
"accounted_tokens": 120,
|
|
"accounted_cost_micro_usd": 1000,
|
|
"accounted_cost_usd": 0.001,
|
|
"run_budget_required": True,
|
|
"run_budget_id": budget_id,
|
|
"run_budget_max_micro_usd": 250_000,
|
|
}
|
|
|
|
async def _run_budget_readback(*, run_id: str) -> dict:
|
|
from src.services.ai_rate_limiter import AIRateLimiter
|
|
|
|
budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013")
|
|
return {
|
|
"schema_version": "ai_paid_run_budget_receipt_v1",
|
|
"run_budget_id": f"paid-run-{budget_hash[:32]}",
|
|
"found": True,
|
|
"status": "finalized",
|
|
"max_cost_micro_usd": 250_000,
|
|
"reserved_cost_micro_usd": 0,
|
|
"accounted_cost_micro_usd": 2000,
|
|
"accounted_cost_usd": 0.002,
|
|
"reservation_count": 2,
|
|
"finalized_count": 2,
|
|
"correlation_verified": True,
|
|
"cap_verified": True,
|
|
"terminal_verified": True,
|
|
}
|
|
|
|
async def _authorization_readback(authorization_ref: str) -> dict:
|
|
assert authorization_ref == _AUTHORIZATION_REF
|
|
return {
|
|
"schema_version": "paid_provider_canary_authorization_receipt_v1",
|
|
"status": "authorized",
|
|
"authorized": True,
|
|
"error_code": None,
|
|
"authorization_ref": authorization_ref,
|
|
"run_id": authorization_ref,
|
|
"trace_id": _AUTHORIZATION_TRACE_ID,
|
|
"project_id": "awoooi",
|
|
"work_item_id": "AIA-SRE-013",
|
|
"checked_at": "2026-07-16T00:00:00+00:00",
|
|
"expires_at": "2026-07-16T00:15:00+00:00",
|
|
"valid_for_seconds": 900,
|
|
"contract": {
|
|
"agent_id": "awoooi-paid-provider-canary",
|
|
"state": "running",
|
|
"trigger_type": "api",
|
|
"trigger_ref": "paid-provider-canary:AIA-SRE-013",
|
|
"input_sha256": "fixture-contract-sha256",
|
|
"operator_approval_step": "operator_console.approve",
|
|
"operator_approval_audit": "run.approval.approve",
|
|
},
|
|
"prior_canary_start_count": 0,
|
|
"reuse_check": "clear_at_read_time",
|
|
"read_only_verification": True,
|
|
"authorization_consumed_by_verifier": False,
|
|
"raw_input_persisted_or_returned": False,
|
|
"operator_identity_returned": False,
|
|
"secret_value_read_or_returned": False,
|
|
}
|
|
|
|
async def _transport_receipt_issuer(**kwargs) -> dict:
|
|
provider = kwargs["provider"]
|
|
return {
|
|
"schema_version": "cloud_transport_preflight_receipt_v1",
|
|
"receipt_id": (
|
|
"cloud-transport:" + ("a" if provider.endswith("_a") else "b") * 32
|
|
),
|
|
"source": "cloud_transport_preflight.network_probe",
|
|
"status": "degraded_transport_reachable",
|
|
"transport_boundary": "public_http_sanitized_candidate_only",
|
|
"transport_security_status": "degraded_public_http",
|
|
"provider": provider,
|
|
"endpoint_sha256": "a" * 64,
|
|
"trace_id": kwargs["trace_id"],
|
|
"run_id": kwargs["run_id"],
|
|
"work_item_id": kwargs["work_item_id"],
|
|
"check_completed": True,
|
|
"http_status": 200,
|
|
"ollama_contract_verified": True,
|
|
"observed_at": "2026-07-16T00:00:00+00:00",
|
|
"expires_at": "2026-07-16T00:01:30+00:00",
|
|
"ttl_seconds": 90,
|
|
"durable_write_ack": True,
|
|
"verifier_status": "verified",
|
|
"verified_by": "cloud_transport_preflight.redis_readback",
|
|
"raw_response_persisted": False,
|
|
"endpoint_value_persisted": False,
|
|
"secret_value_persisted_or_returned": False,
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"_require_paid_canary_authorization",
|
|
_authorization_readback,
|
|
)
|
|
monkeypatch.setattr(
|
|
module,
|
|
"issue_cloud_transport_receipt",
|
|
_transport_receipt_issuer,
|
|
)
|
|
monkeypatch.setattr(module, "_paid_canary_control_readback", _control_readback)
|
|
monkeypatch.setattr(module, "_acquire_paid_canary_lease", _acquire)
|
|
monkeypatch.setattr(
|
|
module,
|
|
"_acquire_paid_canary_execution_claim",
|
|
_acquire_execution_claim,
|
|
)
|
|
monkeypatch.setattr(
|
|
module,
|
|
"_release_paid_canary_execution_claim",
|
|
_release_execution_claim,
|
|
)
|
|
monkeypatch.setattr(module, "_rollback_paid_canary", _rollback)
|
|
monkeypatch.setattr(module, "_generation_receipt_readback", _receipt_readback)
|
|
monkeypatch.setattr(module, "_run_budget_readback", _run_budget_readback)
|
|
return state
|
|
|
|
|
|
def test_canary_run_selection_is_deterministic_and_inside_percent() -> None:
|
|
first = select_claude_canary_run_id("source-sha", 5)
|
|
second = select_claude_canary_run_id("source-sha", 5)
|
|
|
|
assert first == second
|
|
assert first[1] < 5
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_durable_authorization_blocks_before_receipt_or_provider_call(
|
|
monkeypatch,
|
|
) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
|
|
async def _blocked(_authorization_ref: str) -> dict:
|
|
raise RuntimeError("paid_canary_authorization_expired")
|
|
|
|
monkeypatch.setattr(module, "_require_paid_canary_authorization", _blocked)
|
|
providers = [
|
|
_Provider("ollama", include_receipt=False),
|
|
_Provider("ollama_gcp_b", include_receipt=False),
|
|
_Provider("ollama_local", include_receipt=False),
|
|
_Provider("claude"),
|
|
_Provider("gemini"),
|
|
]
|
|
repository = _Repository()
|
|
|
|
with pytest.raises(RuntimeError, match="paid_canary_authorization_expired"):
|
|
await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-auth-blocked",
|
|
operator_id="test-operator",
|
|
authorization_ref="00000000-0000-4000-8000-000000000001",
|
|
ollama_gcp_a_provider=providers[0],
|
|
ollama_gcp_b_provider=providers[1],
|
|
ollama_local_provider=providers[2],
|
|
claude_provider=providers[3],
|
|
gemini_provider=providers[4],
|
|
operation_repository=repository,
|
|
)
|
|
|
|
assert repository.rows == []
|
|
assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_atomic_execution_claim_blocks_concurrent_authorization_reuse(
|
|
monkeypatch,
|
|
) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
_configure_runtime_control(monkeypatch)
|
|
|
|
async def _claim_held(_run_id: str, _owner: str) -> bool:
|
|
return False
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"_acquire_paid_canary_execution_claim",
|
|
_claim_held,
|
|
)
|
|
providers = [_Provider(name) for name in ("a", "b", "c", "d", "e")]
|
|
repository = _Repository()
|
|
|
|
with pytest.raises(
|
|
RuntimeError,
|
|
match="paid_canary_authorization_execution_claim_held",
|
|
):
|
|
await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-concurrent",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=providers[0],
|
|
ollama_gcp_b_provider=providers[1],
|
|
ollama_local_provider=providers[2],
|
|
claude_provider=providers[3],
|
|
gemini_provider=providers[4],
|
|
operation_repository=repository,
|
|
)
|
|
|
|
assert repository.rows == []
|
|
assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_canary_discards_content_and_writes_bounded_receipt(monkeypatch) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
gcp_a = _Provider("ollama", include_receipt=False)
|
|
gcp_b = _Provider("ollama_gcp_b", include_receipt=False)
|
|
local = _Provider("ollama_local", include_receipt=False)
|
|
claude = _Provider("claude")
|
|
gemini = _Provider("gemini")
|
|
repository = _Repository()
|
|
|
|
receipt = await run_paid_provider_canary_validation(
|
|
run_ref="source-sha",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=gcp_a,
|
|
ollama_gcp_b_provider=gcp_b,
|
|
ollama_local_provider=local,
|
|
claude_provider=claude,
|
|
gemini_provider=gemini,
|
|
operation_repository=repository,
|
|
)
|
|
|
|
assert receipt["independent_verifier"]["passed"] is True
|
|
assert receipt["independent_verifier"]["paid_activation_verifier_passed"] is True
|
|
assert receipt["independent_verifier"]["five_lane_verifier_passed"] is True
|
|
assert receipt["independent_verifier"]["paid_provider_comparison"] == (
|
|
"initial_canary_tie_no_overall_winner"
|
|
)
|
|
assert (
|
|
receipt["policy_decision"]["durable_authorization_receipt"]["authorized"]
|
|
is True
|
|
)
|
|
assert receipt["bounded_execution"]["provider_order"] == [
|
|
"ollama_gcp_a",
|
|
"ollama_gcp_b",
|
|
"ollama_local",
|
|
"claude",
|
|
"gemini",
|
|
]
|
|
assert receipt["source_receipt"]["gcp_transport"] == {
|
|
"status": "degraded_public_http",
|
|
"sanitized_only": True,
|
|
"receipt_ids": [
|
|
"cloud-transport:" + "a" * 32,
|
|
"cloud-transport:" + "b" * 32,
|
|
],
|
|
"preflight": [
|
|
{
|
|
"provider": "ollama_gcp_a",
|
|
"status": "verified",
|
|
"receipt_id": "cloud-transport:" + "a" * 32,
|
|
},
|
|
{
|
|
"provider": "ollama_gcp_b",
|
|
"status": "verified",
|
|
"receipt_id": "cloud-transport:" + "b" * 32,
|
|
},
|
|
],
|
|
"same_run_bound": True,
|
|
"durable_readback_verified": True,
|
|
"encrypted_transport_claimed": False,
|
|
}
|
|
assert [
|
|
row["contract_score_percent"] for row in receipt["bounded_execution"]["results"]
|
|
] == [100, 100, 100, 100, 100]
|
|
assert all(
|
|
"raw_response" not in row for row in receipt["bounded_execution"]["results"]
|
|
)
|
|
assert receipt["durable_operation_writeback"] == {
|
|
"start_recorded": True,
|
|
"lease_change_recorded": True,
|
|
"lease_change_record_id": "row-2",
|
|
"comparison_ready_recorded": True,
|
|
"comparison_ready_record_id": "row-3",
|
|
"rollback_change_recorded": True,
|
|
"rollback_change_record_id": "row-4",
|
|
"activation_failure_recorded": False,
|
|
"activation_failure_record_id": None,
|
|
"completion_recorded": True,
|
|
"completion_record_id": "row-5",
|
|
}
|
|
assert [row["event_type"] for row in repository.rows] == [
|
|
"EXECUTION_STARTED",
|
|
"CHANGE_APPLIED",
|
|
"PRE_FLIGHT_PASSED",
|
|
"CHANGE_APPLIED",
|
|
"EXECUTION_COMPLETED",
|
|
]
|
|
assert claude.closed is True
|
|
assert gemini.closed is True
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
assert state["lock_owner"] is None
|
|
assert state["active_owner"] is None
|
|
assert state["execution_claim_owner"] is None
|
|
assert receipt["policy_decision"]["authorization_execution_claim"] == {
|
|
"acquired": True,
|
|
"released_after_durable_start": True,
|
|
"same_run_bound": True,
|
|
}
|
|
assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == (
|
|
"validated_comparison_rollback_verified"
|
|
)
|
|
assert receipt["policy_decision"]["persistent_promotion_authorized"] is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_canary_verifier_rejects_a_mutating_false_positive(monkeypatch) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
receipt = await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-failed",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider("ollama", include_receipt=False),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude", action="RESTART_DEPLOYMENT"),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=_Repository(),
|
|
)
|
|
|
|
assert receipt["independent_verifier"]["passed"] is False
|
|
assert (
|
|
receipt["bounded_execution"]["results"][3]["contract_checks"][
|
|
"expected_no_action"
|
|
]
|
|
is False
|
|
)
|
|
assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
assert state["lock_owner"] is None
|
|
assert state["active_owner"] is None
|
|
assert state["execution_claim_owner"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ollama_baseline_failure_forces_paired_paid_canary_rollback(
|
|
monkeypatch,
|
|
) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
receipt = await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-partial-baseline",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider("ollama", success=False),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude"),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=_Repository(),
|
|
)
|
|
|
|
assert receipt["independent_verifier"]["passed"] is False
|
|
assert receipt["independent_verifier"]["paid_activation_verifier_passed"] is False
|
|
assert (
|
|
receipt["independent_verifier"]["paid_provider_contract_verifier_passed"]
|
|
is True
|
|
)
|
|
assert receipt["independent_verifier"]["five_lane_verifier_passed"] is False
|
|
assert receipt["independent_verifier"]["five_lane_comparison_ready"] is False
|
|
assert receipt["independent_verifier"]["ollama_baseline_available_count"] == 2
|
|
assert receipt["bounded_execution"]["results"][0]["provider"] == ("ollama_gcp_a")
|
|
assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True
|
|
assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == (
|
|
"failed_rollback_verified"
|
|
)
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
assert state["active_owner"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_missing_paid_generation_receipt_forces_atomic_rollback(
|
|
monkeypatch,
|
|
) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
receipt = await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-missing-receipt",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider("ollama", include_receipt=False),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude", include_receipt=False),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=_Repository(),
|
|
)
|
|
|
|
assert receipt["independent_verifier"]["passed"] is False
|
|
assert receipt["rollback_or_no_write_terminal"]["paid_canary_state"] == (
|
|
"failed_rollback_verified"
|
|
)
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
assert state["lock_owner"] is None
|
|
assert state["active_owner"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_paid_or_baseline_call_without_durable_start_receipt(
|
|
monkeypatch,
|
|
) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
providers = [_Provider(name) for name in ("a", "b", "c", "d", "e")]
|
|
|
|
with pytest.raises(
|
|
RuntimeError,
|
|
match="paid_provider_canary_start_receipt_not_recorded",
|
|
):
|
|
await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-no-start",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=providers[0],
|
|
ollama_gcp_b_provider=providers[1],
|
|
ollama_local_provider=providers[2],
|
|
claude_provider=providers[3],
|
|
gemini_provider=providers[4],
|
|
operation_repository=_Repository(omit_start=True),
|
|
)
|
|
|
|
assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0]
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
assert state["lock_owner"] is None
|
|
assert state["active_owner"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_failed_exact_receipt_cannot_promote_and_rolls_back(monkeypatch) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
|
|
async def _failed_receipt(**kwargs) -> dict:
|
|
return {
|
|
"provider": kwargs["provider"],
|
|
"receipt_id": kwargs["receipt_id"],
|
|
"found": True,
|
|
"correlation_verified": True,
|
|
"finalized_accounted": True,
|
|
"reservation_released": True,
|
|
"success": False,
|
|
"status": "failed_estimate_charged",
|
|
"accounted_tokens": 120,
|
|
"accounted_cost_usd": 0.001,
|
|
}
|
|
|
|
monkeypatch.setattr(module, "_generation_receipt_readback", _failed_receipt)
|
|
receipt = await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-failed-exact-receipt",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider("ollama", include_receipt=False),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude"),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=_Repository(),
|
|
)
|
|
|
|
assert receipt["independent_verifier"]["passed"] is False
|
|
assert (
|
|
receipt["independent_verifier"]["generation_receipt_readback_passed"] is False
|
|
)
|
|
assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aggregate_run_budget_must_be_finalized_under_exact_cap(
|
|
monkeypatch,
|
|
) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
|
|
async def _pending_budget(*, run_id: str) -> dict:
|
|
from src.services.ai_rate_limiter import AIRateLimiter
|
|
|
|
budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013")
|
|
return {
|
|
"schema_version": "ai_paid_run_budget_receipt_v1",
|
|
"run_budget_id": f"paid-run-{budget_hash[:32]}",
|
|
"found": True,
|
|
"status": "pending_finalize",
|
|
"max_cost_micro_usd": 250_000,
|
|
"reserved_cost_micro_usd": 1000,
|
|
"accounted_cost_micro_usd": 1000,
|
|
"reservation_count": 2,
|
|
"finalized_count": 1,
|
|
"correlation_verified": True,
|
|
"cap_verified": True,
|
|
"terminal_verified": False,
|
|
}
|
|
|
|
monkeypatch.setattr(module, "_run_budget_readback", _pending_budget)
|
|
receipt = await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-pending-run-budget",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider("ollama", include_receipt=False),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude"),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=_Repository(),
|
|
)
|
|
|
|
assert receipt["independent_verifier"]["passed"] is False
|
|
assert receipt["independent_verifier"]["run_budget_readback_passed"] is False
|
|
assert (
|
|
receipt["independent_verifier"]["generation_receipt_readback_passed"] is False
|
|
)
|
|
assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_canary_rejects_generation_and_aggregate_accounting_mismatch(
|
|
monkeypatch,
|
|
) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
|
|
async def _mismatched_budget(*, run_id: str) -> dict:
|
|
from src.services.ai_rate_limiter import AIRateLimiter
|
|
|
|
budget_hash = AIRateLimiter._identity_hash(run_id, "AIA-SRE-013")
|
|
return {
|
|
"schema_version": "ai_paid_run_budget_receipt_v1",
|
|
"run_budget_id": f"paid-run-{budget_hash[:32]}",
|
|
"found": True,
|
|
"status": "finalized",
|
|
"max_cost_micro_usd": 250_000,
|
|
"reserved_cost_micro_usd": 0,
|
|
"accounted_cost_micro_usd": 1999,
|
|
"reservation_count": 2,
|
|
"finalized_count": 2,
|
|
"correlation_verified": True,
|
|
"cap_verified": True,
|
|
"terminal_verified": True,
|
|
}
|
|
|
|
monkeypatch.setattr(module, "_run_budget_readback", _mismatched_budget)
|
|
repository = _Repository()
|
|
receipt = await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-accounting-mismatch",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider("ollama", include_receipt=False),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude"),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=repository,
|
|
)
|
|
|
|
assert receipt["independent_verifier"]["run_budget_readback_passed"] is False
|
|
assert receipt["independent_verifier"]["passed"] is False
|
|
assert "PRE_FLIGHT_PASSED" not in {row["event_type"] for row in repository.rows}
|
|
assert receipt["rollback_or_no_write_terminal"]["rollback_verified"] is True
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cancelled_paid_call_runs_verified_compensation(monkeypatch) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
repository = _Repository()
|
|
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-cancelled",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider("ollama", include_receipt=False),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude", cancel=True),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=repository,
|
|
)
|
|
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
assert state["lock_owner"] is None
|
|
assert state["active_owner"] is None
|
|
assert repository.rows[-1]["event_type"] == "EXECUTION_COMPLETED"
|
|
assert repository.rows[-1]["success"] is False
|
|
assert repository.rows[-1]["context"]["rollback_verified"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ambiguous_lease_failure_is_compensated_and_verified(monkeypatch) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
state = _configure_runtime_control(monkeypatch)
|
|
|
|
async def _ambiguous_acquire(owner: str) -> bool:
|
|
state.update(
|
|
claude=False,
|
|
gemini=False,
|
|
lock_owner=owner,
|
|
active_owner=None,
|
|
lease_ttl=300,
|
|
)
|
|
return False
|
|
|
|
monkeypatch.setattr(module, "_acquire_paid_canary_lease", _ambiguous_acquire)
|
|
with pytest.raises(
|
|
RuntimeError,
|
|
match="paid_provider_canary_activation_lease_failed",
|
|
):
|
|
await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-ambiguous-lease",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider("ollama", include_receipt=False),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude"),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=_Repository(),
|
|
)
|
|
|
|
assert state["claude"] is True
|
|
assert state["gemini"] is True
|
|
assert state["lock_owner"] is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_provider_error_text_is_never_written_to_safe_receipt(
|
|
monkeypatch,
|
|
) -> None:
|
|
from src.services import paid_provider_canary_validation as module
|
|
|
|
monkeypatch.setattr(
|
|
module,
|
|
"settings",
|
|
SimpleNamespace(
|
|
CLAUDE_EXECUTION_MODE="canary",
|
|
CLAUDE_CANARY_PERCENT=5,
|
|
GEMINI_EXECUTION_MODE="canary",
|
|
GEMINI_CANARY_PERCENT=5,
|
|
ALERT_OLLAMA_MODEL="qwen3:14b",
|
|
),
|
|
)
|
|
_configure_runtime_control(monkeypatch)
|
|
unsafe_error = "http://192.168.0.111:11434/?key=secret-like-value"
|
|
receipt = await run_paid_provider_canary_validation(
|
|
run_ref="source-sha-sanitized-error",
|
|
operator_id="test-operator",
|
|
authorization_ref=_AUTHORIZATION_REF,
|
|
ollama_gcp_a_provider=_Provider(
|
|
"ollama",
|
|
success=False,
|
|
include_receipt=False,
|
|
error=unsafe_error,
|
|
),
|
|
ollama_gcp_b_provider=_Provider("ollama_gcp_b", include_receipt=False),
|
|
ollama_local_provider=_Provider("ollama_local", include_receipt=False),
|
|
claude_provider=_Provider("claude"),
|
|
gemini_provider=_Provider("gemini"),
|
|
operation_repository=_Repository(),
|
|
)
|
|
|
|
serialized = json.dumps(receipt, sort_keys=True)
|
|
assert unsafe_error not in serialized
|
|
assert "192.168.0.111" not in serialized
|
|
assert "secret-like-value" not in serialized
|
|
assert receipt["bounded_execution"]["results"][0]["error_code"] == (
|
|
"provider_error_redacted"
|
|
)
|