fix(platform): bind AI runs to canonical correlation
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 51s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m45s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 33s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 51s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m45s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 33s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
This commit is contained in:
162
apps/api/tests/integration/test_platform_run_idempotency_race.py
Normal file
162
apps/api/tests/integration/test_platform_run_idempotency_race.py
Normal file
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import make_url, text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from src.services import platform_runtime
|
||||
|
||||
_TEST_DATABASE_ENV = "AWOOOI_RUNTIME_IDENTITY_TEST_DATABASE_URL"
|
||||
_TEST_DATABASE_NAME = "awoooi_runtime_identity_test"
|
||||
_LOCAL_DATABASE_HOSTS = frozenset({"127.0.0.1", "localhost"})
|
||||
|
||||
|
||||
def _validated_test_database_url() -> str:
|
||||
raw_url = os.getenv(_TEST_DATABASE_ENV, "").strip()
|
||||
if not raw_url:
|
||||
pytest.skip(f"{_TEST_DATABASE_ENV} is required for the real PostgreSQL race test")
|
||||
|
||||
parsed = make_url(raw_url)
|
||||
if (
|
||||
parsed.get_backend_name() != "postgresql"
|
||||
or parsed.get_driver_name() != "asyncpg"
|
||||
or parsed.host not in _LOCAL_DATABASE_HOSTS
|
||||
or parsed.database != _TEST_DATABASE_NAME
|
||||
):
|
||||
pytest.fail(
|
||||
f"{_TEST_DATABASE_ENV} must use postgresql+asyncpg on localhost "
|
||||
f"with database {_TEST_DATABASE_NAME!r}"
|
||||
)
|
||||
return raw_url
|
||||
|
||||
|
||||
_RUN_STATE_DDL = """
|
||||
CREATE TABLE awooop_run_state (
|
||||
run_id UUID PRIMARY KEY,
|
||||
project_id VARCHAR(64) NOT NULL,
|
||||
agent_id VARCHAR(128) NOT NULL,
|
||||
state VARCHAR(32) NOT NULL DEFAULT 'pending',
|
||||
lease_until TIMESTAMP,
|
||||
next_attempt_at TIMESTAMP,
|
||||
heartbeat_at TIMESTAMP,
|
||||
worker_id VARCHAR(128),
|
||||
attempt_count SMALLINT NOT NULL DEFAULT 0,
|
||||
max_attempts SMALLINT NOT NULL DEFAULT 3,
|
||||
trace_id VARCHAR(128),
|
||||
trigger_type VARCHAR(32),
|
||||
trigger_ref VARCHAR(256),
|
||||
is_shadow BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
input_sha256 VARCHAR(64),
|
||||
output_sha256 VARCHAR(64),
|
||||
cost_usd NUMERIC(10, 4) NOT NULL DEFAULT 0,
|
||||
step_count SMALLINT NOT NULL DEFAULT 0,
|
||||
error_code VARCHAR(64),
|
||||
error_detail TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
started_at TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
timeout_at TIMESTAMP
|
||||
)
|
||||
"""
|
||||
|
||||
_RUN_IDEMPOTENCY_DDL = """
|
||||
CREATE TABLE awooop_run_idempotency (
|
||||
idempotency_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
project_id VARCHAR(64) NOT NULL,
|
||||
channel_type VARCHAR(32) NOT NULL,
|
||||
provider_event_id VARCHAR(256) NOT NULL,
|
||||
run_id UUID NOT NULL REFERENCES awooop_run_state(run_id),
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
CONSTRAINT uix_run_idempotency_key
|
||||
UNIQUE (project_id, channel_type, provider_event_id)
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_provider_event_has_one_postgresql_winner(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
engine = create_async_engine(_validated_test_database_url(), poolclass=NullPool)
|
||||
factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False)
|
||||
tables_created = False
|
||||
|
||||
@asynccontextmanager
|
||||
async def test_db_context(project_id: str):
|
||||
assert project_id == "awoooi"
|
||||
async with factory() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
|
||||
async def skip_redis_cache(*_args: object) -> None:
|
||||
return None
|
||||
|
||||
try:
|
||||
async with engine.begin() as connection:
|
||||
existing_tables = (
|
||||
await connection.execute(
|
||||
text(
|
||||
"SELECT to_regclass('awooop_run_state'), "
|
||||
"to_regclass('awooop_run_idempotency')"
|
||||
)
|
||||
)
|
||||
).one()
|
||||
if any(existing_tables):
|
||||
pytest.fail(
|
||||
"dedicated runtime identity test database must not contain "
|
||||
"pre-existing run tables"
|
||||
)
|
||||
await connection.execute(text(_RUN_STATE_DDL))
|
||||
await connection.execute(text(_RUN_IDEMPOTENCY_DDL))
|
||||
tables_created = True
|
||||
|
||||
monkeypatch.setattr(platform_runtime, "get_db_context", test_db_context)
|
||||
monkeypatch.setattr(platform_runtime, "_cache_idempotency", skip_redis_cache)
|
||||
|
||||
first, second = await asyncio.gather(
|
||||
platform_runtime.create_run(
|
||||
project_id="awoooi",
|
||||
agent_id="runtime-agent",
|
||||
trigger_type="channel_event",
|
||||
channel_type="telegram",
|
||||
provider_event_id="same-real-postgresql-event",
|
||||
),
|
||||
platform_runtime.create_run(
|
||||
project_id="awoooi",
|
||||
agent_id="runtime-agent",
|
||||
trigger_type="channel_event",
|
||||
channel_type="telegram",
|
||||
provider_event_id="same-real-postgresql-event",
|
||||
),
|
||||
)
|
||||
|
||||
async with engine.connect() as connection:
|
||||
run_count = await connection.scalar(
|
||||
text("SELECT COUNT(*) FROM awooop_run_state")
|
||||
)
|
||||
mapping_count = await connection.scalar(
|
||||
text("SELECT COUNT(*) FROM awooop_run_idempotency")
|
||||
)
|
||||
durable_run_id = await connection.scalar(
|
||||
text("SELECT run_id FROM awooop_run_idempotency")
|
||||
)
|
||||
|
||||
assert first[0] == second[0] == durable_run_id
|
||||
assert sorted((first[1], second[1])) == [False, True]
|
||||
assert run_count == 1
|
||||
assert mapping_count == 1
|
||||
finally:
|
||||
if tables_created:
|
||||
async with engine.begin() as connection:
|
||||
await connection.execute(text("DROP TABLE awooop_run_idempotency"))
|
||||
await connection.execute(text("DROP TABLE awooop_run_state"))
|
||||
await engine.dispose()
|
||||
@@ -2,8 +2,11 @@ from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import json
|
||||
from uuid import uuid4
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services import channel_hub
|
||||
from src.services.channel_hub import (
|
||||
_db_timestamp_now,
|
||||
_send_telegram_interim,
|
||||
@@ -17,9 +20,11 @@ from src.services.channel_hub import (
|
||||
format_alertmanager_event_content,
|
||||
format_grouped_alert_digest_text,
|
||||
format_grouped_alert_event_content,
|
||||
handle_telegram_inbound,
|
||||
mirror_inbound_event,
|
||||
record_outbound_message,
|
||||
)
|
||||
from src.services.runtime_correlation import correlation_for_run
|
||||
from src.services.telegram_gateway import _telegram_destination_binding
|
||||
|
||||
_FAKE_SECRET_SHAPED_VALUE = "1234567890:" + "abcdefghijklmnopqrstuvwxyzABCDEFGH"
|
||||
@@ -357,6 +362,69 @@ async def test_completed_shadow_run_sets_run_state_not_null_defaults() -> None:
|
||||
assert "0, 3, 0.0000, 0" in session.statement
|
||||
assert session.params["project_id"] == "awoooi"
|
||||
assert session.params["run_id"] == run_id
|
||||
correlation = correlation_for_run("awoooi", run_id)
|
||||
assert session.params["trace_id"] == correlation.trace_id
|
||||
assert "trace_id, trigger_type" in session.statement
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_telegram_inbound_persists_and_returns_same_run_correlation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
|
||||
identity = correlation_for_run("awoooi", run_id)
|
||||
mirrored: dict[str, object] = {}
|
||||
|
||||
async def fake_create_run(**_kwargs: object) -> tuple[UUID, bool]:
|
||||
return run_id, False
|
||||
|
||||
async def fake_read_run_status(
|
||||
received_run_id: UUID,
|
||||
project_id: str,
|
||||
) -> dict[str, object]:
|
||||
assert received_run_id == run_id
|
||||
assert project_id == "awoooi"
|
||||
return {
|
||||
**identity.public_dict(),
|
||||
"correlation_status": "canonical",
|
||||
}
|
||||
|
||||
async def fake_mirror(_db: object, **kwargs: object) -> UUID:
|
||||
mirrored.update(kwargs)
|
||||
return UUID("019f88d5-f6c9-7a18-b764-df0df0cbe223")
|
||||
|
||||
async def fake_schedule(**_kwargs: object) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(channel_hub, "create_run", fake_create_run)
|
||||
monkeypatch.setattr(
|
||||
channel_hub,
|
||||
"read_platform_run_status",
|
||||
fake_read_run_status,
|
||||
)
|
||||
monkeypatch.setattr(channel_hub, "mirror_inbound_event", fake_mirror)
|
||||
monkeypatch.setattr(channel_hub, "schedule_interim_feedback", fake_schedule)
|
||||
|
||||
result = await handle_telegram_inbound(
|
||||
object(), # type: ignore[arg-type]
|
||||
project_id="awoooi",
|
||||
agent_id="openclaw-sre",
|
||||
message_id="42",
|
||||
user_id="7",
|
||||
chat_id="-1007",
|
||||
text="status",
|
||||
)
|
||||
|
||||
assert result["run_id"] == str(run_id)
|
||||
assert result["trace_id"] == identity.trace_id
|
||||
assert result["work_item_id"] == identity.work_item_id
|
||||
assert result["correlation_status"] == "canonical"
|
||||
source_envelope = mirrored["source_envelope"]
|
||||
assert isinstance(source_envelope, dict)
|
||||
assert source_envelope["runtime_correlation"] == {
|
||||
**identity.public_dict(),
|
||||
"correlation_status": "canonical",
|
||||
}
|
||||
|
||||
|
||||
async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None:
|
||||
@@ -385,6 +453,10 @@ async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None:
|
||||
|
||||
assert "content_redacted" in session.statement
|
||||
assert "source_envelope" in session.statement
|
||||
assert (
|
||||
"COALESCE(\n awooop_conversation_event.run_id,"
|
||||
in session.statement
|
||||
)
|
||||
assert session.params["redaction_version"] == "audit_sink_v1"
|
||||
assert len(json.loads(session.params["source_envelope"])["content_sha256"]) == 64
|
||||
assert "1234567890:" not in session.params["content_redacted"]
|
||||
|
||||
@@ -22,9 +22,10 @@ from src.services.paid_provider_canary_authorization import ( # noqa: E402
|
||||
require_paid_provider_canary_authorization,
|
||||
verify_paid_provider_canary_authorization,
|
||||
)
|
||||
from src.services.runtime_correlation import canonical_traceparent # noqa: E402
|
||||
|
||||
_SELECTED_RUN_ID = UUID("267a3d26-3c29-470f-8fe3-388a2f408f39")
|
||||
_TRACE_ID = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01"
|
||||
_TRACE_ID = canonical_traceparent(_SELECTED_RUN_ID)
|
||||
|
||||
|
||||
class _Reader:
|
||||
@@ -56,9 +57,10 @@ def _evidence(
|
||||
now: datetime,
|
||||
run_id: UUID | None = None,
|
||||
) -> PaidProviderCanaryAuthorizationEvidence:
|
||||
selected_run_id = run_id or _SELECTED_RUN_ID
|
||||
return PaidProviderCanaryAuthorizationEvidence(
|
||||
run_id=run_id or _SELECTED_RUN_ID,
|
||||
trace_id=_TRACE_ID,
|
||||
run_id=selected_run_id,
|
||||
trace_id=canonical_traceparent(selected_run_id),
|
||||
project_id=PAID_CANARY_PROJECT_ID,
|
||||
agent_id=PAID_CANARY_AGENT_ID,
|
||||
state="running",
|
||||
@@ -167,7 +169,7 @@ async def test_authorization_ref_must_be_canonical_uuid_without_db_read() -> Non
|
||||
("trigger_type", "schedule", "trigger_type_exact"),
|
||||
("trigger_ref", "paid-provider-canary:OTHER", "trigger_ref_exact"),
|
||||
("input_sha256", "0" * 64, "input_sha256_exact"),
|
||||
("trace_id", None, "trace_id_present"),
|
||||
("trace_id", None, "trace_id_run_bound"),
|
||||
("approval_step_count", 0, "approval_step_exactly_once"),
|
||||
("approval_step_status", "failed", "approval_step_success"),
|
||||
("approval_step_was_blocked", True, "approval_step_not_blocked"),
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import UUID
|
||||
from uuid import RFC_4122, UUID
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
@@ -25,6 +25,7 @@ from src.services.paid_provider_canary_authorization import ( # noqa: E402
|
||||
PAID_CANARY_TRIGGER_REF,
|
||||
PAID_CANARY_TRIGGER_TYPE,
|
||||
paid_provider_canary_authorization_input_sha256,
|
||||
paid_provider_canary_run_selected,
|
||||
)
|
||||
from src.services.paid_provider_canary_gate5_run import ( # noqa: E402
|
||||
PAID_CANARY_AUTHORIZATION_SCOPE,
|
||||
@@ -32,14 +33,28 @@ from src.services.paid_provider_canary_gate5_run import ( # noqa: E402
|
||||
create_paid_provider_canary_gate5_run,
|
||||
paid_provider_canary_approval_contract_errors,
|
||||
)
|
||||
from src.services.runtime_correlation import ( # noqa: E402
|
||||
canonical_traceparent,
|
||||
is_canonical_traceparent,
|
||||
)
|
||||
|
||||
_SELECTED_RUN_ID = UUID("267a3d26-3c29-470f-8fe3-388a2f408f39")
|
||||
_UNSELECTED_RUN_ID = UUID("00000000-0000-4000-8000-000000000001")
|
||||
|
||||
|
||||
class _Repository:
|
||||
def __init__(self, *, duplicate: bool = False, state: str = "waiting_approval") -> None:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
duplicate: bool = False,
|
||||
state: str = "waiting_approval",
|
||||
mismatched_duplicate_trace: bool = False,
|
||||
unselected_duplicate_run: bool = False,
|
||||
) -> None:
|
||||
self.duplicate = duplicate
|
||||
self.state = state
|
||||
self.mismatched_duplicate_trace = mismatched_duplicate_trace
|
||||
self.unselected_duplicate_run = unselected_duplicate_run
|
||||
self.calls: list[dict[str, object]] = []
|
||||
|
||||
async def create_or_get(self, **kwargs: object) -> tuple[
|
||||
@@ -55,8 +70,15 @@ class _Repository:
|
||||
assert isinstance(timeout_at, datetime)
|
||||
assert isinstance(input_sha256, str)
|
||||
assert isinstance(trace_id, str)
|
||||
assert is_canonical_traceparent(trace_id, run_id=run_id) is True
|
||||
if self.duplicate:
|
||||
run_id = _SELECTED_RUN_ID
|
||||
run_id = (
|
||||
_UNSELECTED_RUN_ID
|
||||
if self.unselected_duplicate_run
|
||||
else _SELECTED_RUN_ID
|
||||
)
|
||||
if not self.mismatched_duplicate_trace:
|
||||
trace_id = canonical_traceparent(run_id)
|
||||
return (
|
||||
PaidProviderCanaryGate5RunRecord(
|
||||
run_id=run_id,
|
||||
@@ -160,6 +182,45 @@ async def test_duplicate_active_authorization_is_reused_without_scope_expansion(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplicate_active_authorization_rejects_unbound_trace() -> None:
|
||||
repository = _Repository(
|
||||
duplicate=True,
|
||||
state="running",
|
||||
mismatched_duplicate_trace=True,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="paid_provider_canary_gate5_run_trace_unbound",
|
||||
):
|
||||
await create_paid_provider_canary_gate5_run(
|
||||
operator_id="ops@example.com",
|
||||
repository=repository,
|
||||
now=datetime(2026, 7, 16, 9, 0, tzinfo=UTC),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_injected_repository_rejects_unselected_execution_run() -> None:
|
||||
assert paid_provider_canary_run_selected(_UNSELECTED_RUN_ID) is False
|
||||
repository = _Repository(
|
||||
duplicate=True,
|
||||
state="running",
|
||||
unselected_duplicate_run=True,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="paid_provider_canary_gate5_run_not_selected",
|
||||
):
|
||||
await create_paid_provider_canary_gate5_run(
|
||||
operator_id="ops@example.com",
|
||||
repository=repository,
|
||||
now=datetime(2026, 7, 16, 9, 0, tzinfo=UTC),
|
||||
)
|
||||
|
||||
|
||||
def _run(
|
||||
*,
|
||||
now: datetime,
|
||||
@@ -171,7 +232,7 @@ def _run(
|
||||
project_id=PAID_CANARY_PROJECT_ID,
|
||||
agent_id=agent_id,
|
||||
state="waiting_approval",
|
||||
trace_id="00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
|
||||
trace_id=canonical_traceparent(_SELECTED_RUN_ID),
|
||||
trigger_type=PAID_CANARY_TRIGGER_TYPE,
|
||||
trigger_ref=trigger_ref,
|
||||
is_shadow=False,
|
||||
@@ -473,21 +534,51 @@ async def test_generic_runtime_service_persists_shadow_true(
|
||||
assert len(captured) == 1
|
||||
assert captured[0].is_shadow is True
|
||||
assert captured[0].state == "pending"
|
||||
assert captured[0].run_id.version == 7
|
||||
assert captured[0].run_id.variant == RFC_4122
|
||||
assert is_canonical_traceparent(
|
||||
str(captured[0].trace_id),
|
||||
run_id=captured[0].run_id,
|
||||
) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generic_public_run_creation_remains_shadow_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
run_id = UUID("018f2d04-4c37-7a18-b764-df0df0cbe222")
|
||||
|
||||
async def fake_create_run(**_: object) -> tuple[UUID, bool]:
|
||||
return UUID("018f2d04-4c37-7a18-b764-df0df0cbe222"), False
|
||||
return run_id, False
|
||||
|
||||
async def fake_read_run_status(
|
||||
received_run_id: UUID,
|
||||
project_id: str,
|
||||
) -> dict[str, object]:
|
||||
assert received_run_id == run_id
|
||||
assert project_id == "awoooi"
|
||||
return {
|
||||
"trace_id": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
|
||||
"work_item_id": f"platform-run:awoooi:{run_id}",
|
||||
"correlation_status": "canonical",
|
||||
}
|
||||
|
||||
async def fake_write_audit(**kwargs: object) -> None:
|
||||
details = kwargs["details"]
|
||||
assert isinstance(details, dict)
|
||||
assert details["is_shadow"] is True
|
||||
assert details["trace_id"] == (
|
||||
"00-0123456789abcdef0123456789abcdef-0123456789abcdef-01"
|
||||
)
|
||||
assert details["work_item_id"] == f"platform-run:awoooi:{run_id}"
|
||||
assert details["correlation_status"] == "canonical"
|
||||
|
||||
monkeypatch.setattr(public_runs, "create_run", fake_create_run)
|
||||
monkeypatch.setattr(
|
||||
public_runs,
|
||||
"read_platform_run_status",
|
||||
fake_read_run_status,
|
||||
)
|
||||
monkeypatch.setattr(public_runs, "write_audit", fake_write_audit)
|
||||
|
||||
response = await public_runs.create_platform_run(
|
||||
@@ -499,4 +590,56 @@ async def test_generic_public_run_creation_remains_shadow_only(
|
||||
)
|
||||
|
||||
assert response.is_shadow is True
|
||||
assert response.trace_id.startswith("00-")
|
||||
assert response.work_item_id == f"platform-run:awoooi:{run_id}"
|
||||
assert response.correlation_status == "canonical"
|
||||
assert response.message == "Run 已接受(shadow mode)"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"identity_readback",
|
||||
[
|
||||
None,
|
||||
{
|
||||
"trace_id": "legacy-trace",
|
||||
"work_item_id": (
|
||||
"platform-run:awoooi:019f88d5-f6c9-7a18-b764-df0df0cbe222"
|
||||
),
|
||||
"correlation_status": "legacy_or_invalid_trace",
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_generic_public_run_creation_fails_closed_without_canonical_identity_readback(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
identity_readback: dict[str, str] | None,
|
||||
) -> None:
|
||||
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
|
||||
|
||||
async def fake_create_run(**_: object) -> tuple[UUID, bool]:
|
||||
return run_id, False
|
||||
|
||||
async def fake_read_run_status(
|
||||
_run_id: UUID,
|
||||
_project_id: str,
|
||||
) -> dict[str, str] | None:
|
||||
return identity_readback
|
||||
|
||||
monkeypatch.setattr(public_runs, "create_run", fake_create_run)
|
||||
monkeypatch.setattr(
|
||||
public_runs,
|
||||
"read_platform_run_status",
|
||||
fake_read_run_status,
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await public_runs.create_platform_run(
|
||||
public_runs.CreateRunRequest(
|
||||
project_id="awoooi",
|
||||
agent_id="generic-agent",
|
||||
trigger_type="api",
|
||||
)
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 500
|
||||
assert "correlation readback missing" in str(exc_info.value.detail)
|
||||
|
||||
@@ -19,9 +19,10 @@ from src.services.paid_provider_canary_validation import (
|
||||
run_paid_provider_canary_validation,
|
||||
select_claude_canary_run_id,
|
||||
)
|
||||
from src.services.runtime_correlation import canonical_traceparent
|
||||
|
||||
_AUTHORIZATION_REF = "267a3d26-3c29-470f-8fe3-388a2f408f39"
|
||||
_AUTHORIZATION_TRACE_ID = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01"
|
||||
_AUTHORIZATION_TRACE_ID = canonical_traceparent(UUID(_AUTHORIZATION_REF))
|
||||
|
||||
|
||||
def test_ollama_canary_uses_bounded_production_diagnose_timeout(
|
||||
@@ -592,6 +593,59 @@ async def test_optional_terminal_aol_projection_cannot_reopen_terminal_run() ->
|
||||
assert record_id is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_injected_authorization_receipt_rejects_trace_unbound_to_run(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from src.services import paid_provider_canary_validation as module
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"settings",
|
||||
SimpleNamespace(
|
||||
CLAUDE_EXECUTION_MODE="canary",
|
||||
CLAUDE_CANARY_PERCENT=5,
|
||||
GEMINI_EXECUTION_MODE="canary",
|
||||
GEMINI_CANARY_PERCENT=5,
|
||||
ALERT_OLLAMA_MODEL="qwen3:14b",
|
||||
),
|
||||
)
|
||||
execution_claim_attempted = False
|
||||
|
||||
async def _unexpected_claim(_run_id: str, _owner: str) -> bool:
|
||||
nonlocal execution_claim_attempted
|
||||
execution_claim_attempted = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(
|
||||
module,
|
||||
"_acquire_paid_canary_execution_claim",
|
||||
_unexpected_claim,
|
||||
)
|
||||
unbound_trace = canonical_traceparent(
|
||||
UUID("267a3d26-3c29-470f-8fe3-388a2f408f3a")
|
||||
)
|
||||
assert unbound_trace != _AUTHORIZATION_TRACE_ID
|
||||
|
||||
with pytest.raises(
|
||||
RuntimeError,
|
||||
match="paid_canary_authorization_same_run_binding_failed",
|
||||
):
|
||||
await module._run_paid_provider_canary_validation_core(
|
||||
run_ref="source-sha-unbound-trace",
|
||||
operator_id="test-operator",
|
||||
authorization_ref=_AUTHORIZATION_REF,
|
||||
verified_authorization_receipt={
|
||||
"authorization_ref": _AUTHORIZATION_REF,
|
||||
"run_id": _AUTHORIZATION_REF,
|
||||
"work_item_id": "AIA-SRE-013",
|
||||
"trace_id": unbound_trace,
|
||||
},
|
||||
)
|
||||
|
||||
assert execution_claim_attempted is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_durable_authorization_blocks_before_receipt_or_provider_call(
|
||||
monkeypatch,
|
||||
|
||||
257
apps/api/tests/test_runtime_correlation.py
Normal file
257
apps/api/tests/test_runtime_correlation.py
Normal file
@@ -0,0 +1,257 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import UTC, datetime
|
||||
from decimal import Decimal
|
||||
from types import SimpleNamespace
|
||||
from uuid import RFC_4122, UUID
|
||||
|
||||
import pytest
|
||||
|
||||
from src.db.awooop_models import AwoooPRunIdempotency, AwoooPRunState
|
||||
from src.services import platform_runtime
|
||||
from src.services.runtime_correlation import (
|
||||
canonical_traceparent,
|
||||
canonical_work_item_id,
|
||||
correlation_for_run,
|
||||
is_canonical_traceparent,
|
||||
new_uuid7,
|
||||
)
|
||||
|
||||
|
||||
def test_uuid7_has_rfc_version_variant_and_timestamp_layout() -> None:
|
||||
timestamp_ms = 1_753_155_200_123
|
||||
generated = new_uuid7(timestamp_ms=timestamp_ms, entropy=(1 << 74) - 1)
|
||||
|
||||
assert generated.version == 7
|
||||
assert generated.variant == RFC_4122
|
||||
assert generated.int >> 80 == timestamp_ms
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("timestamp_ms", "entropy"),
|
||||
[(-1, 0), (1 << 48, 0), (0, -1), (0, 1 << 74)],
|
||||
)
|
||||
def test_uuid7_rejects_out_of_range_inputs(
|
||||
timestamp_ms: int,
|
||||
entropy: int,
|
||||
) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
new_uuid7(timestamp_ms=timestamp_ms, entropy=entropy)
|
||||
|
||||
|
||||
def test_runtime_correlation_is_stable_and_bound_to_run() -> None:
|
||||
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
|
||||
correlation = correlation_for_run("awoooi", run_id)
|
||||
|
||||
assert correlation.run_id == run_id
|
||||
assert correlation.trace_id == canonical_traceparent(run_id)
|
||||
assert correlation.work_item_id == f"platform-run:awoooi:{run_id}"
|
||||
assert is_canonical_traceparent(correlation.trace_id, run_id=run_id) is True
|
||||
assert (
|
||||
is_canonical_traceparent(
|
||||
correlation.trace_id,
|
||||
run_id=UUID("019f88d5-f6c9-7a18-b764-df0df0cbe223"),
|
||||
)
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_correlation_rejects_invalid_project_id() -> None:
|
||||
with pytest.raises(ValueError, match="project_id_invalid"):
|
||||
canonical_work_item_id(
|
||||
"awoooi:forged",
|
||||
UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_status_exposes_canonical_identity_readback(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
|
||||
trace_id = canonical_traceparent(run_id)
|
||||
row = SimpleNamespace(
|
||||
run_id=run_id,
|
||||
project_id="awoooi",
|
||||
agent_id="runtime-agent",
|
||||
state="running",
|
||||
is_shadow=True,
|
||||
trace_id=trace_id,
|
||||
attempt_count=1,
|
||||
cost_usd=Decimal("0.0000"),
|
||||
step_count=2,
|
||||
error_code=None,
|
||||
created_at=datetime(2026, 7, 22, tzinfo=UTC),
|
||||
started_at=datetime(2026, 7, 22, tzinfo=UTC),
|
||||
completed_at=None,
|
||||
)
|
||||
|
||||
class _Result:
|
||||
def scalar_one_or_none(self) -> SimpleNamespace:
|
||||
return row
|
||||
|
||||
class _Db:
|
||||
async def execute(self, _statement: object) -> _Result:
|
||||
return _Result()
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_db_context(project_id: str):
|
||||
assert project_id == "awoooi"
|
||||
yield _Db()
|
||||
|
||||
monkeypatch.setattr(platform_runtime, "get_db_context", fake_db_context)
|
||||
|
||||
payload = await platform_runtime.get_run_status(run_id, "awoooi")
|
||||
|
||||
assert payload is not None
|
||||
assert payload["run_id"] == str(run_id)
|
||||
assert payload["trace_id"] == trace_id
|
||||
assert payload["work_item_id"] == f"platform-run:awoooi:{run_id}"
|
||||
assert payload["correlation_status"] == "canonical"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_status_marks_legacy_unbound_trace_without_rewriting_it(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
|
||||
row = SimpleNamespace(
|
||||
run_id=run_id,
|
||||
project_id="awoooi",
|
||||
agent_id="legacy-agent",
|
||||
state="completed",
|
||||
is_shadow=True,
|
||||
trace_id="legacy-trace",
|
||||
attempt_count=1,
|
||||
cost_usd=Decimal("0.0000"),
|
||||
step_count=0,
|
||||
error_code=None,
|
||||
created_at=None,
|
||||
started_at=None,
|
||||
completed_at=None,
|
||||
)
|
||||
|
||||
class _Result:
|
||||
def scalar_one_or_none(self) -> SimpleNamespace:
|
||||
return row
|
||||
|
||||
class _Db:
|
||||
async def execute(self, _statement: object) -> _Result:
|
||||
return _Result()
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_db_context(_project_id: str):
|
||||
yield _Db()
|
||||
|
||||
monkeypatch.setattr(platform_runtime, "get_db_context", fake_db_context)
|
||||
|
||||
payload = await platform_runtime.get_run_status(run_id, "awoooi")
|
||||
|
||||
assert payload is not None
|
||||
assert payload["trace_id"] == "legacy-trace"
|
||||
assert payload["correlation_status"] == "legacy_or_invalid_trace"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_provider_event_creation_returns_one_durable_winner(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
class _Result:
|
||||
def __init__(self, row: tuple[UUID] | None = None) -> None:
|
||||
self._row = row
|
||||
|
||||
def fetchone(self) -> tuple[UUID] | None:
|
||||
return self._row
|
||||
|
||||
class _Store:
|
||||
def __init__(self) -> None:
|
||||
self.claim_lock = asyncio.Lock()
|
||||
self.mapping: UUID | None = None
|
||||
self.runs: dict[UUID, AwoooPRunState] = {}
|
||||
self.lock_claim_count = 0
|
||||
|
||||
store = _Store()
|
||||
|
||||
class _Db:
|
||||
def __init__(self) -> None:
|
||||
self.lock_held = False
|
||||
self.pending_run: AwoooPRunState | None = None
|
||||
self.pending_mapping: AwoooPRunIdempotency | None = None
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
statement: object,
|
||||
_params: dict[str, str] | None = None,
|
||||
) -> _Result:
|
||||
sql = str(statement)
|
||||
if "pg_advisory_xact_lock" in sql:
|
||||
await store.claim_lock.acquire()
|
||||
self.lock_held = True
|
||||
store.lock_claim_count += 1
|
||||
await asyncio.sleep(0)
|
||||
return _Result()
|
||||
if "awooop_run_idempotency" in sql:
|
||||
return _Result(
|
||||
(store.mapping,) if store.mapping is not None else None
|
||||
)
|
||||
raise AssertionError(f"unexpected statement: {sql}")
|
||||
|
||||
def add(self, row: object) -> None:
|
||||
if isinstance(row, AwoooPRunState):
|
||||
self.pending_run = row
|
||||
elif isinstance(row, AwoooPRunIdempotency):
|
||||
self.pending_mapping = row
|
||||
else:
|
||||
raise AssertionError(f"unexpected row: {type(row).__name__}")
|
||||
|
||||
async def flush(self) -> None:
|
||||
assert self.lock_held is True
|
||||
assert self.pending_run is not None
|
||||
if self.pending_mapping is None:
|
||||
assert store.runs == {}
|
||||
store.runs[self.pending_run.run_id] = self.pending_run
|
||||
return
|
||||
assert store.mapping is None
|
||||
assert self.pending_mapping.run_id in store.runs
|
||||
store.mapping = self.pending_mapping.run_id
|
||||
|
||||
@asynccontextmanager
|
||||
async def fake_db_context(project_id: str):
|
||||
assert project_id == "awoooi"
|
||||
db = _Db()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
if db.lock_held:
|
||||
store.claim_lock.release()
|
||||
|
||||
async def fake_cache(*_args: object) -> None:
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(platform_runtime, "get_db_context", fake_db_context)
|
||||
monkeypatch.setattr(platform_runtime, "_cache_idempotency", fake_cache)
|
||||
|
||||
first, second = await asyncio.gather(
|
||||
platform_runtime.create_run(
|
||||
project_id="awoooi",
|
||||
agent_id="runtime-agent",
|
||||
trigger_type="channel_event",
|
||||
channel_type="telegram",
|
||||
provider_event_id="same-provider-event",
|
||||
),
|
||||
platform_runtime.create_run(
|
||||
project_id="awoooi",
|
||||
agent_id="runtime-agent",
|
||||
trigger_type="channel_event",
|
||||
channel_type="telegram",
|
||||
provider_event_id="same-provider-event",
|
||||
),
|
||||
)
|
||||
|
||||
assert first[0] == second[0] == store.mapping
|
||||
assert sorted((first[1], second[1])) == [False, True]
|
||||
assert list(store.runs) == [store.mapping]
|
||||
assert next(iter(store.runs.values())).timeout_at.tzinfo is None
|
||||
assert store.lock_claim_count == 2
|
||||
Reference in New Issue
Block a user