fix(platform): bind AI runs to canonical correlation
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 51s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m45s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 33s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled

This commit is contained in:
Your Name
2026-07-22 16:23:11 +08:00
parent 1cec67295b
commit 18b5ba25fa
13 changed files with 1051 additions and 105 deletions

View File

@@ -0,0 +1,162 @@
from __future__ import annotations
import asyncio
import os
from contextlib import asynccontextmanager
import pytest
from sqlalchemy import make_url, text
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.pool import NullPool
from src.services import platform_runtime
_TEST_DATABASE_ENV = "AWOOOI_RUNTIME_IDENTITY_TEST_DATABASE_URL"
_TEST_DATABASE_NAME = "awoooi_runtime_identity_test"
_LOCAL_DATABASE_HOSTS = frozenset({"127.0.0.1", "localhost"})
def _validated_test_database_url() -> str:
raw_url = os.getenv(_TEST_DATABASE_ENV, "").strip()
if not raw_url:
pytest.skip(f"{_TEST_DATABASE_ENV} is required for the real PostgreSQL race test")
parsed = make_url(raw_url)
if (
parsed.get_backend_name() != "postgresql"
or parsed.get_driver_name() != "asyncpg"
or parsed.host not in _LOCAL_DATABASE_HOSTS
or parsed.database != _TEST_DATABASE_NAME
):
pytest.fail(
f"{_TEST_DATABASE_ENV} must use postgresql+asyncpg on localhost "
f"with database {_TEST_DATABASE_NAME!r}"
)
return raw_url
_RUN_STATE_DDL = """
CREATE TABLE awooop_run_state (
run_id UUID PRIMARY KEY,
project_id VARCHAR(64) NOT NULL,
agent_id VARCHAR(128) NOT NULL,
state VARCHAR(32) NOT NULL DEFAULT 'pending',
lease_until TIMESTAMP,
next_attempt_at TIMESTAMP,
heartbeat_at TIMESTAMP,
worker_id VARCHAR(128),
attempt_count SMALLINT NOT NULL DEFAULT 0,
max_attempts SMALLINT NOT NULL DEFAULT 3,
trace_id VARCHAR(128),
trigger_type VARCHAR(32),
trigger_ref VARCHAR(256),
is_shadow BOOLEAN NOT NULL DEFAULT TRUE,
input_sha256 VARCHAR(64),
output_sha256 VARCHAR(64),
cost_usd NUMERIC(10, 4) NOT NULL DEFAULT 0,
step_count SMALLINT NOT NULL DEFAULT 0,
error_code VARCHAR(64),
error_detail TEXT,
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
started_at TIMESTAMP,
completed_at TIMESTAMP,
timeout_at TIMESTAMP
)
"""
_RUN_IDEMPOTENCY_DDL = """
CREATE TABLE awooop_run_idempotency (
idempotency_id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
project_id VARCHAR(64) NOT NULL,
channel_type VARCHAR(32) NOT NULL,
provider_event_id VARCHAR(256) NOT NULL,
run_id UUID NOT NULL REFERENCES awooop_run_state(run_id),
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
CONSTRAINT uix_run_idempotency_key
UNIQUE (project_id, channel_type, provider_event_id)
)
"""
@pytest.mark.asyncio
async def test_concurrent_provider_event_has_one_postgresql_winner(
monkeypatch: pytest.MonkeyPatch,
) -> None:
engine = create_async_engine(_validated_test_database_url(), poolclass=NullPool)
factory = async_sessionmaker(engine, expire_on_commit=False, autoflush=False)
tables_created = False
@asynccontextmanager
async def test_db_context(project_id: str):
assert project_id == "awoooi"
async with factory() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
async def skip_redis_cache(*_args: object) -> None:
return None
try:
async with engine.begin() as connection:
existing_tables = (
await connection.execute(
text(
"SELECT to_regclass('awooop_run_state'), "
"to_regclass('awooop_run_idempotency')"
)
)
).one()
if any(existing_tables):
pytest.fail(
"dedicated runtime identity test database must not contain "
"pre-existing run tables"
)
await connection.execute(text(_RUN_STATE_DDL))
await connection.execute(text(_RUN_IDEMPOTENCY_DDL))
tables_created = True
monkeypatch.setattr(platform_runtime, "get_db_context", test_db_context)
monkeypatch.setattr(platform_runtime, "_cache_idempotency", skip_redis_cache)
first, second = await asyncio.gather(
platform_runtime.create_run(
project_id="awoooi",
agent_id="runtime-agent",
trigger_type="channel_event",
channel_type="telegram",
provider_event_id="same-real-postgresql-event",
),
platform_runtime.create_run(
project_id="awoooi",
agent_id="runtime-agent",
trigger_type="channel_event",
channel_type="telegram",
provider_event_id="same-real-postgresql-event",
),
)
async with engine.connect() as connection:
run_count = await connection.scalar(
text("SELECT COUNT(*) FROM awooop_run_state")
)
mapping_count = await connection.scalar(
text("SELECT COUNT(*) FROM awooop_run_idempotency")
)
durable_run_id = await connection.scalar(
text("SELECT run_id FROM awooop_run_idempotency")
)
assert first[0] == second[0] == durable_run_id
assert sorted((first[1], second[1])) == [False, True]
assert run_count == 1
assert mapping_count == 1
finally:
if tables_created:
async with engine.begin() as connection:
await connection.execute(text("DROP TABLE awooop_run_idempotency"))
await connection.execute(text("DROP TABLE awooop_run_state"))
await engine.dispose()

View File

@@ -2,8 +2,11 @@ from __future__ import annotations
import inspect
import json
from uuid import uuid4
from uuid import UUID, uuid4
import pytest
from src.services import channel_hub
from src.services.channel_hub import (
_db_timestamp_now,
_send_telegram_interim,
@@ -17,9 +20,11 @@ from src.services.channel_hub import (
format_alertmanager_event_content,
format_grouped_alert_digest_text,
format_grouped_alert_event_content,
handle_telegram_inbound,
mirror_inbound_event,
record_outbound_message,
)
from src.services.runtime_correlation import correlation_for_run
from src.services.telegram_gateway import _telegram_destination_binding
_FAKE_SECRET_SHAPED_VALUE = "1234567890:" + "abcdefghijklmnopqrstuvwxyzABCDEFGH"
@@ -357,6 +362,69 @@ async def test_completed_shadow_run_sets_run_state_not_null_defaults() -> None:
assert "0, 3, 0.0000, 0" in session.statement
assert session.params["project_id"] == "awoooi"
assert session.params["run_id"] == run_id
correlation = correlation_for_run("awoooi", run_id)
assert session.params["trace_id"] == correlation.trace_id
assert "trace_id, trigger_type" in session.statement
@pytest.mark.asyncio
async def test_telegram_inbound_persists_and_returns_same_run_correlation(
monkeypatch: pytest.MonkeyPatch,
) -> None:
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
identity = correlation_for_run("awoooi", run_id)
mirrored: dict[str, object] = {}
async def fake_create_run(**_kwargs: object) -> tuple[UUID, bool]:
return run_id, False
async def fake_read_run_status(
received_run_id: UUID,
project_id: str,
) -> dict[str, object]:
assert received_run_id == run_id
assert project_id == "awoooi"
return {
**identity.public_dict(),
"correlation_status": "canonical",
}
async def fake_mirror(_db: object, **kwargs: object) -> UUID:
mirrored.update(kwargs)
return UUID("019f88d5-f6c9-7a18-b764-df0df0cbe223")
async def fake_schedule(**_kwargs: object) -> None:
return None
monkeypatch.setattr(channel_hub, "create_run", fake_create_run)
monkeypatch.setattr(
channel_hub,
"read_platform_run_status",
fake_read_run_status,
)
monkeypatch.setattr(channel_hub, "mirror_inbound_event", fake_mirror)
monkeypatch.setattr(channel_hub, "schedule_interim_feedback", fake_schedule)
result = await handle_telegram_inbound(
object(), # type: ignore[arg-type]
project_id="awoooi",
agent_id="openclaw-sre",
message_id="42",
user_id="7",
chat_id="-1007",
text="status",
)
assert result["run_id"] == str(run_id)
assert result["trace_id"] == identity.trace_id
assert result["work_item_id"] == identity.work_item_id
assert result["correlation_status"] == "canonical"
source_envelope = mirrored["source_envelope"]
assert isinstance(source_envelope, dict)
assert source_envelope["runtime_correlation"] == {
**identity.public_dict(),
"correlation_status": "canonical",
}
async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None:
@@ -385,6 +453,10 @@ async def test_mirror_inbound_event_writes_redacted_replay_fields() -> None:
assert "content_redacted" in session.statement
assert "source_envelope" in session.statement
assert (
"COALESCE(\n awooop_conversation_event.run_id,"
in session.statement
)
assert session.params["redaction_version"] == "audit_sink_v1"
assert len(json.loads(session.params["source_envelope"])["content_sha256"]) == 64
assert "1234567890:" not in session.params["content_redacted"]

View File

@@ -22,9 +22,10 @@ from src.services.paid_provider_canary_authorization import ( # noqa: E402
require_paid_provider_canary_authorization,
verify_paid_provider_canary_authorization,
)
from src.services.runtime_correlation import canonical_traceparent # noqa: E402
_SELECTED_RUN_ID = UUID("267a3d26-3c29-470f-8fe3-388a2f408f39")
_TRACE_ID = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01"
_TRACE_ID = canonical_traceparent(_SELECTED_RUN_ID)
class _Reader:
@@ -56,9 +57,10 @@ def _evidence(
now: datetime,
run_id: UUID | None = None,
) -> PaidProviderCanaryAuthorizationEvidence:
selected_run_id = run_id or _SELECTED_RUN_ID
return PaidProviderCanaryAuthorizationEvidence(
run_id=run_id or _SELECTED_RUN_ID,
trace_id=_TRACE_ID,
run_id=selected_run_id,
trace_id=canonical_traceparent(selected_run_id),
project_id=PAID_CANARY_PROJECT_ID,
agent_id=PAID_CANARY_AGENT_ID,
state="running",
@@ -167,7 +169,7 @@ async def test_authorization_ref_must_be_canonical_uuid_without_db_read() -> Non
("trigger_type", "schedule", "trigger_type_exact"),
("trigger_ref", "paid-provider-canary:OTHER", "trigger_ref_exact"),
("input_sha256", "0" * 64, "input_sha256_exact"),
("trace_id", None, "trace_id_present"),
("trace_id", None, "trace_id_run_bound"),
("approval_step_count", 0, "approval_step_exactly_once"),
("approval_step_status", "failed", "approval_step_success"),
("approval_step_was_blocked", True, "approval_step_not_blocked"),

View File

@@ -3,7 +3,7 @@ from __future__ import annotations
import os
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
from uuid import UUID
from uuid import RFC_4122, UUID
import pytest
from fastapi import HTTPException
@@ -25,6 +25,7 @@ from src.services.paid_provider_canary_authorization import ( # noqa: E402
PAID_CANARY_TRIGGER_REF,
PAID_CANARY_TRIGGER_TYPE,
paid_provider_canary_authorization_input_sha256,
paid_provider_canary_run_selected,
)
from src.services.paid_provider_canary_gate5_run import ( # noqa: E402
PAID_CANARY_AUTHORIZATION_SCOPE,
@@ -32,14 +33,28 @@ from src.services.paid_provider_canary_gate5_run import ( # noqa: E402
create_paid_provider_canary_gate5_run,
paid_provider_canary_approval_contract_errors,
)
from src.services.runtime_correlation import ( # noqa: E402
canonical_traceparent,
is_canonical_traceparent,
)
_SELECTED_RUN_ID = UUID("267a3d26-3c29-470f-8fe3-388a2f408f39")
_UNSELECTED_RUN_ID = UUID("00000000-0000-4000-8000-000000000001")
class _Repository:
def __init__(self, *, duplicate: bool = False, state: str = "waiting_approval") -> None:
def __init__(
self,
*,
duplicate: bool = False,
state: str = "waiting_approval",
mismatched_duplicate_trace: bool = False,
unselected_duplicate_run: bool = False,
) -> None:
self.duplicate = duplicate
self.state = state
self.mismatched_duplicate_trace = mismatched_duplicate_trace
self.unselected_duplicate_run = unselected_duplicate_run
self.calls: list[dict[str, object]] = []
async def create_or_get(self, **kwargs: object) -> tuple[
@@ -55,8 +70,15 @@ class _Repository:
assert isinstance(timeout_at, datetime)
assert isinstance(input_sha256, str)
assert isinstance(trace_id, str)
assert is_canonical_traceparent(trace_id, run_id=run_id) is True
if self.duplicate:
run_id = _SELECTED_RUN_ID
run_id = (
_UNSELECTED_RUN_ID
if self.unselected_duplicate_run
else _SELECTED_RUN_ID
)
if not self.mismatched_duplicate_trace:
trace_id = canonical_traceparent(run_id)
return (
PaidProviderCanaryGate5RunRecord(
run_id=run_id,
@@ -160,6 +182,45 @@ async def test_duplicate_active_authorization_is_reused_without_scope_expansion(
)
@pytest.mark.asyncio
async def test_duplicate_active_authorization_rejects_unbound_trace() -> None:
repository = _Repository(
duplicate=True,
state="running",
mismatched_duplicate_trace=True,
)
with pytest.raises(
RuntimeError,
match="paid_provider_canary_gate5_run_trace_unbound",
):
await create_paid_provider_canary_gate5_run(
operator_id="ops@example.com",
repository=repository,
now=datetime(2026, 7, 16, 9, 0, tzinfo=UTC),
)
@pytest.mark.asyncio
async def test_injected_repository_rejects_unselected_execution_run() -> None:
assert paid_provider_canary_run_selected(_UNSELECTED_RUN_ID) is False
repository = _Repository(
duplicate=True,
state="running",
unselected_duplicate_run=True,
)
with pytest.raises(
RuntimeError,
match="paid_provider_canary_gate5_run_not_selected",
):
await create_paid_provider_canary_gate5_run(
operator_id="ops@example.com",
repository=repository,
now=datetime(2026, 7, 16, 9, 0, tzinfo=UTC),
)
def _run(
*,
now: datetime,
@@ -171,7 +232,7 @@ def _run(
project_id=PAID_CANARY_PROJECT_ID,
agent_id=agent_id,
state="waiting_approval",
trace_id="00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
trace_id=canonical_traceparent(_SELECTED_RUN_ID),
trigger_type=PAID_CANARY_TRIGGER_TYPE,
trigger_ref=trigger_ref,
is_shadow=False,
@@ -473,21 +534,51 @@ async def test_generic_runtime_service_persists_shadow_true(
assert len(captured) == 1
assert captured[0].is_shadow is True
assert captured[0].state == "pending"
assert captured[0].run_id.version == 7
assert captured[0].run_id.variant == RFC_4122
assert is_canonical_traceparent(
str(captured[0].trace_id),
run_id=captured[0].run_id,
) is True
@pytest.mark.asyncio
async def test_generic_public_run_creation_remains_shadow_only(
monkeypatch: pytest.MonkeyPatch,
) -> None:
run_id = UUID("018f2d04-4c37-7a18-b764-df0df0cbe222")
async def fake_create_run(**_: object) -> tuple[UUID, bool]:
return UUID("018f2d04-4c37-7a18-b764-df0df0cbe222"), False
return run_id, False
async def fake_read_run_status(
received_run_id: UUID,
project_id: str,
) -> dict[str, object]:
assert received_run_id == run_id
assert project_id == "awoooi"
return {
"trace_id": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
"work_item_id": f"platform-run:awoooi:{run_id}",
"correlation_status": "canonical",
}
async def fake_write_audit(**kwargs: object) -> None:
details = kwargs["details"]
assert isinstance(details, dict)
assert details["is_shadow"] is True
assert details["trace_id"] == (
"00-0123456789abcdef0123456789abcdef-0123456789abcdef-01"
)
assert details["work_item_id"] == f"platform-run:awoooi:{run_id}"
assert details["correlation_status"] == "canonical"
monkeypatch.setattr(public_runs, "create_run", fake_create_run)
monkeypatch.setattr(
public_runs,
"read_platform_run_status",
fake_read_run_status,
)
monkeypatch.setattr(public_runs, "write_audit", fake_write_audit)
response = await public_runs.create_platform_run(
@@ -499,4 +590,56 @@ async def test_generic_public_run_creation_remains_shadow_only(
)
assert response.is_shadow is True
assert response.trace_id.startswith("00-")
assert response.work_item_id == f"platform-run:awoooi:{run_id}"
assert response.correlation_status == "canonical"
assert response.message == "Run 已接受shadow mode"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"identity_readback",
[
None,
{
"trace_id": "legacy-trace",
"work_item_id": (
"platform-run:awoooi:019f88d5-f6c9-7a18-b764-df0df0cbe222"
),
"correlation_status": "legacy_or_invalid_trace",
},
],
)
async def test_generic_public_run_creation_fails_closed_without_canonical_identity_readback(
monkeypatch: pytest.MonkeyPatch,
identity_readback: dict[str, str] | None,
) -> None:
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
async def fake_create_run(**_: object) -> tuple[UUID, bool]:
return run_id, False
async def fake_read_run_status(
_run_id: UUID,
_project_id: str,
) -> dict[str, str] | None:
return identity_readback
monkeypatch.setattr(public_runs, "create_run", fake_create_run)
monkeypatch.setattr(
public_runs,
"read_platform_run_status",
fake_read_run_status,
)
with pytest.raises(HTTPException) as exc_info:
await public_runs.create_platform_run(
public_runs.CreateRunRequest(
project_id="awoooi",
agent_id="generic-agent",
trigger_type="api",
)
)
assert exc_info.value.status_code == 500
assert "correlation readback missing" in str(exc_info.value.detail)

View File

@@ -19,9 +19,10 @@ from src.services.paid_provider_canary_validation import (
run_paid_provider_canary_validation,
select_claude_canary_run_id,
)
from src.services.runtime_correlation import canonical_traceparent
_AUTHORIZATION_REF = "267a3d26-3c29-470f-8fe3-388a2f408f39"
_AUTHORIZATION_TRACE_ID = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01"
_AUTHORIZATION_TRACE_ID = canonical_traceparent(UUID(_AUTHORIZATION_REF))
def test_ollama_canary_uses_bounded_production_diagnose_timeout(
@@ -592,6 +593,59 @@ async def test_optional_terminal_aol_projection_cannot_reopen_terminal_run() ->
assert record_id is None
@pytest.mark.asyncio
async def test_injected_authorization_receipt_rejects_trace_unbound_to_run(
monkeypatch,
) -> None:
from src.services import paid_provider_canary_validation as module
monkeypatch.setattr(
module,
"settings",
SimpleNamespace(
CLAUDE_EXECUTION_MODE="canary",
CLAUDE_CANARY_PERCENT=5,
GEMINI_EXECUTION_MODE="canary",
GEMINI_CANARY_PERCENT=5,
ALERT_OLLAMA_MODEL="qwen3:14b",
),
)
execution_claim_attempted = False
async def _unexpected_claim(_run_id: str, _owner: str) -> bool:
nonlocal execution_claim_attempted
execution_claim_attempted = True
return True
monkeypatch.setattr(
module,
"_acquire_paid_canary_execution_claim",
_unexpected_claim,
)
unbound_trace = canonical_traceparent(
UUID("267a3d26-3c29-470f-8fe3-388a2f408f3a")
)
assert unbound_trace != _AUTHORIZATION_TRACE_ID
with pytest.raises(
RuntimeError,
match="paid_canary_authorization_same_run_binding_failed",
):
await module._run_paid_provider_canary_validation_core(
run_ref="source-sha-unbound-trace",
operator_id="test-operator",
authorization_ref=_AUTHORIZATION_REF,
verified_authorization_receipt={
"authorization_ref": _AUTHORIZATION_REF,
"run_id": _AUTHORIZATION_REF,
"work_item_id": "AIA-SRE-013",
"trace_id": unbound_trace,
},
)
assert execution_claim_attempted is False
@pytest.mark.asyncio
async def test_durable_authorization_blocks_before_receipt_or_provider_call(
monkeypatch,

View File

@@ -0,0 +1,257 @@
from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
from datetime import UTC, datetime
from decimal import Decimal
from types import SimpleNamespace
from uuid import RFC_4122, UUID
import pytest
from src.db.awooop_models import AwoooPRunIdempotency, AwoooPRunState
from src.services import platform_runtime
from src.services.runtime_correlation import (
canonical_traceparent,
canonical_work_item_id,
correlation_for_run,
is_canonical_traceparent,
new_uuid7,
)
def test_uuid7_has_rfc_version_variant_and_timestamp_layout() -> None:
timestamp_ms = 1_753_155_200_123
generated = new_uuid7(timestamp_ms=timestamp_ms, entropy=(1 << 74) - 1)
assert generated.version == 7
assert generated.variant == RFC_4122
assert generated.int >> 80 == timestamp_ms
@pytest.mark.parametrize(
("timestamp_ms", "entropy"),
[(-1, 0), (1 << 48, 0), (0, -1), (0, 1 << 74)],
)
def test_uuid7_rejects_out_of_range_inputs(
timestamp_ms: int,
entropy: int,
) -> None:
with pytest.raises(ValueError):
new_uuid7(timestamp_ms=timestamp_ms, entropy=entropy)
def test_runtime_correlation_is_stable_and_bound_to_run() -> None:
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
correlation = correlation_for_run("awoooi", run_id)
assert correlation.run_id == run_id
assert correlation.trace_id == canonical_traceparent(run_id)
assert correlation.work_item_id == f"platform-run:awoooi:{run_id}"
assert is_canonical_traceparent(correlation.trace_id, run_id=run_id) is True
assert (
is_canonical_traceparent(
correlation.trace_id,
run_id=UUID("019f88d5-f6c9-7a18-b764-df0df0cbe223"),
)
is False
)
def test_runtime_correlation_rejects_invalid_project_id() -> None:
with pytest.raises(ValueError, match="project_id_invalid"):
canonical_work_item_id(
"awoooi:forged",
UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222"),
)
@pytest.mark.asyncio
async def test_run_status_exposes_canonical_identity_readback(
monkeypatch: pytest.MonkeyPatch,
) -> None:
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
trace_id = canonical_traceparent(run_id)
row = SimpleNamespace(
run_id=run_id,
project_id="awoooi",
agent_id="runtime-agent",
state="running",
is_shadow=True,
trace_id=trace_id,
attempt_count=1,
cost_usd=Decimal("0.0000"),
step_count=2,
error_code=None,
created_at=datetime(2026, 7, 22, tzinfo=UTC),
started_at=datetime(2026, 7, 22, tzinfo=UTC),
completed_at=None,
)
class _Result:
def scalar_one_or_none(self) -> SimpleNamespace:
return row
class _Db:
async def execute(self, _statement: object) -> _Result:
return _Result()
@asynccontextmanager
async def fake_db_context(project_id: str):
assert project_id == "awoooi"
yield _Db()
monkeypatch.setattr(platform_runtime, "get_db_context", fake_db_context)
payload = await platform_runtime.get_run_status(run_id, "awoooi")
assert payload is not None
assert payload["run_id"] == str(run_id)
assert payload["trace_id"] == trace_id
assert payload["work_item_id"] == f"platform-run:awoooi:{run_id}"
assert payload["correlation_status"] == "canonical"
@pytest.mark.asyncio
async def test_run_status_marks_legacy_unbound_trace_without_rewriting_it(
monkeypatch: pytest.MonkeyPatch,
) -> None:
run_id = UUID("019f88d5-f6c9-7a18-b764-df0df0cbe222")
row = SimpleNamespace(
run_id=run_id,
project_id="awoooi",
agent_id="legacy-agent",
state="completed",
is_shadow=True,
trace_id="legacy-trace",
attempt_count=1,
cost_usd=Decimal("0.0000"),
step_count=0,
error_code=None,
created_at=None,
started_at=None,
completed_at=None,
)
class _Result:
def scalar_one_or_none(self) -> SimpleNamespace:
return row
class _Db:
async def execute(self, _statement: object) -> _Result:
return _Result()
@asynccontextmanager
async def fake_db_context(_project_id: str):
yield _Db()
monkeypatch.setattr(platform_runtime, "get_db_context", fake_db_context)
payload = await platform_runtime.get_run_status(run_id, "awoooi")
assert payload is not None
assert payload["trace_id"] == "legacy-trace"
assert payload["correlation_status"] == "legacy_or_invalid_trace"
@pytest.mark.asyncio
async def test_concurrent_provider_event_creation_returns_one_durable_winner(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class _Result:
def __init__(self, row: tuple[UUID] | None = None) -> None:
self._row = row
def fetchone(self) -> tuple[UUID] | None:
return self._row
class _Store:
def __init__(self) -> None:
self.claim_lock = asyncio.Lock()
self.mapping: UUID | None = None
self.runs: dict[UUID, AwoooPRunState] = {}
self.lock_claim_count = 0
store = _Store()
class _Db:
def __init__(self) -> None:
self.lock_held = False
self.pending_run: AwoooPRunState | None = None
self.pending_mapping: AwoooPRunIdempotency | None = None
async def execute(
self,
statement: object,
_params: dict[str, str] | None = None,
) -> _Result:
sql = str(statement)
if "pg_advisory_xact_lock" in sql:
await store.claim_lock.acquire()
self.lock_held = True
store.lock_claim_count += 1
await asyncio.sleep(0)
return _Result()
if "awooop_run_idempotency" in sql:
return _Result(
(store.mapping,) if store.mapping is not None else None
)
raise AssertionError(f"unexpected statement: {sql}")
def add(self, row: object) -> None:
if isinstance(row, AwoooPRunState):
self.pending_run = row
elif isinstance(row, AwoooPRunIdempotency):
self.pending_mapping = row
else:
raise AssertionError(f"unexpected row: {type(row).__name__}")
async def flush(self) -> None:
assert self.lock_held is True
assert self.pending_run is not None
if self.pending_mapping is None:
assert store.runs == {}
store.runs[self.pending_run.run_id] = self.pending_run
return
assert store.mapping is None
assert self.pending_mapping.run_id in store.runs
store.mapping = self.pending_mapping.run_id
@asynccontextmanager
async def fake_db_context(project_id: str):
assert project_id == "awoooi"
db = _Db()
try:
yield db
finally:
if db.lock_held:
store.claim_lock.release()
async def fake_cache(*_args: object) -> None:
return None
monkeypatch.setattr(platform_runtime, "get_db_context", fake_db_context)
monkeypatch.setattr(platform_runtime, "_cache_idempotency", fake_cache)
first, second = await asyncio.gather(
platform_runtime.create_run(
project_id="awoooi",
agent_id="runtime-agent",
trigger_type="channel_event",
channel_type="telegram",
provider_event_id="same-provider-event",
),
platform_runtime.create_run(
project_id="awoooi",
agent_id="runtime-agent",
trigger_type="channel_event",
channel_type="telegram",
provider_event_id="same-provider-event",
),
)
assert first[0] == second[0] == store.mapping
assert sorted((first[1], second[1])) == [False, True]
assert list(store.runs) == [store.mapping]
assert next(iter(store.runs.values())).timeout_at.tzinfo is None
assert store.lock_claim_count == 2