fix(platform): bind AI runs to canonical correlation
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 51s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m45s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 33s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
Some checks failed
CD Pipeline / select-latest-carrier (push) Successful in 51s
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m45s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 33s
CD Pipeline / revalidate-post-deploy-carrier (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
This commit is contained in:
@@ -24,7 +24,12 @@ from fastapi import APIRouter, HTTPException, status
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from src.services.audit_sink import write_audit
|
||||
from src.services.platform_runtime import create_run
|
||||
from src.services.platform_runtime import (
|
||||
create_run,
|
||||
)
|
||||
from src.services.platform_runtime import (
|
||||
get_run_status as read_platform_run_status,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -57,6 +62,9 @@ class CreateRunResponse(BaseModel):
|
||||
"""POST /v1/platform/runs response"""
|
||||
|
||||
run_id: str
|
||||
trace_id: str
|
||||
work_item_id: str
|
||||
correlation_status: str
|
||||
is_duplicate: bool = Field(description="True = 冪等命中,返回既有 run_id")
|
||||
is_shadow: bool = Field(True, description="Phase 4 固定 True")
|
||||
message: str
|
||||
@@ -93,6 +101,17 @@ async def create_platform_run(
|
||||
provider_event_id=request.provider_event_id,
|
||||
timeout_seconds=request.timeout_seconds,
|
||||
)
|
||||
identity = await read_platform_run_status(run_id, request.project_id)
|
||||
if (
|
||||
identity is None
|
||||
or not identity.get("trace_id")
|
||||
or not identity.get("work_item_id")
|
||||
or (
|
||||
not is_duplicate
|
||||
and identity.get("correlation_status") != "canonical"
|
||||
)
|
||||
):
|
||||
raise RuntimeError("Run correlation readback missing")
|
||||
except Exception as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
@@ -110,11 +129,17 @@ async def create_platform_run(
|
||||
"trigger_type": request.trigger_type,
|
||||
"is_duplicate": is_duplicate,
|
||||
"is_shadow": True,
|
||||
"trace_id": identity["trace_id"],
|
||||
"work_item_id": identity["work_item_id"],
|
||||
"correlation_status": identity["correlation_status"],
|
||||
},
|
||||
)
|
||||
|
||||
return CreateRunResponse(
|
||||
run_id=str(run_id),
|
||||
trace_id=str(identity["trace_id"]),
|
||||
work_item_id=str(identity["work_item_id"]),
|
||||
correlation_status=str(identity["correlation_status"]),
|
||||
is_duplicate=is_duplicate,
|
||||
is_shadow=True,
|
||||
message="Run 已接受(shadow mode)" if not is_duplicate else "冪等命中,返回既有 run_id",
|
||||
|
||||
@@ -41,7 +41,13 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from src.db.awooop_models import AwoooPRunState
|
||||
from src.services.audit_sink import _redact_string, sanitize
|
||||
from src.services.platform_runtime import create_run
|
||||
from src.services.platform_runtime import (
|
||||
create_run,
|
||||
)
|
||||
from src.services.platform_runtime import (
|
||||
get_run_status as read_platform_run_status,
|
||||
)
|
||||
from src.services.runtime_correlation import correlation_for_run
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -160,17 +166,18 @@ async def ensure_completed_shadow_run(
|
||||
資料。這些事件不應重新觸發 runtime,但需要 run_state 當 Console 的
|
||||
聚合錨點;因此這裡建立的是已完成的 shadow run,不會被 worker pick up。
|
||||
"""
|
||||
correlation = correlation_for_run(project_id, run_id)
|
||||
result = await db.execute(
|
||||
text("""
|
||||
INSERT INTO awooop_run_state (
|
||||
run_id, project_id, agent_id, state,
|
||||
trigger_type, trigger_ref, is_shadow,
|
||||
trace_id, trigger_type, trigger_ref, is_shadow,
|
||||
input_sha256,
|
||||
attempt_count, max_attempts, cost_usd, step_count,
|
||||
created_at, completed_at, timeout_at
|
||||
) VALUES (
|
||||
:run_id, :project_id, :agent_id, 'completed',
|
||||
:trigger_type, :trigger_ref, TRUE,
|
||||
:trace_id, :trigger_type, :trigger_ref, TRUE,
|
||||
:input_sha256,
|
||||
0, 3, 0.0000, 0,
|
||||
NOW(), NOW(), NOW()
|
||||
@@ -180,6 +187,7 @@ async def ensure_completed_shadow_run(
|
||||
"""),
|
||||
{
|
||||
"run_id": run_id,
|
||||
"trace_id": correlation.trace_id,
|
||||
"project_id": project_id,
|
||||
"agent_id": agent_id,
|
||||
"trigger_type": trigger_type,
|
||||
@@ -193,6 +201,8 @@ async def ensure_completed_shadow_run(
|
||||
"completed_shadow_run_created",
|
||||
project_id=project_id,
|
||||
run_id=str(run_id),
|
||||
trace_id=correlation.trace_id,
|
||||
work_item_id=correlation.work_item_id,
|
||||
agent_id=agent_id,
|
||||
trigger_type=trigger_type,
|
||||
)
|
||||
@@ -303,7 +313,10 @@ async def mirror_inbound_event(
|
||||
)
|
||||
ON CONFLICT (project_id, channel_type, provider_event_id) DO UPDATE SET
|
||||
is_duplicate = TRUE,
|
||||
run_id = COALESCE(EXCLUDED.run_id, awooop_conversation_event.run_id),
|
||||
run_id = COALESCE(
|
||||
awooop_conversation_event.run_id,
|
||||
EXCLUDED.run_id
|
||||
),
|
||||
content_redacted = COALESCE(
|
||||
awooop_conversation_event.content_redacted,
|
||||
EXCLUDED.content_redacted
|
||||
@@ -1209,6 +1222,20 @@ async def handle_telegram_inbound(
|
||||
channel_type="telegram",
|
||||
provider_event_id=message_id,
|
||||
)
|
||||
identity = await read_platform_run_status(run_id, project_id)
|
||||
if (
|
||||
identity is None
|
||||
or not identity.get("trace_id")
|
||||
or not identity.get("work_item_id")
|
||||
or (not is_duplicate and identity.get("correlation_status") != "canonical")
|
||||
):
|
||||
raise RuntimeError("telegram_inbound_run_correlation_missing")
|
||||
runtime_correlation = {
|
||||
"run_id": str(run_id),
|
||||
"trace_id": str(identity["trace_id"]),
|
||||
"work_item_id": str(identity["work_item_id"]),
|
||||
"correlation_status": str(identity["correlation_status"]),
|
||||
}
|
||||
|
||||
# Step 2: Mirror event(含 run_id)
|
||||
event_id = await mirror_inbound_event(
|
||||
@@ -1220,6 +1247,7 @@ async def handle_telegram_inbound(
|
||||
channel_chat_id=chat_id,
|
||||
content_type="text" if text else "callback_query",
|
||||
raw_content=text,
|
||||
source_envelope={"runtime_correlation": runtime_correlation},
|
||||
run_id=run_id,
|
||||
is_duplicate=is_duplicate,
|
||||
)
|
||||
@@ -1243,5 +1271,8 @@ async def handle_telegram_inbound(
|
||||
return {
|
||||
"event_id": str(event_id),
|
||||
"run_id": str(run_id),
|
||||
"trace_id": runtime_correlation["trace_id"],
|
||||
"work_item_id": runtime_correlation["work_item_id"],
|
||||
"correlation_status": runtime_correlation["correlation_status"],
|
||||
"is_duplicate": is_duplicate,
|
||||
}
|
||||
|
||||
@@ -16,7 +16,6 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any, Protocol
|
||||
@@ -31,6 +30,7 @@ from src.services.audit_sink import (
|
||||
AWOOOP_AUDIT_SCHEMA_VERSION,
|
||||
audit_idempotency_key_sha256,
|
||||
)
|
||||
from src.services.runtime_correlation import is_canonical_traceparent
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -47,7 +47,6 @@ PAID_CANARY_APPROVAL_AUDIT_IDEMPOTENCY_PREFIX = (
|
||||
PAID_CANARY_START_ACTION = "claude_gemini_sanitized_sre_canary"
|
||||
PAID_CANARY_APPROVAL_TTL_SECONDS = 900
|
||||
PAID_CANARY_PERCENT = 5
|
||||
_SAFE_TRACE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,191}$")
|
||||
|
||||
|
||||
def paid_provider_canary_approval_audit_idempotency_key(
|
||||
@@ -371,8 +370,9 @@ async def verify_paid_provider_canary_authorization(
|
||||
expected_input_sha256 = paid_provider_canary_authorization_input_sha256()
|
||||
checks = {
|
||||
"run_id_exact": evidence.run_id == run_id,
|
||||
"trace_id_present": bool(
|
||||
_SAFE_TRACE_ID.fullmatch(str(evidence.trace_id or "").strip())
|
||||
"trace_id_run_bound": is_canonical_traceparent(
|
||||
str(evidence.trace_id or ""),
|
||||
run_id=evidence.run_id,
|
||||
),
|
||||
"authorization_run_selected": paid_provider_canary_run_selected(
|
||||
evidence.run_id
|
||||
|
||||
@@ -34,6 +34,7 @@ from src.services.paid_provider_canary_authorization import (
|
||||
paid_provider_canary_run_selected,
|
||||
)
|
||||
from src.services.platform_runtime import _new_trace_id, _uuid7
|
||||
from src.services.runtime_correlation import is_canonical_traceparent
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -95,6 +96,8 @@ def _utc_naive(value: datetime) -> datetime:
|
||||
def _record_from_run(run: AwoooPRunState) -> PaidProviderCanaryGate5RunRecord:
|
||||
if run.timeout_at is None or run.input_sha256 is None or run.trace_id is None:
|
||||
raise RuntimeError("paid_provider_canary_gate5_run_incomplete")
|
||||
if not is_canonical_traceparent(str(run.trace_id), run_id=run.run_id):
|
||||
raise RuntimeError("paid_provider_canary_gate5_run_trace_unbound")
|
||||
return PaidProviderCanaryGate5RunRecord(
|
||||
run_id=run.run_id,
|
||||
project_id=run.project_id,
|
||||
@@ -190,6 +193,10 @@ class _SqlPaidProviderCanaryGate5RunRepository:
|
||||
"authorization_run_selected": (
|
||||
paid_provider_canary_run_selected(existing.run_id)
|
||||
),
|
||||
"trace_run_bound": is_canonical_traceparent(
|
||||
str(existing.trace_id or ""),
|
||||
run_id=existing.run_id,
|
||||
),
|
||||
}
|
||||
active_contract_errors = [
|
||||
name
|
||||
@@ -254,6 +261,10 @@ def paid_provider_canary_approval_contract_errors(
|
||||
"authorization_run_selected": paid_provider_canary_run_selected(
|
||||
run.run_id
|
||||
),
|
||||
"trace_run_bound": is_canonical_traceparent(
|
||||
str(run.trace_id or ""),
|
||||
run_id=run.run_id,
|
||||
),
|
||||
"timeout_present": run.timeout_at is not None,
|
||||
"timeout_not_expired": bool(
|
||||
run.timeout_at is not None
|
||||
@@ -285,7 +296,7 @@ async def create_paid_provider_canary_gate5_run(
|
||||
authorization_input = paid_provider_canary_authorization_input()
|
||||
input_sha256 = paid_provider_canary_authorization_input_sha256()
|
||||
run_id = _new_selected_paid_canary_run_id()
|
||||
trace_id = _new_trace_id()
|
||||
trace_id = _new_trace_id(run_id)
|
||||
|
||||
writer = repository or _SqlPaidProviderCanaryGate5RunRepository()
|
||||
record, is_duplicate = await writer.create_or_get(
|
||||
@@ -295,6 +306,10 @@ async def create_paid_provider_canary_gate5_run(
|
||||
timeout_at=timeout_at,
|
||||
input_sha256=input_sha256,
|
||||
)
|
||||
if not paid_provider_canary_run_selected(record.run_id):
|
||||
raise RuntimeError("paid_provider_canary_gate5_run_not_selected")
|
||||
if not is_canonical_traceparent(record.trace_id, run_id=record.run_id):
|
||||
raise RuntimeError("paid_provider_canary_gate5_run_trace_unbound")
|
||||
|
||||
write_creation_audit = audit_writer or write_audit
|
||||
await write_creation_audit(
|
||||
|
||||
@@ -43,9 +43,9 @@ from src.services.paid_provider_canary_authorization import (
|
||||
paid_provider_canary_bucket,
|
||||
paid_provider_canary_run_selected,
|
||||
)
|
||||
from src.services.runtime_correlation import is_canonical_traceparent
|
||||
|
||||
_SAFE_RUN_REF = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
|
||||
_SAFE_TRACE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,191}$")
|
||||
_WORK_ITEM_ID = "AIA-SRE-013"
|
||||
_ACTIVATION_LEASE_SECONDS = 300
|
||||
_PAID_CANARY_MAX_OUTPUT_TOKENS = 512
|
||||
@@ -461,6 +461,12 @@ async def _run_paid_provider_canary_validation_core(
|
||||
# The executable Gate 5 row is the controlled-apply run. A caller-supplied
|
||||
# source label must never mint a second run identity after approval.
|
||||
run_id = clean_authorization_ref
|
||||
try:
|
||||
durable_run_id = UUID(run_id)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(
|
||||
"paid_canary_authorization_same_run_binding_failed"
|
||||
) from exc
|
||||
canary_percent = min(
|
||||
settings.CLAUDE_CANARY_PERCENT,
|
||||
settings.GEMINI_CANARY_PERCENT,
|
||||
@@ -473,7 +479,7 @@ async def _run_paid_provider_canary_validation_core(
|
||||
or authorization_receipt.get("work_item_id") != _WORK_ITEM_ID
|
||||
or not paid_provider_canary_run_selected(run_id)
|
||||
or canary_bucket >= canary_percent
|
||||
or not _SAFE_TRACE_ID.fullmatch(trace_id)
|
||||
or not is_canonical_traceparent(trace_id, run_id=durable_run_id)
|
||||
):
|
||||
raise RuntimeError("paid_canary_authorization_same_run_binding_failed")
|
||||
activation_owner = f"{run_id}.attempt-{uuid4().hex[:12]}"
|
||||
|
||||
@@ -19,17 +19,27 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
import structlog
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from src.db.awooop_models import AwoooPRunIdempotency, AwoooPRunState, AwoooPRunStepJournal
|
||||
from src.db.awooop_models import (
|
||||
AwoooPRunIdempotency,
|
||||
AwoooPRunState,
|
||||
AwoooPRunStepJournal,
|
||||
)
|
||||
from src.db.base import get_db_context
|
||||
from src.services.run_state_machine import LEASE_TTL_SECONDS, transition
|
||||
from src.services.runtime_correlation import (
|
||||
canonical_traceparent,
|
||||
canonical_work_item_id,
|
||||
is_canonical_traceparent,
|
||||
new_runtime_correlation,
|
||||
new_uuid7,
|
||||
)
|
||||
|
||||
logger = structlog.get_logger(__name__)
|
||||
|
||||
@@ -46,26 +56,20 @@ _DESTRUCTIVE_TOOL_KEYWORDS = frozenset({
|
||||
# UUID v7(時間有序)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _uuid7() -> uuid.UUID:
|
||||
"""
|
||||
生成 UUID v7(時間有序,適合資料庫 PK)。
|
||||
格式:48-bit Unix timestamp ms + version(7) + 74-bit random
|
||||
"""
|
||||
now_ms = int(datetime.now(timezone.utc).timestamp() * 1000)
|
||||
rand_bits = int.from_bytes(uuid.uuid4().bytes[6:], "big") & 0x3FFFFFFFFFFFFFFF
|
||||
val = (now_ms << 80) | (0x7 << 76) | (0x8 << 72) | rand_bits
|
||||
return uuid.UUID(int=val)
|
||||
def _uuid7() -> UUID:
|
||||
"""Compatibility wrapper for the canonical RFC 9562 generator."""
|
||||
|
||||
return new_uuid7()
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# W3C traceparent 生成
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _new_trace_id() -> str:
|
||||
"""生成 W3C traceparent-compatible trace_id"""
|
||||
trace_id_hex = uuid.uuid4().hex + uuid.uuid4().hex[:16] # 128-bit
|
||||
span_id_hex = uuid.uuid4().hex[:16] # 64-bit
|
||||
return f"00-{trace_id_hex}-{span_id_hex}-01"
|
||||
def _new_trace_id(run_id: UUID | None = None) -> str:
|
||||
"""Return the canonical W3C traceparent bound to one durable run."""
|
||||
|
||||
return canonical_traceparent(run_id or _uuid7())
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -74,13 +78,14 @@ def _new_trace_id() -> str:
|
||||
|
||||
_IDEMPOTENCY_REDIS_PREFIX = "awooop:run:idem:"
|
||||
_IDEMPOTENCY_REDIS_TTL = 86400 # 24h
|
||||
_IDEMPOTENCY_LOCK_PREFIX = "awooop:run:idem:claim:"
|
||||
|
||||
|
||||
async def check_idempotency(
|
||||
project_id: str,
|
||||
channel_type: str,
|
||||
provider_event_id: str,
|
||||
) -> uuid.UUID | None:
|
||||
) -> UUID | None:
|
||||
"""
|
||||
檢查 (project_id, channel_type, provider_event_id) 是否已有對應 run_id。
|
||||
|
||||
@@ -91,24 +96,20 @@ async def check_idempotency(
|
||||
"""
|
||||
idem_key = f"{_IDEMPOTENCY_REDIS_PREFIX}{project_id}:{channel_type}:{provider_event_id}"
|
||||
|
||||
# Layer 1: Redis
|
||||
cached_run_id: UUID | None = None
|
||||
# Redis is only a lookup hint. PostgreSQL remains the durable authority so
|
||||
# an old or interrupted cache write can never select a different run.
|
||||
try:
|
||||
from src.core.redis_client import get_redis
|
||||
redis = get_redis()
|
||||
cached = await redis.get(idem_key)
|
||||
if cached:
|
||||
run_id_str = cached.decode() if isinstance(cached, bytes) else cached
|
||||
logger.info(
|
||||
"idempotency_hit_redis",
|
||||
project_id=project_id,
|
||||
provider_event_id=provider_event_id,
|
||||
run_id=run_id_str,
|
||||
)
|
||||
return uuid.UUID(run_id_str)
|
||||
cached_run_id = UUID(run_id_str)
|
||||
except Exception as exc:
|
||||
logger.warning("idempotency_redis_check_failed", error=str(exc))
|
||||
|
||||
# Layer 2: PostgreSQL
|
||||
# PostgreSQL is the only authoritative mapping.
|
||||
try:
|
||||
async with get_db_context(project_id) as db:
|
||||
result = await db.execute(
|
||||
@@ -120,44 +121,43 @@ async def check_idempotency(
|
||||
)
|
||||
row = result.fetchone()
|
||||
if row:
|
||||
return uuid.UUID(str(row[0]))
|
||||
durable_run_id = UUID(str(row[0]))
|
||||
if cached_run_id is not None and cached_run_id != durable_run_id:
|
||||
logger.warning(
|
||||
"idempotency_redis_durable_mismatch_ignored",
|
||||
project_id=project_id,
|
||||
provider_event_id=provider_event_id,
|
||||
)
|
||||
return durable_run_id
|
||||
except Exception as exc:
|
||||
logger.warning("idempotency_pg_check_failed", error=str(exc))
|
||||
|
||||
if cached_run_id is not None:
|
||||
logger.warning(
|
||||
"idempotency_redis_orphan_hint_ignored",
|
||||
project_id=project_id,
|
||||
provider_event_id=provider_event_id,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
async def _register_idempotency(
|
||||
async def _cache_idempotency(
|
||||
project_id: str,
|
||||
channel_type: str,
|
||||
provider_event_id: str,
|
||||
run_id: uuid.UUID,
|
||||
run_id: UUID,
|
||||
) -> None:
|
||||
"""寫入 idempotency 記錄(Redis + PostgreSQL)"""
|
||||
"""Cache a PostgreSQL-selected winner; never create durable authority."""
|
||||
idem_key = f"{_IDEMPOTENCY_REDIS_PREFIX}{project_id}:{channel_type}:{provider_event_id}"
|
||||
run_id_str = str(run_id)
|
||||
|
||||
# Redis NX(若已有其他 worker 寫入,NX 失敗,無害)
|
||||
try:
|
||||
from src.core.redis_client import get_redis
|
||||
redis = get_redis()
|
||||
await redis.set(idem_key, run_id_str, ex=_IDEMPOTENCY_REDIS_TTL, nx=True)
|
||||
await redis.set(idem_key, run_id_str, ex=_IDEMPOTENCY_REDIS_TTL)
|
||||
except Exception as exc:
|
||||
logger.warning("idempotency_redis_write_failed", error=str(exc))
|
||||
|
||||
# PostgreSQL(INSERT OR IGNORE)
|
||||
try:
|
||||
async with get_db_context(project_id) as db:
|
||||
stmt = pg_insert(AwoooPRunIdempotency).values(
|
||||
project_id=project_id,
|
||||
channel_type=channel_type,
|
||||
provider_event_id=provider_event_id,
|
||||
run_id=run_id,
|
||||
).on_conflict_do_nothing(constraint="uix_run_idempotency_key")
|
||||
await db.execute(stmt)
|
||||
except Exception as exc:
|
||||
logger.warning("idempotency_pg_write_failed", error=str(exc))
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Shadow destructive tool check
|
||||
@@ -192,7 +192,7 @@ async def create_run(
|
||||
channel_type: str | None = None,
|
||||
provider_event_id: str | None = None,
|
||||
timeout_seconds: int = 600,
|
||||
) -> tuple[uuid.UUID, bool]:
|
||||
) -> tuple[UUID, bool]:
|
||||
"""
|
||||
建立新 run(或返回既有 run,若重複事件)。
|
||||
|
||||
@@ -201,22 +201,14 @@ async def create_run(
|
||||
|
||||
Shadow mode:is_shadow=True,不產生 user response。
|
||||
"""
|
||||
# Idempotency 檢查
|
||||
if channel_type and provider_event_id:
|
||||
existing_run_id = await check_idempotency(project_id, channel_type, provider_event_id)
|
||||
if existing_run_id:
|
||||
logger.info(
|
||||
"run_creation_idempotent",
|
||||
project_id=project_id,
|
||||
channel_type=channel_type,
|
||||
provider_event_id=provider_event_id,
|
||||
existing_run_id=str(existing_run_id),
|
||||
)
|
||||
return existing_run_id, True
|
||||
|
||||
run_id = _uuid7()
|
||||
trace_id = _new_trace_id()
|
||||
timeout_at = datetime.now(timezone.utc) + timedelta(seconds=timeout_seconds)
|
||||
correlation = new_runtime_correlation(project_id)
|
||||
run_id = correlation.run_id
|
||||
trace_id = correlation.trace_id
|
||||
# AwoooPRunState timestamps use PostgreSQL TIMESTAMP WITHOUT TIME ZONE.
|
||||
# Normalize UTC before binding so asyncpg does not reject an aware value.
|
||||
timeout_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(
|
||||
seconds=timeout_seconds
|
||||
)
|
||||
|
||||
# 計算 input_sha256
|
||||
input_sha256 = None
|
||||
@@ -224,24 +216,85 @@ async def create_run(
|
||||
canonical = json.dumps(input_payload, sort_keys=True, separators=(",", ":"))
|
||||
input_sha256 = hashlib.sha256(canonical.encode()).hexdigest()
|
||||
|
||||
async with get_db_context(project_id) as db:
|
||||
run = AwoooPRunState(
|
||||
run_id=run_id,
|
||||
project_id=project_id,
|
||||
agent_id=agent_id,
|
||||
state="pending",
|
||||
trace_id=trace_id,
|
||||
trigger_type=trigger_type,
|
||||
trigger_ref=trigger_ref,
|
||||
is_shadow=_SHADOW_ENABLED,
|
||||
input_sha256=input_sha256,
|
||||
timeout_at=timeout_at,
|
||||
)
|
||||
db.add(run)
|
||||
run = AwoooPRunState(
|
||||
run_id=run_id,
|
||||
project_id=project_id,
|
||||
agent_id=agent_id,
|
||||
state="pending",
|
||||
trace_id=trace_id,
|
||||
trigger_type=trigger_type,
|
||||
trigger_ref=trigger_ref,
|
||||
is_shadow=_SHADOW_ENABLED,
|
||||
input_sha256=input_sha256,
|
||||
timeout_at=timeout_at,
|
||||
)
|
||||
|
||||
# 寫入 idempotency 記錄
|
||||
if channel_type and provider_event_id:
|
||||
await _register_idempotency(project_id, channel_type, provider_event_id, run_id)
|
||||
lock_key = (
|
||||
f"{_IDEMPOTENCY_LOCK_PREFIX}{project_id}:"
|
||||
f"{channel_type}:{provider_event_id}"
|
||||
)
|
||||
async with get_db_context(project_id) as db:
|
||||
await db.execute(
|
||||
text("SELECT pg_advisory_xact_lock(hashtext(:lock_key))"),
|
||||
{"lock_key": lock_key},
|
||||
)
|
||||
existing_result = await db.execute(
|
||||
select(AwoooPRunIdempotency.run_id).where(
|
||||
AwoooPRunIdempotency.project_id == project_id,
|
||||
AwoooPRunIdempotency.channel_type == channel_type,
|
||||
AwoooPRunIdempotency.provider_event_id
|
||||
== provider_event_id,
|
||||
)
|
||||
)
|
||||
existing_row = existing_result.fetchone()
|
||||
if existing_row is not None:
|
||||
selected_run_id = UUID(str(existing_row[0]))
|
||||
is_duplicate = True
|
||||
else:
|
||||
db.add(run)
|
||||
# These mapped classes intentionally have no ORM relationship,
|
||||
# so SQLAlchemy cannot infer the foreign-key insert order. Keep
|
||||
# both writes in this transaction, but materialize the run
|
||||
# before its idempotency mapping.
|
||||
await db.flush()
|
||||
db.add(
|
||||
AwoooPRunIdempotency(
|
||||
project_id=project_id,
|
||||
channel_type=channel_type,
|
||||
provider_event_id=provider_event_id,
|
||||
run_id=run_id,
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
selected_run_id = run_id
|
||||
is_duplicate = False
|
||||
|
||||
await _cache_idempotency(
|
||||
project_id,
|
||||
channel_type,
|
||||
provider_event_id,
|
||||
selected_run_id,
|
||||
)
|
||||
if is_duplicate:
|
||||
logger.info(
|
||||
"run_creation_idempotent",
|
||||
project_id=project_id,
|
||||
channel_type=channel_type,
|
||||
provider_event_id=provider_event_id,
|
||||
existing_run_id=str(selected_run_id),
|
||||
)
|
||||
return selected_run_id, True
|
||||
else:
|
||||
async with get_db_context(project_id) as db:
|
||||
db.add(run)
|
||||
|
||||
if channel_type and provider_event_id:
|
||||
# The atomic path can reach here only for the transaction winner.
|
||||
assert selected_run_id == run_id and is_duplicate is False
|
||||
else:
|
||||
selected_run_id = run_id
|
||||
is_duplicate = False
|
||||
|
||||
logger.info(
|
||||
"run_created",
|
||||
@@ -250,9 +303,10 @@ async def create_run(
|
||||
agent_id=agent_id,
|
||||
is_shadow=_SHADOW_ENABLED,
|
||||
trace_id=trace_id,
|
||||
work_item_id=correlation.work_item_id,
|
||||
trigger_type=trigger_type,
|
||||
)
|
||||
return run_id, False
|
||||
return selected_run_id, is_duplicate
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -341,7 +395,7 @@ async def shadow_execute(run: AwoooPRunState) -> None:
|
||||
)
|
||||
|
||||
|
||||
async def get_run_status(run_id: uuid.UUID, project_id: str) -> dict[str, Any] | None:
|
||||
async def get_run_status(run_id: UUID, project_id: str) -> dict[str, Any] | None:
|
||||
"""
|
||||
查詢單一 run 的 FSM 狀態。回傳 None 表示不存在。
|
||||
Router 層透過此 service 函數存取,不直接操作 DB。
|
||||
@@ -358,6 +412,8 @@ async def get_run_status(run_id: uuid.UUID, project_id: str) -> dict[str, Any] |
|
||||
if run is None:
|
||||
return None
|
||||
|
||||
work_item_id = canonical_work_item_id(run.project_id, run.run_id)
|
||||
trace_id = str(run.trace_id or "")
|
||||
return {
|
||||
"run_id": str(run.run_id),
|
||||
"project_id": run.project_id,
|
||||
@@ -365,6 +421,12 @@ async def get_run_status(run_id: uuid.UUID, project_id: str) -> dict[str, Any] |
|
||||
"state": run.state,
|
||||
"is_shadow": run.is_shadow,
|
||||
"trace_id": run.trace_id,
|
||||
"work_item_id": work_item_id,
|
||||
"correlation_status": (
|
||||
"canonical"
|
||||
if is_canonical_traceparent(trace_id, run_id=run.run_id)
|
||||
else "legacy_or_invalid_trace"
|
||||
),
|
||||
"attempt_count": run.attempt_count,
|
||||
"cost_usd": float(run.cost_usd),
|
||||
"step_count": run.step_count,
|
||||
|
||||
117
apps/api/src/services/runtime_correlation.py
Normal file
117
apps/api/src/services/runtime_correlation.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""Canonical identity helpers for durable AwoooP runtime runs.
|
||||
|
||||
The durable ``awooop_run_state.run_id`` is the root identity. Trace and work
|
||||
item identifiers are derived from it so retries, API readback, channel mirrors,
|
||||
and downstream receipts cannot silently invent unrelated correlation values.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from uuid import UUID
|
||||
|
||||
_PROJECT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
|
||||
_TRACEPARENT_RE = re.compile(
|
||||
r"^00-(?!0{32})[0-9a-f]{32}-(?!0{16})[0-9a-f]{16}-01$"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RuntimeCorrelation:
|
||||
"""Public-safe identity shared by every node in one runtime run."""
|
||||
|
||||
run_id: UUID
|
||||
trace_id: str
|
||||
work_item_id: str
|
||||
|
||||
def public_dict(self) -> dict[str, str]:
|
||||
return {
|
||||
"run_id": str(self.run_id),
|
||||
"trace_id": self.trace_id,
|
||||
"work_item_id": self.work_item_id,
|
||||
}
|
||||
|
||||
|
||||
def new_uuid7(
|
||||
*,
|
||||
timestamp_ms: int | None = None,
|
||||
entropy: int | None = None,
|
||||
) -> UUID:
|
||||
"""Return an RFC 9562 UUIDv7 with the RFC 4122 variant bits set.
|
||||
|
||||
Optional inputs make the bit layout deterministic in focused tests. They
|
||||
are not accepted from API callers.
|
||||
"""
|
||||
|
||||
observed_ms = (
|
||||
int(time.time_ns() // 1_000_000)
|
||||
if timestamp_ms is None
|
||||
else int(timestamp_ms)
|
||||
)
|
||||
if not 0 <= observed_ms < (1 << 48):
|
||||
raise ValueError("uuid7_timestamp_out_of_range")
|
||||
|
||||
random_bits = secrets.randbits(74) if entropy is None else int(entropy)
|
||||
if not 0 <= random_bits < (1 << 74):
|
||||
raise ValueError("uuid7_entropy_out_of_range")
|
||||
|
||||
rand_a = random_bits >> 62
|
||||
rand_b = random_bits & ((1 << 62) - 1)
|
||||
value = (
|
||||
(observed_ms << 80)
|
||||
| (0x7 << 76)
|
||||
| (rand_a << 64)
|
||||
| (0b10 << 62)
|
||||
| rand_b
|
||||
)
|
||||
return UUID(int=value)
|
||||
|
||||
|
||||
def canonical_traceparent(run_id: UUID) -> str:
|
||||
"""Derive a stable W3C traceparent from one durable run identifier."""
|
||||
|
||||
trace_hex = hashlib.sha256(
|
||||
f"awooop-runtime-trace:{run_id}".encode("utf-8")
|
||||
).hexdigest()[:32]
|
||||
span_hex = hashlib.sha256(
|
||||
f"awooop-runtime-root-span:{run_id}".encode("utf-8")
|
||||
).hexdigest()[:16]
|
||||
return f"00-{trace_hex}-{span_hex}-01"
|
||||
|
||||
|
||||
def is_canonical_traceparent(value: str, *, run_id: UUID | None = None) -> bool:
|
||||
"""Validate shape and, when supplied, binding to the durable run."""
|
||||
|
||||
candidate = str(value or "").strip()
|
||||
if _TRACEPARENT_RE.fullmatch(candidate) is None:
|
||||
return False
|
||||
return run_id is None or candidate == canonical_traceparent(run_id)
|
||||
|
||||
|
||||
def canonical_work_item_id(project_id: str, run_id: UUID) -> str:
|
||||
"""Return the only generic work-item identity for one platform run."""
|
||||
|
||||
project = str(project_id or "").strip()
|
||||
if _PROJECT_ID_RE.fullmatch(project) is None:
|
||||
raise ValueError("runtime_correlation_project_id_invalid")
|
||||
return f"platform-run:{project}:{run_id}"
|
||||
|
||||
|
||||
def correlation_for_run(project_id: str, run_id: UUID) -> RuntimeCorrelation:
|
||||
"""Build the canonical public-safe correlation projection for a run."""
|
||||
|
||||
return RuntimeCorrelation(
|
||||
run_id=run_id,
|
||||
trace_id=canonical_traceparent(run_id),
|
||||
work_item_id=canonical_work_item_id(project_id, run_id),
|
||||
)
|
||||
|
||||
|
||||
def new_runtime_correlation(project_id: str) -> RuntimeCorrelation:
|
||||
"""Allocate one new run root and all identifiers derived from it."""
|
||||
|
||||
return correlation_for_run(project_id, new_uuid7())
|
||||
Reference in New Issue
Block a user