fix(agents): bound runtime receipt db entry
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m11s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
ogt
2026-07-09 18:35:55 +08:00
parent c53e9f610f
commit 763bd5b97a
2 changed files with 139 additions and 4 deletions

View File

@@ -46,6 +46,9 @@ _LIVE_READBACK_SCHEMA_VERSION = "ai_agent_autonomous_runtime_receipt_readback_v1
_DEFAULT_PROJECT_ID = "awoooi"
_DEFAULT_LOOKBACK_HOURS = 24
_LIVE_RUNTIME_RECEIPT_TIMEOUT_SECONDS = 5.0
_RUNTIME_RECEIPT_LOCK_TIMEOUT_SECONDS = 0.25
_RUNTIME_RECEIPT_DB_CONTEXT_TIMEOUT_SECONDS = 0.7
_RUNTIME_RECEIPT_DB_CONTEXT_EXIT_TIMEOUT_SECONDS = 0.35
_RUNTIME_RECEIPT_QUERY_BUDGET_SECONDS = 2.75
_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS = 0.7
_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS = 650
@@ -4090,7 +4093,14 @@ async def load_ai_agent_autonomous_runtime_receipt_readback(
if cached is not None:
return cached
async with _get_runtime_receipt_readback_lock():
lock = _get_runtime_receipt_readback_lock()
lock_acquired = False
try:
await asyncio.wait_for(
lock.acquire(),
timeout=_RUNTIME_RECEIPT_LOCK_TIMEOUT_SECONDS,
)
lock_acquired = True
cached = _runtime_receipt_readback_cache_get(cache_key)
if cached is not None:
return cached
@@ -4101,6 +4111,21 @@ async def load_ai_agent_autonomous_runtime_receipt_readback(
)
_runtime_receipt_readback_cache_store(cache_key, readback)
return readback
except TimeoutError:
logger.warning(
"ai_agent_autonomous_runtime_receipt_readback_lock_timeout",
project_id=project_id,
timeout_seconds=_RUNTIME_RECEIPT_LOCK_TIMEOUT_SECONDS,
)
return build_runtime_receipt_readback_from_rows(
project_id=project_id,
lookback_hours=normalized_lookback_hours,
db_read_status="unavailable",
error_type="RuntimeReceiptReadbackLockTimeout",
)
finally:
if lock_acquired:
lock.release()
async def _load_ai_agent_autonomous_runtime_receipt_readback_uncached(
@@ -4118,8 +4143,17 @@ async def _load_ai_agent_autonomous_runtime_receipt_readback_uncached(
}
log_controlled_writeback_consumer: dict[str, Any] | None = None
partial_query_failures: list[dict[str, str]] = []
db_context = get_db_context(project_id)
db_context_entered = False
db_context_stage = "db_context_enter"
try:
async with get_db_context(project_id) as db:
db = await asyncio.wait_for(
db_context.__aenter__(),
timeout=_RUNTIME_RECEIPT_DB_CONTEXT_TIMEOUT_SECONDS,
)
db_context_entered = True
db_context_stage = "db_queries"
try:
query_deadline = time.monotonic() + min(
_RUNTIME_RECEIPT_QUERY_BUDGET_SECONDS,
max(0.5, _LIVE_RUNTIME_RECEIPT_TIMEOUT_SECONDS - 0.75),
@@ -4300,11 +4334,35 @@ async def _load_ai_agent_autonomous_runtime_receipt_readback_uncached(
"grouped_alert_event_counts",
_RUNTIME_GROUPED_ALERT_EVENT_COUNTS_SQL,
)
finally:
if db_context_entered:
db_context_stage = "db_context_exit"
try:
await asyncio.wait_for(
db_context.__aexit__(None, None, None),
timeout=_RUNTIME_RECEIPT_DB_CONTEXT_EXIT_TIMEOUT_SECONDS,
)
except Exception as exit_exc: # pragma: no cover - live DB state
partial_query_failures.append({
"query_name": "db_context_exit",
"error_type": type(exit_exc).__name__,
})
logger.warning(
"ai_agent_autonomous_runtime_trace_db_context_exit_failed",
project_id=project_id,
error_type=type(exit_exc).__name__,
)
except Exception as exc:
error_type = (
"RuntimeReceiptDbContextTimeout"
if isinstance(exc, TimeoutError) and db_context_stage == "db_context_enter"
else type(exc).__name__
)
logger.warning(
"ai_agent_autonomous_runtime_receipt_readback_failed",
project_id=project_id,
error_type=type(exc).__name__,
error_type=error_type,
db_context_stage=db_context_stage,
)
log_controlled_writeback_consumer = (
await _load_log_controlled_writeback_consumer_readback(
@@ -4315,7 +4373,7 @@ async def _load_ai_agent_autonomous_runtime_receipt_readback_uncached(
project_id=project_id,
lookback_hours=params["lookback_hours"],
db_read_status="unavailable",
error_type=type(exc).__name__,
error_type=error_type,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
)