fix(agents): stabilize runtime readback under transient pressure
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -53,6 +53,7 @@ _RUNTIME_RECEIPT_QUERY_BUDGET_SECONDS = 2.0
|
||||
_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS = 0.45
|
||||
_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS = 400
|
||||
_LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS = 4.0
|
||||
_LOG_CONTROLLED_WRITEBACK_CONSUMER_READBACK_CACHE_TTL_SECONDS = 30.0
|
||||
_RUNTIME_RECEIPT_READBACK_CACHE_TTL_SECONDS = 20.0
|
||||
_AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES = {
|
||||
"alert_operation_counts",
|
||||
@@ -60,6 +61,7 @@ _AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES = {
|
||||
"grouped_alert_event_counts",
|
||||
"db_context_exit",
|
||||
"legacy_mcp_counts",
|
||||
"playbook_trust_counts",
|
||||
}
|
||||
_CONSUMER_RECEIPT_FALLBACK_ERROR_TYPES = {
|
||||
"RuntimeReceiptDbContextTimeout",
|
||||
@@ -102,6 +104,10 @@ _runtime_receipt_readback_cache: dict[
|
||||
tuple[str, int, int],
|
||||
tuple[float, dict[str, Any]],
|
||||
] = {}
|
||||
_log_controlled_writeback_consumer_readback_cache: dict[
|
||||
str,
|
||||
tuple[float, dict[str, Any]],
|
||||
] = {}
|
||||
_runtime_receipt_readback_lock: asyncio.Lock | None = None
|
||||
|
||||
|
||||
@@ -213,6 +219,7 @@ def _runtime_receipt_readback_cache_store(
|
||||
|
||||
def _clear_runtime_receipt_readback_cache() -> None:
|
||||
_runtime_receipt_readback_cache.clear()
|
||||
_log_controlled_writeback_consumer_readback_cache.clear()
|
||||
|
||||
|
||||
def _sanitize_latest_rows(
|
||||
@@ -471,18 +478,33 @@ async def _load_log_controlled_writeback_consumer_readback(
|
||||
"""Attach LOG consumer bindings without writing KM/RAG/PlayBook/MCP targets."""
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
readback = await asyncio.wait_for(
|
||||
load_latest_ai_agent_log_controlled_writeback_consumer_readback(
|
||||
project_id=project_id,
|
||||
),
|
||||
timeout=_LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS,
|
||||
)
|
||||
_log_controlled_writeback_consumer_readback_cache_store(
|
||||
project_id=project_id,
|
||||
readback=readback,
|
||||
)
|
||||
return readback
|
||||
except Exception as exc: # pragma: no cover - keeps runtime control API visible
|
||||
logger.warning(
|
||||
"log_controlled_writeback_consumer_readback_failed",
|
||||
project_id=project_id,
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
cached_readback = _log_controlled_writeback_consumer_readback_cache_get(
|
||||
project_id=project_id,
|
||||
)
|
||||
if cached_readback is not None:
|
||||
cached_readback["cache_fallback_active"] = True
|
||||
cached_readback["record_quality"] = "cached_live_consumer_readback"
|
||||
readback_info = cached_readback.get("readback")
|
||||
if isinstance(readback_info, dict):
|
||||
readback_info["cache_fallback_error_type"] = type(exc).__name__
|
||||
return cached_readback
|
||||
return _fallback_log_controlled_writeback_consumer_readback(
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
@@ -602,6 +624,35 @@ def _log_controlled_writeback_consumer_ready(
|
||||
)
|
||||
|
||||
|
||||
def _log_controlled_writeback_consumer_readback_cache_get(
|
||||
*,
|
||||
project_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
cached = _log_controlled_writeback_consumer_readback_cache.get(project_id)
|
||||
if cached is None:
|
||||
return None
|
||||
stored_at, readback = cached
|
||||
if (
|
||||
time.monotonic() - stored_at
|
||||
> _LOG_CONTROLLED_WRITEBACK_CONSUMER_READBACK_CACHE_TTL_SECONDS
|
||||
):
|
||||
_log_controlled_writeback_consumer_readback_cache.pop(project_id, None)
|
||||
return None
|
||||
return copy.deepcopy(readback)
|
||||
|
||||
|
||||
def _log_controlled_writeback_consumer_readback_cache_store(
|
||||
*,
|
||||
project_id: str,
|
||||
readback: Mapping[str, Any],
|
||||
) -> None:
|
||||
if _log_controlled_writeback_consumer_ready(readback):
|
||||
_log_controlled_writeback_consumer_readback_cache[project_id] = (
|
||||
time.monotonic(),
|
||||
copy.deepcopy(dict(readback)),
|
||||
)
|
||||
|
||||
|
||||
def _consumer_receipt_fallback_active(
|
||||
*,
|
||||
db_read_status: str,
|
||||
|
||||
Reference in New Issue
Block a user