fix(agents): stabilize runtime readback under transient pressure
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m7s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled

This commit is contained in:
ogt
2026-07-10 00:12:51 +08:00
parent 9f862bd254
commit a1ec18b9ba
2 changed files with 108 additions and 4 deletions

View File

@@ -532,6 +532,8 @@ async def test_live_runtime_receipt_readback_returns_partial_when_query_budget_e
async def test_log_controlled_writeback_consumer_timeout_returns_fallback(
monkeypatch,
):
runtime_control_module._clear_runtime_receipt_readback_cache()
async def _slow_consumer_readback(*, project_id: str):
assert project_id == "awoooi"
await asyncio.sleep(1)
@@ -548,9 +550,12 @@ async def test_log_controlled_writeback_consumer_timeout_returns_fallback(
_slow_consumer_readback,
)
readback = await runtime_control_module._load_log_controlled_writeback_consumer_readback(
project_id="awoooi",
)
try:
readback = await runtime_control_module._load_log_controlled_writeback_consumer_readback(
project_id="awoooi",
)
finally:
runtime_control_module._clear_runtime_receipt_readback_cache()
assert readback["status"] == "blocked_waiting_controlled_writeback_consumer_receipts"
assert "log_controlled_writeback_consumer_readback_unavailable" in readback[
@@ -559,6 +564,50 @@ async def test_log_controlled_writeback_consumer_timeout_returns_fallback(
assert readback["readback"]["error_type"] == "TimeoutError"
@pytest.mark.asyncio
async def test_log_controlled_writeback_consumer_timeout_uses_cached_ready_readback(
monkeypatch,
):
runtime_control_module._clear_runtime_receipt_readback_cache()
calls = 0
async def _sometimes_slow_consumer_readback(*, project_id: str):
nonlocal calls
assert project_id == "awoooi"
calls += 1
if calls == 1:
return _log_controlled_writeback_consumer_readback()
await asyncio.sleep(1)
return _log_controlled_writeback_consumer_readback()
monkeypatch.setattr(
runtime_control_module,
"_LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS",
0.01,
)
monkeypatch.setattr(
runtime_control_module,
"load_latest_ai_agent_log_controlled_writeback_consumer_readback",
_sometimes_slow_consumer_readback,
)
try:
first = await runtime_control_module._load_log_controlled_writeback_consumer_readback(
project_id="awoooi",
)
second = await runtime_control_module._load_log_controlled_writeback_consumer_readback(
project_id="awoooi",
)
finally:
runtime_control_module._clear_runtime_receipt_readback_cache()
assert first["status"] == "controlled_writeback_consumer_readback_ready"
assert second["status"] == "controlled_writeback_consumer_readback_ready"
assert second["cache_fallback_active"] is True
assert second["record_quality"] == "cached_live_consumer_readback"
assert second["readback"]["cache_fallback_error_type"] == "TimeoutError"
@pytest.mark.asyncio
async def test_live_runtime_receipt_readback_uses_short_cache_to_protect_db_pool(
monkeypatch,
@@ -1676,6 +1725,10 @@ def test_runtime_receipt_auxiliary_alert_query_partial_does_not_block_runtime_tr
],
partial_query_failures=[
{"query_name": "legacy_mcp_counts", "error_type": "DBAPIError"},
{
"query_name": "playbook_trust_counts",
"error_type": "RuntimeReceiptQueryBudgetExceeded",
},
{"query_name": "alert_operation_counts", "error_type": "DBAPIError"},
{
"query_name": "alertmanager_event_counts",