fix(agent): preserve verified runtime closure readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m18s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m18s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
This commit is contained in:
@@ -781,6 +781,133 @@ async def test_live_runtime_receipt_readback_does_not_cache_pool_unavailable(
|
||||
assert second["db_read_status"] == "ok"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_runtime_receipt_readback_uses_last_verified_cache_when_db_degrades(
|
||||
monkeypatch,
|
||||
):
|
||||
runtime_control_module._clear_runtime_receipt_readback_cache()
|
||||
calls = 0
|
||||
|
||||
async def _fake_uncached_loader(
|
||||
*,
|
||||
project_id: str,
|
||||
lookback_hours: int,
|
||||
limit: int,
|
||||
) -> dict:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
return build_runtime_receipt_readback_from_rows(
|
||||
project_id=project_id,
|
||||
lookback_hours=lookback_hours,
|
||||
db_read_status="ok",
|
||||
service_log_count_rows=[
|
||||
{"status": "sanitized_recent_logs", "total": 3, "recent": 1},
|
||||
],
|
||||
)
|
||||
return build_runtime_receipt_readback_from_rows(
|
||||
project_id=project_id,
|
||||
lookback_hours=lookback_hours,
|
||||
db_read_status="unavailable",
|
||||
error_type="RuntimeReceiptDbContextTimeout",
|
||||
log_controlled_writeback_consumer=(
|
||||
_log_controlled_writeback_consumer_readback()
|
||||
),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
runtime_control_module,
|
||||
"_load_ai_agent_autonomous_runtime_receipt_readback_uncached",
|
||||
_fake_uncached_loader,
|
||||
)
|
||||
|
||||
try:
|
||||
first = await runtime_control_module.load_ai_agent_autonomous_runtime_receipt_readback()
|
||||
cache_key = ("awoooi", 24, 20)
|
||||
stored_at, cached_readback = (
|
||||
runtime_control_module._runtime_receipt_readback_cache[cache_key]
|
||||
)
|
||||
runtime_control_module._runtime_receipt_readback_cache[cache_key] = (
|
||||
stored_at - 21.0,
|
||||
cached_readback,
|
||||
)
|
||||
second = await runtime_control_module.load_ai_agent_autonomous_runtime_receipt_readback()
|
||||
finally:
|
||||
runtime_control_module._clear_runtime_receipt_readback_cache()
|
||||
|
||||
assert calls == 2
|
||||
assert first["db_read_status"] == "ok"
|
||||
assert second["db_read_status"] == "ok"
|
||||
assert second["cache_fallback_active"] is True
|
||||
assert second["record_quality"] == "cached_verified_runtime_receipt_readback"
|
||||
fallback = second["runtime_receipt_cache_fallback"]
|
||||
assert fallback["active"] is True
|
||||
assert fallback["reason"] == "live_runtime_receipt_readback_degraded"
|
||||
assert fallback["live_db_read_status"] == "unavailable"
|
||||
assert fallback["live_error_type"] == "RuntimeReceiptDbContextTimeout"
|
||||
assert fallback["cache_age_seconds"] >= 21.0
|
||||
assert fallback["max_stale_seconds"] == 300.0
|
||||
assert fallback["source"] == "in_process_last_verified_db_receipt_readback"
|
||||
assert fallback["durable_receipts_reused"] is True
|
||||
assert fallback["writes_on_read"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_runtime_receipt_lock_timeout_uses_last_verified_cache(
|
||||
monkeypatch,
|
||||
):
|
||||
runtime_control_module._clear_runtime_receipt_readback_cache()
|
||||
|
||||
async def _fake_uncached_loader(
|
||||
*,
|
||||
project_id: str,
|
||||
lookback_hours: int,
|
||||
limit: int,
|
||||
) -> dict:
|
||||
return build_runtime_receipt_readback_from_rows(
|
||||
project_id=project_id,
|
||||
lookback_hours=lookback_hours,
|
||||
db_read_status="ok",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
runtime_control_module,
|
||||
"_load_ai_agent_autonomous_runtime_receipt_readback_uncached",
|
||||
_fake_uncached_loader,
|
||||
)
|
||||
first = await runtime_control_module.load_ai_agent_autonomous_runtime_receipt_readback()
|
||||
assert first["db_read_status"] == "ok"
|
||||
|
||||
monkeypatch.setattr(
|
||||
runtime_control_module,
|
||||
"_RUNTIME_RECEIPT_READBACK_CACHE_TTL_SECONDS",
|
||||
-1.0,
|
||||
)
|
||||
lock = asyncio.Lock()
|
||||
await lock.acquire()
|
||||
monkeypatch.setattr(runtime_control_module, "_runtime_receipt_readback_lock", lock)
|
||||
monkeypatch.setattr(
|
||||
runtime_control_module,
|
||||
"_RUNTIME_RECEIPT_LOCK_TIMEOUT_SECONDS",
|
||||
0.01,
|
||||
)
|
||||
|
||||
try:
|
||||
second = await runtime_control_module.load_ai_agent_autonomous_runtime_receipt_readback()
|
||||
finally:
|
||||
lock.release()
|
||||
runtime_control_module._clear_runtime_receipt_readback_cache()
|
||||
|
||||
assert second["db_read_status"] == "ok"
|
||||
assert second["cache_fallback_active"] is True
|
||||
assert second["runtime_receipt_cache_fallback"]["reason"] == (
|
||||
"runtime_receipt_readback_lock_timeout"
|
||||
)
|
||||
assert second["runtime_receipt_cache_fallback"]["live_error_type"] == (
|
||||
"RuntimeReceiptReadbackLockTimeout"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_control_live_readback_timeout_returns_degraded_payload(
|
||||
monkeypatch,
|
||||
|
||||
Reference in New Issue
Block a user