fix(agents): stabilize alert automation readbacks
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled

This commit is contained in:
ogt
2026-07-10 01:17:21 +08:00
parent b7d1b17924
commit 403c0a3d05
4 changed files with 269 additions and 14 deletions

View File

@@ -180,6 +180,20 @@ def _coverage_payload() -> dict:
}
def _closed_coverage_payload() -> dict:
payload = _coverage_payload()
payload["status"] = "telegram_alert_monitoring_coverage_ready"
payload["summary"] = {
**payload["summary"],
"monitoring_live_receipt_gap_count": 0,
"ai_controlled_live_receipt_batch_count": 0,
"full_coverage_ready": True,
}
payload["ai_controlled_live_receipt_batches"] = []
payload["active_blockers"] = []
return payload
def _row_for(batch: dict, *, op_id: str) -> dict:
return {
"op_id": op_id,
@@ -243,6 +257,56 @@ def test_telegram_monitoring_live_receipt_apply_builder_creates_receipts():
assert marker not in serialized
@pytest.mark.asyncio
async def test_telegram_monitoring_live_receipt_apply_noops_when_gap_closed(
monkeypatch,
):
async def fake_coverage(*, project_id: str):
assert project_id == "awoooi"
return _closed_coverage_payload()
monkeypatch.setattr(
apply_module,
"load_latest_telegram_alert_monitoring_coverage_readback",
fake_coverage,
)
monkeypatch.setattr(
apply_module,
"_write_live_receipts_direct",
lambda *, project_id, receipt: None,
)
payload = await apply_latest_telegram_alert_monitoring_live_receipts(
project_id="awoooi"
)
assert payload["status"] == "telegram_monitoring_live_receipt_apply_noop_ready"
assert payload["active_blockers"] == []
assert payload["rollups"]["source_live_receipt_batch_count"] == 0
assert payload["rollups"]["controlled_live_receipt_apply_ready"] is True
assert payload["apply_result"]["db_write_status"] == "not_needed"
assert payload["apply_result"]["reason"] == (
"coverage_has_no_open_monitoring_live_receipt_gap"
)
assert payload["operation_boundaries"]["runtime_target_write_performed"] is False
def test_telegram_monitoring_live_receipt_apply_readback_ready_when_gap_closed():
payload = build_telegram_alert_monitoring_live_receipt_apply_readback(
coverage=_closed_coverage_payload(),
rows=[],
)
assert payload["status"] == "telegram_monitoring_live_receipt_apply_readback_ready"
assert payload["active_blockers"] == []
assert payload["operator_answer"][
"all_ai_controlled_live_receipt_batches_have_apply_receipt"
] is True
assert payload["operator_answer"]["metadata_live_receipt_apply_performed"] is True
assert payload["rollups"]["expected_live_receipt_batch_count"] == 0
assert payload["rollups"]["metadata_live_receipt_apply_readback_ready"] is True
@pytest.mark.asyncio
async def test_telegram_monitoring_live_receipt_apply_writes_idempotent_rows(
monkeypatch,
@@ -258,11 +322,21 @@ async def test_telegram_monitoring_live_receipt_apply_writes_idempotent_rows(
assert project_id == "awoooi"
return _coverage_payload()
async def unavailable_direct_write(*, project_id: str, receipt: dict):
assert project_id == "awoooi"
assert receipt["live_receipt_apply_receipts"]
return None
monkeypatch.setattr(
apply_module,
"load_latest_telegram_alert_monitoring_coverage_readback",
fake_coverage,
)
monkeypatch.setattr(
apply_module,
"_write_live_receipts_direct",
unavailable_direct_write,
)
payload = await apply_latest_telegram_alert_monitoring_live_receipts(
project_id="awoooi"
@@ -275,6 +349,7 @@ async def test_telegram_monitoring_live_receipt_apply_writes_idempotent_rows(
assert payload["operation_boundaries"]["runtime_target_write_performed"] is True
assert payload["apply_result"]["operation_type"] == OPERATION_TYPE
assert payload["apply_result"]["ledger_operation_type"] == "km_linked"
assert payload["apply_result"]["db_write_status"] == "ok"
write_params = [params for params in fake_db.params if "input" in params]
assert len(write_params) == 2
@@ -289,6 +364,66 @@ async def test_telegram_monitoring_live_receipt_apply_writes_idempotent_rows(
assert '"telegram_send_performed": false' in params["output"]
@pytest.mark.asyncio
async def test_telegram_monitoring_live_receipt_apply_uses_direct_after_write_timeout(
monkeypatch,
):
coverage = _coverage_payload()
direct_rows = [
{
"live_receipt_id": f"{OPERATION_TYPE}::{batch['batch_id']}",
"batch_id": batch["batch_id"],
"domain": batch["domain"],
"created": False,
"op_id": f"direct-live-op-{index}",
}
for index, batch in enumerate(
coverage["ai_controlled_live_receipt_batches"],
start=1,
)
]
async def fake_coverage(*, project_id: str):
assert project_id == "awoooi"
return coverage
async def failing_write(*, project_id: str, receipt: dict):
assert project_id == "awoooi"
assert receipt["live_receipt_apply_receipts"]
raise TimeoutError("pool checkout unavailable before direct fallback")
async def fake_direct_write(*, project_id: str, receipt: dict):
assert project_id == "awoooi"
assert receipt["live_receipt_apply_receipts"]
return "km_linked", direct_rows
monkeypatch.setattr(
apply_module,
"load_latest_telegram_alert_monitoring_coverage_readback",
fake_coverage,
)
monkeypatch.setattr(
apply_module,
"_write_live_receipts_with_retry",
failing_write,
)
monkeypatch.setattr(
apply_module,
"_write_live_receipts_direct",
fake_direct_write,
)
payload = await apply_latest_telegram_alert_monitoring_live_receipts(
project_id="awoooi"
)
assert payload["status"] == "telegram_monitoring_live_receipt_apply_ready"
assert payload["active_blockers"] == []
assert payload["apply_result"]["db_write_status"] == "ok"
assert payload["apply_result"]["existing_count"] == 2
assert payload["rollups"]["runtime_target_write_performed"] is True
def test_telegram_monitoring_live_receipt_apply_readback_preserves_gap_truth():
coverage = _coverage_payload()
rows = [
@@ -321,6 +456,57 @@ def test_telegram_monitoring_live_receipt_apply_readback_preserves_gap_truth():
assert payload["operation_boundaries"]["telegram_send_performed"] is False
@pytest.mark.asyncio
async def test_telegram_monitoring_live_receipt_apply_readback_uses_direct_rows_after_timeout(
monkeypatch,
):
coverage = _coverage_payload()
rows = [
_row_for(batch, op_id=f"direct-live-op-{index}")
for index, batch in enumerate(
coverage["ai_controlled_live_receipt_batches"],
start=1,
)
]
async def fake_coverage(*, project_id: str):
assert project_id == "awoooi"
return coverage
async def failing_read(*, project_id: str):
assert project_id == "awoooi"
raise TimeoutError("session readback timed out before direct fallback")
async def fake_direct_read(*, project_id: str):
assert project_id == "awoooi"
return rows
monkeypatch.setattr(
apply_module,
"load_latest_telegram_alert_monitoring_coverage_readback",
fake_coverage,
)
monkeypatch.setattr(
apply_module,
"_load_live_receipt_rows_with_retry",
failing_read,
)
monkeypatch.setattr(
apply_module,
"_load_live_receipt_rows_direct",
fake_direct_read,
)
payload = await load_latest_telegram_alert_monitoring_live_receipt_apply_readback(
project_id="awoooi"
)
assert payload["status"] == "telegram_monitoring_live_receipt_apply_readback_ready"
assert payload["active_blockers"] == []
assert payload["rollups"]["applied_live_receipt_batch_count"] == 2
assert payload["rollups"]["metadata_live_receipt_apply_readback_ready"] is True
@pytest.mark.asyncio
async def test_telegram_monitoring_live_receipt_apply_degrades_on_db_write_pressure(
monkeypatch,