fix(automation): reconcile new backup alert cards

This commit is contained in:
ogt
2026-07-15 04:59:24 +08:00
parent 1190c737c5
commit 098bdd81fe
2 changed files with 428 additions and 5 deletions

View File

@@ -527,6 +527,102 @@ async def test_db_contract_repair_requeues_only_one_bounded_generation(
assert "attempt_count = 0" in sql
@pytest.mark.asyncio
async def test_live_reconciler_reserves_only_unowned_incomplete_cards(
monkeypatch: pytest.MonkeyPatch,
) -> None:
rows = [_source_row(201), _source_row(202)]
inserted_agents: list[str] = []
trigger_refs: list[str] = []
class FakeDb:
async def execute(
self,
statement: Any,
params: dict[str, Any],
) -> _Result:
if statement is job._LIVE_SOURCE_SCAN_SQL:
assert params["project_id"] == "awoooi"
assert params["backfill_agent_id"] == job.BACKFILL_ITEM_AGENT_ID
assert params["live_agent_id"] == job.LIVE_RECONCILE_AGENT_ID
assert params["limit"] == job.MAX_BATCH_LIMIT
return _Result(rows=rows)
if statement is job._ITEM_RUN_INSERT_SQL:
inserted_agents.append(str(params["agent_id"]))
trigger_refs.append(str(params["trigger_ref"]))
return _Result()
if statement is job._ITEM_IDEMPOTENCY_INSERT_SQL:
assert params["channel_type"] == job.LIVE_RECONCILE_IDEMPOTENCY_CHANNEL
return _Result(value=params["run_id"])
raise AssertionError(f"unexpected statement: {statement}")
monkeypatch.setattr(
job,
"get_db_context",
lambda _project_id: _DbContext(FakeDb()),
)
result = await job._reserve_live_backup_cards(
project_id="awoooi",
limit=job.MAX_BATCH_LIMIT + 10,
)
assert result == {"scanned": 2, "reserved": 2, "deduplicated": 0}
assert inserted_agents == [job.LIVE_RECONCILE_AGENT_ID] * 2
assert all(value.startswith("backup-signal:") for value in trigger_refs)
sql = str(job._LIVE_SOURCE_SCAN_SQL)
assert "NOT EXISTS" in sql
assert "legacy-backup:" in sql
assert "backup-signal:" in sql
@pytest.mark.asyncio
async def test_live_reconciler_projects_bounded_agent99_receipt(
monkeypatch: pytest.MonkeyPatch,
) -> None:
claim = job.LegacyBackupClaim(
**{
**_claim_from_row(203).__dict__,
"agent_id": job.LIVE_RECONCILE_AGENT_ID,
}
)
reserve = AsyncMock(
return_value={"scanned": 1, "reserved": 1, "deduplicated": 0}
)
claims = AsyncMock(return_value=[claim])
replay = AsyncMock(return_value="completed")
counts = AsyncMock(
return_value={
"item_total": 1,
"remaining_total": 0,
"completed_total": 1,
"failed_total": 0,
}
)
processor = AsyncMock()
monkeypatch.setattr(job, "_reserve_live_backup_cards", reserve)
monkeypatch.setattr(job, "_claim_legacy_backup_items", claims)
monkeypatch.setattr(job, "_replay_backfill_claim", replay)
monkeypatch.setattr(job, "_live_reconcile_state_counts", counts)
result = await job.run_backup_restore_outbound_reconciler_once(
project_id="awoooi",
limit=7,
processor=processor,
)
assert result["status"] == "reconciled"
assert result["dispatch_total"] == 1
assert result["runtime_execution_authorized"] is False
assert result["telegram_dispatch_performed"] is False
claims.assert_awaited_once_with(
project_id="awoooi",
limit=7,
agent_id=job.LIVE_RECONCILE_AGENT_ID,
)
replay.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize(
("item_total", "completed_total", "expected_status", "expected_cursor_state"),
@@ -1437,6 +1533,7 @@ async def test_one_time_loop_exits_after_durable_snapshot_completes(
}
)
sleep = AsyncMock()
live_loop = AsyncMock()
monkeypatch.setattr(job.settings, "ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL", True)
monkeypatch.setattr(
job.settings,
@@ -1450,11 +1547,17 @@ async def test_one_time_loop_exits_after_durable_snapshot_completes(
0.0,
)
monkeypatch.setattr(job, "run_backup_restore_legacy_backfill_once", tick)
monkeypatch.setattr(
job,
"run_backup_restore_outbound_reconciler_loop",
live_loop,
)
monkeypatch.setattr(job.asyncio, "sleep", sleep)
await job.run_backup_restore_legacy_backfill_loop()
tick.assert_awaited_once()
live_loop.assert_awaited_once()
sleep.assert_awaited_once_with(0.0)