fix(automation): recover verified backup backfill failures

apps/api/tests/test_backup_restore_legacy_backfill_job.py

@@ -482,6 +482,51 @@ async def test_claim_sweeps_expired_max_attempt_before_selecting_retryable_work(
     assert statements == [job._EXHAUST_EXPIRED_CLAIMS_SQL, job._CLAIM_SQL]
 
 
+@pytest.mark.asyncio
+async def test_db_contract_repair_requeues_only_one_bounded_generation(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    returned = [
+        {"run_id": _claim_from_row(index).run_id}
+        for index in range(3)
+    ]
+
+    class FakeDb:
+        async def execute(
+            self,
+            statement: Any,
+            params: dict[str, Any],
+        ) -> _Result:
+            assert statement is job._REQUEUE_REPAIRED_DBAPI_FAILURES_SQL
+            assert params == {
+                "project_id": "awoooi",
+                "agent_id": job.BACKFILL_ITEM_AGENT_ID,
+                "repair_generation": job.BACKFILL_DB_CONTRACT_REPAIR_GENERATION,
+                "limit": job.MAX_SCOPE_ROWS,
+            }
+            return _Result(rows=returned)
+
+    monkeypatch.setattr(
+        job,
+        "get_db_context",
+        lambda _project_id: _DbContext(FakeDb()),
+    )
+
+    requeued = await job._requeue_repaired_dbapi_failures(
+        project_id="awoooi",
+        limit=job.MAX_SCOPE_ROWS + 500,
+    )
+
+    assert requeued == 3
+    sql = str(job._REQUEUE_REPAIRED_DBAPI_FAILURES_SQL)
+    assert "E-BACKFILL-EXHAUSTED" in sql
+    assert "processor_error:DBAPIError" in sql
+    assert "runtime_execution_authorized" in sql
+    assert "recovery_generation" in sql
+    assert "FOR UPDATE SKIP LOCKED" in sql
+    assert "attempt_count = 0" in sql
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     ("item_total", "completed_total", "expected_status", "expected_cursor_state"),
@@ -1356,6 +1401,8 @@ async def test_default_processor_accepts_repo_known_lan_without_token(
     update_cursor = AsyncMock(
         return_value={"status": "completed", "remaining_item_total": 0}
     )
+    requeue = AsyncMock(return_value=0)
+    monkeypatch.setattr(job, "_requeue_repaired_dbapi_failures", requeue)
     monkeypatch.setattr(job, "_reserve_legacy_backup_page", reserve)
     monkeypatch.setattr(job, "_claim_legacy_backup_items", claim)
     monkeypatch.setattr(job, "_update_cursor_counters", update_cursor)
@@ -1371,6 +1418,9 @@ async def test_default_processor_accepts_repo_known_lan_without_token(
     assert result["status"] == "completed"
     assert result["auth_mode"] == "lan_allowlist"
     assert result["relay_preflight"]["ready"] is True
+    assert result["requeued_repaired_dbapi"] == 0
+    assert result["backfill_state_write_performed"] is False
+    requeue.assert_awaited_once_with(project_id="awoooi", limit=100)
     reserve.assert_awaited_once()
     claim.assert_awaited_once()