Pagination with keyset (Sourcery refactored)

docs/custom_queryset.md

@@ -6,7 +6,11 @@ flask-mongoengine attaches the following methods to Mongoengine's default QueryS
   Optional arguments: *message* - custom message to display.
 * **first_or_404**: same as above, except for .first().
   Optional arguments: *message* - custom message to display.
-* **paginate**: paginates the QuerySet. Takes two arguments, *page* and *per_page*.
+* **paginate**: paginates the QuerySet. Takes two required arguments, *page* and *per_page*.
+   And one optional arguments *max_depth*.
+* **paginate_by_keyset**: paginates the QuerySet. Takes two required arguments, *per_page* and *field_filter_by*.
+  from the second page you need also the last id of the previous page.
+  Arguments: *per_page*, *field_filter_by*, *last_field_value*.
 * **paginate_field**: paginates a field from one document in the QuerySet.
   Arguments: *field_name*, *doc_id*, *page*, *per_page*.
 
@@ -21,6 +25,10 @@ def view_todo(todo_id):
 def view_todos(page=1):
     paginated_todos = Todo.objects.paginate(page=page, per_page=10)
 
+# Paginate by keyset through todo
+def view_todos_by_keyset(page=0):
+    paginated_todos = Todo.objects.paginate(page=page, per_page=10)
+
 # Paginate through tags of todo
 def view_todo_tags(todo_id, page=1):
     todo = Todo.objects.get_or_404(_id=todo_id)

flask_mongoengine/documents.py

@@ -8,7 +8,7 @@
 from mongoengine.queryset import QuerySet
 
 from flask_mongoengine.decorators import wtf_required
-from flask_mongoengine.pagination import ListFieldPagination, Pagination
+from flask_mongoengine.pagination import ListFieldPagination, Pagination, KeysetPagination
 
 try:
     from flask_mongoengine.wtf.models import ModelForm
@@ -60,6 +60,13 @@ def paginate(self, page, per_page):
         """
         return Pagination(self, page, per_page)
 
+    def paginate_by_keyset(self, per_page, field_filter_by, last_field_value):
+        """
+        Paginate the QuerySet with a certain number of docs per page
+        and return docs for a given page.
+        """
+        return KeysetPagination(self, per_page, field_filter_by, last_field_value)
+
     def paginate_field(self, field_name, doc_id, page, per_page, total=None):
         """
         Paginate items within a list field from one document in the

flask_mongoengine/pagination/__init__.py

@@ -0,0 +1,5 @@
+from flask_mongoengine.pagination.basic_pagination import Pagination
+from flask_mongoengine.pagination.list_field_pagination import ListFieldPagination
+from flask_mongoengine.pagination.keyset_pagination import KeysetPagination
+
+__all__ = ("Pagination", "ListFieldPagination", "KeysetPagination")

flask_mongoengine/pagination.py → flask_mongoengine/pagination/basic_pagination.py

@@ -1,14 +1,17 @@
-"""Module responsible for custom pagination."""
 import math
 
 from flask import abort
 from mongoengine.queryset import QuerySet
 
-__all__ = ("Pagination", "ListFieldPagination")
-
 
 class Pagination(object):
-    def __init__(self, iterable, page, per_page):
+    def __init__(self, iterable, page: int, per_page: int, max_depth: int = None):
+        """
+        :param iterable: iterable object .
+        :param page: Required page number start from 1.
+        :param per_page: Required number of documents per page.
+        :param max_depth: Option for limit number of dereference documents.
+        """
 
         if page < 1:
             abort(404)
@@ -19,11 +22,9 @@ def __init__(self, iterable, page, per_page):
 
         if isinstance(self.iterable, QuerySet):
             self.total = iterable.count()
-            self.items = (
-                self.iterable.skip(self.per_page * (self.page - 1))
-                .limit(self.per_page)
-                .select_related()
-            )
+            self.items = self.iterable.skip(self.per_page * (self.page - 1)).limit(self.per_page)
+            if max_depth is not None:
+                self.items = self.items.select_related(max_depth)
         else:
             start_index = (page - 1) * per_page
             end_index = page * per_page
@@ -119,66 +120,3 @@ def iter_pages(self, left_edge=2, left_current=2, right_current=5, right_edge=2)
                 last = num
         if last != self.pages:
             yield None
-
-
-class ListFieldPagination(Pagination):
-    def __init__(self, queryset, doc_id, field_name, page, per_page, total=None):
-        """Allows an array within a document to be paginated.
-
-        Queryset must contain the document which has the array we're
-        paginating, and doc_id should be it's _id.
-        Field name is the name of the array we're paginating.
-        Page and per_page work just like in Pagination.
-        Total is an argument because it can be computed more efficiently
-        elsewhere, but we still use array.length as a fallback.
-        """
-        if page < 1:
-            abort(404)
-
-        self.page = page
-        self.per_page = per_page
-
-        self.queryset = queryset
-        self.doc_id = doc_id
-        self.field_name = field_name
-
-        start_index = (page - 1) * per_page
-
-        field_attrs = {field_name: {"$slice": [start_index, per_page]}}
-
-        qs = queryset(pk=doc_id)
-        self.items = getattr(qs.fields(**field_attrs).first(), field_name)
-        self.total = total or len(
-            getattr(qs.fields(**{field_name: 1}).first(), field_name)
-        )
-
-        if not self.items and page != 1:
-            abort(404)
-
-    def prev(self, error_out=False):
-        """Returns a :class:`Pagination` object for the previous page."""
-        assert (
-            self.items is not None
-        ), "a query object is required for this method to work"
-        return self.__class__(
-            self.queryset,
-            self.doc_id,
-            self.field_name,
-            self.page - 1,
-            self.per_page,
-            self.total,
-        )
-
-    def next(self, error_out=False):
-        """Returns a :class:`Pagination` object for the next page."""
-        assert (
-            self.items is not None
-        ), "a query object is required for this method to work"
-        return self.__class__(
-            self.queryset,
-            self.doc_id,
-            self.field_name,
-            self.page + 1,
-            self.per_page,
-            self.total,
-        )

flask_mongoengine/pagination/keyset_pagination.py

@@ -0,0 +1,88 @@
+from mongoengine.queryset import QuerySet
+
+from flask_mongoengine.pagination.basic_pagination import Pagination
+
+
+class KeysetPagination(Pagination):
+    def __init__(self, iterable, per_page: int, field_filter_by: str = '_id', last_field_value=None):
+        """
+        :param iterable: iterable object .
+        :param page: Required page number start from 1.
+        :param per_page: Required number of documents per page.
+        :param max_depth: Option for limit number of dereference documents.
+        """
+        self.get_page(iterable, per_page, field_filter_by, last_field_value)
+
+    def get_page(self, iterable, per_page: int, field_filter_by: str = '_id', last_field_value=None,
+                 direction='forward'):
+        if last_field_value is None:
+            self.page = 0
+        elif getattr(self, 'page', False):
+            self.page += 1
+        else:
+            self.page = 1
+
+        if direction == 'forward':
+            op = 'gt'
+            order_by = field_filter_by
+        elif direction == 'backward':
+            op = 'lt'
+            order_by = f'-{field_filter_by}'
+
+        else:
+            raise ValueError
+
+        self.iterable = iterable
+        self.field_filter_by = field_filter_by
+        # self.page = page
+        self.per_page = per_page
+
+        if isinstance(self.iterable, QuerySet):
+            self.total = iterable.count()
+            if self.page:
+                self.items = self.iterable.filter(**{f'{field_filter_by}__{op}': last_field_value})\
+                    .order_by(order_by)\
+                    .limit(self.per_page)
+
+            else:
+                self.items = self.iterable.order_by(f'{field_filter_by}').limit(self.per_page)
+
+    def prev(self, error_out=False):
+        assert NotImplementedError
+        """Returns a :class:`Pagination` object for the previous page."""
+        assert (
+            self.iterable is not None
+        ), "an object is required for this method to work"
+        iterable = self.iterable
+        if isinstance(iterable, QuerySet):
+            iterable._skip = None
+            iterable._limit = None
+        self.get_page(iterable, self.per_page, self.field_filter_by,
+                      last_field_value=self.items[0][self.field_filter_by],
+                      direction='backward')
+
+        return self
+
+    def next(self, error_out=False):
+        """Returns a :class:`Pagination` object for the next page."""
+        assert (
+            self.iterable is not None
+        ), "an object is required for this method to work"
+        iterable = self.iterable
+        if self.per_page > self.items.count():
+            raise StopIteration
+        if isinstance(iterable, QuerySet):
+            iterable._skip = None
+            iterable._limit = None
+        self.get_page(iterable, self.per_page, self.field_filter_by,
+                      last_field_value=self.items[self.per_page - 1][self.field_filter_by])
+        return self
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if getattr(self, 'first_page_read', False):
+            return self.next()
+        self.first_page_read = True
+        return self

flask_mongoengine/pagination/list_field_pagination.py

@@ -0,0 +1,67 @@
+"""Module responsible for custom pagination."""
+
+from flask import abort
+from flask_mongoengine.pagination.basic_pagination import Pagination
+
+
+class ListFieldPagination(Pagination):
+    def __init__(self, queryset, doc_id, field_name, page, per_page, total=None):
+        """Allows an array within a document to be paginated.
+
+        Queryset must contain the document which has the array we're
+        paginating, and doc_id should be it's _id.
+        Field name is the name of the array we're paginating.
+        Page and per_page work just like in Pagination.
+        Total is an argument because it can be computed more efficiently
+        elsewhere, but we still use array.length as a fallback.
+        """
+        if page < 1:
+            abort(404)
+
+        self.page = page
+        self.per_page = per_page
+
+        self.queryset = queryset
+        self.doc_id = doc_id
+        self.field_name = field_name
+
+        start_index = (page - 1) * per_page
+
+        field_attrs = {field_name: {"$slice": [start_index, per_page]}}
+
+        qs = queryset(pk=doc_id)
+        self.items = getattr(qs.fields(**field_attrs).first(), field_name)
+        self.total = total or len(
+            getattr(qs.fields(**{field_name: 1}).first(), field_name)
+        )
+
+        if not self.items and page != 1:
+            abort(404)
+
+    def prev(self, error_out=False):
+        """Returns a :class:`Pagination` object for the previous page."""
+        assert (
+            self.items is not None
+        ), "a query object is required for this method to work"
+        return self.__class__(
+            self.queryset,
+            self.doc_id,
+            self.field_name,
+            self.page - 1,
+            self.per_page,
+            self.total,
+        )
+
+    def next(self, error_out=False):
+        """Returns a :class:`Pagination` object for the next page."""
+        assert (
+            self.items is not None
+        ), "a query object is required for this method to work"
+        return self.__class__(
+            self.queryset,
+            self.doc_id,
+            self.field_name,
+            self.page + 1,
+            self.per_page,
+            self.total,
+        )