From 11ab490ba723361b80c4fc66c003dc47ffeeb9a7 Mon Sep 17 00:00:00 2001
From: Ido Shraga <ido@cyabra.com>
Date: Fri, 16 Jun 2023 18:26:06 +0300
Subject: [PATCH 1/3] pagination max_depth

---
 docs/custom_queryset.md         |  3 ++-
 flask_mongoengine/pagination.py | 16 ++++++++-----
 tests/test_pagination.py        | 40 ++++++++++++++++++++++++++++++++-
 3 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/docs/custom_queryset.md b/docs/custom_queryset.md
index f011b79c..0abcbb1f 100644
--- a/docs/custom_queryset.md
+++ b/docs/custom_queryset.md
@@ -6,7 +6,8 @@ flask-mongoengine attaches the following methods to Mongoengine's default QueryS
   Optional arguments: *message* - custom message to display.
 * **first_or_404**: same as above, except for .first().
   Optional arguments: *message* - custom message to display.
-* **paginate**: paginates the QuerySet. Takes two arguments, *page* and *per_page*.
+* **paginate**: paginates the QuerySet. Takes two required arguments, *page* and *per_page*.
+   And one optional arguments *max_depth*.
 * **paginate_field**: paginates a field from one document in the QuerySet.
   Arguments: *field_name*, *doc_id*, *page*, *per_page*.
 
diff --git a/flask_mongoengine/pagination.py b/flask_mongoengine/pagination.py
index 01b8b2ea..838ca8c8 100644
--- a/flask_mongoengine/pagination.py
+++ b/flask_mongoengine/pagination.py
@@ -8,7 +8,13 @@
 
 
 class Pagination(object):
-    def __init__(self, iterable, page, per_page):
+    def __init__(self, iterable, page: int, per_page: int, max_depth: int = None):
+        """
+        :param iterable: iterable object .
+        :param page: Required page number start from 1.
+        :param per_page: Required number of documents per page.
+        :param max_depth: Option for limit number of dereference documents.
+        """
 
         if page < 1:
             abort(404)
@@ -19,11 +25,9 @@ def __init__(self, iterable, page, per_page):
 
         if isinstance(self.iterable, QuerySet):
             self.total = iterable.count()
-            self.items = (
-                self.iterable.skip(self.per_page * (self.page - 1))
-                .limit(self.per_page)
-                .select_related()
-            )
+            self.items = self.iterable.skip(self.per_page * (self.page - 1)).limit(self.per_page)
+            if max_depth is not None:
+                self.items = self.items.select_related(max_depth)
         else:
             start_index = (page - 1) * per_page
             end_index = page * per_page
diff --git a/tests/test_pagination.py b/tests/test_pagination.py
index e9d3f4fe..27f279e0 100644
--- a/tests/test_pagination.py
+++ b/tests/test_pagination.py
@@ -1,14 +1,29 @@
+import flask
 import pytest
 from werkzeug.exceptions import NotFound
 
 from flask_mongoengine import ListFieldPagination, Pagination
 
 
-def test_queryset_paginator(app, todo):
+@pytest.fixture(autouse=True)
+def setup_endpoints(app, todo):
     Todo = todo
     for i in range(42):
         Todo(title=f"post: {i}").save()
 
+    @app.route("/")
+    def index():
+        page = int(flask.request.form.get("page"))
+        per_page = int(flask.request.form.get("per_page"))
+        query_set = Todo.objects().paginate(page=page, per_page=per_page)
+        return {'data': [_ for _ in query_set.items],
+                'total': query_set.total,
+                'has_next': query_set.has_next,
+                }
+
+
+def test_queryset_paginator(app, todo):
+    Todo = todo
     with pytest.raises(NotFound):
         Pagination(iterable=Todo.objects, page=0, per_page=10)
 
@@ -90,3 +105,26 @@ def _test_paginator(paginator):
         # Paginate to the next page
         if i < 5:
             paginator = paginator.next()
+
+
+
+def test_flask_pagination(app, todo):
+    client = app.test_client()
+    response = client.get(f"/", data={"page": 0, "per_page": 10})
+    print(response.status_code)
+    assert response.status_code == 404
+
+    response = client.get(f"/", data={"page": 6, "per_page": 10})
+    print(response.status_code)
+    assert response.status_code == 404
+
+
+def test_flask_pagination_next(app, todo):
+    client = app.test_client()
+    has_next = True
+    page = 1
+    while has_next:
+        response = client.get(f"/", data={"page": page, "per_page": 10})
+        assert response.status_code == 200
+        has_next = response.json['has_next']
+        page += 1

From 17a8a859455c2f542f08aca582521ce88351d0aa Mon Sep 17 00:00:00 2001
From: Ido Shraga <ido@cyabra.com>
Date: Sun, 17 Sep 2023 01:07:17 +0300
Subject: [PATCH 2/3] paginate by keyset bit more complicated than the regular
 but should work better in scale

un-breaking change for other pagination type
---
 docs/custom_queryset.md                       |  7 ++
 flask_mongoengine/documents.py                |  9 +-
 flask_mongoengine/pagination/__init__.py      |  5 ++
 .../basic_pagination.py}                      | 66 --------------
 .../pagination/keyset_pagination.py           | 89 +++++++++++++++++++
 .../pagination/list_field_pagination.py       | 67 ++++++++++++++
 tests/test_pagination.py                      | 28 +++++-
 7 files changed, 203 insertions(+), 68 deletions(-)
 create mode 100644 flask_mongoengine/pagination/__init__.py
 rename flask_mongoengine/{pagination.py => pagination/basic_pagination.py} (65%)
 create mode 100644 flask_mongoengine/pagination/keyset_pagination.py
 create mode 100644 flask_mongoengine/pagination/list_field_pagination.py

diff --git a/docs/custom_queryset.md b/docs/custom_queryset.md
index 0abcbb1f..73cd2e96 100644
--- a/docs/custom_queryset.md
+++ b/docs/custom_queryset.md
@@ -8,6 +8,9 @@ flask-mongoengine attaches the following methods to Mongoengine's default QueryS
   Optional arguments: *message* - custom message to display.
 * **paginate**: paginates the QuerySet. Takes two required arguments, *page* and *per_page*.
    And one optional arguments *max_depth*.
+* **paginate_by_keyset**: paginates the QuerySet. Takes two required arguments, *per_page* and *field_filter_by*.
+  from the second page you need also the last id of the previous page.
+  Arguments: *per_page*, *field_filter_by*, *last_field_value*.
 * **paginate_field**: paginates a field from one document in the QuerySet.
   Arguments: *field_name*, *doc_id*, *page*, *per_page*.
 
@@ -22,6 +25,10 @@ def view_todo(todo_id):
 def view_todos(page=1):
     paginated_todos = Todo.objects.paginate(page=page, per_page=10)
 
+# Paginate by keyset through todo
+def view_todos_by_keyset(page=0):
+    paginated_todos = Todo.objects.paginate(page=page, per_page=10)
+
 # Paginate through tags of todo
 def view_todo_tags(todo_id, page=1):
     todo = Todo.objects.get_or_404(_id=todo_id)
diff --git a/flask_mongoengine/documents.py b/flask_mongoengine/documents.py
index 3a2e26f7..4cbd49d1 100644
--- a/flask_mongoengine/documents.py
+++ b/flask_mongoengine/documents.py
@@ -8,7 +8,7 @@
 from mongoengine.queryset import QuerySet
 
 from flask_mongoengine.decorators import wtf_required
-from flask_mongoengine.pagination import ListFieldPagination, Pagination
+from flask_mongoengine.pagination import ListFieldPagination, Pagination, KeysetPagination
 
 try:
     from flask_mongoengine.wtf.models import ModelForm
@@ -60,6 +60,13 @@ def paginate(self, page, per_page):
         """
         return Pagination(self, page, per_page)
 
+    def paginate_by_keyset(self, per_page, field_filter_by, last_field_value):
+        """
+        Paginate the QuerySet with a certain number of docs per page
+        and return docs for a given page.
+        """
+        return KeysetPagination(self, per_page, field_filter_by, last_field_value)
+
     def paginate_field(self, field_name, doc_id, page, per_page, total=None):
         """
         Paginate items within a list field from one document in the
diff --git a/flask_mongoengine/pagination/__init__.py b/flask_mongoengine/pagination/__init__.py
new file mode 100644
index 00000000..8e808a4c
--- /dev/null
+++ b/flask_mongoengine/pagination/__init__.py
@@ -0,0 +1,5 @@
+from flask_mongoengine.pagination.basic_pagination import Pagination
+from flask_mongoengine.pagination.list_field_pagination import ListFieldPagination
+from flask_mongoengine.pagination.keyset_pagination import KeysetPagination
+
+__all__ = ("Pagination", "ListFieldPagination", "KeysetPagination")
diff --git a/flask_mongoengine/pagination.py b/flask_mongoengine/pagination/basic_pagination.py
similarity index 65%
rename from flask_mongoengine/pagination.py
rename to flask_mongoengine/pagination/basic_pagination.py
index 838ca8c8..4432826c 100644
--- a/flask_mongoengine/pagination.py
+++ b/flask_mongoengine/pagination/basic_pagination.py
@@ -1,11 +1,8 @@
-"""Module responsible for custom pagination."""
 import math
 
 from flask import abort
 from mongoengine.queryset import QuerySet
 
-__all__ = ("Pagination", "ListFieldPagination")
-
 
 class Pagination(object):
     def __init__(self, iterable, page: int, per_page: int, max_depth: int = None):
@@ -123,66 +120,3 @@ def iter_pages(self, left_edge=2, left_current=2, right_current=5, right_edge=2)
                 last = num
         if last != self.pages:
             yield None
-
-
-class ListFieldPagination(Pagination):
-    def __init__(self, queryset, doc_id, field_name, page, per_page, total=None):
-        """Allows an array within a document to be paginated.
-
-        Queryset must contain the document which has the array we're
-        paginating, and doc_id should be it's _id.
-        Field name is the name of the array we're paginating.
-        Page and per_page work just like in Pagination.
-        Total is an argument because it can be computed more efficiently
-        elsewhere, but we still use array.length as a fallback.
-        """
-        if page < 1:
-            abort(404)
-
-        self.page = page
-        self.per_page = per_page
-
-        self.queryset = queryset
-        self.doc_id = doc_id
-        self.field_name = field_name
-
-        start_index = (page - 1) * per_page
-
-        field_attrs = {field_name: {"$slice": [start_index, per_page]}}
-
-        qs = queryset(pk=doc_id)
-        self.items = getattr(qs.fields(**field_attrs).first(), field_name)
-        self.total = total or len(
-            getattr(qs.fields(**{field_name: 1}).first(), field_name)
-        )
-
-        if not self.items and page != 1:
-            abort(404)
-
-    def prev(self, error_out=False):
-        """Returns a :class:`Pagination` object for the previous page."""
-        assert (
-            self.items is not None
-        ), "a query object is required for this method to work"
-        return self.__class__(
-            self.queryset,
-            self.doc_id,
-            self.field_name,
-            self.page - 1,
-            self.per_page,
-            self.total,
-        )
-
-    def next(self, error_out=False):
-        """Returns a :class:`Pagination` object for the next page."""
-        assert (
-            self.items is not None
-        ), "a query object is required for this method to work"
-        return self.__class__(
-            self.queryset,
-            self.doc_id,
-            self.field_name,
-            self.page + 1,
-            self.per_page,
-            self.total,
-        )
diff --git a/flask_mongoengine/pagination/keyset_pagination.py b/flask_mongoengine/pagination/keyset_pagination.py
new file mode 100644
index 00000000..ace08330
--- /dev/null
+++ b/flask_mongoengine/pagination/keyset_pagination.py
@@ -0,0 +1,89 @@
+from mongoengine.queryset import QuerySet
+
+from flask_mongoengine.pagination.basic_pagination import Pagination
+
+
+class KeysetPagination(Pagination):
+    def __init__(self, iterable, per_page: int, field_filter_by: str = '_id', last_field_value=None):
+        """
+        :param iterable: iterable object .
+        :param page: Required page number start from 1.
+        :param per_page: Required number of documents per page.
+        :param max_depth: Option for limit number of dereference documents.
+        """
+        self.get_page(iterable, per_page, field_filter_by, last_field_value)
+
+    def get_page(self, iterable, per_page: int, field_filter_by: str = '_id', last_field_value=None,
+                 direction='forward'):
+        if last_field_value is None:
+            self.page = 0
+        elif getattr(self, 'page', False):
+            self.page += 1
+        else:
+            self.page = 1
+
+        if direction == 'forward':
+            op = 'gt'
+            order_by = field_filter_by
+        elif direction == 'backward':
+            op = 'lt'
+            order_by = f'-{field_filter_by}'
+
+        else:
+            raise ValueError
+
+        self.iterable = iterable
+        self.field_filter_by = field_filter_by
+        # self.page = page
+        self.per_page = per_page
+
+        if isinstance(self.iterable, QuerySet):
+            self.total = iterable.count()
+            if self.page:
+                self.items = self.iterable.filter(**{f'{field_filter_by}__{op}': last_field_value})\
+                    .order_by(order_by)\
+                    .limit(self.per_page)
+
+            else:
+                self.items = self.iterable.order_by(f'{field_filter_by}').limit(self.per_page)
+
+    def prev(self, error_out=False):
+        assert NotImplementedError
+        """Returns a :class:`Pagination` object for the previous page."""
+        assert (
+            self.iterable is not None
+        ), "an object is required for this method to work"
+        iterable = self.iterable
+        if isinstance(iterable, QuerySet):
+            iterable._skip = None
+            iterable._limit = None
+        self.get_page(iterable, self.per_page, self.field_filter_by,
+                      last_field_value=self.items[0][self.field_filter_by],
+                      direction='backward')
+
+        return self
+
+    def next(self, error_out=False):
+        """Returns a :class:`Pagination` object for the next page."""
+        assert (
+            self.iterable is not None
+        ), "an object is required for this method to work"
+        iterable = self.iterable
+        if self.per_page > self.items.count():
+            raise StopIteration
+        if isinstance(iterable, QuerySet):
+            iterable._skip = None
+            iterable._limit = None
+        self.get_page(iterable, self.per_page, self.field_filter_by,
+                      last_field_value=self.items[self.per_page - 1][self.field_filter_by])
+        return self
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        if getattr(self, 'first_page_read', False):
+            return self.next()
+        else:
+            self.first_page_read = True
+            return self
diff --git a/flask_mongoengine/pagination/list_field_pagination.py b/flask_mongoengine/pagination/list_field_pagination.py
new file mode 100644
index 00000000..8808c226
--- /dev/null
+++ b/flask_mongoengine/pagination/list_field_pagination.py
@@ -0,0 +1,67 @@
+"""Module responsible for custom pagination."""
+
+from flask import abort
+from flask_mongoengine.pagination.basic_pagination import Pagination
+
+
+class ListFieldPagination(Pagination):
+    def __init__(self, queryset, doc_id, field_name, page, per_page, total=None):
+        """Allows an array within a document to be paginated.
+
+        Queryset must contain the document which has the array we're
+        paginating, and doc_id should be it's _id.
+        Field name is the name of the array we're paginating.
+        Page and per_page work just like in Pagination.
+        Total is an argument because it can be computed more efficiently
+        elsewhere, but we still use array.length as a fallback.
+        """
+        if page < 1:
+            abort(404)
+
+        self.page = page
+        self.per_page = per_page
+
+        self.queryset = queryset
+        self.doc_id = doc_id
+        self.field_name = field_name
+
+        start_index = (page - 1) * per_page
+
+        field_attrs = {field_name: {"$slice": [start_index, per_page]}}
+
+        qs = queryset(pk=doc_id)
+        self.items = getattr(qs.fields(**field_attrs).first(), field_name)
+        self.total = total or len(
+            getattr(qs.fields(**{field_name: 1}).first(), field_name)
+        )
+
+        if not self.items and page != 1:
+            abort(404)
+
+    def prev(self, error_out=False):
+        """Returns a :class:`Pagination` object for the previous page."""
+        assert (
+            self.items is not None
+        ), "a query object is required for this method to work"
+        return self.__class__(
+            self.queryset,
+            self.doc_id,
+            self.field_name,
+            self.page - 1,
+            self.per_page,
+            self.total,
+        )
+
+    def next(self, error_out=False):
+        """Returns a :class:`Pagination` object for the next page."""
+        assert (
+            self.items is not None
+        ), "a query object is required for this method to work"
+        return self.__class__(
+            self.queryset,
+            self.doc_id,
+            self.field_name,
+            self.page + 1,
+            self.per_page,
+            self.total,
+        )
diff --git a/tests/test_pagination.py b/tests/test_pagination.py
index 27f279e0..c702e0a0 100644
--- a/tests/test_pagination.py
+++ b/tests/test_pagination.py
@@ -1,8 +1,9 @@
 import flask
 import pytest
+import copy
 from werkzeug.exceptions import NotFound
 
-from flask_mongoengine import ListFieldPagination, Pagination
+from flask_mongoengine import ListFieldPagination, Pagination, KeysetPagination
 
 
 @pytest.fixture(autouse=True)
@@ -40,6 +41,31 @@ def test_queryset_paginator(app, todo):
             assert todo.title == f"post: {(page-1) * 5 + index}"
 
 
+def test_keyset_queryset_paginator(app, todo):
+    Todo = todo
+
+    last_field_value = None
+    for page in range(1, 10):
+        p = Todo.objects.paginate_by_keyset(per_page=5, field_filter_by='id', last_field_value=last_field_value)
+        for index, todo in enumerate(p.items):
+            assert todo.title == f"post: {(page-1) * 5 + index}"
+        last_field_value = list(p.items)[-1].pk
+
+    # Pagination
+    paginator = KeysetPagination(Todo.objects, per_page=5, field_filter_by='id')
+    for page_index, page in enumerate(paginator):
+        for index, todo in enumerate(page.items):
+            assert todo.title == f"post: {(page_index) * 5 + index}"
+
+    # Pagination with prev function
+    paginator_2 = KeysetPagination(Todo.objects, per_page=5, field_filter_by='id')
+    a = copy.deepcopy(paginator_2.next().items)
+    paginator_2.next()
+    a2 = paginator_2.prev().items
+    for index, item in enumerate(a2):
+        assert a[4-index].title == item.title
+
+
 def test_paginate_plain_list():
     with pytest.raises(NotFound):
         Pagination(iterable=range(1, 42), page=0, per_page=10)

From 4fe7e30bfea9fca295ac56d5031b529f8f042ae8 Mon Sep 17 00:00:00 2001
From: Sourcery AI <>
Date: Sat, 16 Sep 2023 22:36:44 +0000
Subject: [PATCH 3/3] 'Refactored by Sourcery'

---
 flask_mongoengine/pagination/keyset_pagination.py |  5 ++---
 tests/test_pagination.py                          | 15 ++++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/flask_mongoengine/pagination/keyset_pagination.py b/flask_mongoengine/pagination/keyset_pagination.py
index ace08330..21397407 100644
--- a/flask_mongoengine/pagination/keyset_pagination.py
+++ b/flask_mongoengine/pagination/keyset_pagination.py
@@ -84,6 +84,5 @@ def __iter__(self):
     def __next__(self):
         if getattr(self, 'first_page_read', False):
             return self.next()
-        else:
-            self.first_page_read = True
-            return self
+        self.first_page_read = True
+        return self
diff --git a/tests/test_pagination.py b/tests/test_pagination.py
index c702e0a0..0df67bb3 100644
--- a/tests/test_pagination.py
+++ b/tests/test_pagination.py
@@ -17,10 +17,11 @@ def index():
         page = int(flask.request.form.get("page"))
         per_page = int(flask.request.form.get("per_page"))
         query_set = Todo.objects().paginate(page=page, per_page=per_page)
-        return {'data': [_ for _ in query_set.items],
-                'total': query_set.total,
-                'has_next': query_set.has_next,
-                }
+        return {
+            'data': list(query_set.items),
+            'total': query_set.total,
+            'has_next': query_set.has_next,
+        }
 
 
 def test_queryset_paginator(app, todo):
@@ -136,11 +137,11 @@ def _test_paginator(paginator):
 
 def test_flask_pagination(app, todo):
     client = app.test_client()
-    response = client.get(f"/", data={"page": 0, "per_page": 10})
+    response = client.get("/", data={"page": 0, "per_page": 10})
     print(response.status_code)
     assert response.status_code == 404
 
-    response = client.get(f"/", data={"page": 6, "per_page": 10})
+    response = client.get("/", data={"page": 6, "per_page": 10})
     print(response.status_code)
     assert response.status_code == 404
 
@@ -150,7 +151,7 @@ def test_flask_pagination_next(app, todo):
     has_next = True
     page = 1
     while has_next:
-        response = client.get(f"/", data={"page": page, "per_page": 10})
+        response = client.get("/", data={"page": page, "per_page": 10})
         assert response.status_code == 200
         has_next = response.json['has_next']
         page += 1