Merge pull request #28 from piyush-jaiswal/feature/cursor-based-pagination

Feature/cursor based pagination

Author: piyush-jaiswal

README.md

@@ -10,7 +10,7 @@ Ability to create, read, update, and delete products, categories and subcategori
 <br></br>
 Fetching a product fetches the details of categories and subcategories it belongs to. Provides the ability to search for products by name, category and subcategories.
 <br></br>
-Paginates result when products are fetched by categories or subcategories.
+Paginates result using cursor based pagination when products are fetched by categories, subcategories or themselves.
 
 Deployed as a vercel function with Postgres: [ecommerce-rest-api-five.vercel.app](https://ecommerce-rest-api-five.vercel.app)
 <br> Documented with Swagger UI.
@@ -64,9 +64,10 @@ Test the API using Swagger UI (`/` route), Postman, cURL or your preferred HTTP
 
 #### Fetch products using name, category, subcategory
 - [GET] `/products?name=<name: string>` - Get product with name: `name` <br/><br/>
-- [GET] `/subcategories/<subcategory_id: int>/products?page=<page_no>` - Get product with within subcategory `subcategory`. Returns `page_no` of the paginated results. <br/><br/>
-- [GET] `/categories/<category_id: int>/products` - Get product with within category `category`. Returns first page of the paginated results. <br/><br/>
-- [GET] `/categories/<category_id: int>/products?page=<page_no>` - Get product with within category `category`. Returns `page_no` of the paginated results. <br/><br/>
+- [GET] `/subcategories/<subcategory_id: int>/products` - Get first page of products within subcategory `subcategory_id`. <br/><br/>
+- [GET] `/subcategories/<subcategory_id: int>/products?cursor=<cursor: str>` - Get products paginated using cursor within subcategory `subcategory_id`. Next and previous page `cursors` provided in responses. <br/><br/>
+- [GET] `/categories/<category_id: int>/products` - Get first page of products within category `category_id`. <br/><br/>
+- [GET] `/categories/<category_id: int>/products?cursor=<cursor: str>` - Get products paginated using cursor within category `category_id`. Next and previous page `cursors` provided in responses. <br/><br/>
 
 
 #### Authorization
@@ -150,7 +151,8 @@ Test the API using Swagger UI (`/` route), Postman, cURL or your preferred HTTP
 <br/>
 
 #### Product
-- [GET] `/products` - Get all products
+- [GET] `/products` - Get first page of products
+- [GET] `/products?cursor=<cursor: str>` - Get products paginated using cursor. Next and previous page `cursors` provided in responses.
 - [GET] `/products/(int: product_id)` - Get product with product_id
 - [GET] `/products/(int: product_id)/subcategories` - Get subcategories related to product_id
 - [DELETE] `/products/(int: product_id)` (Protected) - Delete product with product_id

app/routes/category.py

@@ -2,6 +2,7 @@
 from flask_jwt_extended import jwt_required
 from flask_smorest import Blueprint, abort
 from psycopg2.errors import UniqueViolation
+from sqlakeyset import get_page
 from sqlalchemy import UniqueConstraint, exists
 from sqlalchemy.exc import IntegrityError
 
@@ -154,17 +155,14 @@ class CategoryProducts(MethodView):
     @bp.doc(summary="Get Products within a Category")
     @bp.arguments(PaginationArgs, location="query", as_kwargs=True)
     @bp.response(200, ProductsOut)
-    def get(self, id, page):
+    def get(self, id, cursor):
         category_exists = db.session.query(exists().where(Category.id == id)).scalar()
         if not category_exists:
             abort(404)
 
-        products = (
-            Product.query.filter(
-                Product.subcategories.any(Subcategory.categories.any(id=id))
-            )
-            .order_by(Product.id.asc())
-            .paginate(page=page, per_page=CategoryProducts._PER_PAGE, error_out=False)
-        )
+        products = Product.query.filter(
+            Product.subcategories.any(Subcategory.categories.any(id=id))
+        ).order_by(Product.id.asc())
+        page = get_page(products, per_page=CategoryProducts._PER_PAGE, page=cursor)
 
-        return {"products": products}
+        return { "products": page, "cursor": page.paging }

app/routes/product.py

@@ -2,6 +2,7 @@
 from flask_jwt_extended import jwt_required
 from flask_smorest import Blueprint, abort
 from psycopg2.errors import UniqueViolation
+from sqlakeyset import get_page
 from sqlalchemy import UniqueConstraint
 from sqlalchemy.exc import IntegrityError
 
@@ -51,15 +52,14 @@ def _get_by_name(self, name):
     @bp.arguments(NameArgs, location="query", as_kwargs=True)
     @bp.arguments(PaginationArgs, location="query", as_kwargs=True)
     @bp.response(200, ProductsOut)
-    def get(self, name, page):
+    def get(self, name, cursor):
         if name is not None:
             products = self._get_by_name(name)
+            return {"products": products}
         else:
-            products = Product.query.order_by(Product.id.asc()).paginate(
-                page=page, per_page=ProductCollection._PER_PAGE, error_out=False
-            )
-
-        return {"products": products}
+            products = Product.query.order_by(Product.id.asc())
+            page = get_page(products, per_page=ProductCollection._PER_PAGE, page=cursor)
+            return {"products": page, "cursor": page.paging}
 
     @jwt_required()
     @bp.doc(summary="Create Product", security=[{"access_token": []}])

app/routes/subcategory.py

@@ -2,6 +2,7 @@
 from flask_jwt_extended import jwt_required
 from flask_smorest import Blueprint, abort
 from psycopg2.errors import UniqueViolation
+from sqlakeyset import get_page
 from sqlalchemy import UniqueConstraint
 from sqlalchemy.exc import IntegrityError
 
@@ -169,11 +170,9 @@ class SubcategoryProducts(MethodView):
     @bp.doc(summary="Get Products within a Subcategory")
     @bp.arguments(PaginationArgs, location="query", as_kwargs=True)
     @bp.response(200, ProductsOut)
-    def get(self, id, page):
+    def get(self, id, cursor):
         subcategory = Subcategory.query.get_or_404(id)
+        products = subcategory.products.order_by(Product.id.asc())
+        page = get_page(products, per_page=SubcategoryProducts._PER_PAGE, page=cursor)
 
-        products = subcategory.products.order_by(Product.id.asc()).paginate(
-            page=page, per_page=SubcategoryProducts._PER_PAGE, error_out=False
-        )
-
-        return {"products": products}
+        return {"products": page, "cursor": page.paging}

app/schemas.py

@@ -1,9 +1,38 @@
+import base64
+
 from marshmallow import Schema, ValidationError, fields, pre_load, validate, validates
 from marshmallow_sqlalchemy import SQLAlchemyAutoSchema, SQLAlchemySchema, auto_field
+from sqlakeyset import BadBookmark, unserialize_bookmark
 
 from app.models import Category, Product, Subcategory, User
 
 
+class Cursor(fields.Field[dict]):
+    def _serialize(self, paging, attr, obj, **kwargs):
+        def encode(s):
+            bytes = s.encode("utf-8")
+            return base64.urlsafe_b64encode(bytes).decode("utf-8")
+
+        if paging is None:
+            return None
+
+        return {
+            "next": encode(paging.bookmark_next) if paging.has_next else None,
+            "prev": encode(paging.bookmark_previous) if paging.has_previous else None,
+        }
+
+    def _deserialize(self, cursor, attr, data, **kwargs):
+        if cursor is None:
+            return None
+
+        try:
+            decoded_bytes = base64.urlsafe_b64decode(cursor.encode("utf-8"))
+            marker_serialized = decoded_bytes.decode("utf-8")
+            return unserialize_bookmark(marker_serialized)
+        except (TypeError, ValueError, KeyError, BadBookmark) as ex:
+            raise ValidationError("Invalid cursor") from ex
+
+
 class CategoryOut(SQLAlchemyAutoSchema):
     class Meta:
         model = Category
@@ -70,6 +99,7 @@ class Meta:
 
 class ProductsOut(Schema):
     products = fields.List(fields.Nested(ProductOut))
+    cursor = Cursor(required=False)  # require false for getting product by name
 
 
 class ProductIn(SQLAlchemySchema):
@@ -100,7 +130,7 @@ class NameArgs(Schema):
 
 
 class PaginationArgs(Schema):
-    page = fields.Int(load_default=1)
+    cursor = Cursor(load_default=None)
 
 
 class AuthIn(SQLAlchemySchema):

requirements.txt

@@ -9,3 +9,4 @@ email-normalize==0.2.1
 Flask-JWT-Extended==4.7.1
 flask-smorest==0.46.2
 marshmallow-sqlalchemy==1.4.2
+sqlakeyset==2.0.1746777265

tests/test_product.py

@@ -222,7 +222,8 @@ def test_products_pagination(self, create_product):
         assert len(data1["products"]) == 10
 
         # Page 2
-        resp2 = self.client.get("/products?page=2")
+        next_cursor = data1["cursor"]["next"]
+        resp2 = self.client.get(f"/products?cursor={next_cursor}")
         assert resp2.status_code == 200
         data2 = resp2.get_json()
         assert "products" in data2

tests/test_relationships.py

@@ -218,8 +218,9 @@ def test_get_category_products_populated_with_pagination(self, create_category,
             product_resp = create_product(f"P{index}", "desc", subcategories=[subcategory["id"]])
             product_ids.add(product_resp.get_json().get("id"))
 
-        page1 = self.client.get(f"/categories/{category['id']}/products?page=1").get_json()
-        page2 = self.client.get(f"/categories/{category['id']}/products?page=2").get_json()
+        page1 = self.client.get(f"/categories/{category['id']}/products").get_json()
+        next_cursor = page1["cursor"]["next"]
+        page2 = self.client.get(f"/categories/{category['id']}/products?cursor={next_cursor}").get_json()
         assert len(page1["products"]) == 10
         assert len(page2["products"]) == 2
 
@@ -252,8 +253,9 @@ def test_get_subcategory_products_populated_with_pagination(self, create_subcate
             product_resp = create_product(f"SP{index}", "desc", subcategories=[subcategory["id"]])
             product_ids.add(product_resp.get_json().get("id"))
 
-        page1 = self.client.get(f"/subcategories/{subcategory['id']}/products?page=1").get_json()
-        page2 = self.client.get(f"/subcategories/{subcategory['id']}/products?page=2").get_json()
+        page1 = self.client.get(f"/subcategories/{subcategory['id']}/products").get_json()
+        next_cursor = page1["cursor"]["next"]
+        page2 = self.client.get(f"/subcategories/{subcategory['id']}/products?cursor={next_cursor}").get_json()
         assert len(page1["products"]) == 10
         assert len(page2["products"]) == 1