Merge pull request #90 from cloudblue/feature/LITE-28409-Add-pagination-to-endpoints

LITE-28409: Adding pagination capability

Author: Pavel

connect_ext_ppr/pagination.py

@@ -0,0 +1,31 @@
+from fastapi import Query
+from fastapi_pagination import LimitOffsetPage, LimitOffsetParams, set_page
+from fastapi_pagination.api import resolve_params
+from fastapi_pagination.ext.sqlalchemy import paginate
+
+
+set_page(LimitOffsetPage)
+
+
+class PaginationParams(LimitOffsetParams):
+    """Here we can redefine default size value"""
+    limit: int = Query(1000, ge=1, le=1000, description="Page size")
+
+
+def apply_pagination(query, db, params, response):
+    """Apply pagination for the query
+    * Paginate query according to applied parameters (size and page)
+    * Add pagination headers to the response
+    Don't filter or remove elements from query after the pagination
+    """
+    paginated = paginate(db, query, params)
+    resolve_params(params)
+    init = paginated.offset
+    # If we are selecting a page that it's out of range, we return the same start and end for on
+    # header range, copying the same behavior as connect.
+    end = init
+    if paginated.items:
+        end += len(paginated.items) - 1
+
+    response.headers['Content-Range'] = f'items {init}-{end}/{paginated.total}'
+    return paginated.items

connect_ext_ppr/webapp.py

@@ -48,6 +48,7 @@
 from connect_ext_ppr.models.ppr import PPRVersion
 from connect_ext_ppr.models.task import Task
 from connect_ext_ppr.models.replicas import Product
+from connect_ext_ppr.pagination import apply_pagination, PaginationParams
 from connect_ext_ppr.service import (
     add_deployments,
     add_new_deployment_request,
@@ -182,6 +183,8 @@ def add_dep_request(
     )
     def list_deployment_requests(
         self,
+        pagination_params: PaginationParams = Depends(),
+        response: Response = None,
         client: ConnectClient = Depends(get_installation_client),
         db: VerboseBaseSession = Depends(get_db),
         installation: dict = Depends(get_installation),
@@ -203,6 +206,7 @@ def list_deployment_requests(
         ).filter(
             DeploymentRequest.deployment_id.in_(deployments),
         )
+        deployment_requests = apply_pagination(deployment_requests, db, pagination_params, response)
 
         response_list = []
         for dr in deployment_requests:
@@ -272,6 +276,8 @@ def list_deployment_request_tasks(
     def list_deployment_request_marketplaces(
         self,
         depl_req_id: str,
+        pagination_params: PaginationParams = Depends(),
+        response: Response = None,
         db: VerboseBaseSession = Depends(get_db),
         client: ConnectClient = Depends(get_installation_client),
         installation: dict = Depends(get_installation),
@@ -282,6 +288,7 @@ def list_deployment_request_marketplaces(
         marketplaces = db.query(MarketplaceConfiguration).options(
             selectinload(MarketplaceConfiguration.ppr),
         ).filter_by(deployment_request=dr.id)
+        marketplaces = apply_pagination(marketplaces, db, pagination_params, response)
 
         marketplaces_pprs = {m.marketplace: m.ppr for m in marketplaces}
         marketplaces_data = get_marketplaces(client, list(marketplaces_pprs.keys()))
@@ -369,6 +376,8 @@ def get_deployment(
     def list_requests_for_deployment(
         self,
         deployment_id: str,
+        pagination_params: PaginationParams = Depends(),
+        response: Response = None,
         client: ConnectClient = Depends(get_installation_client),
         db: VerboseBaseSession = Depends(get_db),
         installation: dict = Depends(get_installation),
@@ -384,7 +393,8 @@ def list_requests_for_deployment(
             .filter_by(deployment_id=deployment_id)
             .order_by(desc(DeploymentRequest.id))
         )
-        for dr in qs:
+
+        for dr in apply_pagination(qs, db, pagination_params, response):
             response_list.append(get_deployment_request_schema(dr, hub))
         return response_list
 
@@ -396,12 +406,15 @@ def list_requests_for_deployment(
     def get_deployments(
         self,
         deployment_filter: DeploymentFilter = FilterDepends(DeploymentFilter),
+        pagination_params: PaginationParams = Depends(),
+        response: Response = None,
         client: ConnectClient = Depends(get_installation_client),
         db: VerboseBaseSession = Depends(get_db),
         installation: dict = Depends(get_installation),
     ):
         deployments = db.query(Deployment).filter_by(account_id=installation['owner']['id'])
         deployments = deployment_filter.filter(deployments)
+        deployments = apply_pagination(deployments, db, pagination_params, response)
         listings = get_all_listing_info(client)
         vendors = [li['vendor'] for li in listings]
         hubs = [hub['hub'] for li in listings for hub in li['contract']['marketplace']['hubs']]
@@ -422,17 +435,19 @@ def get_deployments(
     def get_configurations(
         self,
         deployment_id: str,
+        pagination_params: PaginationParams = Depends(),
+        response: Response = None,
         db: VerboseBaseSession = Depends(get_db),
         installation: dict = Depends(get_installation),
     ):
         get_deployment_by_id(deployment_id, db, installation)
 
-        conf_file_list = (
+        conf_file_qs = (
             db.query(Configuration, File)
             .filter_by(deployment=deployment_id)
             .join(File, Configuration.file == File.id)
-            .all()
         )
+        conf_file_list = apply_pagination(conf_file_qs, db, pagination_params, response)
         response_list = []
         for conf, file in conf_file_list:
             response_list.append(
@@ -569,20 +584,21 @@ def remove_configuration(
     def get_pprs(
         self,
         deployment_id: str,
+        pagination_params: PaginationParams = Depends(),
+        response: Response = None,
         db: VerboseBaseSession = Depends(get_db),
         installation: dict = Depends(get_installation),
     ):
         get_deployment_by_id(deployment_id, db, installation)
 
-        ppr_file_conf_list = (
+        ppr_file_conf_qs = (
             db.query(PPRVersion, File, Configuration)
             .filter_by(deployment=deployment_id)
             .join(File, PPRVersion.file == File.id)
             .outerjoin(Configuration, PPRVersion.configuration == Configuration.id)
             .order_by(desc(PPRVersion.version))
-            .all()
         )
-
+        ppr_file_conf_list = apply_pagination(ppr_file_conf_qs, db, pagination_params, response)
         response_list = []
         for ppr, file, conf in ppr_file_conf_list:
             response_list.append(
@@ -642,6 +658,8 @@ def add_ppr(
     def get_marketplaces_by_deployment(
         self,
         deployment_id: str,
+        pagination_params: PaginationParams = Depends(),
+        response: Response = None,
         client: ConnectClient = Depends(get_installation_client),
         db: VerboseBaseSession = Depends(get_db),
         installation: dict = Depends(get_installation),
@@ -651,6 +669,7 @@ def get_marketplaces_by_deployment(
         mkplc_configs = db.query(MarketplaceConfiguration).options(
             selectinload(MarketplaceConfiguration.ppr),
         ).filter_by(deployment_id=deployment_id, active=True)
+        mkplc_configs = apply_pagination(mkplc_configs, db, pagination_params, response)
 
         mkplc_ids = [m.marketplace for m in mkplc_configs]
 
@@ -668,19 +687,23 @@ def get_marketplaces_by_deployment(
     )
     def list_products(
         self,
+        pagination_params: PaginationParams = Depends(),
+        response: Response = None,
         db: VerboseBaseSession = Depends(get_db),
         installation: dict = Depends(get_installation),
     ):
         products_ids = db.query(Deployment.product_id).filter_by(
             account_id=installation['owner']['id'],
         ).distinct()
-
-        response_list = []
         products = db.query(Product).filter(Product.id.in_(products_ids)).options(
             selectinload(Product.owner),
         )
+        products = apply_pagination(products, db, pagination_params, response)
+
+        response_list = []
         for product in products:
             response_list.append(get_product_schema(product))
+
         return response_list
 
     @router.get(

poetry.lock

@@ -23,6 +23,7 @@ pandas = "^2.0.3"
 openpyxl = "^3.1.2"
 PyJWT = "^2.8.0"
 fastapi_filter = "^0.6.1"
+fastapi-pagination = "^0.12.8"
 
 [tool.poetry.dev-dependencies]
 pytest = ">=6.1.2,<8"

pyproject.toml

@@ -32,6 +32,38 @@ def test_get_configurations(
     assert data['state'] == ConfigurationStateChoices.inactive
 
 
+@pytest.mark.parametrize(
+    ('pagination', 'expected_amount', 'expected_header'),
+    (
+        ('limit=10&offset=0', 10, 'items 0-9/12'),
+        ('limit=5&offset=10', 2, 'items 10-11/12'),
+        ('limit=5&offset=21', 0, 'items 21-21/12'),
+    ),
+)
+def test_get_configurations_with_pagination(
+    pagination,
+    expected_amount,
+    expected_header,
+    deployment_factory,
+    file_factory,
+    configuration_factory,
+    installation,
+    api_client,
+):
+    deployment = deployment_factory(account_id=installation['owner']['id'])
+    for i in range(12):
+        ppr_file = file_factory(id=f'ML-{i}')
+        configuration_factory(file=ppr_file.id, deployment=deployment.id)
+
+    response = api_client.get(
+        f'/api/deployments/{deployment.id}/configurations?{pagination}',
+        installation=installation,
+    )
+    assert response.status_code == 200
+    assert len(response.json()) == expected_amount
+    assert response.headers['Content-Range'] == expected_header
+
+
 def test_get_configurations_empty(
     deployment,
     installation,

tests/api/test_configurations.py

@@ -63,6 +63,46 @@ def test_list_deployments_requests(
         assert list(events['created'].keys()) == ['at', 'by']
 
 
+@pytest.mark.parametrize(
+    ('pagination', 'expected_amount', 'expected_header'),
+    (
+        ('limit=10&offset=0', 10, 'items 0-9/20'),
+        ('limit=9&offset=18', 2, 'items 18-19/20'),
+        ('limit=5&offset=21', 0, 'items 21-21/20'),
+    ),
+)
+def test_list_deployments_requests_pagination(
+    pagination,
+    expected_amount,
+    expected_header,
+    mocker,
+    deployment_factory,
+    deployment_request_factory,
+    installation,
+    api_client,
+):
+    hub_data = {
+        'id': 'HB-0000-0000',
+        'name': 'Another Hub for the best',
+    }
+    mocker.patch(
+        'connect_ext_ppr.webapp.get_hubs',
+        side_effect=[[hub_data]],
+    )
+
+    for _ in range(20):
+        dep1 = deployment_factory(account_id=installation['owner']['id'])
+        deployment_request_factory(deployment=dep1)
+
+    response = api_client.get(
+        f'/api/deployments/requests?{pagination}',
+        installation=installation,
+    )
+    assert response.status_code == 200
+    assert len(response.json()) == expected_amount
+    assert response.headers['Content-Range'] == expected_header
+
+
 def test_get_deployment_request(
     mocker,
     deployment_factory,