feat: add pagination setup for catalog search

Author: wtlow003

backend/firestore.indexes.json

@@ -12,7 +12,21 @@
           }
         }
       ]
+    },
+    {
+      "collectionGroup": "schemes",
+      "queryScope": "COLLECTION",
+      "fields": [
+        {
+          "fieldPath": "agency",
+          "order": "ASCENDING"
+        },
+        {
+          "fieldPath": "last_scraped_update",
+          "order": "DESCENDING"
+        }
+      ]
     }
   ],
   "fieldOverrides": []
-}
+}

backend/functions/schemes/catalog.py

@@ -1,51 +1,54 @@
 """
-url for local testing:
+Handler for catalog endpoint
+
+URL for local testing:
 http://127.0.0.1:5001/schemessg-v3-dev/asia-southeast1/catalog
-http://127.0.0.1:5001/schemessg-v3-dev/asia-southeast1/catalog?category=<category>
 http://127.0.0.1:5001/schemessg-v3-dev/asia-southeast1/catalog?agency=<agency>
 """
 
 import json
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
+from enum import Enum
 from typing import Literal, Union
 
 from fb_manager.firebaseManager import FirebaseManager
 from firebase_functions import https_fn, options
-from firebase_functions.firestore_fn import DocumentSnapshot
 from google.cloud.firestore_v1 import FieldFilter
-from google.cloud.firestore_v1.query_results import QueryResultsList
 from loguru import logger
 from utils.auth import verify_auth_token
+from utils.catalog_pagination import PaginationResult, get_paginated_results
 from utils.cors_config import get_cors_headers, handle_cors_preflight
 from utils.json_utils import safe_json_dumps
 from werkzeug.datastructures import MultiDict
 
 
-@dataclass
-class CatalogRequestTarget:
-    ALL: Literal["all"] = "all"
-    CATEGORY: Literal["category"] = "category"
-    AGENCY: Literal["agency"] = "agency"
+ALLOWED_QUERY_PARAMS = {"agency", "limit", "cursor", "is_warmup"}
 
 
-@dataclass
-class AllCatalogRequestParams:
-    target: CatalogRequestTarget.ALL
+class CatalogTarget(Enum):
+    ALL = "all"
+    AGENCY = "agency"
+
+
+@dataclass(kw_only=True)
+class BaseCatalogRequestParams:
+    target: CatalogTarget
+    limit: int = 10
+    cursor: str | None = None
 
 
 @dataclass
-class CategoryCatalogRequestParams:
-    target: CatalogRequestTarget.CATEGORY
-    category_name: str
+class AllCatalogRequestParams(BaseCatalogRequestParams):
+    target: Literal[CatalogTarget.ALL]
 
 
 @dataclass
-class AgencyCatalogRequestParams:
-    target: CatalogRequestTarget.AGENCY
+class AgencyCatalogRequestParams(BaseCatalogRequestParams):
+    target: Literal[CatalogTarget.AGENCY]
     agency_name: str
 
 
-CatalogRequestParams = Union[AllCatalogRequestParams, CategoryCatalogRequestParams, AgencyCatalogRequestParams]
+CatalogRequestParams = Union[AllCatalogRequestParams, AgencyCatalogRequestParams]
 
 
 def create_firebase_manager() -> FirebaseManager:
@@ -106,27 +109,22 @@ def catalog(req: https_fn.Request) -> https_fn.Response:
     try:
         query_params = _parse_query_params(req.args)
     except ValueError as e:
-        logger.error("Error parsing query parameters", e)
+        logger.exception("Error parsing query parameters", e)
         return https_fn.Response(
             response=json.dumps(
-                {
-                    "error": "Error parsing query parameters; only /catalog, /catalog?category=<category> and /catalog?agency=<agency> are supported"
-                }
+                {"error": "Error parsing query parameters; only /catalog and /catalog?agency=<agency> are supported"}
             ),
             status=400,
             mimetype="application/json",
             headers=headers,
         )
 
     try:
-        docs: QueryResultsList[DocumentSnapshot] | None = None
         match query_params.target:
-            case CatalogRequestTarget.ALL:
-                docs = _handle_all_catalog_request(firebase_manager)
-            case CatalogRequestTarget.CATEGORY:
-                docs = _handle_category_catalog_request(firebase_manager, query_params)
-            case CatalogRequestTarget.AGENCY:
-                docs = _handle_agency_catalog_request(firebase_manager, query_params)
+            case CatalogTarget.ALL:
+                results = _handle_all_catalog_request(firebase_manager, query_params)
+            case CatalogTarget.AGENCY:
+                results = _handle_agency_catalog_request(firebase_manager, query_params)
     except Exception as e:
         logger.exception("Unable to fetch scheme from firestore", e)
         return https_fn.Response(
@@ -136,81 +134,105 @@ def catalog(req: https_fn.Request) -> https_fn.Response:
             headers=headers,
         )
 
-    if not docs:
+    if results.data is None or len(results.data) == 0:
         return https_fn.Response(
             response=json.dumps({"error": "No scheme found"}),
             status=404,
             mimetype="application/json",
             headers=headers,
         )
 
-    results = {"data": [doc.to_dict() for doc in docs]}
     return https_fn.Response(
-        response=safe_json_dumps(results),
+        response=safe_json_dumps(asdict(results)),
         status=200,
         mimetype="application/json",
         headers=headers,
     )
 
 
-# TODO: Add support for pagination parsing
 def _parse_query_params(query_params: MultiDict[str, str]) -> CatalogRequestParams:
     """
     Parse request query parameters into CatalogRequestParams.
 
     Supported:
-      - /catalog                         -> AllCatalogRequestParams
-      - /catalog?category=<name>         -> CategoryCatalogRequestParams
-      - /catalog?agency=<name>           -> AgencyCatalogRequestParams
+      - /catalog                     -> AllCatalogRequestParams
+      - /catalog?agency=<name>      -> AgencyCatalogRequestParams
+
+    Raises:
+        ValueError: If unsupported query parameters are provided.
     """
 
-    category = query_params.get("category")
-    agency = query_params.get("agency")
+    # Validate unknown query parameters
+    unknown_params = set(query_params.keys()) - ALLOWED_QUERY_PARAMS
+    if unknown_params:
+        raise ValueError(f"Unsupported query parameter(s): {', '.join(sorted(unknown_params))}")
 
-    # Disallow ambiguous requests
-    if category and agency:
-        raise ValueError("Invalid query; provide only one of 'category' or 'agency'")
+    # Retrieve relevant sub-catalog query parameters
+    agency = query_params.get("agency")
 
-    # /catalog?category=<>
-    if category:
-        if not category.strip():
-            raise ValueError("Invalid query; 'category' must be a non-empty value")
-        return CategoryCatalogRequestParams(target=CatalogRequestTarget.CATEGORY, category_name=category)
+    # Retrieve limit and cursor from query parameters
+    limit = int(query_params.get("limit", "10"))
+    cursor = query_params.get("cursor", "")
 
     # /catalog?agency=<>
     if agency:
         if not agency.strip():
             raise ValueError("Invalid query; 'agency' must be a non-empty value")
-        return AgencyCatalogRequestParams(target=CatalogRequestTarget.AGENCY, agency_name=agency)
+        return AgencyCatalogRequestParams(target=CatalogTarget.AGENCY, agency_name=agency, limit=limit, cursor=cursor)
 
     # /catalog
-    return AllCatalogRequestParams(target=CatalogRequestTarget.ALL)
-
+    return AllCatalogRequestParams(target=CatalogTarget.ALL, limit=limit, cursor=cursor)
 
-def _handle_all_catalog_request(firebase_manager: FirebaseManager) -> QueryResultsList[DocumentSnapshot] | None:
-    """Handle all catalog request."""
 
-    ref = firebase_manager.firestore_client.collection("schemes")
-    return ref.get()
+def _handle_all_catalog_request(
+    firebase_manager: FirebaseManager,
+    query_params: AllCatalogRequestParams,
+) -> PaginationResult:
+    """Retrieve all catalog entries using cursor pagination.
 
+    Args:
+        firebase_manager: Firebase manager providing Firestore access.
 
-def _handle_category_catalog_request(
-    firebase_manager: FirebaseManager, query_params: CategoryCatalogRequestParams
-) -> QueryResultsList[DocumentSnapshot] | None:
-    """Handle category catalog request."""
+    Returns:
+        PaginationResult:
+            data: Documents for the current page.
+            next_cursor: Cursor for the next page, or None if exhausted.
+            has_more: Whether more results exist.
+    """
+    col = firebase_manager.firestore_client.collection("schemes")
 
-    ref = firebase_manager.firestore_client.collection("schemes").where(
-        filter=FieldFilter("scheme_type", "==", query_params.category_name)
+    return get_paginated_results(
+        collection_ref=col,
+        cursor=query_params.cursor,
+        limit=query_params.limit,
     )
-    return ref.get()
 
 
 def _handle_agency_catalog_request(
     firebase_manager: FirebaseManager, query_params: AgencyCatalogRequestParams
-) -> QueryResultsList[DocumentSnapshot] | None:
-    """Handle agency catalog request."""
+) -> PaginationResult:
+    """Retrieve agency-filtered catalog entries using cursor pagination.
+
+    Args:
+        firebase_manager: Firebase manager providing Firestore access.
+        query_params: Request parameters containing agency name, limit,
+            and optional pagination cursor.
+
+    Returns:
+        PaginationResult:
+            data: Documents for the current page.
+            next_cursor: Cursor for the next page, or None if exhausted.
+            has_more: Whether more results exist.
+    """
+    col = firebase_manager.firestore_client.collection("schemes")
+
+    # Apply base filters for filtering within catalog
+    query = col.where(filter=FieldFilter("agency", "==", query_params.agency_name))
+    print("Query:", query)
 
-    ref = firebase_manager.firestore_client.collection("schemes").where(
-        filter=FieldFilter("agency", "==", query_params.agency_name)
+    return get_paginated_results(
+        collection_ref=col,
+        base_query=query,
+        cursor=query_params.cursor,
+        limit=query_params.limit,
     )
-    return ref.get()