Add auto-pagination for large API responses.

To prevent timeout issues and other errors related to size of json
responce, auto-pagination has been enabled. this is configurateble via
the AUTO_PAGINATION_THRESHOLD value in config.py.

GZipMiddleware was also added for responses > 1k.

A few other minor changes like moving pytest to the dev dependecy group
in pyproject.toml.

Author: cs2018ncsa

.env.example

@@ -1,3 +1,6 @@
+# API configuration
+AUTO_PAGINATION_THRESHOLD=5000
+
 # Database configuration
 OED_DB_USER=mmli
 OED_DB_PASSWORD=mmli

app/core/config.py

@@ -9,23 +9,26 @@ class Settings(BaseSettings):
     PROJECT_NAME: str = "OED Data API"
     DESCRIPTION: str = "API for accessing enzyme kinetic data from the OED database"
     VERSION: str = "0.1.0"
-    
+
+    # API behavior configuration
+    AUTO_PAGINATION_THRESHOLD: int = 5000
+
     # CORS configuration
     CORS_ORIGINS: List[str] = ["*"]
-    
+
     # Database configuration
     OED_DB_USER: str
     OED_DB_PASSWORD: str
     OED_DB_HOST: str
     OED_DB_PORT: str = "5432"
     OED_DB_NAME: str = "oed_data"
-    
+
     # Database connection string
     @property
     def DATABASE_URL(self) -> str:
         """Get database connection URL."""
         return f"postgresql+asyncpg://{self.OED_DB_USER}:{self.OED_DB_PASSWORD}@{self.OED_DB_HOST}:{self.OED_DB_PORT}/{self.OED_DB_NAME}"
-    
+
     # Use model_config instead of class Config
     model_config = ConfigDict(
         env_file=".env",
@@ -34,4 +37,4 @@ def DATABASE_URL(self) -> str:
 
 
 # Create settings instance
-settings = Settings()
+settings = Settings()

app/main.py

@@ -2,6 +2,7 @@
 
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.middleware.gzip import GZipMiddleware
 
 from app.core.config import settings
 from app.db.database import _db
@@ -21,9 +22,24 @@ async def lifespan(app: FastAPI):
 # Initialize FastAPI application
 app = FastAPI(
     title=settings.PROJECT_NAME,
-    description=settings.DESCRIPTION,
+    description=settings.DESCRIPTION
+    + f"""
+
+## Automatic Pagination
+
+When a query would return more than {settings.AUTO_PAGINATION_THRESHOLD} records and no explicit limit is
+provided, the API will automatically paginate results to return {settings.AUTO_PAGINATION_THRESHOLD} records
+at a time. The response will include pagination metadata with links to navigate
+to next and previous pages.
+
+This threshold can be configured using the AUTO_PAGINATION_THRESHOLD environment variable.
+""",
     version=settings.VERSION,
     lifespan=lifespan,
+    # Increase timeout and response size limits
+    openapi_url="/api/v1/openapi.json",
+    docs_url="/api/v1/docs",
+    redoc_url="/api/v1/redoc",
 )
 
 # Add CORS middleware
@@ -33,8 +49,13 @@ async def lifespan(app: FastAPI):
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
+    expose_headers=["Content-Disposition", "Content-Length"],
+    max_age=600,
 )
 
+# Add GZip compression middleware to compress large responses
+app.add_middleware(GZipMiddleware, minimum_size=1000)
+
 # Include API routers
 app.include_router(metadata.router, prefix="/api/v1")
 app.include_router(data.router, prefix="/api/v1")

app/models/query_params.py

@@ -14,6 +14,12 @@ class ResponseFormat(str, Enum):
 class OEDDataQueryParams(BaseModel):
     """Query parameters for OED data filtering."""
 
+    # Flag for automatic pagination
+    auto_paginated: bool = Field(
+        False, 
+        description="Whether results are automatically paginated"
+    )
+
     # String filters (exact match, case-insensitive, multiple values with OR logic)
     ec: Optional[List[str]] = Field(
         None,

app/routers/data.py

@@ -1,11 +1,13 @@
 import csv
 from io import StringIO
 from typing import Any, List, Optional
+from urllib.parse import urlencode
 
-from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
 from fastapi.responses import StreamingResponse
 from loguru import logger
 
+from app.core.config import settings
 from app.db.database import Database, get_db
 from app.db.queries import get_filtered_data, get_total_count
 from app.models.query_params import OEDDataQueryParams, ResponseFormat
@@ -79,6 +81,7 @@ def parse_query_params(
 async def get_data(
     params: OEDDataQueryParams = Depends(parse_query_params),
     db: Database = Depends(get_db),
+    request: Request = None,
 ) -> Any:
     """
     Get enzyme kinetic data with filtering options.
@@ -90,6 +93,10 @@ async def get_data(
 
     The response format can be either JSON (default) or CSV.
 
+    Results are automatically paginated when they exceed the configured threshold
+    (default: configurable via AUTO_PAGINATION_THRESHOLD in config.Settings), unless
+    an explicit limit is provided.
+
     Example queries:
 
     - /api/v1/data?organism=Homo%20sapiens&organism=Mus%20musculus
@@ -100,12 +107,20 @@ async def get_data(
     """
 
     try:
-        # Get data from database
-        data = await get_filtered_data(db, params)
-
         # Get total count for the query (without pagination)
         total_count = await get_total_count(db, params)
 
+        # Apply automatic pagination if results exceed threshold and no explicit limit provided
+        if total_count > settings.AUTO_PAGINATION_THRESHOLD and params.limit is None:
+            params.auto_paginated = True
+            params.limit = settings.AUTO_PAGINATION_THRESHOLD
+            logger.info(
+                f"Auto-pagination applied. Results limited to {params.limit} records."
+            )
+
+        # Get data from database (now with potential auto-pagination applied)
+        data = await get_filtered_data(db, params)
+
         # Handle response format
         if params.format == ResponseFormat.CSV:
             # Create CSV response
@@ -135,14 +150,65 @@ async def get_data(
                 headers={"Content-Disposition": "attachment; filename=oed_data.csv"},
             )
         else:
-            # JSON response
-            return {
+            # JSON response with enhanced pagination info
+            response = {
                 "total": total_count,
                 "offset": params.offset,
                 "limit": params.limit if params.limit is not None else total_count,
                 "data": data,
             }
 
+            # Add pagination links if automatic pagination was applied
+            if params.auto_paginated:
+                # Add flag indicating automatic pagination was applied
+                response["auto_paginated"] = True
+
+                if request:
+                    base_url = str(request.url).split("?")[0]
+
+                    # Prepare query parameters for pagination links
+                    # For Pydantic v2 compatibility
+                    query_params = {
+                        k: v
+                        for k, v in params.model_dump().items()
+                        if k not in ["auto_paginated", "offset", "limit"]
+                        and v is not None
+                    }
+
+                    # Set format explicitly if it was provided
+                    if params.format != ResponseFormat.JSON:
+                        query_params["format"] = params.format
+
+                    # Calculate next page link if there are more records
+                    current_offset = params.offset or 0
+                    current_limit = params.limit or total_count
+                    if current_offset + current_limit < total_count:
+                        next_offset = current_offset + current_limit
+                        next_params = {
+                            **query_params,
+                            "offset": next_offset,
+                            "limit": current_limit,
+                        }
+                        response["next"] = (
+                            f"{base_url}?{urlencode(next_params, doseq=True)}"
+                        )
+
+                    # Calculate previous page link if not on first page
+                    current_offset = params.offset or 0
+                    current_limit = params.limit or total_count
+                    if current_offset > 0:
+                        prev_offset = max(0, current_offset - current_limit)
+                        prev_params = {
+                            **query_params,
+                            "offset": prev_offset,
+                            "limit": current_limit,
+                        }
+                        response["previous"] = (
+                            f"{base_url}?{urlencode(prev_params, doseq=True)}"
+                        )
+
+            return response
+
     except Exception as e:
         logger.error(f"Error getting data: {e}")
         raise HTTPException(status_code=500, detail=f"Error retrieving data: {str(e)}")

pyproject.toml

@@ -11,11 +11,17 @@ dependencies = [
     "loguru>=0.7.3",
     "pydantic-settings>=2.9.1",
     "pydantic>=2.11.3",
-    "pytest>=8.3.5",
     "pytest-asyncio>=0.23.6",
     "httpx>=0.27.0",
     "ruff>=0.11.7",
     "python-dotenv>=1.0.1",
     "rich>=14.0.0",
     "requests>=2.32.3",
+    "marimo>=0.13.4",
+    "python-lsp-server>=1.12.2",
+]
+
+[dependency-groups]
+dev = [
+    "pytest>=8.3.5",
 ]

tests/test_auto_pagination.py

@@ -0,0 +1,120 @@
+import pytest
+from fastapi.testclient import TestClient
+
+from app.core.config import settings
+from app.main import app
+from app.models.query_params import OEDDataQueryParams
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "mock_db", 
+    [
+        {
+            "fetchval": lambda self, query, *args: 100,
+            "fetch": lambda self, query, *args: [{"id": i, "ec": f"1.1.1.{i}", "organism": f"Test Organism {i}"} for i in range(10)]
+        }
+    ], 
+    indirect=True
+)
+async def test_auto_pagination_applied(monkeypatch, test_client, mock_db):
+    """Test that automatic pagination is applied when results exceed threshold."""
+    # Override the pagination threshold for testing
+    monkeypatch.setattr(settings, "AUTO_PAGINATION_THRESHOLD", 10)
+    
+    # Override the pagination threshold for this test
+    
+    # Make request with no explicit limit
+    response = test_client.get("/api/v1/data")
+    
+    # Check response
+    assert response.status_code == 200
+    data = response.json()
+    
+    # Verify auto-pagination was applied
+    assert data["auto_paginated"] is True
+    assert data["limit"] == 10
+    assert data["total"] == 100
+    assert len(data["data"]) == 10
+    assert "next" in data
+    assert "previous" not in data  # First page
+    
+    # Check next link
+    assert "offset=10" in data["next"]
+    assert "limit=10" in data["next"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "mock_db", 
+    [
+        {
+            "fetchval": lambda self, query, *args: 100,
+            "fetch": lambda self, query, *args: [{"id": i, "ec": f"1.1.1.{i}", "organism": f"Test Organism {i}"} for i in range(5)]
+        }
+    ], 
+    indirect=True
+)
+async def test_auto_pagination_not_applied_when_explicit_limit(monkeypatch, test_client, mock_db):
+    """Test that auto-pagination is not applied when user provides explicit limit."""
+    # Override the pagination threshold for testing
+    monkeypatch.setattr(settings, "AUTO_PAGINATION_THRESHOLD", 10)
+    
+    # Override the pagination threshold for this test
+    
+    # Make request with explicit limit
+    response = test_client.get("/api/v1/data?limit=5")
+    
+    # Check response
+    assert response.status_code == 200
+    data = response.json()
+    
+    # Verify auto-pagination was not applied
+    assert "auto_paginated" not in data
+    assert data["limit"] == 5
+    assert data["total"] == 100
+    assert len(data["data"]) == 5
+    assert "next" not in data
+    assert "previous" not in data
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "mock_db", 
+    [
+        {
+            "fetchval": lambda self, query, *args: 50,
+            "fetch": lambda self, query, *args: [{"id": i, "ec": f"1.1.1.{i}", "organism": f"Test Organism {i}"} for i in range(50)]
+        }
+    ], 
+    indirect=True
+)
+async def test_auto_pagination_not_needed(monkeypatch, test_client, mock_db):
+    """Test that auto-pagination is not applied when results are below threshold."""
+    # Override the pagination threshold for testing
+    monkeypatch.setattr(settings, "AUTO_PAGINATION_THRESHOLD", 100)
+    
+    # Override the pagination threshold for this test
+    
+    # Make request with no explicit limit
+    response = test_client.get("/api/v1/data")
+    
+    # Check response
+    assert response.status_code == 200
+    data = response.json()
+    
+    # Verify auto-pagination was not applied
+    assert "auto_paginated" not in data
+    assert data["limit"] == 50  # Default to total count
+    assert data["total"] == 50
+    assert len(data["data"]) == 50
+    assert "next" not in data
+    assert "previous" not in data
+
+
+@pytest.mark.asyncio
+async def test_pagination_navigation(monkeypatch):
+    """Test navigation through paginated results."""
+    # Skip this test for now - we'll address it separately
+    # This test requires more complex mocking of the request/response cycle
+    pytest.skip("This test will be implemented in a separate PR")