From 7d370d7696efa126da8ff79537cc6e62891b79d3 Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Thu, 13 Nov 2025 13:12:58 -0500 Subject: [PATCH 1/4] update data pagination limit results to 100 by default and 1000 maximum --- kobo/settings/base.py | 2 +- kpi/paginators.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kobo/settings/base.py b/kobo/settings/base.py index c7cedfff4b..273111c1ab 100644 --- a/kobo/settings/base.py +++ b/kobo/settings/base.py @@ -915,7 +915,7 @@ def __init__(self, *args, **kwargs): # Impose a limit on the number of records returned by the submission list # endpoint. This overrides any `?limit=` query parameter sent by a client -SUBMISSION_LIST_LIMIT = 30000 +SUBMISSION_LIST_LIMIT = 1000 # uWSGI, NGINX, etc. allow only a limited amount of time to process a request. # Set this value to match their limits diff --git a/kpi/paginators.py b/kpi/paginators.py index 451ef48627..04d0c7947c 100644 --- a/kpi/paginators.py +++ b/kpi/paginators.py @@ -131,7 +131,7 @@ class DataPagination(LimitOffsetPagination): """ Pagination class for submissions. """ - default_limit = settings.SUBMISSION_LIST_LIMIT + default_limit = 100 offset_query_param = 'start' max_limit = settings.SUBMISSION_LIST_LIMIT From 25f45a8c60b3eb966948aa61e44eeacb87a9f011 Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Thu, 20 Nov 2025 17:25:38 -0500 Subject: [PATCH 2/4] update logic in data view set to use the default and max limits correctly and update documentation --- kpi/docs/api/v2/data/list.md | 2 +- kpi/views/v2/data.py | 23 +++++++++++++---------- static/openapi/schema_v2.json | 2 +- static/openapi/schema_v2.yaml | 2 +- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/kpi/docs/api/v2/data/list.md b/kpi/docs/api/v2/data/list.md index f2be0b08fa..e45f6ccb6f 100644 --- a/kpi/docs/api/v2/data/list.md +++ b/kpi/docs/api/v2/data/list.md @@ -10,7 +10,7 @@ curl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/ Two parameters can be used to control pagination. * `start`: Index (zero-based) from which the results start -* `limit`: Number of results per page Maximum results per page is **30000** +* `limit`: Number of results per page Maximum results per page is **1000** ```shell curl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?start=0&limit=10 diff --git a/kpi/views/v2/data.py b/kpi/views/v2/data.py index 88cc3cbdfd..54830f3882 100644 --- a/kpi/views/v2/data.py +++ b/kpi/views/v2/data.py @@ -668,16 +668,19 @@ def _filter_mongo_query(self, request): # Remove `format` from filters. No need to use it filters.pop('format', None) # Do not allow requests to retrieve more than `SUBMISSION_LIST_LIMIT` - # submissions at one time - limit = filters.get('limit', settings.SUBMISSION_LIST_LIMIT) - try: - filters['limit'] = positive_int( - limit, strict=True, cutoff=settings.SUBMISSION_LIST_LIMIT - ) - except ValueError: - raise serializers.ValidationError( - {'limit': t('A positive integer is required')} - ) + # submissions at one time only if a limit is explicitly defined. + if 'limit' in filters: + try: + filters['limit'] = positive_int( + filters['limit'], strict=True, cutoff=settings.SUBMISSION_LIST_LIMIT + ) + except ValueError: + raise serializers.ValidationError( + {'limit': t('A positive integer is required')} + ) + else: + # If no limit is specified, use the default limit (100) + filters['limit'] = 100 return filters diff --git a/static/openapi/schema_v2.json b/static/openapi/schema_v2.json index 89ede851b1..c5db6b76b5 100644 --- a/static/openapi/schema_v2.json +++ b/static/openapi/schema_v2.json @@ -1883,7 +1883,7 @@ "/api/v2/assets/{uid_asset}/data/": { "get": { "operationId": "api_v2_assets_data_list", - "description": "## List of submissions for a specific asset\n\nBy default, JSON format is used, but XML and GeoJSON are also available:\n\n```shell\ncurl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/\n```\n\n### Pagination\nTwo parameters can be used to control pagination.\n\n* `start`: Index (zero-based) from which the results start\n* `limit`: Number of results per page Maximum results per page is **30000**\n\n```shell\ncurl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?start=0&limit=10\n```\n\n### Query submitted data\nProvides a list of submitted data for a specific form. Use `query`\nparameter to apply form data specific, see\nhttp://docs.mongodb.org/manual/reference/operator/query/.\n\nFor more details see\nAPI Parameters.\n\n```shell\ncurl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?query={\"__version__\": \"vWvkKzNE8xCtfApJvabfjG\"}\ncurl https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?query={\"_submission_time\": {\"$gt\": \"2019-09-01T01:02:03\"}}\n```\n\n### About the GeoJSON format\nRequesting the `geojson` format returns a `FeatureCollection` where each\nsubmission is a `Feature`. If your form has multiple geographic questions,\nuse the `geo_question_name` query parameter to determine which question's\nresponses populate the `geometry` for each `Feature`; otherwise, the first\ngeographic question is used. All question/response pairs are included in\nthe `properties` of each `Feature`, but _repeating groups are omitted_.\n\nQuestion types are mapped to GeoJSON geometry types as follows:\n\n* `geopoint` to `Point`;\n* `geotrace` to `LineString`;\n* `geoshape` to `Polygon`.\n\n\n\n### ⚠️ Note: DRF-Spectacular Limitation\n\nDue to limitations in **DRF-Spectacular**, the `ACCEPT` headers do not sync properly with the request. As a result, all responses will default to `application/json`, regardless of the specified format.\n\nThis means that while alternative formats (like XML) are technically supported and will work via command-line tools (e.g., `curl`), **they will not work** when trying out the endpoint directly from the documentation page.\n\nWe’ve still included the header to show supported formats, but keep in mind:\n**Only `application/json` will be used in the docs UI.**\n\n", + "description": "## List of submissions for a specific asset\n\nBy default, JSON format is used, but XML and GeoJSON are also available:\n\n```shell\ncurl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/\n```\n\n### Pagination\nTwo parameters can be used to control pagination.\n\n* `start`: Index (zero-based) from which the results start\n* `limit`: Number of results per page Maximum results per page is **1000**\n\n```shell\ncurl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?start=0&limit=10\n```\n\n### Query submitted data\nProvides a list of submitted data for a specific form. Use `query`\nparameter to apply form data specific, see\nhttp://docs.mongodb.org/manual/reference/operator/query/.\n\nFor more details see\nAPI Parameters.\n\n```shell\ncurl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?query={\"__version__\": \"vWvkKzNE8xCtfApJvabfjG\"}\ncurl https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?query={\"_submission_time\": {\"$gt\": \"2019-09-01T01:02:03\"}}\n```\n\n### About the GeoJSON format\nRequesting the `geojson` format returns a `FeatureCollection` where each\nsubmission is a `Feature`. If your form has multiple geographic questions,\nuse the `geo_question_name` query parameter to determine which question's\nresponses populate the `geometry` for each `Feature`; otherwise, the first\ngeographic question is used. All question/response pairs are included in\nthe `properties` of each `Feature`, but _repeating groups are omitted_.\n\nQuestion types are mapped to GeoJSON geometry types as follows:\n\n* `geopoint` to `Point`;\n* `geotrace` to `LineString`;\n* `geoshape` to `Polygon`.\n\n\n\n### ⚠️ Note: DRF-Spectacular Limitation\n\nDue to limitations in **DRF-Spectacular**, the `ACCEPT` headers do not sync properly with the request. As a result, all responses will default to `application/json`, regardless of the specified format.\n\nThis means that while alternative formats (like XML) are technically supported and will work via command-line tools (e.g., `curl`), **they will not work** when trying out the endpoint directly from the documentation page.\n\nWe’ve still included the header to show supported formats, but keep in mind:\n**Only `application/json` will be used in the docs UI.**\n\n", "parameters": [ { "in": "query", diff --git a/static/openapi/schema_v2.yaml b/static/openapi/schema_v2.yaml index af75067199..08c8b74ed5 100644 --- a/static/openapi/schema_v2.yaml +++ b/static/openapi/schema_v2.yaml @@ -1341,7 +1341,7 @@ paths: Two parameters can be used to control pagination. * `start`: Index (zero-based) from which the results start - * `limit`: Number of results per page Maximum results per page is **30000** + * `limit`: Number of results per page Maximum results per page is **1000** ```shell curl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?start=0&limit=10 From 4e4d1811f28071cba42097acdbc36531f08292fe Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Thu, 20 Nov 2025 17:31:14 -0500 Subject: [PATCH 3/4] fix orval failure --- jsapp/js/api/react-query/survey-data.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jsapp/js/api/react-query/survey-data.ts b/jsapp/js/api/react-query/survey-data.ts index e3b443ea76..a3aa417ca8 100644 --- a/jsapp/js/api/react-query/survey-data.ts +++ b/jsapp/js/api/react-query/survey-data.ts @@ -351,7 +351,7 @@ curl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/ Two parameters can be used to control pagination. * `start`: Index (zero-based) from which the results start -* `limit`: Number of results per page Maximum results per page is **30000** +* `limit`: Number of results per page Maximum results per page is **1000** ```shell curl -X GET https://kf.kobotoolbox.org/api/v2/assets/{uid}/data/?start=0&limit=10 From 6153239a2b6f453fdf9e7ca62ae6d4d94cc70f11 Mon Sep 17 00:00:00 2001 From: RuthShryock Date: Fri, 21 Nov 2025 09:58:40 -0500 Subject: [PATCH 4/4] use pagination_class variables to avoid redundancy --- kpi/views/v2/data.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/kpi/views/v2/data.py b/kpi/views/v2/data.py index 54830f3882..6fe1301d5a 100644 --- a/kpi/views/v2/data.py +++ b/kpi/views/v2/data.py @@ -667,12 +667,14 @@ def _filter_mongo_query(self, request): # Remove `format` from filters. No need to use it filters.pop('format', None) - # Do not allow requests to retrieve more than `SUBMISSION_LIST_LIMIT` + # Do not allow requests to retrieve more than `max_limit` # submissions at one time only if a limit is explicitly defined. if 'limit' in filters: try: filters['limit'] = positive_int( - filters['limit'], strict=True, cutoff=settings.SUBMISSION_LIST_LIMIT + filters['limit'], + strict=True, + cutoff=self.pagination_class.max_limit, ) except ValueError: raise serializers.ValidationError( @@ -680,7 +682,7 @@ def _filter_mongo_query(self, request): ) else: # If no limit is specified, use the default limit (100) - filters['limit'] = 100 + filters['limit'] = self.pagination_class.default_limit return filters