param page size

Author: v-rocheleau

submission-snapshot/README.md

@@ -102,6 +102,7 @@ List of supported environement variables:
 | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- |
 | `SOURCES__API_BEARER_TOKEN`                                   | Bearer token for Submission API                                                                |
 | `SOURCES__BASE_URL`                                           | Base URL for Submission API                                                                    |
+| `SOURCES__PAGE_SIZE`                                          | Page size for paginated Submission API requests                                                |
 | `DESTINATION__FILESYSTEM__BUCKET_URL`                         | S3 bucket URL                                                                                  |
 | `DESTINATION__FILESYSTEM__LAYOUT`                             | [DLT files layout](https://dlthub.com/docs/dlt-ecosystem/destinations/filesystem#files-layout) |
 | `DESTINATION__FILESYSTEM__CREDENTIALS__ENDPOINT_URL`          | S3 service endpoint URL                                                                        |

submission-snapshot/main.py

@@ -8,7 +8,7 @@
 
 @dlt.source(max_table_nesting=0, parallelized=True)
 def submission_source(
-    api_bearer_token: str = dlt.secrets.value, base_url: str = dlt.secrets.value
+    api_bearer_token: str = dlt.secrets.value, base_url: str = dlt.secrets.value, page_size: int = dlt.secrets.value
 ):
     config: RESTAPIConfig = {
         "client": {
@@ -37,7 +37,7 @@ def submission_source(
                     },
                     # Only store the "records" JSON, ignore pagination
                     "data_selector": "records",
-                    "params": {"pageSize": 500},
+                    "params": {"pageSize": page_size},
                 },
                 # Includes the Category.id and Category.studyId in all records for filtering downstream
                 "include_from_parent": ["id", "studyId"],