feat: expose include, prefix and exclude filter for data connections

Author: baptiste-olivier

src/kili/adapters/kili_api_gateway/cloud_storage/mappers.py

@@ -33,10 +33,13 @@ def data_connection_where_mapper(filters: DataConnectionFilters) -> Dict:
 def add_data_connection_data_mapper(data: AddDataConnectionKiliAPIGatewayInput) -> Dict:
     """Build the GraphQL DataConnectionInput variable to be sent in an operation."""
     return {
-        "projectId": data.project_id,
+        "exclude": data.exclude,
+        "include": data.include,
         "integrationId": data.integration_id,
         "isChecking": data.is_checking,
         "lastChecked": data.last_checked.isoformat(sep="T", timespec="milliseconds") + "Z",
+        "prefix": data.prefix,
+        "projectId": data.project_id,
         "selectedFolders": data.selected_folders,
     }
 

src/kili/adapters/kili_api_gateway/cloud_storage/types.py

@@ -17,9 +17,12 @@
 class AddDataConnectionKiliAPIGatewayInput:
     """Add data connection input data for Kili API Gateway."""
 
-    is_checking: bool
+    exclude: Optional[List[str]]
+    include: Optional[List[str]]
     integration_id: DataIntegrationId
+    is_checking: bool
     last_checked: datetime
+    prefix: Optional[str]
     project_id: ProjectId
     selected_folders: Optional[List[str]]
 
@@ -52,10 +55,10 @@ class DataIntegrationData:
     name: Optional[str]
     organization_id: Optional[OrganizationId]
     platform: Optional[DataIntegrationPlatform]
-    status: Optional[DataIntegrationStatus]
     s3_access_key: Optional[str]
     s3_bucket_name: Optional[str]
     s3_endpoint: Optional[str]
     s3_region: Optional[str]
     s3_secret_key: Optional[str]
     s3_session_token: Optional[str]
+    status: Optional[DataIntegrationStatus]

src/kili/presentation/client/cloud_storage.py

@@ -296,23 +296,39 @@ def add_cloud_storage_connection(
         project_id: str,
         cloud_storage_integration_id: str,
         selected_folders: Optional[List[str]] = None,
+        prefix: Optional[str] = None,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
     ) -> Dict:
-        """Connect a cloud storage to a project.
+        """Connect a cloud storage to a project. More details about parameters
+        can be found in the [documentation](https://docs.kili-technology.com/docs/filtering-assets-from-cloud-storage).
 
         Args:
             project_id: Id of the project.
             cloud_storage_integration_id: Id of the cloud storage integration.
             selected_folders: List of folders of the data integration to connect to the project.
                 If not provided, all folders of the data integration will be connected.
+                This option is deprecated and will be removed in the future.
+            prefix: Filter files to synchronize based on their base path.
+            include: List of pattern used to include files based on their path.
+            exclude: List of pattern used to exclude files based on their path.
 
         Returns:
             A dict with the DataConnection Id.
         """
+        if selected_folders is not None:
+            logger.warning(
+                "The selected_folders argument is deprecated and will be removed in the future."
+            )
+
         data_connection_id = CloudStorageUseCases(self.kili_api_gateway).add_data_connection(
             project_id=ProjectId(project_id),
             data_integration_id=DataIntegrationId(cloud_storage_integration_id),
             selected_folders=selected_folders,
             fields=("id",),
+            prefix=prefix,
+            include=include,
+            exclude=exclude,
         )["id"]
 
         return {"id": data_connection_id}

src/kili/use_cases/cloud_storage/__init__.py

@@ -78,6 +78,9 @@ def add_data_connection(
         project_id: ProjectId,
         selected_folders: Optional[List[str]],
         fields: ListOrTuple[str],
+        prefix: Optional[str] = None,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
     ) -> Dict:
         """Add data connection to a project."""
         if (
@@ -90,9 +93,12 @@ def add_data_connection(
         return self._kili_api_gateway.add_data_connection(
             fields=fields,
             data=AddDataConnectionKiliAPIGatewayInput(
-                is_checking=False,
+                exclude=exclude,
+                include=include,
                 integration_id=data_integration_id,
+                is_checking=False,
                 last_checked=datetime.now(),
+                prefix=prefix,
                 project_id=project_id,
                 selected_folders=selected_folders,
             ),