[SYNPY-1426] Deprecate getChildren method and extend sync_from_synapse method (#1222)

* Enhance Folder and Project Models to Support Additional Entity Types

- Updated Folder and Project models to include support for Tables, EntityViews, SubmissionViews, Datasets, DatasetCollections, MaterializedViews, and VirtualTables.
- Modified the StorableContainer mixin to handle the new entity types during synchronization.
- Added tests to verify the synchronization of all supported entity types within both Folder and Project models.
- Refactored relevant methods to accommodate the new entity types and ensure proper handling during sync operations.
- Deprecate the getChildren method call in the Synapse class

Author: BryanFauble

synapseclient/api/__init__.py

@@ -31,6 +31,7 @@
     delete_entity_generated_by,
     delete_entity_provenance,
     get_activity,
+    get_children,
     get_entities_by_md5,
     get_entity,
     get_entity_acl,
@@ -136,6 +137,7 @@
     "get_activity",
     "create_activity",
     "update_activity",
+    "get_children",
     # configuration_services
     "get_config_file",
     "get_config_section_dict",

synapseclient/api/api_client.py

@@ -1,3 +1,4 @@
+import json
 import sys
 from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, Optional
 
@@ -46,7 +47,7 @@ async def rest_post_paginated_async(
             body["nextPageToken"] = next_page_token
         response = await client.rest_post_async(
             uri=uri,
-            body=body,
+            body=json.dumps(body),
             endpoint=endpoint,
             headers=headers,
             retry_policy=retry_policy,

synapseclient/api/entity_services.py

@@ -4,10 +4,11 @@
 
 import json
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union
 
 from async_lru import alru_cache
 
+from synapseclient.api.api_client import rest_post_paginated_async
 from synapseclient.core.exceptions import SynapseHTTPError
 from synapseclient.core.utils import get_synid_and_version
 
@@ -867,3 +868,99 @@ async def main():
 
     client = Synapse.get_client(synapse_client=synapse_client)
     return await client.rest_get_async(uri=f"/activity/{activity_id}")
+
+
+async def get_children(
+    parent: Optional[str] = None,
+    include_types: List[str] = None,
+    sort_by: str = "NAME",
+    sort_direction: str = "ASC",
+    *,
+    synapse_client: Optional["Synapse"] = None,
+) -> AsyncGenerator[Dict[str, Any], None]:
+    """
+    Retrieve all entities stored within a parent such as folder or project.
+
+    Arguments:
+        parent: The ID of a Synapse container (folder or project) or None to retrieve all projects
+        include_types: List of entity types to include (e.g., ["folder", "file"]).
+                      Available types can be found at:
+                      https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/EntityType.html
+        sort_by: How results should be sorted. Can be "NAME" or "CREATED_ON"
+        sort_direction: The direction of the result sort. Can be "ASC" or "DESC"
+        synapse_client: If not passed in and caching was not disabled by
+                       `Synapse.allow_client_caching(False)` this will use the last created
+                       instance from the Synapse class constructor.
+
+    Yields:
+        An async generator that yields entity children dictionaries.
+
+    Example: Getting children of a folder
+        Retrieve all children of a folder:
+
+        ```python
+        import asyncio
+        from synapseclient import Synapse
+        from synapseclient.api import get_children
+
+        syn = Synapse()
+        syn.login()
+
+        async def main():
+            async for child in get_children(parent="syn123456"):
+                print(f"Child: {child['name']} (ID: {child['id']})")
+
+        asyncio.run(main())
+        ```
+
+    Example: Getting children with specific types
+        Retrieve only files and folders:
+
+        ```python
+        import asyncio
+        from synapseclient import Synapse
+        from synapseclient.api import get_children
+
+        syn = Synapse()
+        syn.login()
+
+        async def main():
+            async for child in get_children(
+                parent="syn123456",
+                include_types=["file", "folder"],
+                sort_by="NAME",
+                sort_direction="ASC"
+            ):
+                print(f"Child: {child['name']} (Type: {child['type']})")
+
+        asyncio.run(main())
+        ```
+    """
+    if include_types is None:
+        include_types = [
+            "folder",
+            "file",
+            "table",
+            "link",
+            "entityview",
+            "dockerrepo",
+            "submissionview",
+            "dataset",
+            "materializedview",
+        ]
+
+    request_body = {
+        "parentId": parent,
+        "includeTypes": include_types,
+        "sortBy": sort_by,
+        "sortDirection": sort_direction,
+    }
+
+    response = rest_post_paginated_async(
+        uri="/entity/children",
+        body=request_body,
+        synapse_client=synapse_client,
+    )
+
+    async for child in response:
+        yield child

synapseclient/api/json_schema_services.py

@@ -164,7 +164,7 @@ async def get_invalid_json_schema_validation(
     request_body = {"containerId": synapse_id}
     response = rest_post_paginated_async(
         f"/entity/{synapse_id}/schema/validation/invalid",
-        body=json.dumps(request_body),
+        body=request_body,
         synapse_client=synapse_client,
     )
     async for item in response:

synapseclient/client.py

@@ -3395,6 +3395,13 @@ def set_annotations(self, annotations: Annotations):
     #                         Querying                         #
     ############################################################
 
+    @deprecated(
+        version="4.9.0",
+        reason="To be removed in 5.0.0. "
+        "Use the new dataclass models `from synapseclient.models import Project, Folder` "
+        "with their `sync_from_synapse` method for most use cases, "
+        "or the `synapseclient.api.get_children` function for direct API access with sorting and filtering.",
+    )
     def getChildren(
         self,
         parent,
@@ -3427,6 +3434,87 @@ def getChildren(
         Also see:
 
         - [synapseutils.walk][]
+
+        Example: Migrating from this method to new approaches
+             
+
+            **Legacy approach (deprecated):**
+            ```python
+            # Using the deprecated getChildren method
+            for child in syn.getChildren("syn12345", includeTypes=["file", "folder"]):
+                print(f"Child: {child['name']} (ID: {child['id']})")
+            ```
+
+            **New approach using dataclass models with sync_from_synapse:**
+            ```python
+            import synapseclient
+            from synapseclient.models import Project, Folder
+
+            # Create client and login
+            syn = synapseclient.Synapse()
+            syn.login()
+
+            # For projects - get all children automatically (recursive by default)
+            project = Project(id="syn12345")
+            project = project.sync_from_synapse(download_file=False)
+
+            # Access different types of children
+            print(f"Files: {len(project.files)}")
+            for file in project.files:
+                print(f"  File: {file.name} (ID: {file.id})")
+
+            print(f"Folders: {len(project.folders)}")
+            for folder in project.folders:
+                print(f"  Folder: {folder.name} (ID: {folder.id})")
+
+            print(f"Tables: {len(project.tables)}")
+            for table in project.tables:
+                print(f"  Table: {table.name} (ID: {table.id})")
+
+            # For folders - get all children automatically (recursive by default)
+            folder = Folder(id="syn67890")
+            folder = folder.sync_from_synapse(download_file=False)
+
+            # Access children in the same way
+            for file in folder.files:
+                print(f"  File: {file.name} (ID: {file.id})")
+
+            # For non-recursive behavior (equivalent to single getChildren call)
+            folder = Folder(id="syn67890")
+            folder = folder.sync_from_synapse(download_file=False, recursive=False)
+
+            # This will only get immediate children, not subfolders' contents
+            for file in folder.files:
+                print(f"  File: {file.name} (ID: {file.id})")
+            for subfolder in folder.folders:
+                print(f"  Subfolder: {subfolder.name} (ID: {subfolder.id})")
+                # Note: subfolder.files and subfolder.folders will be empty
+                # because recursive=False
+            ```
+
+            **New approach using the API directly (for advanced sorting/filtering):**
+            ```python
+            import asyncio
+            import synapseclient
+            from synapseclient.api import get_children
+
+            # Create client and login
+            syn = synapseclient.Synapse()
+            syn.login()
+
+            # Using the new async API function directly
+            async def get_sorted_children():
+                async for child in get_children(
+                    parent="syn12345",
+                    include_types=["file", "folder"],
+                    sort_by="NAME",
+                    sort_direction="ASC"
+                ):
+                    print(f"Child: {child['name']} (ID: {child['id']}, Type: {child['type']})")
+
+            # Run the async function
+            asyncio.run(get_sorted_children())
+            ```
         """
         parentId = id_of(parent) if parent is not None else None
 

synapseclient/models/dataset.py

@@ -9,7 +9,7 @@
 
 from synapseclient import Synapse
 from synapseclient.api.table_services import ViewEntityType, ViewTypeMask
-from synapseclient.core.async_utils import async_to_sync
+from synapseclient.core.async_utils import async_to_sync, wrap_async_to_sync
 from synapseclient.core.constants import concrete_types
 from synapseclient.core.utils import MB, delete_none_keys
 from synapseclient.models import Activity, Annotations
@@ -1022,7 +1022,9 @@ def add_item(
                 entity_ref=EntityRef(id=item.id, version=item.version_number)
             )
         elif isinstance(item, Folder):
-            children = item._retrieve_children(follow_link=True)
+            children = wrap_async_to_sync(
+                item._retrieve_children(follow_link=True), client
+            )
             for child in children:
                 if child["type"] == concrete_types.FILE_ENTITY:
                     self._append_entity_ref(
@@ -1127,7 +1129,9 @@ def remove_item(
                 ).get()
             self._remove_entity_ref(EntityRef(id=item.id, version=item.version_number))
         elif isinstance(item, Folder):
-            children = item._retrieve_children(follow_link=True)
+            children = wrap_async_to_sync(
+                item._retrieve_children(follow_link=True), client
+            )
             for child in children:
                 if child["type"] == concrete_types.FILE_ENTITY:
                     self._remove_entity_ref(

synapseclient/models/folder.py

@@ -27,7 +27,16 @@
 from synapseutils import copy
 
 if TYPE_CHECKING:
-    from synapseclient.models import Project
+    from synapseclient.models import (
+        Dataset,
+        DatasetCollection,
+        EntityView,
+        MaterializedView,
+        Project,
+        SubmissionView,
+        Table,
+        VirtualTable,
+    )
 
 
 @dataclass()
@@ -59,6 +68,13 @@ class Folder(
         modified_by: (Read Only) The ID of the user that last modified this entity.
         files: Files that exist within this folder.
         folders: Folders that exist within this folder.
+        tables: Tables that exist within this folder.
+        entityviews: Entity views that exist within this folder.
+        submissionviews: Submission views that exist within this folder.
+        datasets: Datasets that exist within this folder.
+        datasetcollections: Dataset collections that exist within this folder.
+        materializedviews: Materialized views that exist within this folder.
+        virtualtables: Virtual tables that exist within this folder.
         annotations: Additional metadata associated with the folder. The key is the name
             of your desired annotations. The value is an object containing a list of
             values (use empty list to represent no values for key) and the value type
@@ -116,6 +132,31 @@ class Folder(
     folders: List["Folder"] = field(default_factory=list, compare=False)
     """Folders that exist within this folder."""
 
+    tables: List["Table"] = field(default_factory=list, compare=False)
+    """Tables that exist within this folder."""
+
+    entityviews: List["EntityView"] = field(default_factory=list, compare=False)
+    """Entity views that exist within this folder."""
+
+    submissionviews: List["SubmissionView"] = field(default_factory=list, compare=False)
+    """Submission views that exist within this folder."""
+
+    datasets: List["Dataset"] = field(default_factory=list, compare=False)
+    """Datasets that exist within this folder."""
+
+    datasetcollections: List["DatasetCollection"] = field(
+        default_factory=list, compare=False
+    )
+    """Dataset collections that exist within this folder."""
+
+    materializedviews: List["MaterializedView"] = field(
+        default_factory=list, compare=False
+    )
+    """Materialized views that exist within this folder."""
+
+    virtualtables: List["VirtualTable"] = field(default_factory=list, compare=False)
+    """Virtual tables that exist within this folder."""
+
     annotations: Optional[
         Dict[
             str,