sonar

Author: sanderegg

services/storage/src/simcore_service_storage/datcore_dsm.py

@@ -17,7 +17,6 @@
 from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID
 from models_library.users import UserID
 from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter, ValidationError
-from servicelib.utils import limited_as_completed
 
 from .constants import DATCORE_ID, DATCORE_STR
 from .dsm_factory import BaseDataManager
@@ -206,39 +205,41 @@ async def compute_path_total_size(self, user_id: UserID, *, path: Path) -> ByteS
             if dataset_size is not None:
                 return dataset_size
 
-        # generic computation
+        # generic computation (slow and unoptimized - could be improved if necessary by using datcore data better)
         try:
-            paths, cursor, total_number = await self.list_paths(
-                user_id, file_filter=path, cursor=None, limit=50
-            )
             accumulated_size = ByteSize(0)
+            paths_to_process = [path]
 
-            next_folders: list[PathMetaData] = []
-            for p in paths:
-                if p.file_meta_data is not None:
-                    # this is a file
-                    assert (
-                        p.file_meta_data.file_size is not UNDEFINED_SIZE_TYPE
-                    )  # nosec
-                    assert isinstance(p.file_meta_data.file_size, ByteSize)  # nosec
-                    accumulated_size = ByteSize(
-                        accumulated_size + p.file_meta_data.file_size
-                    )
-                else:
-                    next_folders.append(p)
-            async for sbfolder_size_future in limited_as_completed(
-                (
-                    self.compute_path_total_size(user_id, path=sub_folder.path)
-                    for sub_folder in next_folders
-                ),
-                limit=3,
-            ):
-                size = await sbfolder_size_future
-                accumulated_size = ByteSize(accumulated_size + size)
+            while paths_to_process:
+                current_path = paths_to_process.pop()
+                paths, cursor, _ = await self.list_paths(
+                    user_id, file_filter=current_path, cursor=None, limit=50
+                )
+
+                while paths:
+                    for p in paths:
+                        if p.file_meta_data is not None:
+                            # this is a file
+                            assert (
+                                p.file_meta_data.file_size is not UNDEFINED_SIZE_TYPE
+                            )  # nosec
+                            assert isinstance(
+                                p.file_meta_data.file_size, ByteSize
+                            )  # nosec
+                            accumulated_size = ByteSize(
+                                accumulated_size + p.file_meta_data.file_size
+                            )
+                            continue
+                        paths_to_process.append(p.path)
+
+                    if cursor:
+                        paths, cursor, _ = await self.list_paths(
+                            user_id, file_filter=current_path, cursor=cursor, limit=50
+                        )
+                    else:
+                        break
 
             return accumulated_size
-            if len(paths) == 0:
-                return ByteSize(0)
 
         except ValidationError:
             # invalid path