Merge branch 'develop'

Author: dolfinus

docs/changelog/0.9.1.rst

@@ -0,0 +1,9 @@
+0.9.1 (2023-08-17)
+==================
+
+Bug Fixes
+---------
+
+- Fixed bug then number of threads created by ``FileDownloader`` / ``FileUploader`` / ``FileMover`` was
+  not ``min(workers, len(files))``, but ``max(workers, len(files))``. leading to create too much workers
+  on large files list.

docs/changelog/index.rst

@@ -4,6 +4,7 @@
 
     DRAFT
     NEXT_RELEASE
+    0.9.1
     0.9.0
     0.8.1
     0.8.0

onetl/VERSION

@@ -1 +1 @@
-0.9.0
+0.9.1

onetl/file/file_downloader/file_downloader.py

@@ -719,7 +719,7 @@ def _bulk_download(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_download)),
+                max_workers=min(workers, len(to_download)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [

onetl/file/file_mover/file_mover.py

@@ -516,7 +516,7 @@ def _bulk_move(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_move)),
+                max_workers=min(workers, len(to_move)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [

onetl/file/file_uploader/file_uploader.py

@@ -544,7 +544,7 @@ def _bulk_upload(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_upload)),
+                max_workers=min(workers, len(to_upload)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [