using faster settings for rclone

Author: Andrei Neagu

packages/settings-library/src/settings_library/r_clone.py

@@ -36,3 +36,29 @@ class RCloneSettings(BaseCustomSettings):
             description="`--buffer-size X`: sets the amount of RAM to use for each individual transfer",
         ),
     ] = "16M"
+
+    R_CLONE_OPTION_CHECKERS: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--checkers X`: sets the number checkers",
+        ),
+    ] = 32
+
+    R_CLONE_S3_UPLOAD_CONCURRENCY: Annotated[
+        NonNegativeInt,
+        Field(
+            description="`--s3-upload-concurrency X`: sets the number of concurrent uploads to S3",
+        ),
+    ] = 8
+
+    R_CLONE_CHUNK_SIZE: Annotated[
+        str,
+        Field(description="`--s3-chunk-size X`: sets the chunk size for S3"),
+    ] = "128M"
+
+    R_CLONE_ORDER_BY: Annotated[
+        str,
+        Field(
+            description="`--order-by X`: sets the order of file upload, e.g., 'size,mixed'",
+        ),
+    ] = "size,mixed"

packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py

@@ -30,8 +30,7 @@
 _logger = logging.getLogger(__name__)
 
 
-class BaseRCloneError(OsparcErrorMixin, RuntimeError):
-    ...
+class BaseRCloneError(OsparcErrorMixin, RuntimeError): ...
 
 
 class RCloneFailedError(BaseRCloneError):
@@ -207,6 +206,16 @@ async def _sync_sources(
             # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
             "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
             r_clone_settings.R_CLONE_OPTION_BUFFER_SIZE,
+            "--checkers",
+            f"{r_clone_settings.R_CLONE_OPTION_CHECKERS}",
+            "--s3-upload-concurrency",
+            f"{r_clone_settings.R_CLONE_S3_UPLOAD_CONCURRENCY}",
+            "--s3-chunk-size",
+            r_clone_settings.R_CLONE_CHUNK_SIZE,
+            # handles the order of file upload
+            "--order-by",
+            r_clone_settings.R_CLONE_ORDER_BY,
+            "--fast-list",
             "--use-json-log",
             # frequent polling for faster progress updates
             "--stats",

services/agent/src/simcore_service_agent/services/backup.py

@@ -162,7 +162,7 @@ async def _store_in_s3(
         # below two options reduce to a minimum the memory footprint
         # https://forum.rclone.org/t/how-to-set-a-memory-limit/10230/4
         "--buffer-size",  # docs https://rclone.org/docs/#buffer-size-size
-        "0M",
+        "16M",
         "--stats",
         "5s",
         "--stats-one-line",