support large file uploading

Author: bingzhang

pyclowder/files.py

@@ -9,6 +9,7 @@
 import tempfile
 
 import requests
+from requests_toolbelt.multipart.encoder import MultipartEncoder
 from urllib3.filepost import encode_multipart_formdata
 
 from pyclowder.datasets import get_file_list
@@ -312,7 +313,11 @@ def upload_to_dataset(connector, host, key, datasetid, filepath, check_duplicate
     url = '%sapi/uploadToDataset/%s?key=%s' % (host, datasetid, key)
 
     if os.path.exists(filepath):
-        result = connector.post(url, files={"File": open(filepath, 'rb')},
+        filename = os.path.basename(filepath)
+        m = MultipartEncoder(
+            fields={'file': (filename, open(filepath, 'rb'))}
+        )
+        result = connector.post(url, data=m, headers={'Content-Type': m.content_type},
                                 verify=connector.ssl_verify if connector else True)
 
         uploadedfileid = result.json()['id']
@@ -345,10 +350,11 @@ def _upload_to_dataset_local(connector, host, key, datasetid, filepath):
                                             source_path)
                 break
 
-        (content, header) = encode_multipart_formdata([
-            ("file", '{"path":"%s"}' % filepath)
-        ])
-        result = connector.post(url, data=content, headers={'Content-Type': header},
+        filename = os.path.basename(filepath)
+        m = MultipartEncoder(
+            fields={'file': (filename, open(filepath, 'rb'))}
+        )
+        result = connector.post(url, data=m, headers={'Content-Type': m.content_type},
                                 verify=connector.ssl_verify if connector else True)
 
         uploadedfileid = result.json()['id']

requirements.txt

@@ -6,3 +6,4 @@ wheel==0.33.1
 urllib3==1.24.1
 pytest==4.3.1
 pytest-pep8==1.0.6
+requests-toolbelt==0.9.1

setup.py

@@ -36,6 +36,7 @@ def description():
           'pika==1.0.0',
           'PyYAML==5.1',
           'requests==2.21.0',
+          'requests-toolbelt==0.9.1',
       ],
 
       include_package_data=True,