Merge branch '0.12.7_minio_proxy' into add_minio_proxy_server

Author: MaxFBurg

datajoint/s3.py

@@ -3,6 +3,7 @@
 """
 from io import BytesIO
 import minio   # https://docs.minio.io/docs/python-client-api-reference
+import urllib3
 import warnings
 import uuid
 import logging
@@ -16,9 +17,29 @@ class Folder:
     """
     A Folder instance manipulates a flat folder of objects within an S3-compatible object store
     """
-    def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False, **_):
-        self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key,
-                                  secure=secure)
+    def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False, proxy_server=None, **_):
+        if proxy_server:
+            # from https://docs.min.io/docs/python-client-api-reference
+            http_client = urllib3.ProxyManager(
+                    proxy_server,
+                    timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
+                    cert_reqs="CERT_REQUIRED",
+                    retries=urllib3.Retry(
+                        total=5,
+                        backoff_factor=0.2,
+                        status_forcelist=[500, 502, 503, 504],
+                    )
+                )
+        else:
+            http_client = None
+
+        self.client = minio.Minio(
+            endpoint,
+            access_key=access_key,
+            secret_key=secret_key,
+            secure=secure,
+            http_client=http_client,
+            )
         self.bucket = bucket
         if not self.client.bucket_exists(bucket):
             raise errors.BucketInaccessible('Inaccessible s3 bucket %s' % bucket)

datajoint/settings.py

@@ -137,7 +137,7 @@ def get_store_spec(self, store):
         spec['subfolding'] = spec.get('subfolding', DEFAULT_SUBFOLDING)
         spec_keys = {  # REQUIRED in uppercase and allowed in lowercase
             'file': ('PROTOCOL', 'LOCATION', 'subfolding', 'stage'),
-            's3': ('PROTOCOL', 'ENDPOINT', 'BUCKET', 'ACCESS_KEY', 'SECRET_KEY', 'LOCATION', 'secure', 'subfolding', 'stage')}
+            's3': ('PROTOCOL', 'ENDPOINT', 'BUCKET', 'ACCESS_KEY', 'SECRET_KEY', 'LOCATION', 'secure', 'subfolding', 'stage', 'proxy_server')}
 
         try:
             spec_keys = spec_keys[spec.get('protocol', '').lower()]