Merge pull request datajoint#962 from MaxFBurg/add_minio_proxy_server

Adds proxy server to minio / aws s3 storage

Author: guzman-raphael

datajoint/s3.py

@@ -3,6 +3,7 @@
 """
 from io import BytesIO
 import minio   # https://docs.minio.io/docs/python-client-api-reference
+import urllib3
 import warnings
 import uuid
 import logging
@@ -16,9 +17,24 @@ class Folder:
     """
     A Folder instance manipulates a flat folder of objects within an S3-compatible object store
     """
-    def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False, **_):
-        self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key,
-                                  secure=secure)
+    def __init__(self, endpoint, bucket, access_key, secret_key, *, secure=False,
+                 proxy_server=None, **_):
+        # from https://docs.min.io/docs/python-client-api-reference
+        self.client = minio.Minio(
+            endpoint,
+            access_key=access_key,
+            secret_key=secret_key,
+            secure=secure,
+            http_client=(
+                urllib3.ProxyManager(proxy_server,
+                                     timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
+                                     cert_reqs="CERT_REQUIRED",
+                                     retries=urllib3.Retry(total=5,
+                                                           backoff_factor=0.2,
+                                                           status_forcelist=[500, 502, 503,
+                                                                             504]))
+                if proxy_server else None),
+            )
         self.bucket = bucket
         if not self.client.bucket_exists(bucket):
             raise errors.BucketInaccessible('Inaccessible s3 bucket %s' % bucket)

datajoint/settings.py

@@ -137,7 +137,8 @@ def get_store_spec(self, store):
         spec['subfolding'] = spec.get('subfolding', DEFAULT_SUBFOLDING)
         spec_keys = {  # REQUIRED in uppercase and allowed in lowercase
             'file': ('PROTOCOL', 'LOCATION', 'subfolding', 'stage'),
-            's3': ('PROTOCOL', 'ENDPOINT', 'BUCKET', 'ACCESS_KEY', 'SECRET_KEY', 'LOCATION', 'secure', 'subfolding', 'stage')}
+            's3': ('PROTOCOL', 'ENDPOINT', 'BUCKET', 'ACCESS_KEY', 'SECRET_KEY', 'LOCATION',
+                   'secure', 'subfolding', 'stage', 'proxy_server')}
 
         try:
             spec_keys = spec_keys[spec.get('protocol', '').lower()]