Merge pull request #293 from jonasfj/direct-s3-download-via-redirect

Direct s3 download via redirect

Author: shin-

README.md

@@ -104,6 +104,9 @@ Available configuration options
    *non*-Amazon S3-compliant object store, in one of the boto config files'
    `[Credentials]` section, set `boto_host`, `boto_port` as appropriate for the
    service you are using.
+1. `storage_redirect`: Redirect resource requested if storage engine supports
+   this, e.g. S3 will redirect signed URLs, this can be used to offload the
+   server.
 
 ### Authentication options
 
@@ -127,6 +130,7 @@ to `s3`.
 1. `s3_access_key`: string, S3 access key
 1. `s3_secret_key`: string, S3 secret key
 1. `s3_bucket`: string, S3 bucket name
+1. `s3_region`: S3 region where the bucket is located
 1. `s3_encrypt`: boolean, if true, the container will be encrypted on the
       server-side by S3 and will be stored in an encrypted form while at rest
       in S3.
@@ -305,7 +309,7 @@ is already taken, find out which container is already using it by running "docke
 Install the system requirements for building a Python library:
 
 ```
-sudo apt-get install build-essential python-dev libevent-dev python-pip libssl-dev liblzma-dev
+sudo apt-get install build-essential python-dev libevent-dev python-pip libssl-dev liblzma-dev libffi-dev
 ```
 
 Then install the Registry app:

config/config_s3.yml

@@ -15,4 +15,5 @@ prod:
     secret_key: REPLACEME
     s3_encrypt: true
     s3_secure: true
-    storage_path: /images
+    storage_path: /images
+    storage_redirect: False  # Redirect signed URL for image files

config/config_test.yml

@@ -10,6 +10,7 @@ test:
     s3_bucket: _env:S3_BUCKET
     s3_encrypt: _env:S3_ENCRYPT
     s3_secure: _env:S3_SECURE
+    storage_redirect: False
 
     gs_access_key: _env:GS_ACCESS_KEY
     gs_secret_key: _env:GS_SECRET_KEY 

lib/storage/__init__.py

@@ -94,6 +94,17 @@ def put_content(self, path, content):
     def stream_read(self, path, bytes_range=None):
         raise NotImplementedError
 
+    def content_redirect_url(self, path):
+        """Get a URL for content at path
+
+        Get a URL to which client can be redirected to get the content from
+        the path. Return None if not supported by this engine.
+
+        Note, this feature will only be used if the `storage_redirect`
+        configuration key is set to `True`.
+        """
+        return None
+
     def stream_write(self, path, fp):
         raise NotImplementedError
 

lib/storage/boto_base.py

@@ -105,6 +105,7 @@ def __init__(self, config):
         self._boto_conn = self.makeConnection()
         self._boto_bucket = self._boto_conn.get_bucket(
             self._config.boto_bucket)
+        logger.info("Boto based storage initialized")
 
     def _build_connection_params(self):
         kwargs = {'is_secure': (self._config.boto_secure is True)}

lib/storage/s3.py

@@ -5,6 +5,7 @@
 import cStringIO as StringIO
 import logging
 
+import boto.s3
 import boto.s3.connection
 import boto.s3.key
 
@@ -29,6 +30,14 @@ def _build_connection_params(self):
 
     def makeConnection(self):
         kwargs = self._build_connection_params()
+        if self._config.s3_region is not None:
+            return boto.s3.connect_to_region(
+                region_name=self._config.s3_region,
+                aws_access_key_id=self._config.s3_access_key,
+                aws_secret_access_key=self._config.s3_secret_key,
+                **kwargs)
+        logger.warn("No S3 region specified, using boto default region, " +
+                    "this may affect performance and stability.")
         return boto.s3.connection.S3Connection(
             self._config.s3_access_key,
             self._config.s3_secret_key,
@@ -66,3 +75,13 @@ def stream_write(self, path, fp):
         except IOError:
             pass
         mp.complete_upload()
+
+    def content_redirect_url(self, path):
+        path = self._init_path(path)
+        key = self.makeKey(path)
+        if not key.exists():
+            raise IOError('No such key: \'{0}\''.format(path))
+        return key.generate_url(
+            expires_in=1200,
+            method='GET',
+            query_auth=True)

registry/images.py

@@ -71,6 +71,14 @@ def _get_image_layer(image_id, headers=None, bytes_range=None):
         else:
             logger.warn('nginx_x_accel_redirect config set,'
                         ' but storage is not LocalStorage')
+
+    # If store allows us to just redirect the client let's do that, we'll
+    # offload a lot of expensive I/O and get faster I/O
+    if cfg.storage_redirect:
+        content_redirect_url = store.content_redirect_url(path)
+        if content_redirect_url:
+            return flask.redirect(content_redirect_url, 302)
+
     status = None
     layer_size = 0
 

requirements.txt

@@ -1,4 +1,4 @@
-boto==2.19.0
+boto==2.27.0
 backports.lzma==0.0.2
 Flask==0.9
 PyYAML==3.10