Add s3 upload storage method

Signed-off-by: Mike Perez <thingee@gmail.com>

Author: Thingee

chacra/controllers/binaries/archs.py

@@ -1,5 +1,7 @@
 import logging
 import os
+import boto3
+from botocore.exceptions import ClientError
 import pecan
 from pecan import response
 from pecan.secure import secure
@@ -89,7 +91,7 @@ def index_post(self):
                 if request.POST.get('force', False) is False:
                     error('/errors/invalid', 'resource already exists and "force" key was not used')
 
-        full_path = self.save_file(file_obj)
+        full_path, size = self.save_file(file_obj)
 
         if self.binary is None:
             path = full_path
@@ -102,14 +104,19 @@ def index_post(self):
             self.binary = Binary(
                 self.binary_name, self.project, arch=arch,
                 distro=distro, distro_version=distro_version,
-                ref=ref, sha1=sha1, path=path, size=os.path.getsize(path)
+                ref=ref, sha1=sha1, path=path, size=size
             )
         else:
             self.binary.path = full_path
 
         # check if this binary is interesting for other configured projects,
         # and if so, then mark those other repos so that they can be re-built
         self.mark_related_repos()
+
+        # Remove the local file after S3 upload
+        if pecan.conf.storage_method == 's3':
+            os.remove(full_path)
+
         return dict()
 
     def mark_related_repos(self):
@@ -175,8 +182,21 @@ def save_file(self, file_obj):
             for chunk in file_iterable:
                 f.write(chunk)
 
+        if pecan.conf.storage_method == 's3':
+            bucket = pecan.conf.bucket
+            object_name = os.path.basename(self.binary_name)
+
+            s3_client = boto3.client('s3')
+            try:
+                with open(destination, 'rb') as f:
+                    s3_client.upload_fileobj(f, bucket, object_name)
+            except ClientError as e:
+                error('/errors/error/', 'file object upload to S3 failed with error %s' % e)
+
+        size = os.path.getsize(destination)
+
         # return the full path to the saved object:
-        return destination
+        return destination, size
 
     @expose()
     def _lookup(self, name, *remainder):

config/dev.py

@@ -63,11 +63,17 @@
     'encoding':      'utf-8'
 }
 
+# Where to store the data. Options are 's3' or 'local'
+storage_method = 'local'
+
 # location for storing uploaded binaries
 binary_root = '%(confdir)s/public'
 repos_root = '%(confdir)s/repos'
 distributions_root = '%(confdir)s/distributions'
 
+# If storage method is s3, provide a bucket name
+bucket = ''
+
 # When True it will set the headers so that Nginx can serve the download
 # instead of Pecan.
 delegate_downloads = False

requirements.txt

@@ -8,4 +8,5 @@ alembic
 ipython
 python-statsd
 requests
+boto3
 importlib_metadata<=3.6; python_version<'3.8'