Factor out the S3 functions to cosalib

Since we use s3_check_exists and s3_copy at other places
we factored out those functions to cosalib

Author: gursewak1997

src/cmd-buildupload

@@ -10,8 +10,7 @@ import sys
 import tempfile
 import subprocess
 import boto3
-from botocore.exceptions import ClientError, NoCredentialsError
-from tenacity import retry
+from cosalib.s3 import s3_copy, s3_check_exists
 
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
@@ -22,13 +21,7 @@ CACHE_MAX_AGE_ARTIFACT = 60 * 60 * 24 * 365
 # set metadata caching to 5m
 CACHE_MAX_AGE_METADATA = 60 * 5
 from cosalib.builds import Builds, BUILDFILES
-from cosalib.cmdlib import (
-    load_json,
-    retry_stop_long,
-    retry_wait_long,
-    retry_boto_exception,
-    retry_callback
-)
+from cosalib.cmdlib import load_json
 
 
 def main():
@@ -194,57 +187,5 @@ def s3_upload_build(s3_client, args, builddir, bucket, prefix):
             dry_run=args.dry_run)
 
 
-@retry(stop=retry_stop_long, wait=retry_wait_long,
-       retry=retry_boto_exception, before_sleep=retry_callback)
-def s3_check_exists(s3_client, bucket, key, dry_run=False):
-    print(f"Checking if bucket '{bucket}' has key '{key}'")
-    try:
-        s3_client.head_object(Bucket=bucket, Key=key)
-    except ClientError as e:
-        if e.response['Error']['Code'] == '404':
-            return False
-        raise e
-    except NoCredentialsError as e:
-        # It's reasonable to run without creds if doing a dry-run
-        if dry_run:
-            return False
-        raise e
-    return True
-
-
-@retry(stop=retry_stop_long, wait=retry_wait_long,
-       retry=retry_boto_exception, retry_error_callback=retry_callback)
-def s3_copy(s3_client, src, bucket, key, max_age, acl, extra_args={}, dry_run=False):
-    extra_args = dict(extra_args)
-    if 'ContentType' not in extra_args:
-        if key.endswith('.json'):
-            extra_args['ContentType'] = 'application/json'
-        elif key.endswith('.tar'):
-            extra_args['ContentType'] = 'application/x-tar'
-        elif key.endswith('.xz'):
-            extra_args['ContentType'] = 'application/x-xz'
-        elif key.endswith('.gz'):
-            extra_args['ContentType'] = 'application/gzip'
-        elif key.endswith('.iso'):
-            extra_args['ContentType'] = 'application/x-iso9660-image'
-        else:
-            # use a standard MIME type for "binary blob" instead of the default
-            # 'binary/octet-stream' AWS slaps on
-            extra_args['ContentType'] = 'application/octet-stream'
-    upload_args = {
-        'CacheControl': f'max-age={max_age}',
-        'ACL': acl
-    }
-    upload_args.update(extra_args)
-
-    print((f"{'Would upload' if dry_run else 'Uploading'} {src} to "
-           f"s3://{bucket}/{key} with args {upload_args}"))
-
-    if dry_run:
-        return
-
-    s3_client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=upload_args)
-
-
 if __name__ == '__main__':
     sys.exit(main())

src/cosalib/s3.py

@@ -1,8 +1,10 @@
 import boto3
 
-from botocore.exceptions import ClientError
+from botocore.exceptions import ClientError, NoCredentialsError
 from cosalib.cmdlib import (
     retry_stop,
+    retry_stop_long,
+    retry_wait_long,
     retry_boto_exception,
     retry_callback
 )
@@ -52,3 +54,55 @@ def delete_object(self, bucket, key):
             print("S3: deleting {sub_objects}")
             self.client.delete_objects(Bucket=bucket, Delete=sub_objects)
         self.client.delete_object(Bucket=bucket, Key=key)
+
+
+@retry(stop=retry_stop_long, wait=retry_wait_long,
+       retry=retry_boto_exception, before_sleep=retry_callback)
+def s3_check_exists(s3_client, bucket, key, dry_run=False):
+    print(f"Checking if bucket '{bucket}' has key '{key}'")
+    try:
+        s3_client.head_object(Bucket=bucket, Key=key)
+    except ClientError as e:
+        if e.response['Error']['Code'] == '404':
+            return False
+        raise e
+    except NoCredentialsError as e:
+        # It's reasonable to run without creds if doing a dry-run
+        if dry_run:
+            return False
+        raise e
+    return True
+
+
+@retry(stop=retry_stop_long, wait=retry_wait_long,
+       retry=retry_boto_exception, retry_error_callback=retry_callback)
+def s3_copy(s3_client, src, bucket, key, max_age, acl, extra_args={}, dry_run=False):
+    extra_args = dict(extra_args)
+    if 'ContentType' not in extra_args:
+        if key.endswith('.json'):
+            extra_args['ContentType'] = 'application/json'
+        elif key.endswith('.tar'):
+            extra_args['ContentType'] = 'application/x-tar'
+        elif key.endswith('.xz'):
+            extra_args['ContentType'] = 'application/x-xz'
+        elif key.endswith('.gz'):
+            extra_args['ContentType'] = 'application/gzip'
+        elif key.endswith('.iso'):
+            extra_args['ContentType'] = 'application/x-iso9660-image'
+        else:
+            # use a standard MIME type for "binary blob" instead of the default
+            # 'binary/octet-stream' AWS slaps on
+            extra_args['ContentType'] = 'application/octet-stream'
+    upload_args = {
+        'CacheControl': f'max-age={max_age}',
+        'ACL': acl
+    }
+    upload_args.update(extra_args)
+
+    print((f"{'Would upload' if dry_run else 'Uploading'} {src} to "
+           f"s3://{bucket}/{key} {extra_args if len(extra_args) else ''}"))
+
+    if dry_run:
+        return
+
+    s3_client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=upload_args)