Add ContentType detection and Tests for upload ExtraArgs (#35)

* enable server-side bucket to bucket copies

The 'copy_object' method permits setting different source and target buckets. When different, a server-side copy takes place between the buckets.

Appropriate ACLs must exist in AWS, but this is a quick method for copying keys between buckets within the same account and region.

Copying across regions and/or accounts is more complex, and is not enabled with this change.

* Allow settable extra_args for uploads and downloads

Allow setting upload and download `extra_args` values for the lifetime of the filesystem instance.

Ticket #32 has details

* Moved args to constructor and removed bucket copy

* load unquoted json if value is a string

* Expose `acl` and `cache-control` directly and via opener

* Small cleanup

* (bugfix) Only set if value exists

* Boto3 changed from `extra_args` to `ExtraArgs` at some point

This has been tested and works.

I uploaded a file to a private bucket, setting the max-age and ACL.

* Add ContentType Detection with fallback

The boto3 `upload_fileobj` never guesses the ContentType (unlike other Boto3 methods). So all files are stored as `binary/octet-stream`.

This can problematic when uploading images or audio, and expecting it to load inline.

`guess_type` is a lightweight guesser, that only looks at filenames. If this implementation fails to identify a type, it falls back to the defaults `binary/octet-stream`

* moved ContentType to ExtraArgs

The `client.upload_fileobj` method has `ContentType` in ExtraArgs.

* moved _upload_args to method to inject ContentType

* Add tests for upload ExtraArgs

* fix typo

* Fix TypeError if ContentType manually set

* fixes #38

Author: geoffjukes

fs_s3fs/_s3fs.py

@@ -12,6 +12,7 @@
 from ssl import SSLError
 import tempfile
 import threading
+import mimetypes
 import json
 
 import boto3
@@ -353,6 +354,13 @@ def _get_object(self, path, key):
         else:
             return obj
 
+    def _upload_args(self, key):
+        if 'ContentType' not in self.upload_args:
+            mimetype = mimetypes.guess_type(key)[0] or 'binary/octet-stream'
+            return dict(ContentType=mimetype, **self.upload_args)
+        else:
+            return self.upload_args
+
     @property
     def s3(self):
         if not hasattr(self._tlocal, 's3'):
@@ -535,7 +543,7 @@ def makedir(self, path, permissions=None, recreate=False):
             else:
                 raise errors.DirectoryExists(path)
         with s3errors(path):
-            self.s3.Object(self._bucket_name, _key).put()
+            self.s3.Object(self._bucket_name, _key).put(**self._upload_args(_key))
         return SubFS(self, path)
 
     def openbin(self, path, mode="r", buffering=-1, **options):
@@ -553,7 +561,7 @@ def on_close_create(s3file):
                     s3file.raw.seek(0)
                     with s3errors(path):
                         self.client.upload_fileobj(
-                            s3file.raw, self._bucket_name, _key, ExtraArgs=self.upload_args
+                            s3file.raw, self._bucket_name, _key, ExtraArgs=self._upload_args(_key)
                         )
                 finally:
                     s3file.raw.close()
@@ -602,7 +610,7 @@ def on_close(s3file):
                     s3file.raw.seek(0, os.SEEK_SET)
                     with s3errors(path):
                         self.client.upload_fileobj(
-                            s3file.raw, self._bucket_name, _key, ExtraArgs=self.upload_args
+                            s3file.raw, self._bucket_name, _key, ExtraArgs=self._upload_args(_key)
                         )
             finally:
                 s3file.raw.close()
@@ -771,7 +779,7 @@ def setbytes(self, path, contents):
         bytes_file = io.BytesIO(contents)
         with s3errors(path):
             self.client.upload_fileobj(
-                bytes_file, self._bucket_name, _key, ExtraArgs=self.upload_args
+                bytes_file, self._bucket_name, _key, ExtraArgs=self._upload_args(_key)
             )
 
     def setbinfile(self, path, file):
@@ -789,7 +797,7 @@ def setbinfile(self, path, file):
                 pass
 
         with s3errors(path):
-            self.client.upload_fileobj(file, self._bucket_name, _key, ExtraArgs=self.upload_args)
+            self.client.upload_fileobj(file, self._bucket_name, _key, self._upload_args(_key))
 
     def copy(self, src_path, dst_path, overwrite=False):
         if not overwrite and self.exists(dst_path):

fs_s3fs/tests/test_s3fs.py

@@ -69,3 +69,13 @@ def test_path_to_key_subdir(self):
         self.assertEqual(s3._path_to_key('foo.bar'), 'dir/foo.bar')
         self.assertEqual(s3._path_to_key('foo/bar'), 'dir/foo/bar')
 
+    def test_upload_args(self):
+        s3 = S3FS('foo', acl='acl', cache_control='cc')
+        self.assertDictEqual(s3._upload_args('test.jpg'),
+                             {'ACL': 'acl', 'CacheControl': 'cc', 'ContentType': 'image/jpeg'})
+        self.assertDictEqual(s3._upload_args('test.mp3'),
+                             {'ACL': 'acl', 'CacheControl': 'cc', 'ContentType': 'audio/mpeg'})
+        self.assertDictEqual(s3._upload_args('test.json'),
+                             {'ACL': 'acl', 'CacheControl': 'cc', 'ContentType': 'application/json'})
+        self.assertDictEqual(s3._upload_args('unknown.ext'),
+                             {'ACL': 'acl', 'CacheControl': 'cc', 'ContentType': 'binary/octet-stream'})