Add ability to handle sse-c with add-header

This will suppress md5-sum checks for sse-c uploaded and downloaded
files. And use extra-headers also for get and info requests to download
sse-c enrypted files again.

Author: geraldhansen

S3/S3.py

@@ -706,10 +706,13 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
         response = self.send_file(request, src_stream, labels)
         return response
 
-    def object_get(self, uri, stream, dest_name, start_position = 0, extra_label = ""):
+    def object_get(self, uri, stream, dest_name, start_position = 0, extra_headers = None, extra_label = ""):
+        headers = SortedDict(ignore_case = True)
+        if extra_headers:
+            headers.update(extra_headers)
         if uri.type != "s3":
             raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
-        request = self.create_request("OBJECT_GET", uri = uri)
+        request = self.create_request("OBJECT_GET", uri = uri, headers = headers)
         labels = { 'source' : uri.uri(), 'destination' : dest_name, 'extra' : extra_label }
         response = self.recv_file(request, stream, labels, start_position)
         return response
@@ -924,8 +927,12 @@ def object_move(self, src_uri, dst_uri, extra_headers = None):
             debug("Object '%s' NOT deleted because of an unexepected response data content.", src_uri)
         return response_copy
 
-    def object_info(self, uri):
-        request = self.create_request("OBJECT_HEAD", uri = uri)
+    def object_info(self, uri, extra_headers = None):
+        headers = SortedDict(ignore_case = True)
+        if extra_headers:
+            headers.update(extra_headers)
+
+        request = self.create_request("OBJECT_HEAD", uri = uri, headers=headers)
         response = self.send_request(request)
         return response
 
@@ -1583,7 +1590,10 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0,
         debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\'')))
         ## when using KMS encryption, MD5 etag value will not match
         md5_from_s3 = response["headers"].get("etag", "").strip('"\'')
-        if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms':
+        if (('-' not in md5_from_s3) and
+                (md5_from_s3 != md5_hash.hexdigest()) and
+                response["headers"].get("x-amz-server-side-encryption") != 'aws:kms' and
+                response["headers"].get("x-amz-server-side​-encryption​-customer-key-md5") == ""):
             warning("MD5 Sums don't match!")
             if retries:
                 warning("Retrying upload of %s" % (filename))
@@ -1821,7 +1831,9 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_
                 start_position + int(response["headers"]["content-length"]), response["size"]))
         debug("ReceiveFile: Computed MD5 = %s" % response.get("md5"))
         # avoid ETags from multipart uploads that aren't the real md5
-        if ('-' not in md5_from_s3 and not response["md5match"]) and (response["headers"].get("x-amz-server-side-encryption") != 'aws:kms'):
+        if (('-' not in md5_from_s3 and not response["md5match"]) and
+                response["headers"].get("x-amz-server-side-encryption") != 'aws:kms' and
+                response["headers"].get("x-amz-server-side​-encryption​-customer-key-md5") == ""):
             warning("MD5 signatures do not match: computed=%s, received=%s" % (
                 response.get("md5"), md5_from_s3))
         return response

s3cmd

@@ -548,6 +548,8 @@ def cmd_object_get(args):
         warning(u"Exiting now because of --dry-run")
         return EX_OK
 
+    extra_headers = copy(cfg.extra_headers)
+
     seq = 0
     ret = EX_OK
     for key in remote_list:
@@ -599,7 +601,10 @@ def cmd_object_get(args):
                 continue
         try:
             try:
-                response = s3.object_get(uri, dst_stream, destination, start_position = start_position, extra_label = seq_label)
+                response = s3.object_get(uri, dst_stream, destination,
+                                         start_position=start_position,
+                                         extra_headers=extra_headers,
+                                         extra_label=seq_label)
             finally:
                 dst_stream.close()
         except S3DownloadError as e:
@@ -916,6 +921,7 @@ def cmd_mv(args):
 def cmd_info(args):
     cfg = Config()
     s3 = S3(cfg)
+    extra_headers = copy(cfg.extra_headers)
 
     while (len(args)):
         uri_arg = args.pop(0)
@@ -925,7 +931,7 @@ def cmd_info(args):
 
         try:
             if uri.has_object():
-                info = s3.object_info(uri)
+                info = s3.object_info(uri, extra_headers=extra_headers)
                 output(u"%s (object):" % uri.uri())
                 output(u"   File size: %s" % info['headers']['content-length'])
                 output(u"   Last mod:  %s" % info['headers']['last-modified'])