Skip to content

Add config to avoid checking MD5 on get and put #928

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gaul wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add config to avoid checking MD5 on get and put #928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions S3/Config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ class Config(object):
# s3 will timeout if a request/transfer is stuck for more than a short time
throttle_max = 100
public_url_use_https = False
put_check_etag = True

## Creating a singleton
def __new__(self, configfile = None, access_key=None, secret_key=None, access_token=None):
Expand Down
4 changes: 2 additions & 2 deletions S3/S3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1583,7 +1583,7 @@ def send_file(self, request, stream, labels, buffer = '', throttle = 0,
debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"].get('etag', '').strip('"\'')))
## when using KMS encryption, MD5 etag value will not match
md5_from_s3 = response["headers"].get("etag", "").strip('"\'')
if ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms':
if self.config.put_check_etag and ('-' not in md5_from_s3) and (md5_from_s3 != md5_hash.hexdigest()) and response["headers"].get("x-amz-server-side-encryption") != 'aws:kms':
warning("MD5 Sums don't match!")
if retries:
warning("Retrying upload of %s" % (filename))
Expand Down Expand Up @@ -1821,7 +1821,7 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_
start_position + int(response["headers"]["content-length"]), response["size"]))
debug("ReceiveFile: Computed MD5 = %s" % response.get("md5"))
# avoid ETags from multipart uploads that aren't the real md5
if ('-' not in md5_from_s3 and not response["md5match"]) and (response["headers"].get("x-amz-server-side-encryption") != 'aws:kms'):
if self.config.put_check_etag and ('-' not in md5_from_s3 and not response["md5match"]) and (response["headers"].get("x-amz-server-side-encryption") != 'aws:kms'):
warning("MD5 signatures do not match: computed=%s, received=%s" % (
response.get("md5"), md5_from_s3))
return response
Expand Down
2 changes: 2 additions & 0 deletions s3cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2856,12 +2856,14 @@ def main():

## Process --(no-)check-md5
if options.check_md5 == False:
cfg.put_check_etag = False
try:
cfg.sync_checks.remove("md5")
cfg.preserve_attrs_list.remove("md5")
except Exception:
pass
if options.check_md5 == True:
cfg.put_check_etag = True
if cfg.sync_checks.count("md5") == 0:
cfg.sync_checks.append("md5")
if cfg.preserve_attrs_list.count("md5") == 0:
Expand Down