From 2d0c64bf270fc7027c90e64fdd772f349008349a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Al=C3=A9p=C3=A9e?= Date: Mon, 20 Jan 2025 12:00:37 +0100 Subject: [PATCH 1/2] add force checksum --- cosmotech/coal/cli/commands/s3_bucket_delete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosmotech/coal/cli/commands/s3_bucket_delete.py b/cosmotech/coal/cli/commands/s3_bucket_delete.py index c5b50905..9c36e5da 100644 --- a/cosmotech/coal/cli/commands/s3_bucket_delete.py +++ b/cosmotech/coal/cli/commands/s3_bucket_delete.py @@ -105,4 +105,4 @@ def s3_bucket_delete( if boto_objects: LOGGER.info(f'Deleting {boto_objects}') boto_delete_request = {'Objects': boto_objects} - bucket.delete_objects(Delete=boto_delete_request) + bucket.delete_objects(Delete=boto_delete_request, ChecksumAlgorithm='SHA256') From 5b59a734b95ea73dc06025e9176de174f710e125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Al=C3=A9p=C3=A9e?= Date: Tue, 21 Jan 2025 10:18:14 +0100 Subject: [PATCH 2/2] fix boto3 version to 1.35. use resource interface Boto3 version 1.36.x add a validation on content-MD5 that is not properly managed. --- cosmotech/coal/cli/commands/s3_bucket_delete.py | 4 +++- cosmotech/coal/cli/commands/s3_bucket_upload.py | 6 +++--- requirements.txt | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cosmotech/coal/cli/commands/s3_bucket_delete.py b/cosmotech/coal/cli/commands/s3_bucket_delete.py index 9c36e5da..1483252c 100644 --- a/cosmotech/coal/cli/commands/s3_bucket_delete.py +++ b/cosmotech/coal/cli/commands/s3_bucket_delete.py @@ -105,4 +105,6 @@ def s3_bucket_delete( if boto_objects: LOGGER.info(f'Deleting {boto_objects}') boto_delete_request = {'Objects': boto_objects} - bucket.delete_objects(Delete=boto_delete_request, ChecksumAlgorithm='SHA256') + bucket.delete_objects(Delete=boto_delete_request) + else: + LOGGER.info('No objects to delete') diff --git a/cosmotech/coal/cli/commands/s3_bucket_upload.py b/cosmotech/coal/cli/commands/s3_bucket_upload.py index 8ff7fadb..f6811e17 100644 --- a/cosmotech/coal/cli/commands/s3_bucket_upload.py +++ b/cosmotech/coal/cli/commands/s3_bucket_upload.py @@ -98,7 +98,7 @@ def s3_bucket_upload( Make use of the boto3 library to access the bucket -More information is available on this page: +More information is available on this page: [https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html) """ source_path = pathlib.Path(source_folder) @@ -115,12 +115,12 @@ def s3_bucket_upload( if ssl_cert_bundle: boto3_parameters["verify"] = ssl_cert_bundle - s3_client = boto3.client("s3", **boto3_parameters) + s3_resource = boto3.resource("s3", **boto3_parameters) def file_upload(file_path: pathlib.Path, file_name: str): uploaded_file_name = file_prefix + file_name LOGGER.info(f"Sending {file_path} as {uploaded_file_name}") - s3_client.upload_file(file_path, bucket_name, uploaded_file_name) + s3_resource.Bucket(bucket_name).upload_file(file_path, uploaded_file_name) if source_path.is_dir(): _source_name = str(source_path) diff --git a/requirements.txt b/requirements.txt index cba267aa..72bc43ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ redisgraph_bulk_loader==0.10.2 cosmotech-api~=3.2 # Commands requirements -boto3~=1.34 +boto3~=1.35.19 requests~=2.32.3 singlestoredb~=1.10.0