From 74f1a4cf75e70abad4536397526353b531cb8a9e Mon Sep 17 00:00:00 2001 From: jorgee Date: Thu, 31 Jul 2025 20:56:30 +0200 Subject: [PATCH] add s3 checksum algorithm Signed-off-by: jorgee --- docs/reference/config.md | 4 +++ .../config/scopes/AwsClientConfig.java | 6 ++++ .../cloud/aws/batch/AwsBatchExecutor.groovy | 6 +++- .../cloud/aws/batch/AwsOptions.groovy | 4 +++ .../cloud/aws/config/AwsS3Config.groovy | 18 +++++++++++ .../main/nextflow/cloud/aws/nio/S3Client.java | 27 ++++++++++++++++ .../cloud/aws/nio/S3FileSystemProvider.java | 2 ++ .../cloud/aws/nio/S3OutputStream.java | 16 ++++++++++ .../nextflow/cloud/aws/util/S3BashLib.groovy | 13 ++++++-- .../batch/AwsBatchFileCopyStrategyTest.groovy | 8 +++-- .../cloud/aws/batch/AwsOptionsTest.groovy | 32 +++++++++++++++---- .../cloud/aws/config/AwsS3ConfigTest.groovy | 3 ++ .../aws/nio/S3FileSystemProviderTest.groovy | 4 ++- .../cloud/aws/util/S3BashLibTest.groovy | 16 ++++++---- .../executor/AwsBatchExecutorTest.groovy | 3 +- 15 files changed, 140 insertions(+), 22 deletions(-) diff --git a/docs/reference/config.md b/docs/reference/config.md index e8857d9fac..97939e74f5 100644 --- a/docs/reference/config.md +++ b/docs/reference/config.md @@ -101,6 +101,7 @@ aws { connectionTimeout = 10000 uploadStorageClass = 'INTELLIGENT_TIERING' storageEncryption = 'AES256' + checksumAlgorithm = 'SHA256' } batch { cliPath = '/home/ec2-user/miniconda/bin/aws' @@ -194,6 +195,9 @@ The following settings are available: `aws.client.anonymous` : Allow the access of public S3 buckets without the need to provide AWS credentials (default: `false`). Any service that does not accept unsigned requests will return a service access error. +`aws.client.checksumAlgorithm` +: The S3 checksum algorithm to be used when saving objects on S3. Can be one of `CRC32`, `CRC32C`, `SHA1`, `SHA256` or `CRC64NVME`. + `aws.client.s3Acl` : Allow the setting of predefined bucket permissions, also known as *canned ACL*. Permitted values are `Private`, `PublicRead`, `PublicReadWrite`, `AuthenticatedRead`, `LogDeliveryWrite`, `BucketOwnerRead`, `BucketOwnerFullControl`, and `AwsExecRead` (default: none). See [Amazon docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl) for details. diff --git a/modules/nf-lang/src/main/java/nextflow/config/scopes/AwsClientConfig.java b/modules/nf-lang/src/main/java/nextflow/config/scopes/AwsClientConfig.java index 7549901033..b0e0ebb0c8 100644 --- a/modules/nf-lang/src/main/java/nextflow/config/scopes/AwsClientConfig.java +++ b/modules/nf-lang/src/main/java/nextflow/config/scopes/AwsClientConfig.java @@ -29,6 +29,12 @@ public class AwsClientConfig implements ConfigScope { """) public boolean anonymous; + @ConfigOption + @Description(""" + The S3 checksum algorithm to be used when saving objects on S3. Can be one of `CRC32`, `CRC32C`, `SHA1`, `SHA256` or `CRC64NVME`. + """) + public String checksumAlgorithm; + @ConfigOption @Description(""" Specify predefined bucket permissions, also known as *canned ACL*. Can be one of `Private`, `PublicRead`, `PublicReadWrite`, `AuthenticatedRead`, `LogDeliveryWrite`, `BucketOwnerRead`, `BucketOwnerFullControl`, or `AwsExecRead`. diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy index 8d1e86a5fc..435bbd0f17 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy @@ -370,13 +370,17 @@ class AwsBatchExecutor extends Executor implements ExtensionPoint, TaskArrayExec final debug = opts.debug ? ' --debug' : '' final sse = opts.storageEncryption ? " --sse $opts.storageEncryption" : '' final kms = opts.storageKmsKeyId ? " --sse-kms-key-id $opts.storageKmsKeyId" : '' + final checksum = opts.checksumAlgorithm ? " --checksum-algorithm $opts.checksumAlgorithm" : '' final requesterPays = opts.requesterPays ? ' --request-payer requester' : '' - final aws = "$cli s3 cp --only-show-errors${sse}${kms}${debug}${requesterPays}" + final aws = "$cli s3 cp --only-show-errors${sse}${kms}${checksum}${debug}${requesterPays}" final cmd = "trap \"{ ret=\$?; $aws ${TaskRun.CMD_LOG} ${workDir}/${TaskRun.CMD_LOG}||true; exit \$ret; }\" EXIT; $aws ${workDir}/${TaskRun.CMD_RUN} - | bash 2>&1 | tee ${TaskRun.CMD_LOG}" return cmd } static String s5Cmd(String workDir, AwsOptions opts) { + if( opts.checksumAlgorithm ){ + log.warn1("Checksum Algorithm is not supported by `s5cmd` command. This option will be ignored in command line operations.") + } final cli = opts.getS5cmdPath() final sse = opts.storageEncryption ? " --sse $opts.storageEncryption" : '' final kms = opts.storageKmsKeyId ? " --sse-kms-key-id $opts.storageKmsKeyId" : '' diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy index 86ea44db9d..80487ec282 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy @@ -116,6 +116,10 @@ class AwsOptions implements CloudTransferOptions { return awsConfig.s3Config.getStorageClass() } + String getChecksumAlgorithm(){ + return awsConfig.s3Config.getChecksumAlgorithm() + } + String getStorageEncryption() { return awsConfig.s3Config.getStorageEncryption() } diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy index 658a533f60..6583e15b99 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy @@ -17,6 +17,8 @@ package nextflow.cloud.aws.config +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm + import static nextflow.cloud.aws.util.AwsHelper.* import software.amazon.awssdk.services.s3.model.ObjectCannedACL @@ -41,6 +43,8 @@ class AwsS3Config { private String storageKmsKeyId + private String checksumAlgorithm + private Boolean debug private ObjectCannedACL s3Acl @@ -59,6 +63,7 @@ class AwsS3Config { this.storageClass = parseStorageClass((opts.storageClass ?: opts.uploadStorageClass) as String) // 'uploadStorageClass' is kept for legacy purposes this.storageEncryption = parseStorageEncryption(opts.storageEncryption as String) this.storageKmsKeyId = opts.storageKmsKeyId + this.checksumAlgorithm = parseChecksumAlgorithm(opts.checksumAlgorithm as String) this.pathStyleAccess = opts.s3PathStyleAccess as Boolean this.anonymous = opts.anonymous as Boolean this.s3Acl = parseS3Acl(opts.s3Acl as String) @@ -85,6 +90,15 @@ class AwsS3Config { return null } + private String parseChecksumAlgorithm(String value) { + if (value) { + if( value in ChecksumAlgorithm.knownValues()*.toString() ) + return value + log.warn "Unsupported AWS checksum algorithm: $value" + } + return null + } + // ==== getters ===== String getEndpoint() { return endpoint @@ -102,6 +116,10 @@ class AwsS3Config { return storageKmsKeyId } + String getChecksumAlgorithm() { + return checksumAlgorithm + } + Boolean getDebug() { return debug } diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Client.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Client.java index 247b7a71f5..abeb54d783 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Client.java +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Client.java @@ -76,6 +76,8 @@ public class S3Client { private boolean global; + private ChecksumAlgorithm checksumAlgorithm; + public S3Client(AwsClientFactory factory, Properties props, boolean global) { S3SyncClientConfiguration clientConfig = S3SyncClientConfiguration.create(props); this.factory = factory; @@ -155,6 +157,9 @@ private PutObjectRequest preparePutObjectRequest(PutObjectRequest.Builder reqBui if( storageEncryption!=null ) { reqBuilder.serverSideEncryption(storageEncryption); } + if( checksumAlgorithm != null ) { + reqBuilder.checksumAlgorithm(checksumAlgorithm); + } if( contentType!=null ) { reqBuilder.contentType(contentType); } @@ -183,6 +188,9 @@ public PutObjectResponse putObject(String bucket, String keyName, InputStream in if( storageEncryption!=null ) { reqBuilder.serverSideEncryption(storageEncryption); } + if( checksumAlgorithm != null ) { + reqBuilder.checksumAlgorithm(checksumAlgorithm); + } if( contentType!=null ) { reqBuilder.contentType(contentType); } @@ -217,6 +225,9 @@ public void copyObject(CopyObjectRequest.Builder reqBuilder, List tags, Str if( kmsKeyId !=null ) { reqBuilder.ssekmsKeyId(kmsKeyId); } + if( checksumAlgorithm != null ) { + reqBuilder.checksumAlgorithm(checksumAlgorithm); + } if( contentType!=null ) { reqBuilder.metadataDirective(MetadataDirective.REPLACE); reqBuilder.contentType(contentType); @@ -281,6 +292,13 @@ public void setTransferManagerThreads(String value) { } } + public void setChecksumAlgorithm(String alg){ + if( alg == null ) + return; + this.checksumAlgorithm = ChecksumAlgorithm.fromValue(alg); + log.debug("Setting S3 ChecksumAlgorithm={}", alg); + } + public ObjectCannedACL getCannedAcl() { return cannedAcl; } @@ -340,6 +358,9 @@ public void multipartCopyObject(S3Path s3Source, S3Path s3Target, Long objectSiz if( kmsKeyId != null ) { reqBuilder.ssekmsKeyId(kmsKeyId); } + if( checksumAlgorithm != null ) { + reqBuilder.checksumAlgorithm(checksumAlgorithm); + } if( tags != null && tags.size()>0 ) { reqBuilder.tagging( Tagging.builder().tagSet(tags).build() ); @@ -548,6 +569,9 @@ private PutObjectRequest.Builder updateBuilder(PutObjectRequest.Builder porBuild porBuilder.serverSideEncryption(storageEncryption); if( kmsKeyId != null ) porBuilder.ssekmsKeyId(kmsKeyId); + if( checksumAlgorithm != null ) { + porBuilder.checksumAlgorithm(checksumAlgorithm); + } if( tags != null && !tags.isEmpty() ) porBuilder.tagging(Tagging.builder().tagSet(tags).build()); return porBuilder; @@ -593,6 +617,9 @@ public void copyFile(CopyObjectRequest.Builder reqBuilder, List tags, Strin if( kmsKeyId !=null ) { reqBuilder.ssekmsKeyId(kmsKeyId); } + if( checksumAlgorithm != null ) { + reqBuilder.checksumAlgorithm(checksumAlgorithm); + } if( contentType!=null ) { reqBuilder.metadataDirective(MetadataDirective.REPLACE); reqBuilder.contentType(contentType); diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java index 4e04fc2b2e..ff7c032573 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java @@ -340,6 +340,7 @@ private S3OutputStream createUploaderOutputStream( S3Path fileToUpload ) { .setCannedAcl(s3.getCannedAcl()) .setStorageClass(storageClass) .setStorageEncryption(props.getProperty("storage_encryption")) + .setChecksumAlgorithm(props.getProperty("checksum_algorithm")) .setKmsKeyId(props.getProperty("storage_kms_key_id")) .setContentType(fileToUpload.getContentType()) .setTags(fileToUpload.getTagsList()); @@ -742,6 +743,7 @@ protected S3FileSystem createFileSystem(URI uri, AwsConfig awsConfig) { // set the client acl client.setCannedAcl(getProp(props, "s_3_acl", "s3_acl", "s3acl", "s3Acl")); client.setStorageEncryption(props.getProperty("storage_encryption")); + client.setChecksumAlgorithm(props.getProperty("checksum_algorithm")); client.setKmsKeyId(props.getProperty("storage_kms_key_id")); client.setTransferManagerThreads(props.getProperty("transfer_manager_threads")); client.setRequesterPaysEnabled(props.getProperty("requester_pays")); diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3OutputStream.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3OutputStream.java index 9caf4bbb48..8a7c4bb9a2 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3OutputStream.java +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3OutputStream.java @@ -87,6 +87,8 @@ public final class S3OutputStream extends OutputStream { private String contentType; + private ChecksumAlgorithm checksumAlgorithm; + /** * Indicates if the stream has been closed. */ @@ -201,6 +203,12 @@ public S3OutputStream setKmsKeyId(String kmsKeyId) { return this; } + public S3OutputStream setChecksumAlgorithm(String checksumAlgorithm){ + if( checksumAlgorithm !=null ) + this.checksumAlgorithm = ChecksumAlgorithm.fromValue(checksumAlgorithm); + return this; + } + public S3OutputStream setContentType(String type) { this.contentType = type; return this; @@ -428,6 +436,10 @@ private CreateMultipartUploadResponse initiateMultipartUpload() throws IOExcepti reqBuilder.serverSideEncryption(storageEncryption); } + if( checksumAlgorithm != null ) { + reqBuilder.checksumAlgorithm(checksumAlgorithm); + } + if( contentType != null ) { reqBuilder.contentType(contentType); } @@ -614,6 +626,10 @@ private void putObject(final InputStream content, final long contentLength, byte reqBuilder.serverSideEncryption( storageEncryption ); } + if( checksumAlgorithm != null ) { + reqBuilder.checksumAlgorithm(checksumAlgorithm); + } + if( contentType != null ) { reqBuilder.contentType(contentType); } diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy index 3f5c569ee3..58f81c4bac 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy @@ -38,6 +38,7 @@ class S3BashLib extends BashFunLib { private String s5cmdPath private String acl = '' private String requesterPays = '' + private String checksumAlgorithm = '' S3BashLib withCliPath(String cliPath) { if( cliPath ) @@ -61,6 +62,11 @@ class S3BashLib extends BashFunLib { this.storageClass = value return this } + S3BashLib withChecksumAlgorithm(String value) { + if( value ) + this.checksumAlgorithm = value ? "--checksum-algorithm $value " : '' + return this + } S3BashLib withStorageEncryption(String value) { if( value ) @@ -112,11 +118,11 @@ class S3BashLib extends BashFunLib { local name=\$1 local s3path=\$2 if [[ "\$name" == - ]]; then - $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass - "\$s3path" + $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass - "\$s3path" elif [[ -d "\$name" ]]; then - $cli s3 cp --only-show-errors --recursive ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name" + $cli s3 cp --only-show-errors --recursive ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name" else - $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name" + $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name" fi } @@ -187,6 +193,7 @@ class S3BashLib extends BashFunLib { .withMaxTransferAttempts( opts.maxTransferAttempts ) .withCliPath( opts.awsCli ) .withStorageClass(opts.storageClass ) + .withChecksumAlgorithm( opts.checksumAlgorithm ) .withStorageEncryption( opts.storageEncryption ) .withStorageKmsKeyId( opts.storageKmsKeyId ) .withRetryMode( opts.retryMode ) diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy index 0e54b42fb8..8f907d3248 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy @@ -126,6 +126,7 @@ class AwsBatchFileCopyStrategyTest extends Specification { 1 * opts.getAwsCli() >> 'aws' 1 * opts.getStorageClass() >> null 1 * opts.getStorageEncryption() >> null + 1 * opts.getChecksumAlgorithm() >> null script == '''\ # bash helper functions @@ -217,6 +218,7 @@ class AwsBatchFileCopyStrategyTest extends Specification { 1 * opts.getAwsCli() >> '/foo/aws' 1 * opts.getStorageClass() >> 'STANDARD_IA' 1 * opts.getStorageEncryption() >> 'AES256' + 1 * opts.getChecksumAlgorithm() >> 'SHA256' script == '''\ # bash helper functions @@ -281,11 +283,11 @@ class AwsBatchFileCopyStrategyTest extends Specification { local name=$1 local s3path=$2 if [[ "$name" == - ]]; then - /foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA - "$s3path" + /foo/aws s3 cp --only-show-errors --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA - "$s3path" elif [[ -d "$name" ]]; then - /foo/aws s3 cp --only-show-errors --recursive --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name" + /foo/aws s3 cp --only-show-errors --recursive --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA "$name" "$s3path/$name" else - /foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name" + /foo/aws s3 cp --only-show-errors --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA "$name" "$s3path/$name" fi } diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsOptionsTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsOptionsTest.groovy index 06b0e645aa..ef24172483 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsOptionsTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsOptionsTest.groovy @@ -85,7 +85,8 @@ class AwsOptionsTest extends Specification { volumes: '/foo,/this:/that'], client: [ uploadStorageClass: 'STANDARD', - storageEncryption: 'AES256'], + storageEncryption: 'AES256', + checksumAlgorithm: 'SHA256'], region: 'aws-west-2' ] ] @@ -103,6 +104,7 @@ class AwsOptionsTest extends Specification { opts.delayBetweenAttempts.seconds == 9 opts.storageClass == 'STANDARD' opts.storageEncryption == 'AES256' + opts.checksumAlgorithm == 'SHA256' opts.region == 'aws-west-2' opts.jobRole == 'aws::foo::bar' opts.volumes == ['/foo','/this:/that'] @@ -137,6 +139,16 @@ class AwsOptionsTest extends Specification { } + def 'should set aws checksum algorithm' () { + when: + def sess1 = Mock(Session) { + getConfig() >> [aws: [client: [checksumAlgorithm: 'SHA256']]] + } + and: + def opts = new AwsOptions(sess1) + then: + opts.checksumAlgorithm == 'SHA256' + } @Unroll @@ -145,7 +157,8 @@ class AwsOptionsTest extends Specification { def cfg = [ aws: [client: [ uploadStorageClass: awsStorClass, - storageEncryption : awsStorEncrypt], + storageEncryption: awsStorEncrypt, + checksumAlgorithm: awsChecksum ], batch: [ cliPath: awscliPath ]] ] def session = new Session(cfg) @@ -156,11 +169,12 @@ class AwsOptionsTest extends Specification { opts.cliPath == awscliPath opts.storageClass == awsStorClass opts.storageEncryption == awsStorEncrypt + opts.checksumAlgorithm == awsChecksum where: - awscliPath | awsStorClass | awsStorEncrypt - null | null | null - '/foo/bin/aws' | 'STANDARD' | 'AES256' + awscliPath | awsStorClass | awsStorEncrypt | awsChecksum + null | null | null | null + '/foo/bin/aws' | 'STANDARD' | 'AES256' | 'SHA256' } @@ -174,11 +188,12 @@ class AwsOptionsTest extends Specification { opts.getStorageEncryption() == null when: - opts = new AwsOptions(awsConfig: new AwsConfig(batch: [cliPath: '/foo/bin/aws'], client: [storageClass: 'STANDARD', storageEncryption: 'AES256'])) + opts = new AwsOptions(awsConfig: new AwsConfig(batch: [cliPath: '/foo/bin/aws'], client: [storageClass: 'STANDARD', storageEncryption: 'AES256', checksumAlgorithm: 'SHA256'])) then: opts.getCliPath() == '/foo/bin/aws' opts.getStorageClass() == 'STANDARD' opts.getStorageEncryption() == 'AES256' + opts.getChecksumAlgorithm() == 'SHA256' when: opts = new AwsOptions(awsConfig: new AwsConfig(client:[storageClass: 'foo'])) @@ -190,6 +205,11 @@ class AwsOptionsTest extends Specification { then: opts.getStorageEncryption() == null + when: + opts = new AwsOptions(awsConfig: new AwsConfig(client:[checksumAlgorithm: 'foo'])) + then: + opts.getChecksumAlgorithm() == null + when: opts = new AwsOptions(awsConfig: new AwsConfig(client:[storageKmsKeyId: 'arn:aws:kms:eu-west-1:1234567890:key/e97ecf28-951e-4700-bf22-1bd416ec519f'])) then: diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsS3ConfigTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsS3ConfigTest.groovy index 9211bc4a4b..2ea60b9123 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsS3ConfigTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsS3ConfigTest.groovy @@ -35,6 +35,7 @@ class AwsS3ConfigTest extends Specification { !client.storageClass !client.storageKmsKeyId !client.storageEncryption + !client.checksumAlgorithm !client.debug !client.s3Acl !client.pathStyleAccess @@ -49,6 +50,7 @@ class AwsS3ConfigTest extends Specification { storageClass: 'STANDARD', storageKmsKeyId: 'key-1', storageEncryption: 'AES256', + checksumAlgorithm: 'SHA256', s3Acl: 'public-read', s3PathStyleAccess: true, anonymous: true @@ -61,6 +63,7 @@ class AwsS3ConfigTest extends Specification { client.storageClass == 'STANDARD' client.storageKmsKeyId == 'key-1' client.storageEncryption == 'AES256' + client.checksumAlgorithm == 'SHA256' client.s3Acl == ObjectCannedACL.PUBLIC_READ client.pathStyleAccess client.anonymous diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/nio/S3FileSystemProviderTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/nio/S3FileSystemProviderTest.groovy index 5b0cfa842c..a9d145825d 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/nio/S3FileSystemProviderTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/nio/S3FileSystemProviderTest.groovy @@ -17,6 +17,7 @@ package nextflow.cloud.aws.nio +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm import software.amazon.awssdk.services.s3.model.ObjectCannedACL import software.amazon.awssdk.services.s3.model.ServerSideEncryption import spock.lang.Specification @@ -33,7 +34,7 @@ class S3FileSystemProviderTest extends Specification { maxConcurrency: 10, maxNativeMemory: '500MB', minimumPartSize: '7MB', multipartThreshold: '32MB', maxConnections: 100, maxErrorRetry: 3, socketTimeout: 20000, requesterPays: true, s3PathStyleAccess: true, proxyHost: 'host.com', proxyPort: 80, proxyScheme: 'https', proxyUsername: 'user', proxyPassword: 'pass', - signerOverride: 'S3SignerType', userAgent: 'Agent1', storageEncryption: 'AES256', storageKmsKeyId: 'arn:key:id', + signerOverride: 'S3SignerType', userAgent: 'Agent1', storageEncryption: 'AES256', storageKmsKeyId: 'arn:key:id', checksumAlgorithm: 'SHA256', transferManagerThreads: 20, uploadMaxThreads: 15, uploadChunkSize: '7MB', uploadMaxAttempts: 4, uploadRetrySleep: '200ms' ], accessKey: '123456abc', secretKey: '78910def', profile: 'test'] @@ -47,6 +48,7 @@ class S3FileSystemProviderTest extends Specification { client.transferManagerThreads == 20 client.cannedAcl == ObjectCannedACL.PRIVATE client.storageEncryption == ServerSideEncryption.AES256 + client.checksumAlgorithm == ChecksumAlgorithm.SHA256 client.isRequesterPaysEnabled == true client.kmsKeyId == 'arn:key:id' client.factory.accessKey() == '123456abc' diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy index d05d4df994..ec5e160706 100644 --- a/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy @@ -25,6 +25,7 @@ class S3BashLibTest extends Specification { 1 * opts.getAwsCli() >> 'aws' 1 * opts.getStorageClass() >> null 1 * opts.getStorageEncryption() >> null + 1 * opts.getChecksumAlgorithm() >> null script == '''\ # bash helper functions @@ -122,6 +123,7 @@ class S3BashLibTest extends Specification { then: opts.getStorageClass() >> 'S-CLAZZ' opts.getStorageEncryption() >> 'S-ENCRYPT' + opts.getChecksumAlgorithm() >> 'S-CHECK' opts.getAwsCli() >> '/foo/bin/aws' opts.getMaxParallelTransfers() >> 33 @@ -188,11 +190,11 @@ class S3BashLibTest extends Specification { local name=$1 local s3path=$2 if [[ "$name" == - ]]; then - /foo/bin/aws s3 cp --only-show-errors --sse S-ENCRYPT --storage-class S-CLAZZ - "$s3path" + /foo/bin/aws s3 cp --only-show-errors --sse S-ENCRYPT --checksum-algorithm S-CHECK --storage-class S-CLAZZ - "$s3path" elif [[ -d "$name" ]]; then - /foo/bin/aws s3 cp --only-show-errors --recursive --sse S-ENCRYPT --storage-class S-CLAZZ "$name" "$s3path/$name" + /foo/bin/aws s3 cp --only-show-errors --recursive --sse S-ENCRYPT --checksum-algorithm S-CHECK --storage-class S-CLAZZ "$name" "$s3path/$name" else - /foo/bin/aws s3 cp --only-show-errors --sse S-ENCRYPT --storage-class S-CLAZZ "$name" "$s3path/$name" + /foo/bin/aws s3 cp --only-show-errors --sse S-ENCRYPT --checksum-algorithm S-CHECK --storage-class S-CLAZZ "$name" "$s3path/$name" fi } @@ -551,7 +553,7 @@ class S3BashLibTest extends Specification { def 'should create with storage encrypt' () { given: def sess1 = Mock(Session) { - getConfig() >> [aws: [ client: [ storageKmsKeyId: 'my-kms-key', storageEncryption: 'aws:kms']]] + getConfig() >> [aws: [ client: [ storageKmsKeyId: 'my-kms-key', storageEncryption: 'aws:kms', checksumAlgorithm: 'SHA256']]] } and: def opts = new AwsOptions(sess1) @@ -623,11 +625,11 @@ class S3BashLibTest extends Specification { local name=$1 local s3path=$2 if [[ "$name" == - ]]; then - aws s3 cp --only-show-errors --sse aws:kms --sse-kms-key-id my-kms-key --storage-class STANDARD - "$s3path" + aws s3 cp --only-show-errors --sse aws:kms --sse-kms-key-id my-kms-key --checksum-algorithm SHA256 --storage-class STANDARD - "$s3path" elif [[ -d "$name" ]]; then - aws s3 cp --only-show-errors --recursive --sse aws:kms --sse-kms-key-id my-kms-key --storage-class STANDARD "$name" "$s3path/$name" + aws s3 cp --only-show-errors --recursive --sse aws:kms --sse-kms-key-id my-kms-key --checksum-algorithm SHA256 --storage-class STANDARD "$name" "$s3path/$name" else - aws s3 cp --only-show-errors --sse aws:kms --sse-kms-key-id my-kms-key --storage-class STANDARD "$name" "$s3path/$name" + aws s3 cp --only-show-errors --sse aws:kms --sse-kms-key-id my-kms-key --checksum-algorithm SHA256 --storage-class STANDARD "$name" "$s3path/$name" fi } diff --git a/plugins/nf-amazon/src/test/nextflow/executor/AwsBatchExecutorTest.groovy b/plugins/nf-amazon/src/test/nextflow/executor/AwsBatchExecutorTest.groovy index 5dbef4439e..7804dcae2e 100644 --- a/plugins/nf-amazon/src/test/nextflow/executor/AwsBatchExecutorTest.groovy +++ b/plugins/nf-amazon/src/test/nextflow/executor/AwsBatchExecutorTest.groovy @@ -120,6 +120,7 @@ class AwsBatchExecutorTest extends Specification { getAwsOptions() >> Mock(AwsOptions) { getS5cmdPath() >> { S5CMD ? 's5cmd' : null } getAwsCli() >> { 'aws' } + getChecksumAlgorithm() >> 'SHA256' } } expect: @@ -127,7 +128,7 @@ class AwsBatchExecutorTest extends Specification { where: FUSION | DEFAULT_FS | S5CMD | TASK_DIR | EXPECTED - false | false | false | 's3://foo/work/dir' | 'bash -o pipefail -c \'trap "{ ret=$?; aws s3 cp --only-show-errors .command.log s3://foo/work/dir/.command.log||true; exit $ret; }" EXIT; aws s3 cp --only-show-errors s3://foo/work/dir/.command.run - | bash 2>&1 | tee .command.log\'' + false | false | false | 's3://foo/work/dir' | 'bash -o pipefail -c \'trap "{ ret=$?; aws s3 cp --only-show-errors --checksum-algorithm SHA256 .command.log s3://foo/work/dir/.command.log||true; exit $ret; }" EXIT; aws s3 cp --only-show-errors --checksum-algorithm SHA256 s3://foo/work/dir/.command.run - | bash 2>&1 | tee .command.log\'' false | false | true | 's3://foo/work/dir' | 'bash -o pipefail -c \'trap "{ ret=$?; s5cmd cp .command.log s3://foo/work/dir/.command.log||true; exit $ret; }" EXIT; s5cmd cat s3://foo/work/dir/.command.run | bash 2>&1 | tee .command.log\'' and: true | false | false | '/fusion/work/dir' | 'bash /fusion/work/dir/.command.run'