nextflow-io · jorgee · Jul 31, 2025
@@ -101,6 +101,7 @@ aws {
         connectionTimeout = 10000
         uploadStorageClass = 'INTELLIGENT_TIERING'
         storageEncryption = 'AES256'
+        checksumAlgorithm = 'SHA256'
     }
     batch {
         cliPath = '/home/ec2-user/miniconda/bin/aws'
@@ -194,6 +195,9 @@ The following settings are available:
 `aws.client.anonymous`
 : Allow the access of public S3 buckets without the need to provide AWS credentials (default: `false`). Any service that does not accept unsigned requests will return a service access error.
 
+`aws.client.checksumAlgorithm`
+: The S3 checksum algorithm to be used when saving objects on S3. Can be one of `CRC32`, `CRC32C`, `SHA1`, `SHA256` or `CRC64NVME`.
+
 `aws.client.s3Acl`
 : Allow the setting of predefined bucket permissions, also known as *canned ACL*. Permitted values are `Private`, `PublicRead`, `PublicReadWrite`, `AuthenticatedRead`, `LogDeliveryWrite`, `BucketOwnerRead`, `BucketOwnerFullControl`, and `AwsExecRead` (default: none). See [Amazon docs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl) for details.
 

@@ -29,6 +29,12 @@ public class AwsClientConfig implements ConfigScope {
     """)
     public boolean anonymous;
 
+    @ConfigOption
+    @Description("""
+        The S3 checksum algorithm to be used when saving objects on S3. Can be one of `CRC32`, `CRC32C`, `SHA1`, `SHA256` or `CRC64NVME`.
+    """)
+    public String checksumAlgorithm;
+
     @ConfigOption
     @Description("""
         Specify predefined bucket permissions, also known as *canned ACL*. Can be one of `Private`, `PublicRead`, `PublicReadWrite`, `AuthenticatedRead`, `LogDeliveryWrite`, `BucketOwnerRead`, `BucketOwnerFullControl`, or `AwsExecRead`.

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsBatchExecutor.groovy
@@ -370,13 +370,17 @@ class AwsBatchExecutor extends Executor implements ExtensionPoint, TaskArrayExec
         final debug = opts.debug ? ' --debug' : ''
         final sse = opts.storageEncryption ? " --sse $opts.storageEncryption" : ''
         final kms = opts.storageKmsKeyId ? " --sse-kms-key-id $opts.storageKmsKeyId" : ''
+        final checksum = opts.checksumAlgorithm ? " --checksum-algorithm $opts.checksumAlgorithm" : ''
         final requesterPays = opts.requesterPays ? ' --request-payer requester' : ''
-        final aws = "$cli s3 cp --only-show-errors${sse}${kms}${debug}${requesterPays}"
+        final aws = "$cli s3 cp --only-show-errors${sse}${kms}${checksum}${debug}${requesterPays}"
         final cmd = "trap \"{ ret=\$?; $aws ${TaskRun.CMD_LOG} ${workDir}/${TaskRun.CMD_LOG}||true; exit \$ret; }\" EXIT; $aws ${workDir}/${TaskRun.CMD_RUN} - | bash 2>&1 | tee ${TaskRun.CMD_LOG}"
         return cmd
     }
 
     static String s5Cmd(String workDir, AwsOptions opts) {
+        if( opts.checksumAlgorithm ){
+            log.warn1("Checksum Algorithm is not supported by `s5cmd` command. This option will be ignored in command line operations.")
+        }
         final cli = opts.getS5cmdPath()
         final sse = opts.storageEncryption ? " --sse $opts.storageEncryption" : ''
         final kms = opts.storageKmsKeyId ? " --sse-kms-key-id $opts.storageKmsKeyId" : ''

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy
@@ -116,6 +116,10 @@ class AwsOptions implements CloudTransferOptions {
         return awsConfig.s3Config.getStorageClass()
     }
 
+    String getChecksumAlgorithm(){
+        return awsConfig.s3Config.getChecksumAlgorithm()
+    }
+
     String getStorageEncryption() {
         return awsConfig.s3Config.getStorageEncryption()
     }

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsS3Config.groovy
@@ -17,6 +17,8 @@
 
 package nextflow.cloud.aws.config
 
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm
+
 import static nextflow.cloud.aws.util.AwsHelper.*
 
 import software.amazon.awssdk.services.s3.model.ObjectCannedACL
@@ -41,6 +43,8 @@ class AwsS3Config {
 
     private String storageKmsKeyId
 
+    private String checksumAlgorithm
+
     private Boolean debug
 
     private ObjectCannedACL s3Acl
@@ -59,6 +63,7 @@ class AwsS3Config {
         this.storageClass = parseStorageClass((opts.storageClass ?: opts.uploadStorageClass) as String)     // 'uploadStorageClass' is kept for legacy purposes
         this.storageEncryption = parseStorageEncryption(opts.storageEncryption as String)
         this.storageKmsKeyId = opts.storageKmsKeyId
+        this.checksumAlgorithm = parseChecksumAlgorithm(opts.checksumAlgorithm as String)
         this.pathStyleAccess = opts.s3PathStyleAccess as Boolean
         this.anonymous = opts.anonymous as Boolean
         this.s3Acl = parseS3Acl(opts.s3Acl as String)
@@ -85,6 +90,15 @@ class AwsS3Config {
         return null
     }
 
+    private String parseChecksumAlgorithm(String value) {
+        if (value) {
+            if( value in ChecksumAlgorithm.knownValues()*.toString() )
+                return value
+            log.warn "Unsupported AWS checksum algorithm: $value"
+        }
+        return null
+    }
+
     // ==== getters =====
     String getEndpoint() {
         return endpoint
@@ -102,6 +116,10 @@ class AwsS3Config {
         return storageKmsKeyId
     }
 
+    String getChecksumAlgorithm() {
+        return checksumAlgorithm
+    }
+
     Boolean getDebug() {
         return debug
     }

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Client.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Client.java
@@ -76,6 +76,8 @@ public class S3Client {
 
 	private boolean global;
 
+    private ChecksumAlgorithm checksumAlgorithm;
+
 	public S3Client(AwsClientFactory factory, Properties props, boolean global) {
 		S3SyncClientConfiguration clientConfig = S3SyncClientConfiguration.create(props);
 		this.factory = factory;
@@ -155,6 +157,9 @@ private PutObjectRequest preparePutObjectRequest(PutObjectRequest.Builder reqBui
 		if( storageEncryption!=null ) {
 			reqBuilder.serverSideEncryption(storageEncryption);
 		}
+        if( checksumAlgorithm != null ) {
+            reqBuilder.checksumAlgorithm(checksumAlgorithm);
+        }
 		if( contentType!=null ) {
 			reqBuilder.contentType(contentType);
 		}
@@ -183,6 +188,9 @@ public PutObjectResponse putObject(String bucket, String keyName, InputStream in
 		if( storageEncryption!=null ) {
 			reqBuilder.serverSideEncryption(storageEncryption);
 		}
+        if( checksumAlgorithm != null ) {
+            reqBuilder.checksumAlgorithm(checksumAlgorithm);
+        }
 		if( contentType!=null ) {
 			reqBuilder.contentType(contentType);
 		}
@@ -217,6 +225,9 @@ public void copyObject(CopyObjectRequest.Builder reqBuilder, List<Tag> tags, Str
 		if( kmsKeyId !=null ) {
 			reqBuilder.ssekmsKeyId(kmsKeyId);
 		}
+        if( checksumAlgorithm != null ) {
+            reqBuilder.checksumAlgorithm(checksumAlgorithm);
+        }
 		if( contentType!=null ) {
 			reqBuilder.metadataDirective(MetadataDirective.REPLACE);
 			reqBuilder.contentType(contentType);
@@ -281,6 +292,13 @@ public void setTransferManagerThreads(String value) {
 		}
 	}
 
+    public void setChecksumAlgorithm(String alg){
+        if( alg == null )
+			return;
+		this.checksumAlgorithm = ChecksumAlgorithm.fromValue(alg);
+		log.debug("Setting S3 ChecksumAlgorithm={}", alg);
+    }
+
 	public ObjectCannedACL getCannedAcl() {
 		return cannedAcl;
 	}
@@ -340,6 +358,9 @@ public void multipartCopyObject(S3Path s3Source, S3Path s3Target, Long objectSiz
 		if( kmsKeyId != null ) {
 			reqBuilder.ssekmsKeyId(kmsKeyId);
 		}
+        if( checksumAlgorithm != null ) {
+            reqBuilder.checksumAlgorithm(checksumAlgorithm);
+        }
 
 		if( tags != null && tags.size()>0 ) {
 			reqBuilder.tagging( Tagging.builder().tagSet(tags).build() );
@@ -548,6 +569,9 @@ private PutObjectRequest.Builder updateBuilder(PutObjectRequest.Builder porBuild
 			porBuilder.serverSideEncryption(storageEncryption);
 		if( kmsKeyId != null )
 			porBuilder.ssekmsKeyId(kmsKeyId);
+        if( checksumAlgorithm != null ) {
+            porBuilder.checksumAlgorithm(checksumAlgorithm);
+        }
 		if( tags != null && !tags.isEmpty() )
 			porBuilder.tagging(Tagging.builder().tagSet(tags).build());
 		return porBuilder;
@@ -593,6 +617,9 @@ public void copyFile(CopyObjectRequest.Builder reqBuilder, List<Tag> tags, Strin
 		if( kmsKeyId !=null ) {
             reqBuilder.ssekmsKeyId(kmsKeyId);
 		}
+        if( checksumAlgorithm != null ) {
+            reqBuilder.checksumAlgorithm(checksumAlgorithm);
+        }
 		if( contentType!=null ) {
 			reqBuilder.metadataDirective(MetadataDirective.REPLACE);
             reqBuilder.contentType(contentType);

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3FileSystemProvider.java
@@ -340,6 +340,7 @@ private S3OutputStream createUploaderOutputStream( S3Path fileToUpload ) {
 				.setCannedAcl(s3.getCannedAcl())
 				.setStorageClass(storageClass)
 				.setStorageEncryption(props.getProperty("storage_encryption"))
+                .setChecksumAlgorithm(props.getProperty("checksum_algorithm"))
 				.setKmsKeyId(props.getProperty("storage_kms_key_id"))
 				.setContentType(fileToUpload.getContentType())
 				.setTags(fileToUpload.getTagsList());
@@ -742,6 +743,7 @@ protected S3FileSystem createFileSystem(URI uri, AwsConfig awsConfig) {
 		// set the client acl
 		client.setCannedAcl(getProp(props, "s_3_acl", "s3_acl", "s3acl", "s3Acl"));
 		client.setStorageEncryption(props.getProperty("storage_encryption"));
+        client.setChecksumAlgorithm(props.getProperty("checksum_algorithm"));
 		client.setKmsKeyId(props.getProperty("storage_kms_key_id"));
 		client.setTransferManagerThreads(props.getProperty("transfer_manager_threads"));
         client.setRequesterPaysEnabled(props.getProperty("requester_pays"));

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3OutputStream.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3OutputStream.java
@@ -87,6 +87,8 @@ public final class S3OutputStream extends OutputStream {
 
     private String contentType;
 
+    private ChecksumAlgorithm checksumAlgorithm;
+
     /**
      * Indicates if the stream has been closed.
      */
@@ -201,6 +203,12 @@ public S3OutputStream setKmsKeyId(String kmsKeyId) {
         return this;
     }
 
+    public S3OutputStream setChecksumAlgorithm(String checksumAlgorithm){
+        if( checksumAlgorithm !=null )
+            this.checksumAlgorithm = ChecksumAlgorithm.fromValue(checksumAlgorithm);
+        return this;
+    }
+
     public S3OutputStream setContentType(String type) {
         this.contentType = type;
         return this;
@@ -428,6 +436,10 @@ private CreateMultipartUploadResponse initiateMultipartUpload() throws IOExcepti
             reqBuilder.serverSideEncryption(storageEncryption);
         }
 
+        if( checksumAlgorithm != null ) {
+            reqBuilder.checksumAlgorithm(checksumAlgorithm);
+        }
+
         if( contentType != null ) {
             reqBuilder.contentType(contentType);
         }
@@ -614,6 +626,10 @@ private void putObject(final InputStream content, final long contentLength, byte
             reqBuilder.serverSideEncryption( storageEncryption );
         }
 
+        if( checksumAlgorithm != null ) {
+            reqBuilder.checksumAlgorithm(checksumAlgorithm);
+        }
+
         if( contentType != null ) {
             reqBuilder.contentType(contentType);
         }

diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy
@@ -38,6 +38,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
     private String s5cmdPath
     private String acl = ''
     private String requesterPays = ''
+    private String checksumAlgorithm = ''
 
     S3BashLib withCliPath(String cliPath) {
         if( cliPath )
@@ -61,6 +62,11 @@ class S3BashLib extends BashFunLib<S3BashLib> {
             this.storageClass = value
         return this
     }
+    S3BashLib withChecksumAlgorithm(String value) {
+        if( value )
+            this.checksumAlgorithm = value ? "--checksum-algorithm $value " : ''
+        return this
+    }
 
     S3BashLib withStorageEncryption(String value) {
         if( value )
@@ -112,11 +118,11 @@ class S3BashLib extends BashFunLib<S3BashLib> {
             local name=\$1
             local s3path=\$2
             if [[ "\$name" == - ]]; then
-              $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass - "\$s3path"
+              $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass - "\$s3path"
             elif [[ -d "\$name" ]]; then
-              $cli s3 cp --only-show-errors --recursive ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name"
+              $cli s3 cp --only-show-errors --recursive ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name"
             else
-              $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name"
+              $cli s3 cp --only-show-errors ${debug}${acl}${storageEncryption}${storageKmsKeyId}${checksumAlgorithm}${requesterPays}--storage-class $storageClass "\$name" "\$s3path/\$name"
             fi
         }
 
@@ -187,6 +193,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
                 .withMaxTransferAttempts( opts.maxTransferAttempts )
                 .withCliPath( opts.awsCli )
                 .withStorageClass(opts.storageClass )
+                .withChecksumAlgorithm( opts.checksumAlgorithm )
                 .withStorageEncryption( opts.storageEncryption )
                 .withStorageKmsKeyId( opts.storageKmsKeyId )
                 .withRetryMode( opts.retryMode )

diff --git a/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy b/plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy
@@ -126,6 +126,7 @@ class AwsBatchFileCopyStrategyTest extends Specification {
         1 * opts.getAwsCli() >> 'aws'
         1 * opts.getStorageClass() >> null
         1 * opts.getStorageEncryption() >> null
+        1 * opts.getChecksumAlgorithm() >> null
 
         script ==   '''\
                     # bash helper functions
@@ -217,6 +218,7 @@ class AwsBatchFileCopyStrategyTest extends Specification {
         1 * opts.getAwsCli() >> '/foo/aws'
         1 * opts.getStorageClass() >> 'STANDARD_IA'
         1 * opts.getStorageEncryption() >> 'AES256'
+        1 * opts.getChecksumAlgorithm() >> 'SHA256'
 
         script == '''\
                 # bash helper functions
@@ -281,11 +283,11 @@ class AwsBatchFileCopyStrategyTest extends Specification {
                     local name=$1
                     local s3path=$2
                     if [[ "$name" == - ]]; then
-                      /foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA - "$s3path"
+                      /foo/aws s3 cp --only-show-errors --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA - "$s3path"
                     elif [[ -d "$name" ]]; then
-                      /foo/aws s3 cp --only-show-errors --recursive --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name"
+                      /foo/aws s3 cp --only-show-errors --recursive --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA "$name" "$s3path/$name"
                     else
-                      /foo/aws s3 cp --only-show-errors --sse AES256 --storage-class STANDARD_IA "$name" "$s3path/$name"
+                      /foo/aws s3 cp --only-show-errors --sse AES256 --checksum-algorithm SHA256 --storage-class STANDARD_IA "$name" "$s3path/$name"
                     fi
                 }