Add aws.batch.forceGlacierTransfer config option

Add configuration option to enable --force-glacier-transfer flag for
AWS CLI S3 download commands. This allows downloading restored
Glacier/Deep Archive objects that retain their storage class metadata.

When enabled via:
  aws.batch.forceGlacierTransfer = true

The nxf_s3_download function will include --force-glacier-transfer
in aws s3 cp commands, enabling downloads of restored Glacier objects
without AWS CLI skipping them due to storage class checks.

Fixes: #4747

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
Signed-off-by: Rob Syme <rob.syme@gmail.com>

Author: robsyme

plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy

@@ -164,4 +164,8 @@ class AwsOptions implements CloudTransferOptions {
         return awsConfig.batchConfig.terminateUnschedulableJobs
     }
 
+    Boolean getForceGlacierTransfer() {
+        return awsConfig.batchConfig.forceGlacierTransfer
+    }
+
 }

plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsBatchConfig.groovy

@@ -125,6 +125,12 @@ class AwsBatchConfig implements CloudTransferOptions, ConfigScope {
     """)
     final List<String> volumes
 
+    @ConfigOption
+    @Description("""
+        When `true`, the `--force-glacier-transfer` flag is added to AWS CLI S3 download commands, allowing restored Glacier/Deep Archive objects to be downloaded (default: `false`).
+    """)
+    final boolean forceGlacierTransfer
+
     /**
      * The path for the `s5cmd` tool as an alternative to `aws s3` CLI to upload/download files
      */
@@ -151,6 +157,7 @@ class AwsBatchConfig implements CloudTransferOptions, ConfigScope {
         schedulingPriority = opts.schedulingPriority as Integer ?: 0
         executionRole = opts.executionRole
         terminateUnschedulableJobs = opts.terminateUnschedulableJobs as boolean
+        forceGlacierTransfer = opts.forceGlacierTransfer as boolean
         if( retryMode == 'built-in' )
             retryMode = null // this force falling back on NF built-in retry mode instead of delegating to AWS CLI tool
         if( retryMode && retryMode !in AwsOptions.VALID_RETRY_MODES )

plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy

@@ -38,6 +38,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
     private String s5cmdPath
     private String acl = ''
     private String requesterPays = ''
+    private String forceGlacierTransfer = ''
 
     S3BashLib withCliPath(String cliPath) {
         if( cliPath )
@@ -90,6 +91,11 @@ class S3BashLib extends BashFunLib<S3BashLib> {
         return this
     }
 
+    S3BashLib withForceGlacierTransfer(Boolean value) {
+        this.forceGlacierTransfer = value ? '--force-glacier-transfer ' : ''
+        return this
+    }
+
     protected String retryEnv() {
         if( !retryMode )
             return ''
@@ -126,9 +132,9 @@ class S3BashLib extends BashFunLib<S3BashLib> {
             local file_name=\$(basename \$1)
             local is_dir=\$($cli s3 ls \$source | grep -F "PRE \${file_name}/" -c)
             if [[ \$is_dir == 1 ]]; then
-                $cli s3 cp --only-show-errors --recursive "\$source" "\$target"
+                $cli s3 cp --only-show-errors --recursive ${forceGlacierTransfer}"\$source" "\$target"
             else
-                $cli s3 cp --only-show-errors "\$source" "\$target"
+                $cli s3 cp --only-show-errors ${forceGlacierTransfer}"\$source" "\$target"
             fi
         }
         """.stripIndent(true)
@@ -194,6 +200,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
                 .withS5cmdPath( opts.s5cmdPath )
                 .withAcl( opts.s3Acl )
                 .withRequesterPays( opts.requesterPays )
+                .withForceGlacierTransfer( opts.forceGlacierTransfer )
     }
 
     static String script(AwsOptions opts) {

plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsBatchConfigTest.groovy

@@ -47,6 +47,7 @@ class AwsBatchConfigTest extends Specification {
         !batch.s5cmdPath
         batch.schedulingPriority == 0
         !batch.terminateUnschedulableJobs
+        !batch.forceGlacierTransfer
     }
 
     def 'should create config with options' () {
@@ -153,4 +154,18 @@ class AwsBatchConfigTest extends Specification {
         [terminateUnschedulableJobs: false]     | false
         [terminateUnschedulableJobs: true]      | true
     }
+
+    def 'should parse forceGlacierTransfer flag' () {
+        given:
+        def opts = new AwsBatchConfig(OPTS)
+
+        expect:
+        opts.forceGlacierTransfer == FORCE_GLACIER
+
+        where:
+        OPTS                                    | FORCE_GLACIER
+        [:]                                     | false
+        [forceGlacierTransfer: false]           | false
+        [forceGlacierTransfer: true]            | true
+    }
 }

plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy

@@ -802,7 +802,7 @@ class S3BashLibTest extends Specification {
                   s5cmd cp --acl public-read --storage-class STANDARD "$name" "$s3path/$name"
                 fi
             }
-            
+
             nxf_s3_download() {
                 local source=$1
                 local target=$2
@@ -816,6 +816,44 @@ class S3BashLibTest extends Specification {
             }
             '''.stripIndent(true)
     }
-    
+
+    def 'should create script with force glacier transfer' () {
+        given:
+        Global.session = Mock(Session) {
+            getConfig() >> [aws:[batch:[forceGlacierTransfer: true]]]
+        }
+
+        expect:
+        S3BashLib.script()  == '''
+            # aws cli retry config
+            export AWS_RETRY_MODE=standard
+            export AWS_MAX_ATTEMPTS=5
+            # aws helper
+            nxf_s3_upload() {
+                local name=$1
+                local s3path=$2
+                if [[ "$name" == - ]]; then
+                  aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path"
+                elif [[ -d "$name" ]]; then
+                  aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name"
+                else
+                  aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name"
+                fi
+            }
+
+            nxf_s3_download() {
+                local source=$1
+                local target=$2
+                local file_name=$(basename $1)
+                local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
+                if [[ $is_dir == 1 ]]; then
+                    aws s3 cp --only-show-errors --recursive --force-glacier-transfer "$source" "$target"
+                else
+                    aws s3 cp --only-show-errors --force-glacier-transfer "$source" "$target"
+                fi
+            }
+            '''.stripIndent(true)
+    }
+
 
 }