Skip to content

Commit 66e43f6

Browse files
robsymebentsherman
authored and
pditommaso
committed
Add aws.batch.forceGlacierTransfer config option (#6700)
Co-authored-by: Ben Sherman <[email protected]>
1 parent d4cf0ca commit 66e43f6

File tree

9 files changed

+110
-33
lines changed

9 files changed

+110
-33
lines changed

‎docs/reference/config.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,12 @@ The following settings are available:
112112
`aws.batch.executionRole`
113113
: The AWS Batch [Execution Role](https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html) ARN that needs to be used to execute the Batch Job. It is mandatory when using AWS Fargate.
114114

115+
`aws.batch.forceGlacierTransfer`
116+
: :::{versionadded 26.04.0}
117+
:::
118+
: When `true`, add the `--force-glacier-transfer` flag to AWS CLI S3 download commands (default: `false`).
119+
: This option is needed when staging directories that have been restored from [S3 Glacier](https://aws.amazon.com/s3/storage-classes/glacier/). It does not restore objects from Glacier.
120+
115121
`aws.batch.jobRole`
116122
: The AWS Batch Job Role ARN that needs to be used to execute the Batch Job.
117123

‎plugins/nf-amazon/src/main/nextflow/cloud/aws/batch/AwsOptions.groovy‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,4 +164,8 @@ class AwsOptions implements CloudTransferOptions {
164164
return awsConfig.batchConfig.terminateUnschedulableJobs
165165
}
166166

167+
Boolean getForceGlacierTransfer() {
168+
return awsConfig.batchConfig.forceGlacierTransfer
169+
}
170+
167171
}

‎plugins/nf-amazon/src/main/nextflow/cloud/aws/config/AwsBatchConfig.groovy‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,12 @@ class AwsBatchConfig implements CloudTransferOptions, ConfigScope {
5959
""")
6060
final String executionRole
6161

62+
@ConfigOption
63+
@Description("""
64+
When `true`, add the `--force-glacier-transfer` flag to AWS CLI S3 download commands (default: `false`).
65+
""")
66+
final boolean forceGlacierTransfer
67+
6268
@ConfigOption
6369
@Description("""
6470
The AWS Batch Job Role ARN that needs to be used to execute the Batch Job.
@@ -151,6 +157,7 @@ class AwsBatchConfig implements CloudTransferOptions, ConfigScope {
151157
schedulingPriority = opts.schedulingPriority as Integer ?: 0
152158
executionRole = opts.executionRole
153159
terminateUnschedulableJobs = opts.terminateUnschedulableJobs as boolean
160+
forceGlacierTransfer = opts.forceGlacierTransfer as boolean
154161
if( retryMode == 'built-in' )
155162
retryMode = null // this force falling back on NF built-in retry mode instead of delegating to AWS CLI tool
156163
if( retryMode && retryMode !in AwsOptions.VALID_RETRY_MODES )

‎plugins/nf-amazon/src/main/nextflow/cloud/aws/util/S3BashLib.groovy‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
3838
private String s5cmdPath
3939
private String acl = ''
4040
private String requesterPays = ''
41+
private String forceGlacierTransfer = ''
4142

4243
S3BashLib withCliPath(String cliPath) {
4344
if( cliPath )
@@ -90,12 +91,17 @@ class S3BashLib extends BashFunLib<S3BashLib> {
9091
return this
9192
}
9293

94+
S3BashLib withForceGlacierTransfer(Boolean value) {
95+
this.forceGlacierTransfer = value ? '--force-glacier-transfer ' : ''
96+
return this
97+
}
98+
9399
protected String retryEnv() {
94100
if( !retryMode )
95101
return ''
96102
"""
97103
# aws cli retry config
98-
export AWS_RETRY_MODE=${retryMode}
104+
export AWS_RETRY_MODE=${retryMode}
99105
export AWS_MAX_ATTEMPTS=${maxTransferAttempts}
100106
""".stripIndent().rightTrim()
101107
}
@@ -126,8 +132,8 @@ class S3BashLib extends BashFunLib<S3BashLib> {
126132
local file_name=\$(basename \$1)
127133
local is_dir=\$($cli s3 ls \$source | grep -F "PRE \${file_name}/" -c)
128134
if [[ \$is_dir == 1 ]]; then
129-
$cli s3 cp --only-show-errors --recursive "\$source" "\$target"
130-
else
135+
$cli s3 cp --only-show-errors --recursive ${forceGlacierTransfer}"\$source" "\$target"
136+
else
131137
$cli s3 cp --only-show-errors "\$source" "\$target"
132138
fi
133139
}
@@ -165,7 +171,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
165171
local is_dir=\$($cli ls \$source | grep -F "DIR \${file_name}/" -c)
166172
if [[ \$is_dir == 1 ]]; then
167173
$cli cp "\$source/*" "\$target"
168-
else
174+
else
169175
$cli cp "\$source" "\$target"
170176
fi
171177
}
@@ -194,6 +200,7 @@ class S3BashLib extends BashFunLib<S3BashLib> {
194200
.withS5cmdPath( opts.s5cmdPath )
195201
.withAcl( opts.s3Acl )
196202
.withRequesterPays( opts.requesterPays )
203+
.withForceGlacierTransfer( opts.forceGlacierTransfer )
197204
}
198205

199206
static String script(AwsOptions opts) {

‎plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchFileCopyStrategyTest.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ class AwsBatchFileCopyStrategyTest extends Specification {
205205
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
206206
if [[ $is_dir == 1 ]]; then
207207
aws s3 cp --only-show-errors --recursive "$source" "$target"
208-
else
208+
else
209209
aws s3 cp --only-show-errors "$source" "$target"
210210
fi
211211
}
@@ -296,7 +296,7 @@ class AwsBatchFileCopyStrategyTest extends Specification {
296296
local is_dir=$(/foo/aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
297297
if [[ $is_dir == 1 ]]; then
298298
/foo/aws s3 cp --only-show-errors --recursive "$source" "$target"
299-
else
299+
else
300300
/foo/aws s3 cp --only-show-errors "$source" "$target"
301301
fi
302302
}

‎plugins/nf-amazon/src/test/nextflow/cloud/aws/batch/AwsBatchScriptLauncherTest.groovy‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ class AwsBatchScriptLauncherTest extends Specification {
139139
local is_dir=$(/conda/bin/aws --region eu-west-1 s3 ls $source | grep -F "PRE ${file_name}/" -c)
140140
if [[ $is_dir == 1 ]]; then
141141
/conda/bin/aws --region eu-west-1 s3 cp --only-show-errors --recursive "$source" "$target"
142-
else
142+
else
143143
/conda/bin/aws --region eu-west-1 s3 cp --only-show-errors "$source" "$target"
144144
fi
145145
}
@@ -318,7 +318,7 @@ class AwsBatchScriptLauncherTest extends Specification {
318318
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
319319
if [[ $is_dir == 1 ]]; then
320320
aws s3 cp --only-show-errors --recursive "$source" "$target"
321-
else
321+
else
322322
aws s3 cp --only-show-errors "$source" "$target"
323323
fi
324324
}
@@ -490,7 +490,7 @@ class AwsBatchScriptLauncherTest extends Specification {
490490
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
491491
if [[ $is_dir == 1 ]]; then
492492
aws s3 cp --only-show-errors --recursive "$source" "$target"
493-
else
493+
else
494494
aws s3 cp --only-show-errors "$source" "$target"
495495
fi
496496
}
@@ -584,7 +584,7 @@ class AwsBatchScriptLauncherTest extends Specification {
584584
}
585585
586586
# aws cli retry config
587-
export AWS_RETRY_MODE=adaptive
587+
export AWS_RETRY_MODE=adaptive
588588
export AWS_MAX_ATTEMPTS=3
589589
# aws helper
590590
nxf_s3_upload() {
@@ -606,7 +606,7 @@ class AwsBatchScriptLauncherTest extends Specification {
606606
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
607607
if [[ $is_dir == 1 ]]; then
608608
aws s3 cp --only-show-errors --recursive "$source" "$target"
609-
else
609+
else
610610
aws s3 cp --only-show-errors "$source" "$target"
611611
fi
612612
}

‎plugins/nf-amazon/src/test/nextflow/cloud/aws/config/AwsBatchConfigTest.groovy‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ class AwsBatchConfigTest extends Specification {
4747
!batch.s5cmdPath
4848
batch.schedulingPriority == 0
4949
!batch.terminateUnschedulableJobs
50+
!batch.forceGlacierTransfer
5051
}
5152

5253
def 'should create config with options' () {
@@ -153,4 +154,18 @@ class AwsBatchConfigTest extends Specification {
153154
[terminateUnschedulableJobs: false] | false
154155
[terminateUnschedulableJobs: true] | true
155156
}
157+
158+
def 'should parse forceGlacierTransfer flag' () {
159+
given:
160+
def opts = new AwsBatchConfig(OPTS)
161+
162+
expect:
163+
opts.forceGlacierTransfer == FORCE_GLACIER
164+
165+
where:
166+
OPTS | FORCE_GLACIER
167+
[:] | false
168+
[forceGlacierTransfer: false] | false
169+
[forceGlacierTransfer: true] | true
170+
}
156171
}

‎plugins/nf-amazon/src/test/nextflow/cloud/aws/util/S3BashLibTest.groovy‎

Lines changed: 57 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ class S3BashLibTest extends Specification {
104104
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
105105
if [[ $is_dir == 1 ]]; then
106106
aws s3 cp --only-show-errors --recursive "$source" "$target"
107-
else
107+
else
108108
aws s3 cp --only-show-errors "$source" "$target"
109109
fi
110110
}
@@ -203,7 +203,7 @@ class S3BashLibTest extends Specification {
203203
local is_dir=$(/foo/bin/aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
204204
if [[ $is_dir == 1 ]]; then
205205
/foo/bin/aws s3 cp --only-show-errors --recursive "$source" "$target"
206-
else
206+
else
207207
/foo/bin/aws s3 cp --only-show-errors "$source" "$target"
208208
fi
209209
}
@@ -223,7 +223,7 @@ class S3BashLibTest extends Specification {
223223
expect:
224224
S3BashLib.script() == '''
225225
# aws cli retry config
226-
export AWS_RETRY_MODE=standard
226+
export AWS_RETRY_MODE=standard
227227
export AWS_MAX_ATTEMPTS=5
228228
# aws helper
229229
nxf_s3_upload() {
@@ -245,7 +245,7 @@ class S3BashLibTest extends Specification {
245245
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
246246
if [[ $is_dir == 1 ]]; then
247247
aws s3 cp --only-show-errors --recursive "$source" "$target"
248-
else
248+
else
249249
aws s3 cp --only-show-errors "$source" "$target"
250250
fi
251251
}
@@ -261,7 +261,7 @@ class S3BashLibTest extends Specification {
261261
expect:
262262
S3BashLib.script() == '''
263263
# aws cli retry config
264-
export AWS_RETRY_MODE=legacy
264+
export AWS_RETRY_MODE=legacy
265265
export AWS_MAX_ATTEMPTS=100
266266
# aws helper
267267
nxf_s3_upload() {
@@ -283,7 +283,7 @@ class S3BashLibTest extends Specification {
283283
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
284284
if [[ $is_dir == 1 ]]; then
285285
aws s3 cp --only-show-errors --recursive "$source" "$target"
286-
else
286+
else
287287
aws s3 cp --only-show-errors "$source" "$target"
288288
fi
289289
}
@@ -318,7 +318,7 @@ class S3BashLibTest extends Specification {
318318
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
319319
if [[ $is_dir == 1 ]]; then
320320
aws s3 cp --only-show-errors --recursive "$source" "$target"
321-
else
321+
else
322322
aws s3 cp --only-show-errors "$source" "$target"
323323
fi
324324
}
@@ -334,7 +334,7 @@ class S3BashLibTest extends Specification {
334334
expect:
335335
S3BashLib.script() == '''
336336
# aws cli retry config
337-
export AWS_RETRY_MODE=legacy
337+
export AWS_RETRY_MODE=legacy
338338
export AWS_MAX_ATTEMPTS=99
339339
# aws helper
340340
nxf_s3_upload() {
@@ -356,7 +356,7 @@ class S3BashLibTest extends Specification {
356356
local is_dir=$(/some/bin/aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
357357
if [[ $is_dir == 1 ]]; then
358358
/some/bin/aws s3 cp --only-show-errors --recursive "$source" "$target"
359-
else
359+
else
360360
/some/bin/aws s3 cp --only-show-errors "$source" "$target"
361361
fi
362362
}
@@ -450,7 +450,7 @@ class S3BashLibTest extends Specification {
450450
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
451451
if [[ $is_dir == 1 ]]; then
452452
aws s3 cp --only-show-errors --recursive "$source" "$target"
453-
else
453+
else
454454
aws s3 cp --only-show-errors "$source" "$target"
455455
fi
456456
}
@@ -541,7 +541,7 @@ class S3BashLibTest extends Specification {
541541
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
542542
if [[ $is_dir == 1 ]]; then
543543
aws s3 cp --only-show-errors --recursive "$source" "$target"
544-
else
544+
else
545545
aws s3 cp --only-show-errors "$source" "$target"
546546
fi
547547
}
@@ -616,7 +616,7 @@ class S3BashLibTest extends Specification {
616616
}
617617
618618
# aws cli retry config
619-
export AWS_RETRY_MODE=standard
619+
export AWS_RETRY_MODE=standard
620620
export AWS_MAX_ATTEMPTS=5
621621
# aws helper
622622
nxf_s3_upload() {
@@ -638,7 +638,7 @@ class S3BashLibTest extends Specification {
638638
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
639639
if [[ $is_dir == 1 ]]; then
640640
aws s3 cp --only-show-errors --recursive "$source" "$target"
641-
else
641+
else
642642
aws s3 cp --only-show-errors "$source" "$target"
643643
fi
644644
}
@@ -714,7 +714,7 @@ class S3BashLibTest extends Specification {
714714
}
715715
716716
# aws cli retry config
717-
export AWS_RETRY_MODE=standard
717+
export AWS_RETRY_MODE=standard
718718
export AWS_MAX_ATTEMPTS=5
719719
# aws helper
720720
nxf_s3_upload() {
@@ -736,7 +736,7 @@ class S3BashLibTest extends Specification {
736736
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
737737
if [[ $is_dir == 1 ]]; then
738738
aws s3 cp --only-show-errors --recursive "$source" "$target"
739-
else
739+
else
740740
aws s3 cp --only-show-errors "$source" "$target"
741741
fi
742742
}
@@ -773,7 +773,7 @@ class S3BashLibTest extends Specification {
773773
local is_dir=$(s5cmd ls $source | grep -F "DIR ${file_name}/" -c)
774774
if [[ $is_dir == 1 ]]; then
775775
s5cmd cp "$source/*" "$target"
776-
else
776+
else
777777
s5cmd cp "$source" "$target"
778778
fi
779779
}
@@ -802,20 +802,58 @@ class S3BashLibTest extends Specification {
802802
s5cmd cp --acl public-read --storage-class STANDARD "$name" "$s3path/$name"
803803
fi
804804
}
805-
805+
806806
nxf_s3_download() {
807807
local source=$1
808808
local target=$2
809809
local file_name=$(basename $1)
810810
local is_dir=$(s5cmd ls $source | grep -F "DIR ${file_name}/" -c)
811811
if [[ $is_dir == 1 ]]; then
812812
s5cmd cp "$source/*" "$target"
813-
else
813+
else
814814
s5cmd cp "$source" "$target"
815815
fi
816816
}
817817
'''.stripIndent(true)
818818
}
819-
819+
820+
def 'should create script with force glacier transfer' () {
821+
given:
822+
Global.session = Mock(Session) {
823+
getConfig() >> [aws:[batch:[forceGlacierTransfer: true]]]
824+
}
825+
826+
expect:
827+
S3BashLib.script() == '''
828+
# aws cli retry config
829+
export AWS_RETRY_MODE=standard
830+
export AWS_MAX_ATTEMPTS=5
831+
# aws helper
832+
nxf_s3_upload() {
833+
local name=$1
834+
local s3path=$2
835+
if [[ "$name" == - ]]; then
836+
aws s3 cp --only-show-errors --storage-class STANDARD - "$s3path"
837+
elif [[ -d "$name" ]]; then
838+
aws s3 cp --only-show-errors --recursive --storage-class STANDARD "$name" "$s3path/$name"
839+
else
840+
aws s3 cp --only-show-errors --storage-class STANDARD "$name" "$s3path/$name"
841+
fi
842+
}
843+
844+
nxf_s3_download() {
845+
local source=$1
846+
local target=$2
847+
local file_name=$(basename $1)
848+
local is_dir=$(aws s3 ls $source | grep -F "PRE ${file_name}/" -c)
849+
if [[ $is_dir == 1 ]]; then
850+
aws s3 cp --only-show-errors --recursive --force-glacier-transfer "$source" "$target"
851+
else
852+
aws s3 cp --only-show-errors "$source" "$target"
853+
fi
854+
}
855+
'''.stripIndent(true)
856+
}
857+
820858

821859
}

0 commit comments

Comments
 (0)