From 9d313767f0612eb58477f983012283dec00036ea Mon Sep 17 00:00:00 2001 From: jorgee Date: Wed, 12 Nov 2025 15:49:20 +0100 Subject: [PATCH 1/3] fix stage file write when google cloud Signed-off-by: jorgee --- .../cloud/google/batch/GoogleBatchScriptLauncher.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy index 176ad46123..d7f902ebb1 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy @@ -184,6 +184,11 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc return remoteWorkDir.resolve(TaskRun.CMD_INFILE) } + @Override + protected Path targetStageFile() { + return remoteWorkDir.resolve(TaskRun.CMD_STAGE) + } + GoogleBatchScriptLauncher withConfig(GoogleOpts config) { this.config = config return this From b02c1cb3b80c985b015de7580db39e942f4656fc Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 14 Nov 2025 20:36:23 +0100 Subject: [PATCH 2/3] add stageFileEnabled flag and set it only for Grid executors Signed-off-by: jorgee --- .../nextflow/executor/BashWrapperBuilder.groovy | 12 +++++++++--- .../groovy/nextflow/executor/GridTaskHandler.groovy | 5 ++++- .../google/batch/GoogleBatchScriptLauncher.groovy | 5 ----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy index e927366386..ab4e18ac36 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy @@ -117,6 +117,14 @@ class BashWrapperBuilder { private BashTemplateEngine engine = new BashTemplateEngine() + // Flag to enable .command.stage file. False by default and enabled for Grid executors + // see https://github.com/nextflow-io/nextflow/issues/4279 and https://github.com/nextflow-io/nextflow/issues/5888 + private Boolean stageFileEnabled = false + + void enableStageFile(){ + stageFileEnabled = true + } + BashWrapperBuilder( TaskRun task ) { this(new TaskBean(task)) } @@ -272,9 +280,7 @@ class BashWrapperBuilder { return null final header = "# stage input files\n" - // enable only when the stage uses the default file system, i.e. it's not a remote object storage file - // see https://github.com/nextflow-io/nextflow/issues/4279 - if( stageFile.fileSystem == FileSystems.default && stagingScript.size() >= stageFileThreshold.bytes ) { + if( stageFileEnabled && stagingScript.size() >= stageFileThreshold.bytes ) { stageScript = stagingScript return header + "bash ${stageFile}" } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy index 63776de50d..601af54522 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/GridTaskHandler.groovy @@ -103,7 +103,10 @@ class GridTaskHandler extends TaskHandler implements FusionAwareTask { @Override void prepareLauncher() { // -- create the wrapper script - createTaskWrapper(task).build() + final builder = createTaskWrapper(task) + // Enable stageFile to avoid problems with submission script limitations + builder.enableStageFile() + builder.build() } protected ProcessBuilder createProcessBuilder() { diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy index d7f902ebb1..176ad46123 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy @@ -184,11 +184,6 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc return remoteWorkDir.resolve(TaskRun.CMD_INFILE) } - @Override - protected Path targetStageFile() { - return remoteWorkDir.resolve(TaskRun.CMD_STAGE) - } - GoogleBatchScriptLauncher withConfig(GoogleOpts config) { this.config = config return this From 37eab748e0581c8789e6bf480346b796dd41ebf7 Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 14 Nov 2025 22:11:46 +0100 Subject: [PATCH 3/3] add flags test Signed-off-by: jorgee --- .../executor/BashWrapperBuilderTest.groovy | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy index 4b48881ffa..d1ea6a66ea 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/BashWrapperBuilderTest.groovy @@ -473,7 +473,7 @@ class BashWrapperBuilderTest extends Specification { binding.stage_inputs == stageScript } - def 'should stage inputs to external file' () { + def 'should stage inputs to external file when enabled' () { given: SysEnv.push([NXF_WRAPPER_STAGE_FILE_THRESHOLD: '100']) and: @@ -494,6 +494,7 @@ class BashWrapperBuilderTest extends Specification { workDir: folder, targetDir: folder, inputFiles: inputs ]) + builder.enableStageFile() when: def binding = builder.makeBinding() @@ -510,6 +511,43 @@ class BashWrapperBuilderTest extends Specification { folder?.deleteDir() } + def 'should not stage inputs to external file when not enabled' () { + given: + SysEnv.push([NXF_WRAPPER_STAGE_FILE_THRESHOLD: '100']) + and: + def folder = Files.createTempDirectory('test') + and: + def inputs = [ + 'sample_1.fq': Paths.get('/some/data/sample_1.fq'), + 'sample_2.fq': Paths.get('/some/data/sample_2.fq'), + ] + def stageScript = '''\ + rm -f sample_1.fq + rm -f sample_2.fq + ln -s /some/data/sample_1.fq sample_1.fq + ln -s /some/data/sample_2.fq sample_2.fq + '''.stripIndent().rightTrim() + and: + def builder = newBashWrapperBuilder([ + workDir: folder, + targetDir: folder, + inputFiles: inputs ]) + + when: + def binding = builder.makeBinding() + then: + binding.stage_inputs == "# stage input files\n"+stageScript + + when: + builder.build() + then: + folder.resolve('.command.stage').exists() == false + + cleanup: + SysEnv.pop() + folder?.deleteDir() + } + def 'should include sync command' () { given: SysEnv.push([NXF_ENABLE_FS_SYNC: 'true'])