Allow collection-type params to be loaded from files (#6675)

Co-authored-by: Jorge Ejarque <[email protected]>

Author: bentsherman

docs/workflow.md

@@ -53,13 +53,7 @@ params {
 }
 ```
 
-The following types can be used for parameters:
-
-- {ref}`stdlib-types-boolean`
-- {ref}`stdlib-types-float`
-- {ref}`stdlib-types-integer`
-- {ref}`stdlib-types-path`
-- {ref}`stdlib-types-string`
+All {ref}`standard types <stdlib-types>` except for the dataflow types (`Channel` and `Value`) can be used for parameters.
 
 Parameters can be used in the entry workflow:
 
@@ -75,7 +69,16 @@ As a best practice, parameters should only be referenced in the entry workflow o
 
 The default value can be overridden by the command line, params file, or config file. Parameters from multiple sources are resolved in the order described in {ref}`cli-params`. Parameters specified on the command line are converted to the appropriate type based on the corresponding type annotation.
 
-A parameter that doesn't specify a default value is a *required* param. If a required param is not given a value at runtime, the run will fail.
+A parameter that doesn't specify a default value is a *required* parameter. If a required parameter is not given a value at runtime, the run will fail.
+
+:::{versionadded} 26.04.0
+:::
+
+Parameters with a collection type (i.e., `List`, `Set`, or `Bag`) can be supplied a file path instead of a literal collection. The file must be CSV, JSON, or YAML. Nextflow will parse the file contents and assign the resuling collection to the parameter. An error is thrown if the file contents do not match the parameter type.
+
+:::{note}
+When supplying a CSV file to a collection parameter, the CSV file must contain a header row and must use a comma (`,`) as the column separator.
+:::
 
 (workflow-params-legacy)=
 

modules/nextflow/src/main/groovy/nextflow/script/ParamsDsl.groovy

@@ -18,13 +18,22 @@ package nextflow.script
 
 import java.nio.file.Path
 
+import groovy.json.JsonSlurper
+import groovy.yaml.YamlSlurper
 import groovy.transform.Canonical
 import groovy.transform.CompileStatic
 import groovy.util.logging.Slf4j
 import nextflow.Session
 import nextflow.file.FileHelper
 import nextflow.exception.ScriptRuntimeException
+import nextflow.script.types.Bag
 import nextflow.script.types.Types
+import nextflow.splitter.CsvSplitter
+import nextflow.util.ArrayBag
+import nextflow.util.Duration
+import nextflow.util.MemoryUnit
+import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation
+import org.codehaus.groovy.runtime.typehandling.GroovyCastException
 /**
  * Implements the DSL for defining workflow params
  *
@@ -102,6 +111,18 @@ class ParamsDsl {
             if( str.isBigDecimal() ) return str.toBigDecimal()
         }
 
+        if( decl.type == Duration ) {
+            return Duration.of(str)
+        }
+
+        if( decl.type == MemoryUnit ) {
+            return MemoryUnit.of(str)
+        }
+
+        if( Collection.class.isAssignableFrom(decl.type) ) {
+            return resolveFromFile(decl.name, decl.type, FileHelper.asPath(str))
+        }
+
         if( decl.type == Path ) {
             return FileHelper.asPath(str)
         }
@@ -113,12 +134,41 @@ class ParamsDsl {
         if( value == null )
             return null
 
-        if( decl.type == Path && value instanceof CharSequence )
-            return FileHelper.asPath(value.toString())
+        if( value !instanceof CharSequence )
+            return value
+
+        final str = value.toString()
+
+        if( Collection.class.isAssignableFrom(decl.type) )
+            return resolveFromFile(decl.name, decl.type, FileHelper.asPath(str))
+
+        if( decl.type == Path )
+            return FileHelper.asPath(str)
 
         return value
     }
 
+    private Object resolveFromFile(String name, Class type, Path file) {
+        final ext = file.getExtension()
+        final value = switch( ext ) {
+            case 'csv' -> new CsvSplitter().options(header: true, sep: ',').target(file).list()
+            case 'json' -> new JsonSlurper().parse(file)
+            case 'yaml' -> new YamlSlurper().parse(file)
+            case 'yml' -> new YamlSlurper().parse(file)
+            default -> throw new ScriptRuntimeException("Unrecognized file format '${ext}' for input file '${file}' supplied for parameter `${name}` -- should be CSV, JSON, or YAML")
+        }
+
+        try {
+            if( Bag.class.isAssignableFrom(type) && value instanceof Collection )
+                return new ArrayBag(value)
+            return DefaultTypeTransformation.castToType(value, type)
+        }
+        catch( GroovyCastException e ) {
+            final actualType = value.getClass()
+            throw new ScriptRuntimeException("Parameter `${name}` with type ${Types.getName(type)} cannot be assigned to contents of '${file}' [${Types.getName(actualType)}]")
+        }
+    }
+
     private boolean isAssignableFrom(Class target, Class source) {
         if( target == Float.class )
             return Number.class.isAssignableFrom(source)

modules/nextflow/src/test/groovy/nextflow/script/ParamsDslTest.groovy

@@ -1,10 +1,12 @@
 package nextflow.script
 
+import java.nio.file.Files
 import java.nio.file.Path
 
 import nextflow.Session
 import nextflow.file.FileHelper
 import nextflow.exception.ScriptRuntimeException
+import nextflow.script.types.Bag
 import spock.lang.Specification
 import spock.lang.Unroll
 /**
@@ -128,4 +130,187 @@ class ParamsDslTest extends Specification {
         DEF_VALUE << [ 100i, 100l, 100g ]
     }
 
+    def 'should load collection param from CSV file'() {
+        given:
+        def csvFile = Files.createTempFile('test', '.csv')
+        csvFile.text = '''\
+            id,name,value
+            1,sample1,100
+            2,sample2,200
+            3,sample3,300
+            '''.stripIndent()
+        def cliParams = [samples: csvFile.toString()]
+        def session = new Session()
+        session.init(null, null, cliParams, [:])
+
+        when:
+        def dsl = new ParamsDsl()
+        dsl.declare('samples', List, false)
+        dsl.apply(session)
+
+        then:
+        def samples = session.binding.getParams().samples
+        samples instanceof List
+        samples.size() == 3
+        samples[0].id == '1'
+        samples[0].name == 'sample1'
+        samples[0].value == '100'
+        samples[1].id == '2'
+        samples[2].id == '3'
+
+        cleanup:
+        csvFile?.delete()
+    }
+
+    def 'should load collection param from JSON file'() {
+        given:
+        def jsonFile = Files.createTempFile('test', '.json')
+        jsonFile.text = '''\
+            [
+              {"id": 1, "name": "sample1", "value": 100},
+              {"id": 2, "name": "sample2", "value": 200},
+              {"id": 3, "name": "sample3", "value": 300}
+            ]
+            '''.stripIndent()
+        def cliParams = [
+            samplesList: jsonFile.toString(),
+            samplesBag: jsonFile.toString(),
+            samplesSet: jsonFile.toString()
+        ]
+        def session = new Session()
+        session.init(null, null, cliParams, [:])
+
+        when:
+        def dsl = new ParamsDsl()
+        dsl.declare('samplesList', List, false)
+        dsl.declare('samplesBag', Bag, false)
+        dsl.declare('samplesSet', Set, false)
+        dsl.apply(session)
+
+        then:
+        def samplesList = session.binding.getParams().samplesList
+        samplesList instanceof List
+        samplesList.size() == 3
+        samplesList[0].id == 1
+        samplesList[0].name == 'sample1'
+        samplesList[0].value == 100
+        samplesList[1].id == 2
+        samplesList[2].id == 3
+
+        def samplesBag = session.binding.getParams().samplesBag
+        samplesBag instanceof Bag
+        samplesBag.size() == 3
+
+        def samplesSet = session.binding.getParams().samplesSet
+        samplesSet instanceof Set
+        samplesSet.size() == 3
+
+        cleanup:
+        jsonFile?.delete()
+    }
+
+    def 'should load collection param from YAML file'() {
+        given:
+        def yamlFile = Files.createTempFile('test', '.yml')
+        yamlFile.text = '''\
+            - id: 1
+              name: sample1
+              value: 100
+            - id: 2
+              name: sample2
+              value: 200
+            - id: 3
+              name: sample3
+              value: 300
+            '''.stripIndent()
+        def cliParams = [samples: yamlFile.toString()]
+        def session = new Session()
+        session.init(null, null, cliParams, [:])
+
+        when:
+        def dsl = new ParamsDsl()
+        dsl.declare('samples', List, false)
+        dsl.apply(session)
+
+        then:
+        def samples = session.binding.getParams().samples
+        samples instanceof List
+        samples.size() == 3
+        samples[0].id == 1
+        samples[0].name == 'sample1'
+        samples[0].value == 100
+        samples[1].id == 2
+        samples[2].id == 3
+
+        cleanup:
+        yamlFile?.delete()
+    }
+
+    def 'should load collection param from file specified in config'() {
+        given:
+        def jsonFile = Files.createTempFile('test', '.json')
+        jsonFile.text = '[{"x": 1}, {"x": 2}]'
+        def configParams = [items: jsonFile.toString()]
+        def session = new Session()
+        session.init(null, null, [:], configParams)
+
+        when:
+        def dsl = new ParamsDsl()
+        dsl.declare('items', List, false)
+        dsl.apply(session)
+
+        then:
+        def items = session.binding.getParams().items
+        items instanceof List
+        items.size() == 2
+        items[0].x == 1
+        items[1].x == 2
+
+        cleanup:
+        jsonFile?.delete()
+    }
+
+    def 'should report error for unrecognized file format'() {
+        given:
+        def txtFile = Files.createTempFile('test', '.txt')
+        txtFile.text = 'some text'
+        def cliParams = [items: txtFile.toString()]
+        def session = new Session()
+        session.init(null, null, cliParams, [:])
+
+        when:
+        def dsl = new ParamsDsl()
+        dsl.declare('items', List, false)
+        dsl.apply(session)
+
+        then:
+        def e = thrown(ScriptRuntimeException)
+        e.message.contains("Unrecognized file format 'txt'")
+        e.message.contains("supplied for parameter `items` -- should be CSV, JSON, or YAML")
+
+        cleanup:
+        txtFile?.delete()
+    }
+
+    def 'should report error for invalid file content type'() {
+        given:
+        def jsonFile = Files.createTempFile('test', '.json')
+        jsonFile.text = '{"not": "a list"}'
+        def cliParams = [items: jsonFile.toString()]
+        def session = new Session()
+        session.init(null, null, cliParams, [:])
+
+        when:
+        def dsl = new ParamsDsl()
+        dsl.declare('items', List, false)
+        dsl.apply(session)
+
+        then:
+        def e = thrown(ScriptRuntimeException)
+        e.message.contains('Parameter `items` with type List cannot be assigned to contents of')
+
+        cleanup:
+        jsonFile?.delete()
+    }
+
 }